|
@@ -667,7 +667,7 @@ def save_docs_to_vector_db(
|
|
add_start_index=True,
|
|
add_start_index=True,
|
|
)
|
|
)
|
|
elif app.state.config.TEXT_SPLITTER == "token":
|
|
elif app.state.config.TEXT_SPLITTER == "token":
|
|
-
|
|
|
|
|
|
+ tiktoken.get_encoding(app.state.config.TIKTOKEN_ENCODING_NAME)
|
|
text_splitter = TokenTextSplitter(
|
|
text_splitter = TokenTextSplitter(
|
|
encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
|
|
encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
|
|
chunk_size=app.state.config.CHUNK_SIZE,
|
|
chunk_size=app.state.config.CHUNK_SIZE,
|