Timothy J. Baek 6 months ago
parent
commit
47e4250f58
1 changed files with 1 additions and 1 deletions
  1. 1 1
      backend/open_webui/apps/retrieval/main.py

+ 1 - 1
backend/open_webui/apps/retrieval/main.py

@@ -667,7 +667,7 @@ def save_docs_to_vector_db(
                 add_start_index=True,
                 add_start_index=True,
             )
             )
         elif app.state.config.TEXT_SPLITTER == "token":
         elif app.state.config.TEXT_SPLITTER == "token":
-
+            tiktoken.get_encoding(app.state.config.TIKTOKEN_ENCODING_NAME)
             text_splitter = TokenTextSplitter(
             text_splitter = TokenTextSplitter(
                 encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
                 encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
                 chunk_size=app.state.config.CHUNK_SIZE,
                 chunk_size=app.state.config.CHUNK_SIZE,