Timothy J. Baek 6 月之前
父节点
当前提交
f2c78ac0fb
共有 1 个文件被更改,包括 5 次插入1 次删除
  1. 5 1
      backend/open_webui/apps/retrieval/main.py

+ 5 - 1
backend/open_webui/apps/retrieval/main.py

@@ -667,7 +667,11 @@ def save_docs_to_vector_db(
                 add_start_index=True,
             )
         elif app.state.config.TEXT_SPLITTER == "token":
-            tiktoken.get_encoding(app.state.config.TIKTOKEN_ENCODING_NAME)
+            log.info(
+                f"Using token text splitter: {app.state.config.TIKTOKEN_ENCODING_NAME}"
+            )
+
+            tiktoken.get_encoding(str(app.state.config.TIKTOKEN_ENCODING_NAME))
             text_splitter = TokenTextSplitter(
                 encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
                 chunk_size=app.state.config.CHUNK_SIZE,