|
@@ -666,7 +666,7 @@ def save_docs_to_vector_db(
|
|
overwrite: bool = False,
|
|
overwrite: bool = False,
|
|
split: bool = True,
|
|
split: bool = True,
|
|
add: bool = False,
|
|
add: bool = False,
|
|
- user = None,
|
|
|
|
|
|
+ user=None,
|
|
) -> bool:
|
|
) -> bool:
|
|
def _get_docs_info(docs: list[Document]) -> str:
|
|
def _get_docs_info(docs: list[Document]) -> str:
|
|
docs_info = set()
|
|
docs_info = set()
|
|
@@ -782,8 +782,7 @@ def save_docs_to_vector_db(
|
|
)
|
|
)
|
|
|
|
|
|
embeddings = embedding_function(
|
|
embeddings = embedding_function(
|
|
- list(map(lambda x: x.replace("\n", " "), texts)),
|
|
|
|
- user = user
|
|
|
|
|
|
+ list(map(lambda x: x.replace("\n", " "), texts)), user=user
|
|
)
|
|
)
|
|
|
|
|
|
items = [
|
|
items = [
|
|
@@ -941,7 +940,7 @@ def process_file(
|
|
"hash": hash,
|
|
"hash": hash,
|
|
},
|
|
},
|
|
add=(True if form_data.collection_name else False),
|
|
add=(True if form_data.collection_name else False),
|
|
- user=user
|
|
|
|
|
|
+ user=user,
|
|
)
|
|
)
|
|
|
|
|
|
if result:
|
|
if result:
|
|
@@ -1032,7 +1031,9 @@ def process_youtube_video(
|
|
content = " ".join([doc.page_content for doc in docs])
|
|
content = " ".join([doc.page_content for doc in docs])
|
|
log.debug(f"text_content: {content}")
|
|
log.debug(f"text_content: {content}")
|
|
|
|
|
|
- save_docs_to_vector_db(request, docs, collection_name, overwrite=True, user=user)
|
|
|
|
|
|
+ save_docs_to_vector_db(
|
|
|
|
+ request, docs, collection_name, overwrite=True, user=user
|
|
|
|
+ )
|
|
|
|
|
|
return {
|
|
return {
|
|
"status": True,
|
|
"status": True,
|
|
@@ -1073,7 +1074,9 @@ def process_web(
|
|
content = " ".join([doc.page_content for doc in docs])
|
|
content = " ".join([doc.page_content for doc in docs])
|
|
|
|
|
|
log.debug(f"text_content: {content}")
|
|
log.debug(f"text_content: {content}")
|
|
- save_docs_to_vector_db(request, docs, collection_name, overwrite=True, user=user)
|
|
|
|
|
|
+ save_docs_to_vector_db(
|
|
|
|
+ request, docs, collection_name, overwrite=True, user=user
|
|
|
|
+ )
|
|
|
|
|
|
return {
|
|
return {
|
|
"status": True,
|
|
"status": True,
|
|
@@ -1289,7 +1292,9 @@ def process_web_search(
|
|
requests_per_second=request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
|
requests_per_second=request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
|
)
|
|
)
|
|
docs = loader.load()
|
|
docs = loader.load()
|
|
- save_docs_to_vector_db(request, docs, collection_name, overwrite=True, user=user)
|
|
|
|
|
|
+ save_docs_to_vector_db(
|
|
|
|
+ request, docs, collection_name, overwrite=True, user=user
|
|
|
|
+ )
|
|
|
|
|
|
return {
|
|
return {
|
|
"status": True,
|
|
"status": True,
|
|
@@ -1323,7 +1328,9 @@ def query_doc_handler(
|
|
return query_doc_with_hybrid_search(
|
|
return query_doc_with_hybrid_search(
|
|
collection_name=form_data.collection_name,
|
|
collection_name=form_data.collection_name,
|
|
query=form_data.query,
|
|
query=form_data.query,
|
|
- embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(query, user=user),
|
|
|
|
|
|
+ embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(
|
|
|
|
+ query, user=user
|
|
|
|
+ ),
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
reranking_function=request.app.state.rf,
|
|
reranking_function=request.app.state.rf,
|
|
r=(
|
|
r=(
|
|
@@ -1331,14 +1338,16 @@ def query_doc_handler(
|
|
if form_data.r
|
|
if form_data.r
|
|
else request.app.state.config.RELEVANCE_THRESHOLD
|
|
else request.app.state.config.RELEVANCE_THRESHOLD
|
|
),
|
|
),
|
|
- user=user
|
|
|
|
|
|
+ user=user,
|
|
)
|
|
)
|
|
else:
|
|
else:
|
|
return query_doc(
|
|
return query_doc(
|
|
collection_name=form_data.collection_name,
|
|
collection_name=form_data.collection_name,
|
|
- query_embedding=request.app.state.EMBEDDING_FUNCTION(form_data.query, user=user),
|
|
|
|
|
|
+ query_embedding=request.app.state.EMBEDDING_FUNCTION(
|
|
|
|
+ form_data.query, user=user
|
|
|
|
+ ),
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
- user=user
|
|
|
|
|
|
+ user=user,
|
|
)
|
|
)
|
|
except Exception as e:
|
|
except Exception as e:
|
|
log.exception(e)
|
|
log.exception(e)
|
|
@@ -1367,7 +1376,9 @@ def query_collection_handler(
|
|
return query_collection_with_hybrid_search(
|
|
return query_collection_with_hybrid_search(
|
|
collection_names=form_data.collection_names,
|
|
collection_names=form_data.collection_names,
|
|
queries=[form_data.query],
|
|
queries=[form_data.query],
|
|
- embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(query, user=user),
|
|
|
|
|
|
+ embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(
|
|
|
|
+ query, user=user
|
|
|
|
+ ),
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
reranking_function=request.app.state.rf,
|
|
reranking_function=request.app.state.rf,
|
|
r=(
|
|
r=(
|
|
@@ -1380,7 +1391,9 @@ def query_collection_handler(
|
|
return query_collection(
|
|
return query_collection(
|
|
collection_names=form_data.collection_names,
|
|
collection_names=form_data.collection_names,
|
|
queries=[form_data.query],
|
|
queries=[form_data.query],
|
|
- embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(query,user=user),
|
|
|
|
|
|
+ embedding_function=lambda query: request.app.state.EMBEDDING_FUNCTION(
|
|
|
|
+ query, user=user
|
|
|
|
+ ),
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
k=form_data.k if form_data.k else request.app.state.config.TOP_K,
|
|
)
|
|
)
|
|
|
|
|