refac: rm sub standard code

backend/apps/rag/main.py

@@ -145,8 +145,8 @@ app.state.config = AppConfig()
 
 app.state.config.TOP_K = RAG_TOP_K
 app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD
-app.state.config.MAX_FILE_SIZE = RAG_FILE_MAX_SIZE
-app.state.config.MAX_FILE_COUNT = RAG_FILE_MAX_COUNT
+app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE
+app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT
 
 app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
 app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
@@ -397,6 +397,10 @@ async def get_rag_config(user=Depends(get_admin_user)):
     return {
         "status": True,
         "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
+        "file": {
+            "max_size": app.state.config.FILE_MAX_SIZE,
+            "max_count": app.state.config.FILE_MAX_COUNT,
+        },
         "content_extraction": {
             "engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
             "tika_server_url": app.state.config.TIKA_SERVER_URL,
@@ -522,6 +526,10 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
 
     return {
         "status": True,
+        "file": {
+            "max_size": app.state.config.FILE_MAX_SIZE,
+            "max_count": app.state.config.FILE_MAX_COUNT,
+        },
         "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
         "content_extraction": {
             "engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
@@ -571,8 +579,6 @@ async def get_query_settings(user=Depends(get_admin_user)):
         "template": app.state.config.RAG_TEMPLATE,
         "k": app.state.config.TOP_K,
         "r": app.state.config.RELEVANCE_THRESHOLD,
-        "max_file_size": app.state.config.MAX_FILE_SIZE,
-        "max_file_count": app.state.config.MAX_FILE_COUNT,
         "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
     }
 
@@ -580,16 +586,16 @@ async def get_query_settings(user=Depends(get_admin_user)):
 @app.get("/file/limit/settings")
 async def get_query_settings(user=Depends(get_verified_user)):
     return {
-        "max_file_size": app.state.config.MAX_FILE_SIZE,
-        "max_file_count": app.state.config.MAX_FILE_COUNT,
+        "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
+        "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
     }
 
 
 class QuerySettingsForm(BaseModel):
     k: Optional[int] = None
     r: Optional[float] = None
-    max_file_size: Optional[int] = None
-    max_file_count: Optional[int] = None
+    FILE_MAX_SIZE: Optional[int] = None
+    FILE_MAX_COUNT: Optional[int] = None
     template: Optional[str] = None
     hybrid: Optional[bool] = None
 
@@ -606,11 +612,11 @@ async def update_query_settings(
     app.state.config.ENABLE_RAG_HYBRID_SEARCH = (
         form_data.hybrid if form_data.hybrid else False
     )
-    app.state.config.MAX_FILE_SIZE = (
-        form_data.max_file_size if form_data.max_file_size else 10
+    app.state.config.FILE_MAX_SIZE = (
+        form_data.FILE_MAX_SIZE if form_data.FILE_MAX_SIZE else 10
     )
-    app.state.config.MAX_FILE_COUNT = (
-        form_data.max_file_count if form_data.max_file_count else 5
+    app.state.config.FILE_MAX_COUNT = (
+        form_data.FILE_MAX_COUNT if form_data.FILE_MAX_COUNT else 5
     )
 
     return {
@@ -618,8 +624,8 @@ async def update_query_settings(
         "template": app.state.config.RAG_TEMPLATE,
         "k": app.state.config.TOP_K,
         "r": app.state.config.RELEVANCE_THRESHOLD,
-        "max_file_size": app.state.config.MAX_FILE_SIZE,
-        "max_file_count": app.state.config.MAX_FILE_COUNT,
+        "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
+        "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
         "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
     }
 

src/lib/apis/rag/index.ts

@@ -134,38 +134,9 @@ export const getQuerySettings = async (token: string) => {
 	return res;
 };
 
-export const getFileLimitSettings = async (token: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/file/limit/settings`, {
-		method: 'GET',
-		headers: {
-			'Content-Type': 'application/json',
-			Authorization: `Bearer ${token}`
-		}
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			console.log(err);
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
 type QuerySettings = {
 	k: number | null;
 	r: number | null;
-	max_file_size: number | null;
-	max_file_count: number | null;
 	template: string | null;
 };
 

src/lib/components/admin/Settings/Documents.svelte

@@ -37,6 +37,9 @@
 	let embeddingModel = '';
 	let rerankingModel = '';
 
+	let fileMaxSize = null;
+	let fileMaxCount = null;
+
 	let contentExtractionEngine = 'default';
 	let tikaServerUrl = '';
 	let showTikaServerUrl = false;
@@ -53,8 +56,6 @@
 		template: '',
 		r: 0.0,
 		k: 4,
-		max_file_size: 10,
-		max_file_count: 5,
 		hybrid: false
 	};
 
@@ -220,7 +221,6 @@
 		await setRerankingConfig();
 
 		querySettings = await getQuerySettings(localStorage.token);
-
 		const res = await getRAGConfig(localStorage.token);
 
 		if (res) {
@@ -232,6 +232,9 @@
 			contentExtractionEngine = res.content_extraction.engine;
 			tikaServerUrl = res.content_extraction.tika_server_url;
 			showTikaServerUrl = contentExtractionEngine === 'tika';
+
+			fileMaxSize = res.file.file_max_size;
+			fileMaxCount = res.file.file_max_count;
 		}
 	});
 </script>
@@ -388,41 +391,6 @@
 				</div>
 			{/if}
 
-			<div class=" my-2 flex gap-1.5">
-				<div class="  w-full justify-between">
-					<div class="self-center text-xs font-medium min-w-fit mb-1">
-						{$i18n.t('Max File Count')}
-					</div>
-					<div class="self-center">
-						<input
-							class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
-							type="number"
-							placeholder={$i18n.t('Enter Max File Count')}
-							bind:value={querySettings.max_file_count}
-							autocomplete="off"
-							min="0"
-						/>
-					</div>
-				</div>
-
-				<div class="w-full">
-					<div class=" self-center text-xs font-medium min-w-fit mb-1">
-						{$i18n.t('Max File Size(MB)')}
-					</div>
-
-					<div class="self-center">
-						<input
-							class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
-							type="number"
-							placeholder={$i18n.t('Enter Max File Size(MB)')}
-							bind:value={querySettings.max_file_size}
-							autocomplete="off"
-							min="0"
-						/>
-					</div>
-				</div>
-			</div>
-
 			<div class=" flex w-full justify-between">
 				<div class=" self-center text-xs font-medium">{$i18n.t('Hybrid Search')}</div>
 
@@ -647,6 +615,48 @@
 				</div>
 			{/if}
 		</div>
+
+		<hr class=" dark:border-gray-850" />
+
+		<div class="">
+			<div class="text-sm font-medium">{$i18n.t('Files')}</div>
+
+			<div class=" my-2 flex gap-1.5">
+				<div class="  w-full justify-between">
+					<div class="self-center text-xs font-medium min-w-fit mb-1">
+						{$i18n.t('Max Upload Count')}
+					</div>
+					<div class="self-center">
+						<input
+							class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
+							type="number"
+							placeholder={$i18n.t('Enter File Count')}
+							bind:value={fileMaxCount}
+							autocomplete="off"
+							min="0"
+						/>
+					</div>
+				</div>
+
+				<div class="w-full">
+					<div class=" self-center text-xs font-medium min-w-fit mb-1">
+						{$i18n.t('Max Upload Size')}
+					</div>
+
+					<div class="self-center">
+						<input
+							class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
+							type="number"
+							placeholder={$i18n.t('Enter File Size (MB)')}
+							bind:value={fileMaxSize}
+							autocomplete="off"
+							min="0"
+						/>
+					</div>
+				</div>
+			</div>
+		</div>
+
 		<hr class=" dark:border-gray-850" />
 
 		<div class=" ">

src/lib/components/chat/MessageInput.svelte

@@ -17,7 +17,7 @@
 	import { blobToFile, findWordIndices } from '$lib/utils';
 	import { transcribeAudio } from '$lib/apis/audio';
 
-	import { getFileLimitSettings, processDocToVectorDB } from '$lib/apis/rag';
+	import { processDocToVectorDB } from '$lib/apis/rag';
 
 	import { uploadFile } from '$lib/apis/files';
 	import {
@@ -56,7 +56,6 @@
 	let commandsElement;
 
 	let inputFiles;
-	let fileLimitSettings;
 	let dragged = false;
 
 	let user = null;
@@ -173,7 +172,7 @@
 	};
 
 	const processFileCountLimit = async (querySettings, inputFiles) => {
-		const maxFiles = querySettings.max_file_count;
+		const maxFiles = querySettings.FILE_MAX_COUNT;
 		const currentFilesCount = files.length;
 		const inputFilesCount = inputFiles.length;
 		const totalFilesCount = currentFilesCount + inputFilesCount;
@@ -190,8 +189,8 @@
 		return [true, inputFiles];
 	};
 
-	const processFileSizeLimit = async (fileLimitSettings, file) => {
-		if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) {
+	const inputFilesHandler = async (inputFiles) => {
+		inputFiles.forEach((file) => {
 			if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
 				if (visionCapableModels.length === 0) {
 					toast.error($i18n.t('Selected model(s) do not support image inputs'));
@@ -211,25 +210,10 @@
 			} else {
 				uploadFileHandler(file);
 			}
-		} else {
-			toast.error(
-				$i18n.t('File size exceeds the limit of {{size}}MB', {
-					size: fileLimitSettings.max_file_size
-				})
-			);
-		}
+		});
 	};
 
 	onMount(() => {
-		const initFileLimitSettings = async () => {
-			try {
-				fileLimitSettings = await getFileLimitSettings(localStorage.token);
-			} catch (error) {
-				console.error('Error fetching query settings:', error);
-			}
-		};
-		initFileLimitSettings();
-
 		window.setTimeout(() => chatTextAreaElement?.focus(), 0);
 
 		const dropZone = document.querySelector('body');
@@ -256,22 +240,10 @@
 
 			if (e.dataTransfer?.files) {
 				const inputFiles = Array.from(e.dataTransfer?.files);
-
+				console.log(file, file.name.split('.').at(-1));
 				if (inputFiles && inputFiles.length > 0) {
 					console.log(inputFiles);
-					const [canProcess, filesToProcess] = await processFileCountLimit(
-						fileLimitSettings,
-						inputFiles
-					);
-					if (!canProcess) {
-						dragged = false;
-						return;
-					}
-					console.log(filesToProcess);
-					filesToProcess.forEach((file) => {
-						console.log(file, file.name.split('.').at(-1));
-						processFileSizeLimit(fileLimitSettings, file);
-					});
+					inputFilesHandler(inputFiles);
 				} else {
 					toast.error($i18n.t(`File not found.`));
 				}
@@ -392,21 +364,7 @@
 					multiple
 					on:change={async () => {
 						if (inputFiles && inputFiles.length > 0) {
-							const _inputFiles = Array.from(inputFiles);
-							console.log(_inputFiles);
-							const [canProcess, filesToProcess] = await processFileCountLimit(
-								fileLimitSettings,
-								_inputFiles
-							);
-							if (!canProcess) {
-								filesInputElement.value = '';
-								return;
-							}
-							console.log(filesToProcess);
-							filesToProcess.forEach((file) => {
-								console.log(file, file.name.split('.').at(-1));
-								processFileSizeLimit(fileLimitSettings, file);
-							});
+							inputFilesHandler(inputFiles);
 						} else {
 							toast.error($i18n.t(`File not found.`));
 						}
@@ -715,17 +673,7 @@
 													.map((item) => item.getAsFile())
 													.filter((file) => file);
 
-												const [canProcess, filesToProcess] = await processFileCountLimit(
-													fileLimitSettings,
-													inputFiles
-												);
-												if (!canProcess) {
-													return;
-												}
-												filesToProcess.forEach((file) => {
-													console.log(file, file.name.split('.').at(-1));
-													processFileSizeLimit(fileLimitSettings, file);
-												});
+												inputFilesHandler(inputFiles);
 											} else {
 												toast.error($i18n.t(`File not found.`));
 											}

src/lib/components/workspace/Documents.svelte

@@ -8,7 +8,7 @@
 	import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents';
 
 	import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants';
-	import { getFileLimitSettings, processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag';
+	import { processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag';
 	import { blobToFile, transformFileName } from '$lib/utils';
 
 	import Checkbox from '$lib/components/common/Checkbox.svelte';
@@ -24,7 +24,7 @@
 	let importFiles = '';
 
 	let inputFiles = '';
-	let fileLimitSettings;
+
 	let query = '';
 	let documentsImportInputElement: HTMLInputElement;
 	let tags = [];
@@ -99,17 +99,7 @@
 		}
 	};
 
-	const initFileLimitSettings = async () => {
-		try {
-			fileLimitSettings = await getFileLimitSettings(localStorage.token);
-		} catch (error) {
-			console.error('Error fetching query settings:', error);
-		}
-	};
-
 	onMount(() => {
-		initFileLimitSettings();
-
 		documents.subscribe((docs) => {
 			tags = docs.reduce((a, e, i, arr) => {
 				return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
@@ -147,24 +137,16 @@
 				if (inputFiles && inputFiles.length > 0) {
 					for (const file of inputFiles) {
 						console.log(file, file.name.split('.').at(-1));
-						if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) {
-							if (
-								SUPPORTED_FILE_TYPE.includes(file['type']) ||
-								SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
-							) {
-								uploadDoc(file);
-							} else {
-								toast.error(
-									`Unknown File Type '${file['type']}', but accepting and treating as plain text`
-								);
-								uploadDoc(file);
-							}
+						if (
+							SUPPORTED_FILE_TYPE.includes(file['type']) ||
+							SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
+						) {
+							uploadDoc(file);
 						} else {
 							toast.error(
-								$i18n.t('File size exceeds the limit of {{size}}MB', {
-									size: fileLimitSettings.max_file_size
-								})
+								`Unknown File Type '${file['type']}', but accepting and treating as plain text`
 							);
+							uploadDoc(file);
 						}
 					}
 				} else {