refac: deprecate docs_dir

backend/open_webui/apps/webui/routers/files.py

@@ -92,6 +92,7 @@ def upload_file(file: UploadFile = File(...), user=Depends(get_verified_user)):
 
 @router.post("/upload/dir")
 def upload_dir(user=Depends(get_admin_user)):
+    file_ids = []
     for path in Path(DOCS_DIR).rglob("./**/*"):
         if path.is_file() and not path.name.startswith("."):
             try:
@@ -126,13 +127,14 @@ def upload_dir(user=Depends(get_admin_user)):
                 try:
                     process_file(ProcessFileForm(file_id=id))
                     log.debug(f"File processed: {path}, {file.id}")
+                    file_ids.append(file.id)
                 except Exception as e:
                     log.exception(e)
                     log.error(f"Error processing file: {file.id}")
             except Exception as e:
                 log.exception(e)
                 pass
-    return True
+    return file_ids
 
 
 ############################

backend/open_webui/apps/webui/routers/knowledge.py

@@ -160,6 +160,10 @@ def add_file_to_knowledge_by_id(
         process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
     except Exception as e:
         log.debug(e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
 
     if knowledge:
         data = knowledge.data or {}

src/lib/components/admin/Settings/Documents.svelte

@@ -62,16 +62,6 @@
 		hybrid: false
 	};
 
-	const scanHandler = async () => {
-		scanDirLoading = true;
-		const res = await uploadDir(localStorage.token);
-		scanDirLoading = false;
-
-		if (res) {
-			toast.success($i18n.t('Scan complete!'));
-		}
-	};
-
 	const embeddingModelUpdateHandler = async () => {
 		if (embeddingEngine === '' && embeddingModel.split('/').length - 1 > 1) {
 			toast.error(
@@ -284,58 +274,6 @@
 		<div class="flex flex-col gap-0.5">
 			<div class=" mb-0.5 text-sm font-medium">{$i18n.t('General Settings')}</div>
 
-			<div class="  flex w-full justify-between">
-				<div class=" self-center text-xs font-medium">
-					{$i18n.t('Scan for documents from {{path}}', { path: 'DOCS_DIR (/data/docs)' })}
-				</div>
-
-				<button
-					class=" self-center text-xs p-1 px-3 bg-gray-50 dark:bg-gray-800 dark:hover:bg-gray-700 rounded-lg flex flex-row space-x-1 items-center {scanDirLoading
-						? ' cursor-not-allowed'
-						: ''}"
-					on:click={() => {
-						scanHandler();
-						console.log('check');
-					}}
-					type="button"
-					disabled={scanDirLoading}
-				>
-					<div class="self-center font-medium">{$i18n.t('Scan')}</div>
-
-					{#if scanDirLoading}
-						<div class="ml-3 self-center">
-							<svg
-								class=" w-3 h-3"
-								viewBox="0 0 24 24"
-								fill="currentColor"
-								xmlns="http://www.w3.org/2000/svg"
-							>
-								<style>
-									.spinner_ajPY {
-										transform-origin: center;
-										animation: spinner_AtaB 0.75s infinite linear;
-									}
-
-									@keyframes spinner_AtaB {
-										100% {
-											transform: rotate(360deg);
-										}
-									}
-								</style>
-								<path
-									d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
-									opacity=".25"
-								/>
-								<path
-									d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
-									class="spinner_ajPY"
-								/>
-							</svg>
-						</div>
-					{/if}
-				</button>
-			</div>
-
 			<div class=" flex w-full justify-between">
 				<div class=" self-center text-xs font-medium">{$i18n.t('Embedding Model Engine')}</div>
 				<div class="flex items-center relative">

src/lib/components/icons/FolderOpen.svelte

@@ -0,0 +1,19 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	fill="none"
+	viewBox="0 0 24 24"
+	stroke-width={strokeWidth}
+	stroke="currentColor"
+	class={className}
+>
+	<path
+		stroke-linecap="round"
+		stroke-linejoin="round"
+		d="M3.75 9.776c.112-.017.227-.026.344-.026h15.812c.117 0 .232.009.344.026m-16.5 0a2.25 2.25 0 0 0-1.883 2.542l.857 6a2.25 2.25 0 0 0 2.227 1.932H19.05a2.25 2.25 0 0 0 2.227-1.932l.857-6a2.25 2.25 0 0 0-1.883-2.542m-16.5 0V6A2.25 2.25 0 0 1 6 3.75h3.879a1.5 1.5 0 0 1 1.06.44l2.122 2.12a1.5 1.5 0 0 0 1.06.44H18A2.25 2.25 0 0 1 20.25 9v.776"
+	/>
+</svg>

src/lib/components/workspace/Knowledge/Collection.svelte

@@ -125,6 +125,81 @@
 		}
 	};
 
+	const uploadDirectoryHandler = async () => {
+		try {
+			// Get directory handle through picker
+			const dirHandle = await window.showDirectoryPicker();
+
+			let totalFiles = 0;
+			let uploadedFiles = 0;
+
+			// Function to update the UI with the progress
+			const updateProgress = () => {
+				const percentage = (uploadedFiles / totalFiles) * 100;
+				toast.info(`Upload Progress: ${uploadedFiles}/${totalFiles} (${percentage.toFixed(2)}%)`);
+			};
+
+			// Recursive function to count all files excluding hidden ones
+			async function countFiles(dirHandle) {
+				for await (const entry of dirHandle.values()) {
+					if (entry.name.startsWith('.')) continue; // Skip hidden files and directories
+
+					if (entry.kind === 'file') {
+						totalFiles++;
+					} else if (entry.kind === 'directory') {
+						await countFiles(entry);
+					}
+				}
+			}
+
+			// Recursive function to process directories excluding hidden files
+			async function processDirectory(dirHandle, path = '') {
+				for await (const entry of dirHandle.values()) {
+					if (entry.name.startsWith('.')) continue; // Skip hidden files and directories
+
+					const entryPath = path ? `${path}/${entry.name}` : entry.name;
+
+					if (entry.kind === 'file') {
+						// Get file from handle
+						const file = await entry.getFile();
+						// Create a new file with the path information
+						const fileWithPath = new File([file], entryPath, { type: file.type });
+
+						await uploadFileHandler(fileWithPath);
+						uploadedFiles++;
+						updateProgress();
+					} else if (entry.kind === 'directory') {
+						// Recursively process subdirectories
+						await processDirectory(entry, entryPath);
+					}
+				}
+			}
+
+			// First count all files excluding hidden ones
+			await countFiles(dirHandle);
+			updateProgress();
+
+			// Start processing from root directory
+			if (totalFiles > 0) {
+				await processDirectory(dirHandle);
+			} else {
+				console.log('No files to upload.');
+			}
+		} catch (error) {
+			if (error.name === 'AbortError') {
+				toast.info('Directory selection was cancelled');
+			} else {
+				toast.error('Error accessing directory');
+				console.error('Directory access error:', error);
+			}
+		}
+	};
+
+	// Helper function to maintain file paths within zip
+	const getRelativePath = (fullPath, basePath) => {
+		return fullPath.substring(basePath.length + 1);
+	};
+
 	const addFileHandler = async (fileId) => {
 		const updatedKnowledge = await addFileToKnowledgeById(localStorage.token, id, fileId).catch(
 			(e) => {
@@ -417,11 +492,14 @@
 
 									<div>
 										<AddContentMenu
-											on:files={() => {
-												document.getElementById('files-input').click();
-											}}
-											on:text={() => {
-												showAddTextContentModal = true;
+											on:upload={(e) => {
+												if (e.detail.type === 'directory') {
+													uploadDirectoryHandler();
+												} else if (e.detail.type === 'text') {
+													showAddTextContentModal = true;
+												} else {
+													document.getElementById('files-input').click();
+												}
 											}}
 										/>
 									</div>

src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte

@@ -16,6 +16,7 @@
 	import ArrowUpCircle from '$lib/components/icons/ArrowUpCircle.svelte';
 	import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
 	import BarsArrowUp from '$lib/components/icons/BarsArrowUp.svelte';
+	import FolderOpen from '$lib/components/icons/FolderOpen.svelte';
 
 	const i18n = getContext('i18n');
 
@@ -65,7 +66,7 @@
 			<DropdownMenu.Item
 				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
 				on:click={() => {
-					dispatch('files');
+					dispatch('upload', { type: 'files' });
 				}}
 			>
 				<ArrowUpCircle strokeWidth="2" />
@@ -75,7 +76,17 @@
 			<DropdownMenu.Item
 				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
 				on:click={() => {
-					dispatch('text');
+					dispatch('upload', { type: 'directory' });
+				}}
+			>
+				<FolderOpen strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Upload directory')}</div>
+			</DropdownMenu.Item>
+
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					dispatch('upload', { type: 'text' });
 				}}
 			>
 				<BarsArrowUp strokeWidth="2" />