Merge pull request #5989 from jannikstdl/dev

enh: citations show relevance score - compact citation view

backend/open_webui/apps/retrieval/utils.py

@@ -411,13 +411,14 @@ def get_rag_context(
                 )
 
                 if "metadatas" in context:
-                    citations.append(
-                        {
-                            "source": context["file"],
-                            "document": context["documents"][0],
-                            "metadata": context["metadatas"][0],
-                        }
-                    )
+                    citation = {
+                        "source": context["file"],
+                        "document": context["documents"][0],
+                        "metadata": context["metadatas"][0],
+                    }
+                    if "distances" in context and context["distances"]:
+                        citation["distances"] = context["distances"][0]
+                    citations.append(citation)
         except Exception as e:
             log.exception(e)
 

backend/open_webui/apps/retrieval/vector/dbs/chroma.py

@@ -109,7 +109,10 @@ class ChromaClient:
 
     def insert(self, collection_name: str, items: list[VectorItem]):
         # Insert the items into the collection, if the collection does not exist, it will be created.
-        collection = self.client.get_or_create_collection(name=collection_name)
+        collection = self.client.get_or_create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"}
+            )
 
         ids = [item["id"] for item in items]
         documents = [item["text"] for item in items]
@@ -127,7 +130,10 @@ class ChromaClient:
 
     def upsert(self, collection_name: str, items: list[VectorItem]):
         # Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created.
-        collection = self.client.get_or_create_collection(name=collection_name)
+        collection = self.client.get_or_create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"}
+            )
 
         ids = [item["id"] for item in items]
         documents = [item["text"] for item in items]

src/lib/components/chat/Messages/Citations.svelte

@@ -1,67 +1,219 @@
 <script lang="ts">
+	import { getContext } from 'svelte';
 	import CitationsModal from './CitationsModal.svelte';
+	import Collapsible from '$lib/components/common/Collapsible.svelte';
+	import ChevronDown from '$lib/components/icons/ChevronDown.svelte';
+	import ChevronUp from '$lib/components/icons/ChevronUp.svelte';
+
+	const i18n = getContext('i18n');
 
 	export let citations = [];
 
 	let _citations = [];
-
-	$: _citations = citations.reduce((acc, citation) => {
-		citation.document.forEach((document, index) => {
-			const metadata = citation.metadata?.[index];
-			const id = metadata?.source ?? 'N/A';
-			let source = citation?.source;
-
-			if (metadata?.name) {
-				source = { ...source, name: metadata.name };
-			}
-
-			// Check if ID looks like a URL
-			if (id.startsWith('http://') || id.startsWith('https://')) {
-				source = { name: id };
-			}
-
-			const existingSource = acc.find((item) => item.id === id);
-
-			if (existingSource) {
-				existingSource.document.push(document);
-				existingSource.metadata.push(metadata);
-			} else {
-				acc.push({
-					id: id,
-					source: source,
-					document: [document],
-					metadata: metadata ? [metadata] : []
-				});
-			}
-		});
-		return acc;
-	}, []);
+	let showPercentage = false;
+	let showRelevance = true;
 
 	let showCitationModal = false;
-	let selectedCitation = null;
+	let selectedCitation: any = null;
+	let isCollapsibleOpen = false;
+
+	function calculateShowRelevance(citations: any[]) {
+		const distances = citations.flatMap((citation) => citation.distances ?? []);
+		const inRange = distances.filter((d) => d !== undefined && d >= -1 && d <= 1).length;
+		const outOfRange = distances.filter((d) => d !== undefined && (d < -1 || d > 1)).length;
+
+		if (distances.length === 0) {
+			return false;
+		}
+
+		if (
+			(inRange === distances.length - 1 && outOfRange === 1) ||
+			(outOfRange === distances.length - 1 && inRange === 1)
+		) {
+			return false;
+		}
+
+		return true;
+	}
+
+	function shouldShowPercentage(citations: any[]) {
+		const distances = citations.flatMap((citation) => citation.distances ?? []);
+		return distances.every((d) => d !== undefined && d >= -1 && d <= 1);
+	}
+
+	$: {
+		_citations = citations.reduce((acc, citation) => {
+			citation.document.forEach((document, index) => {
+				const metadata = citation.metadata?.[index];
+				const distance = citation.distances?.[index];
+				const id = metadata?.source ?? 'N/A';
+				let source = citation?.source;
+
+				if (metadata?.name) {
+					source = { ...source, name: metadata.name };
+				}
+
+				if (id.startsWith('http://') || id.startsWith('https://')) {
+					source = { name: id };
+				}
+
+				const existingSource = acc.find((item) => item.id === id);
+
+				if (existingSource) {
+					existingSource.document.push(document);
+					existingSource.metadata.push(metadata);
+					if (distance !== undefined) existingSource.distances.push(distance);
+				} else {
+					acc.push({
+						id: id,
+						source: source,
+						document: [document],
+						metadata: metadata ? [metadata] : [],
+						distances: distance !== undefined ? [distance] : undefined
+					});
+				}
+			});
+			return acc;
+		}, []);
+
+		showRelevance = calculateShowRelevance(_citations);
+		showPercentage = shouldShowPercentage(_citations);
+	}
 </script>
 
-<CitationsModal bind:show={showCitationModal} citation={selectedCitation} />
+<CitationsModal
+	bind:show={showCitationModal}
+	citation={selectedCitation}
+	{showPercentage}
+	{showRelevance}
+/>
 
 {#if _citations.length > 0}
 	<div class="mt-1 mb-2 w-full flex gap-1 items-center flex-wrap">
-		{#each _citations as citation, idx}
-			<div class="flex gap-1 text-xs font-semibold">
-				<button
-					class="flex dark:text-gray-300 py-1 px-1 bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800 transition rounded-xl max-w-96"
-					on:click={() => {
-						showCitationModal = true;
-						selectedCitation = citation;
-					}}
+		{#if _citations.length <= 3}
+			{#each _citations as citation, idx}
+				<div class="flex gap-1 text-xs font-semibold">
+					<button
+						class="no-toggle flex dark:text-gray-300 py-1 px-1 bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800 transition rounded-xl max-w-96"
+						on:click={() => {
+							showCitationModal = true;
+							selectedCitation = citation;
+						}}
+					>
+						{#if _citations.every((c) => c.distances !== undefined)}
+							<div class="bg-white dark:bg-gray-700 rounded-full size-4">
+								{idx + 1}
+							</div>
+						{/if}
+						<div class="flex-1 mx-2 line-clamp-1 truncate">
+							{citation.source.name}
+						</div>
+					</button>
+				</div>
+			{/each}
+		{:else}
+			<Collapsible bind:open={isCollapsibleOpen} className="w-full">
+				<div
+					class="flex items-center gap-1 text-gray-500 hover:text-gray-600 dark:hover:text-gray-400 transition cursor-pointer"
 				>
-					<div class="bg-white dark:bg-gray-700 rounded-full size-4">
-						{idx + 1}
+					<div class="flex-grow flex items-center gap-1 overflow-hidden">
+						<span class="whitespace-nowrap hidden sm:inline">{$i18n.t('References from')}</span>
+						<div class="flex items-center">
+							{#if _citations.length > 1 && _citations
+									.slice(0, 2)
+									.reduce((acc, citation) => acc + citation.source.name.length, 0) <= 50}
+								{#each _citations.slice(0, 2) as citation, idx}
+									<div class="flex items-center">
+										<button
+											class="no-toggle flex dark:text-gray-300 py-1 px-1 bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800 transition rounded-xl max-w-96 text-xs font-semibold"
+											on:click={() => {
+												showCitationModal = true;
+												selectedCitation = citation;
+											}}
+										>
+											{#if _citations.every((c) => c.distances !== undefined)}
+												<div class="bg-white dark:bg-gray-700 rounded-full size-4">
+													{idx + 1}
+												</div>
+											{/if}
+											<div class="flex-1 mx-2 line-clamp-1">
+												{citation.source.name}
+											</div>
+										</button>
+										{#if idx === 0}<span class="mr-1">,</span>
+										{/if}
+									</div>
+								{/each}
+							{:else}
+								{#each _citations.slice(0, 1) as citation, idx}
+									<div class="flex items-center">
+										<button
+											class="no-toggle flex dark:text-gray-300 py-1 px-1 bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800 transition rounded-xl max-w-96 text-xs font-semibold"
+											on:click={() => {
+												showCitationModal = true;
+												selectedCitation = citation;
+											}}
+										>
+											{#if _citations.every((c) => c.distances !== undefined)}
+												<div class="bg-white dark:bg-gray-700 rounded-full size-4">
+													{idx + 1}
+												</div>
+											{/if}
+											<div class="flex-1 mx-2 line-clamp-1">
+												{citation.source.name}
+											</div>
+										</button>
+									</div>
+								{/each}
+							{/if}
+						</div>
+						<div class="flex items-center gap-1 whitespace-nowrap">
+							<span class="hidden sm:inline">{$i18n.t('and')}</span>
+							<span class="text-gray-600 dark:text-gray-400">
+								{_citations.length -
+									(_citations.length > 1 &&
+									_citations
+										.slice(0, 2)
+										.reduce((acc, citation) => acc + citation.source.name.length, 0) <= 50
+										? 2
+										: 1)}
+							</span>
+							<span>{$i18n.t('more')}</span>
+						</div>
+					</div>
+					<div class="flex-shrink-0">
+						{#if isCollapsibleOpen}
+							<ChevronUp strokeWidth="3.5" className="size-3.5" />
+						{:else}
+							<ChevronDown strokeWidth="3.5" className="size-3.5" />
+						{/if}
 					</div>
-					<div class="flex-1 mx-2 line-clamp-1">
-						{citation.source.name}
+				</div>
+				<div slot="content" class="mt-2">
+					<div class="flex flex-wrap gap-2">
+						{#each _citations as citation, idx}
+							<div class="flex gap-1 text-xs font-semibold">
+								<button
+									class="no-toggle flex dark:text-gray-300 py-1 px-1 bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800 transition rounded-xl max-w-96"
+									on:click={() => {
+										showCitationModal = true;
+										selectedCitation = citation;
+									}}
+								>
+									{#if _citations.every((c) => c.distances !== undefined)}
+										<div class="bg-white dark:bg-gray-700 rounded-full size-4">
+											{idx + 1}
+										</div>
+									{/if}
+									<div class="flex-1 mx-2 line-clamp-1">
+										{citation.source.name}
+									</div>
+								</button>
+							</div>
+						{/each}
 					</div>
-				</button>
-			</div>
-		{/each}
+				</div>
+			</Collapsible>
+		{/if}
 	</div>
 {/if}

src/lib/components/chat/Messages/CitationsModal.svelte

@@ -2,21 +2,44 @@
 	import { getContext, onMount, tick } from 'svelte';
 	import Modal from '$lib/components/common/Modal.svelte';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
+
 	const i18n = getContext('i18n');
 
 	export let show = false;
 	export let citation;
+	export let showPercentage = false;
+	export let showRelevance = true;
 
 	let mergedDocuments = [];
 
+	function calculatePercentage(distance: number) {
+		if (distance < 0) return 100;
+		if (distance > 1) return 0;
+		return Math.round((1 - distance) * 10000) / 100;
+	}
+
+	function getRelevanceColor(percentage: number) {
+		if (percentage >= 80)
+			return 'bg-green-200 dark:bg-green-800 text-green-800 dark:text-green-200';
+		if (percentage >= 60)
+			return 'bg-yellow-200 dark:bg-yellow-800 text-yellow-800 dark:text-yellow-200';
+		if (percentage >= 40)
+			return 'bg-orange-200 dark:bg-orange-800 text-orange-800 dark:text-orange-200';
+		return 'bg-red-200 dark:bg-red-800 text-red-800 dark:text-red-200';
+	}
+
 	$: if (citation) {
 		mergedDocuments = citation.document?.map((c, i) => {
 			return {
 				source: citation.source,
 				document: c,
-				metadata: citation.metadata?.[i]
+				metadata: citation.metadata?.[i],
+				distance: citation.distances?.[i]
 			};
 		});
+		if (mergedDocuments.every((doc) => doc.distance !== undefined)) {
+			mergedDocuments.sort((a, b) => (a.distance ?? Infinity) - (b.distance ?? Infinity));
+		}
 	}
 </script>
 
@@ -59,11 +82,11 @@
 							<Tooltip
 								content={$i18n.t('Open file')}
 								placement="left"
-								tippyOptions={{ duration: [500, 0], animation: 'perspective' }}
+								tippyOptions={{ duration: [500, 0] }}
 							>
-								<div class="text-sm dark:text-gray-400">
+								<div class="text-sm dark:text-gray-400 flex items-center gap-2">
 									<a
-										class="hover:text-gray-500 hover:dark:text-gray-100 underline"
+										class="hover:text-gray-500 hover:dark:text-gray-100 underline flex-grow"
 										href={document?.metadata?.file_id
 											? `/api/v1/files/${document?.metadata?.file_id}/content${document?.metadata?.page !== undefined ? `#page=${document.metadata.page + 1}` : ''}`
 											: document.source.name.includes('http')
@@ -73,11 +96,46 @@
 									>
 										{document?.metadata?.name ?? document.source.name}
 									</a>
-									{document?.metadata?.page
-										? `(${$i18n.t('page')} ${document.metadata.page + 1})`
-										: ''}
+									{#if document?.metadata?.page}
+										<span class="text-xs text-gray-500 dark:text-gray-400">
+											({$i18n.t('page')}
+											{document.metadata.page + 1})
+										</span>
+									{/if}
 								</div>
 							</Tooltip>
+							{#if showRelevance}
+								<div class="text-sm font-medium dark:text-gray-300 mt-2">
+									{$i18n.t('Relevance')}
+								</div>
+								{#if document.distance !== undefined}
+									<Tooltip
+										content={$i18n.t('Semantic distance to query from vector store')}
+										placement="left"
+										tippyOptions={{ duration: [500, 0] }}
+									>
+										<div class="text-sm my-1 dark:text-gray-400 flex items-center gap-2">
+											{#if showPercentage}
+												{@const percentage = calculatePercentage(document.distance)}
+												<span class={`px-1 rounded font-medium ${getRelevanceColor(percentage)}`}>
+													{percentage.toFixed(2)}%
+												</span>
+												<span class="text-gray-500 dark:text-gray-500">
+													({document.distance.toFixed(4)})
+												</span>
+											{:else}
+												<span class="text-gray-500 dark:text-gray-500">
+													{document.distance.toFixed(4)}
+												</span>
+											{/if}
+										</div>
+									</Tooltip>
+								{:else}
+									<div class="text-sm dark:text-gray-400">
+										{$i18n.t('No distance available')}
+									</div>
+								{/if}
+							{/if}
 						{:else}
 							<div class="text-sm dark:text-gray-400">
 								{$i18n.t('No source available')}
@@ -85,7 +143,7 @@
 						{/if}
 					</div>
 					<div class="flex flex-col w-full">
-						<div class=" text-sm font-medium dark:text-gray-300">
+						<div class=" text-sm font-medium dark:text-gray-300 mt-2">
 							{$i18n.t('Content')}
 						</div>
 						<pre class="text-sm dark:text-gray-400 whitespace-pre-line">

src/lib/components/common/Collapsible.svelte

@@ -38,9 +38,9 @@
 
 				<div>
 					{#if open}
-						<ChevronUp strokeWidth="3.5" className="size-3.5 " />
+						<ChevronUp strokeWidth="3.5" className="size-3.5" />
 					{:else}
-						<ChevronDown strokeWidth="3.5" className="size-3.5 " />
+						<ChevronDown strokeWidth="3.5" className="size-3.5" />
 					{/if}
 				</div>
 			</div>
@@ -68,5 +68,5 @@
 		<div transition:slide={{ duration: 300, easing: quintOut, axis: 'y' }}>
 			<slot name="content" />
 		</div>
-	{/if}
+	</div>
 </div>

src/lib/i18n/locales/de-DE/translation.json

@@ -441,6 +441,7 @@
 	"Modelfile Content": "Modelfile-Inhalt",
 	"Models": "Modelle",
 	"More": "Mehr",
+	"more": "mehr",
 	"Move to Top": "",
 	"Name": "Name",
 	"Name your model": "Benennen Sie Ihr Modell",
@@ -788,5 +789,6 @@
 	"Your account status is currently pending activation.": "Ihr Kontostatus ist derzeit ausstehend und wartet auf Aktivierung.",
 	"Your entire contribution will go directly to the plugin developer; Open WebUI does not take any percentage. However, the chosen funding platform might have its own fees.": "",
 	"Youtube": "YouTube",
-	"Youtube Loader Settings": "YouTube-Ladeeinstellungen"
+	"Youtube Loader Settings": "YouTube-Ladeeinstellungen",
+	"References from": "Referenzen aus"
 }