feat: arena models

backend/open_webui/apps/webui/main.py

@@ -1,6 +1,7 @@
 import inspect
 import json
 import logging
+import time
 from typing import AsyncGenerator, Generator, Iterator
 
 from open_webui.apps.socket.main import get_event_call, get_event_emitter
@@ -17,6 +18,7 @@ from open_webui.apps.webui.routers import (
     models,
     knowledge,
     prompts,
+    evaluations,
     tools,
     users,
     utils,
@@ -32,6 +34,9 @@ from open_webui.config import (
     ENABLE_LOGIN_FORM,
     ENABLE_MESSAGE_RATING,
     ENABLE_SIGNUP,
+    ENABLE_EVALUATION_ARENA_MODELS,
+    EVALUATION_ARENA_MODELS,
+    DEFAULT_ARENA_MODEL,
     JWT_EXPIRES_IN,
     ENABLE_OAUTH_ROLE_MANAGEMENT,
     OAUTH_ROLES_CLAIM,
@@ -94,6 +99,9 @@ app.state.config.BANNERS = WEBUI_BANNERS
 app.state.config.ENABLE_COMMUNITY_SHARING = ENABLE_COMMUNITY_SHARING
 app.state.config.ENABLE_MESSAGE_RATING = ENABLE_MESSAGE_RATING
 
+app.state.config.ENABLE_EVALUATION_ARENA_MODELS = ENABLE_EVALUATION_ARENA_MODELS
+app.state.config.EVALUATION_ARENA_MODELS = EVALUATION_ARENA_MODELS
+
 app.state.config.OAUTH_USERNAME_CLAIM = OAUTH_USERNAME_CLAIM
 app.state.config.OAUTH_PICTURE_CLAIM = OAUTH_PICTURE_CLAIM
 app.state.config.OAUTH_EMAIL_CLAIM = OAUTH_EMAIL_CLAIM
@@ -117,20 +125,24 @@ app.add_middleware(
 
 
 app.include_router(configs.router, prefix="/configs", tags=["configs"])
+
 app.include_router(auths.router, prefix="/auths", tags=["auths"])
 app.include_router(users.router, prefix="/users", tags=["users"])
+
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
-app.include_router(folders.router, prefix="/folders", tags=["folders"])
 
 app.include_router(models.router, prefix="/models", tags=["models"])
 app.include_router(knowledge.router, prefix="/knowledge", tags=["knowledge"])
 app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
-
-app.include_router(files.router, prefix="/files", tags=["files"])
 app.include_router(tools.router, prefix="/tools", tags=["tools"])
 app.include_router(functions.router, prefix="/functions", tags=["functions"])
 
 app.include_router(memories.router, prefix="/memories", tags=["memories"])
+app.include_router(evaluations.router, prefix="/evaluations", tags=["evaluations"])
+
+app.include_router(folders.router, prefix="/folders", tags=["folders"])
+app.include_router(files.router, prefix="/files", tags=["files"])
+
 app.include_router(utils.router, prefix="/utils", tags=["utils"])
 
 
@@ -145,8 +157,44 @@ async def get_status():
 
 
 async def get_all_models():
+    models = []
     pipe_models = await get_pipe_models()
-    return pipe_models
+    models = models + pipe_models
+
+    if app.state.config.ENABLE_EVALUATION_ARENA_MODELS:
+        arena_models = []
+        if len(app.state.config.EVALUATION_ARENA_MODELS) > 0:
+            arena_models = [
+                {
+                    "id": model["id"],
+                    "name": model["name"],
+                    "info": {
+                        "meta": model["meta"],
+                    },
+                    "object": "model",
+                    "created": int(time.time()),
+                    "owned_by": "arena",
+                    "arena": True,
+                }
+                for model in app.state.config.EVALUATION_ARENA_MODELS
+            ]
+        else:
+            # Add default arena model
+            arena_models = [
+                {
+                    "id": DEFAULT_ARENA_MODEL["id"],
+                    "name": DEFAULT_ARENA_MODEL["name"],
+                    "info": {
+                        "meta": DEFAULT_ARENA_MODEL["meta"],
+                    },
+                    "object": "model",
+                    "created": int(time.time()),
+                    "owned_by": "arena",
+                    "arena": True,
+                }
+            ]
+        models = models + arena_models
+    return models
 
 
 def get_function_module(pipe_id: str):

backend/open_webui/apps/webui/routers/evaluations.py

@@ -0,0 +1,49 @@
+from typing import Optional
+from fastapi import APIRouter, Depends, HTTPException, status, Request
+from pydantic import BaseModel
+
+
+from open_webui.constants import ERROR_MESSAGES
+from open_webui.utils.utils import get_admin_user, get_verified_user
+
+router = APIRouter()
+
+
+############################
+# GetConfig
+############################
+
+
+@router.get("/config")
+async def get_config(request: Request, user=Depends(get_admin_user)):
+    return {
+        "ENABLE_EVALUATION_ARENA_MODELS": request.app.state.config.ENABLE_EVALUATION_ARENA_MODELS,
+        "EVALUATION_ARENA_MODELS": request.app.state.config.EVALUATION_ARENA_MODELS,
+    }
+
+
+############################
+# UpdateConfig
+############################
+
+
+class UpdateConfigForm(BaseModel):
+    ENABLE_EVALUATION_ARENA_MODELS: Optional[bool] = None
+    EVALUATION_ARENA_MODELS: Optional[list[dict]] = None
+
+
+@router.post("/config")
+async def update_config(
+    request: Request,
+    form_data: UpdateConfigForm,
+    user=Depends(get_admin_user),
+):
+    config = request.app.state.config
+    if form_data.ENABLE_EVALUATION_ARENA_MODELS is not None:
+        config.ENABLE_EVALUATION_ARENA_MODELS = form_data.ENABLE_EVALUATION_ARENA_MODELS
+    if form_data.EVALUATION_ARENA_MODELS is not None:
+        config.EVALUATION_ARENA_MODELS = form_data.EVALUATION_ARENA_MODELS
+    return {
+        "ENABLE_EVALUATION_ARENA_MODELS": config.ENABLE_EVALUATION_ARENA_MODELS,
+        "EVALUATION_ARENA_MODELS": config.EVALUATION_ARENA_MODELS,
+    }

backend/open_webui/config.py

@@ -751,6 +751,28 @@ USER_PERMISSIONS = PersistentConfig(
     },
 )
 
+
+ENABLE_EVALUATION_ARENA_MODELS = PersistentConfig(
+    "ENABLE_EVALUATION_ARENA_MODELS",
+    "evaluation.arena.enable",
+    os.environ.get("ENABLE_EVALUATION_ARENA_MODELS", "True").lower() == "true",
+)
+EVALUATION_ARENA_MODELS = PersistentConfig(
+    "EVALUATION_ARENA_MODELS",
+    "evaluation.arena.models",
+    [],
+)
+
+DEFAULT_ARENA_MODEL = {
+    "id": "arena-model",
+    "name": "Arena Model",
+    "meta": {
+        "profile_image_url": "/favicon.png",
+        "description": "Submit your questions to anonymous AI chatbots and vote on the best response.",
+        "model_ids": None,
+    },
+}
+
 ENABLE_MODEL_FILTER = PersistentConfig(
     "ENABLE_MODEL_FILTER",
     "model_filter.enable",

backend/open_webui/main.py

@@ -7,6 +7,7 @@ import os
 import shutil
 import sys
 import time
+import random
 from contextlib import asynccontextmanager
 from typing import Optional
 
@@ -23,7 +24,7 @@ from fastapi import (
     status,
 )
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
 from sqlalchemy import text
@@ -1093,6 +1094,23 @@ async def generate_chat_completions(form_data: dict, user=Depends(get_verified_u
             )
 
     model = app.state.MODELS[model_id]
+
+    if model["owned_by"] == "arena":
+        model_ids = model.get("info", {}).get("meta", {}).get("model_ids")
+        model_id = None
+        if isinstance(model_ids, list) and model_ids:
+            model_id = random.choice(model_ids)
+        else:
+            model_ids = [
+                model["id"]
+                for model in await get_all_models()
+                if model.get("owned_by") != "arena"
+                and not model.get("info", {}).get("meta", {}).get("hidden", False)
+            ]
+            model_id = random.choice(model_ids)
+
+        form_data["model"] = model_id
+        return await generate_chat_completions(form_data, user)
     if model.get("pipe"):
         return await generate_function_chat_completion(form_data, user=user)
     if model["owned_by"] == "ollama":

backend/open_webui/utils/payload.py

@@ -116,6 +116,9 @@ def convert_messages_openai_to_ollama(messages: list[dict]) -> list[dict]:
                 elif item.get("type") == "image_url":
                     img_url = item.get("image_url", {}).get("url", "")
                     if img_url:
+                        # If the image url starts with data:, it's a base64 image and should be trimmed
+                        if img_url.startswith("data:"):
+                            img_url = img_url.split(",")[-1]
                         images.append(img_url)
 
             # Add content text (if any)

src/lib/apis/evaluations/index.ts

@@ -0,0 +1,63 @@
+import { WEBUI_API_BASE_URL } from '$lib/constants';
+
+export const getConfig = async (token: string = '') => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/config`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateConfig = async (token: string, config: object) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/config`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			...config
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};

src/lib/components/admin/AddUserModal.svelte

@@ -139,7 +139,7 @@
 			</button>
 		</div>
 
-		<div class="flex flex-col md:flex-row w-full px-5 pb-4 md:space-x-4 dark:text-gray-200">
+		<div class="flex flex-col md:flex-row w-full px-4 pb-3 md:space-x-4 dark:text-gray-200">
 			<div class=" flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
 				<form
 					class="flex flex-col w-full"
@@ -147,9 +147,9 @@
 						submitHandler();
 					}}
 				>
-					<div class="flex text-center text-sm font-medium rounded-xl bg-transparent/10 p-1 mb-2">
+					<div class="flex text-center text-sm font-medium rounded-full bg-transparent/10 p-1 mb-2">
 						<button
-							class="w-full rounded-lg p-1.5 {tab === '' ? 'bg-gray-50 dark:bg-gray-850' : ''}"
+							class="w-full rounded-full p-1.5 {tab === '' ? 'bg-gray-50 dark:bg-gray-850' : ''}"
 							type="button"
 							on:click={() => {
 								tab = '';
@@ -157,7 +157,9 @@
 						>
 
 						<button
-							class="w-full rounded-lg p-1 {tab === 'import' ? 'bg-gray-50 dark:bg-gray-850' : ''}"
+							class="w-full rounded-full p-1 {tab === 'import'
+								? 'bg-gray-50 dark:bg-gray-850'
+								: ''}"
 							type="button"
 							on:click={() => {
 								tab = 'import';
@@ -183,7 +185,7 @@
 								</div>
 							</div>
 
-							<div class="flex flex-col w-full mt-2">
+							<div class="flex flex-col w-full mt-1">
 								<div class=" mb-1 text-xs text-gray-500">{$i18n.t('Name')}</div>
 
 								<div class="flex-1">
@@ -198,7 +200,7 @@
 								</div>
 							</div>
 
-							<hr class=" dark:border-gray-800 my-3 w-full" />
+							<hr class=" border-gray-50 dark:border-gray-850 my-2.5 w-full" />
 
 							<div class="flex flex-col w-full">
 								<div class=" mb-1 text-xs text-gray-500">{$i18n.t('Email')}</div>
@@ -209,13 +211,12 @@
 										type="email"
 										bind:value={_user.email}
 										placeholder={$i18n.t('Enter Your Email')}
-										autocomplete="off"
 										required
 									/>
 								</div>
 							</div>
 
-							<div class="flex flex-col w-full mt-2">
+							<div class="flex flex-col w-full mt-1">
 								<div class=" mb-1 text-xs text-gray-500">{$i18n.t('Password')}</div>
 
 								<div class="flex-1">
@@ -271,13 +272,13 @@
 
 					<div class="flex justify-end pt-3 text-sm font-medium">
 						<button
-							class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg flex flex-row space-x-1 items-center {loading
+							class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full flex flex-row space-x-1 items-center {loading
 								? ' cursor-not-allowed'
 								: ''}"
 							type="submit"
 							disabled={loading}
 						>
-							{$i18n.t('Submit')}
+							{$i18n.t('Save')}
 
 							{#if loading}
 								<div class="ml-2 self-center">

src/lib/components/admin/Evaluations.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+	import { onMount, getContext } from 'svelte';
+	import Tooltip from '../common/Tooltip.svelte';
+	import Plus from '../icons/Plus.svelte';
+	import Collapsible from '../common/Collapsible.svelte';
+	import Switch from '../common/Switch.svelte';
+	import ChevronUp from '../icons/ChevronUp.svelte';
+	import ChevronDown from '../icons/ChevronDown.svelte';
+	const i18n = getContext('i18n');
+
+	let loaded = false;
+	let evaluationEnabled = true;
+
+	let showModels = false;
+
+	onMount(() => {
+		loaded = true;
+	});
+</script>
+
+{#if loaded}
+	<div class="my-0.5 gap-1 flex flex-col md:flex-row justify-between">
+		<div class="flex md:self-center text-lg font-medium px-0.5">
+			{$i18n.t('Leaderboard')}
+		</div>
+	</div>
+{/if}

src/lib/components/admin/Settings.svelte

@@ -17,6 +17,9 @@
 	import WebSearch from './Settings/WebSearch.svelte';
 	import { config } from '$lib/stores';
 	import { getBackendConfig } from '$lib/apis';
+	import ChartBar from '../icons/ChartBar.svelte';
+	import DocumentChartBar from '../icons/DocumentChartBar.svelte';
+	import Evaluations from './Settings/Evaluations.svelte';
 
 	const i18n = getContext('i18n');
 
@@ -141,6 +144,21 @@
 			<div class=" self-center">{$i18n.t('Models')}</div>
 		</button>
 
+		<button
+			class="px-2.5 py-2 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
+			'evaluations'
+				? 'bg-gray-100 dark:bg-gray-800'
+				: ' hover:bg-gray-50 dark:hover:bg-gray-850'}"
+			on:click={() => {
+				selectedTab = 'evaluations';
+			}}
+		>
+			<div class=" self-center mr-2">
+				<DocumentChartBar />
+			</div>
+			<div class=" self-center">{$i18n.t('Evaluations')}</div>
+		</button>
+
 		<button
 			class="px-2.5 py-2 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
 			'documents'
@@ -357,6 +375,8 @@
 			/>
 		{:else if selectedTab === 'models'}
 			<Models />
+		{:else if selectedTab === 'evaluations'}
+			<Evaluations />
 		{:else if selectedTab === 'documents'}
 			<Documents
 				on:save={async () => {

src/lib/components/admin/Settings/Evaluations.svelte

@@ -0,0 +1,155 @@
+<script lang="ts">
+	import { toast } from 'svelte-sonner';
+	import { models, user } from '$lib/stores';
+	import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
+
+	const dispatch = createEventDispatcher();
+	import { getModels } from '$lib/apis';
+
+	import Switch from '$lib/components/common/Switch.svelte';
+	import Spinner from '$lib/components/common/Spinner.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import Plus from '$lib/components/icons/Plus.svelte';
+	import Model from './Evaluations/Model.svelte';
+	import ModelModal from './Evaluations/ModelModal.svelte';
+	import { getConfig, updateConfig } from '$lib/apis/evaluations';
+
+	const i18n = getContext('i18n');
+
+	let config = null;
+	let showAddModel = false;
+
+	const submitHandler = async () => {
+		config = await updateConfig(localStorage.token, config).catch((err) => {
+			toast.error(err);
+			return null;
+		});
+
+		if (config) {
+			toast.success('Settings saved successfully');
+		}
+	};
+
+	const addModelHandler = async (model) => {
+		config.EVALUATION_ARENA_MODELS.push(model);
+		config.EVALUATION_ARENA_MODELS = [...config.EVALUATION_ARENA_MODELS];
+
+		await submitHandler();
+		models.set(await getModels(localStorage.token));
+	};
+
+	const editModelHandler = async (model, modelIdx) => {
+		config.EVALUATION_ARENA_MODELS[modelIdx] = model;
+		config.EVALUATION_ARENA_MODELS = [...config.EVALUATION_ARENA_MODELS];
+
+		await submitHandler();
+		models.set(await getModels(localStorage.token));
+	};
+
+	const deleteModelHandler = async (modelIdx) => {
+		config.EVALUATION_ARENA_MODELS = config.EVALUATION_ARENA_MODELS.filter(
+			(m, mIdx) => mIdx !== modelIdx
+		);
+
+		await submitHandler();
+		models.set(await getModels(localStorage.token));
+	};
+
+	onMount(async () => {
+		if ($user.role === 'admin') {
+			config = await getConfig(localStorage.token).catch((err) => {
+				toast.error(err);
+				return null;
+			});
+		}
+	});
+</script>
+
+<ModelModal
+	bind:show={showAddModel}
+	on:submit={async (e) => {
+		addModelHandler(e.detail);
+	}}
+/>
+
+<form
+	class="flex flex-col h-full justify-between text-sm"
+	on:submit|preventDefault={() => {
+		submitHandler();
+		dispatch('save');
+	}}
+>
+	<div class="overflow-y-scroll scrollbar-hidden h-full">
+		{#if config !== null}
+			<div class="">
+				<div class="text-sm font-medium mb-2">{$i18n.t('General Settings')}</div>
+
+				<div class=" mb-2">
+					<div class="flex justify-between items-center text-xs">
+						<div class=" text-xs font-medium">{$i18n.t('Arena Models')}</div>
+						<Switch bind:state={config.ENABLE_EVALUATION_ARENA_MODELS} />
+					</div>
+				</div>
+
+				{#if config.ENABLE_EVALUATION_ARENA_MODELS}
+					<hr class=" border-gray-50 dark:border-gray-700/10 my-2" />
+
+					<div class="flex justify-between items-center mb-2">
+						<div class="text-sm font-medium">{$i18n.t('Manage Arena Models')}</div>
+
+						<div>
+							<Tooltip content={$i18n.t('Add Arena Model')}>
+								<button
+									class="p-1"
+									type="button"
+									on:click={() => {
+										showAddModel = true;
+									}}
+								>
+									<Plus />
+								</button>
+							</Tooltip>
+						</div>
+					</div>
+
+					<div class="flex flex-col gap-2">
+						{#if (config?.EVALUATION_ARENA_MODELS ?? []).length > 0}
+							{#each config.EVALUATION_ARENA_MODELS as model, index}
+								<Model
+									{model}
+									on:edit={(e) => {
+										editModelHandler(e.detail, index);
+									}}
+									on:delete={(e) => {
+										deleteModelHandler(index);
+									}}
+								/>
+							{/each}
+						{:else}
+							<div class=" text-center text-xs text-gray-500">
+								{$i18n.t(
+									`Using the default arena model with all models. Click the plus button to add custom models.`
+								)}
+							</div>
+						{/if}
+					</div>
+				{/if}
+			</div>
+		{:else}
+			<div class="flex h-full justify-center">
+				<div class="my-auto">
+					<Spinner className="size-6" />
+				</div>
+			</div>
+		{/if}
+	</div>
+
+	<div class="flex justify-end pt-3 text-sm font-medium">
+		<button
+			class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full"
+			type="submit"
+		>
+			{$i18n.t('Save')}
+		</button>
+	</div>
+</form>

src/lib/components/admin/Settings/Evaluations/Model.svelte

@@ -0,0 +1,63 @@
+<script lang="ts">
+	import { getContext, createEventDispatcher } from 'svelte';
+	const dispatch = createEventDispatcher();
+	const i18n = getContext('i18n');
+
+	import Cog6 from '$lib/components/icons/Cog6.svelte';
+	import ModelModal from './ModelModal.svelte';
+	export let model;
+
+	let showModel = false;
+</script>
+
+<ModelModal
+	bind:show={showModel}
+	edit={true}
+	{model}
+	on:submit={async (e) => {
+		dispatch('edit', e.detail);
+	}}
+	on:delete={async () => {
+		dispatch('delete');
+	}}
+/>
+
+<div class="py-0.5">
+	<div class="flex justify-between items-center mb-1">
+		<div class="flex flex-col flex-1">
+			<div class="flex gap-2.5 items-center">
+				<img
+					src={model.meta.profile_image_url}
+					alt={model.name}
+					class="size-8 rounded-full object-cover shrink-0"
+				/>
+
+				<div class="w-full flex flex-col">
+					<div class="flex items-center gap-1">
+						<div class="flex-shrink-0 line-clamp-1">
+							{model.name}
+						</div>
+					</div>
+
+					<div class="flex items-center gap-1">
+						<div class=" text-xs w-full text-gray-500 bg-transparent line-clamp-1">
+							{model.meta.description}
+						</div>
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<div class="flex items-center">
+			<button
+				class="self-center w-fit text-sm p-1.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-xl"
+				type="button"
+				on:click={() => {
+					showModel = true;
+				}}
+			>
+				<Cog6 />
+			</button>
+		</div>
+	</div>
+</div>

src/lib/components/admin/Settings/Evaluations/ModelModal.svelte

@@ -0,0 +1,398 @@
+<script>
+	import { createEventDispatcher, getContext, onMount } from 'svelte';
+	const i18n = getContext('i18n');
+	const dispatch = createEventDispatcher();
+
+	import Modal from '$lib/components/common/Modal.svelte';
+	import { models } from '$lib/stores';
+	import Plus from '$lib/components/icons/Plus.svelte';
+	import Minus from '$lib/components/icons/Minus.svelte';
+	import PencilSolid from '$lib/components/icons/PencilSolid.svelte';
+	import { toast } from 'svelte-sonner';
+
+	export let show = false;
+	export let edit = false;
+
+	export let model = null;
+
+	let name = '';
+	let id = '';
+
+	$: if (name) {
+		generateId();
+	}
+
+	const generateId = () => {
+		if (!edit) {
+			id = name
+				.toLowerCase()
+				.replace(/[^a-z0-9]/g, '-')
+				.replace(/-+/g, '-')
+				.replace(/^-|-$/g, '');
+		}
+	};
+
+	let profileImageUrl = '/favicon.png';
+	let description = '';
+
+	let selectedModelId = '';
+	let modelIds = [];
+
+	let imageInputElement;
+	let loading = false;
+
+	const addModelHandler = () => {
+		if (selectedModelId) {
+			modelIds = [...modelIds, selectedModelId];
+			selectedModelId = '';
+		}
+	};
+
+	const submitHandler = () => {
+		loading = true;
+
+		if (!name || !id) {
+			loading = false;
+			toast.error('Name and ID are required, please fill them out');
+			return;
+		}
+
+		if (!edit) {
+			if ($models.find((model) => model.name === name)) {
+				loading = false;
+				name = '';
+				toast.error('Model name already exists, please choose a different one');
+				return;
+			}
+		}
+
+		const model = {
+			id: id,
+			name: name,
+			meta: {
+				profile_image_url: profileImageUrl,
+				description: description || null,
+				model_ids: modelIds.length > 0 ? modelIds : null
+			}
+		};
+
+		dispatch('submit', model);
+		loading = false;
+		show = false;
+
+		name = '';
+		id = '';
+		profileImageUrl = '/favicon.png';
+		description = '';
+		modelIds = [];
+		selectedModelId = '';
+	};
+
+	const initModel = () => {
+		if (model) {
+			name = model.name;
+			id = model.id;
+			profileImageUrl = model.meta.profile_image_url;
+			description = model.meta.description;
+			modelIds = model.meta.model_ids || [];
+		}
+	};
+
+	$: if (show) {
+		initModel();
+	}
+
+	onMount(() => {
+		initModel();
+	});
+</script>
+
+<Modal size="sm" bind:show>
+	<div>
+		<div class=" flex justify-between dark:text-gray-100 px-5 pt-4 pb-2">
+			<div class=" text-lg font-medium self-center font-primary">
+				{#if edit}
+					{$i18n.t('Edit Arena Model')}
+				{:else}
+					{$i18n.t('Add Arena Model')}
+				{/if}
+			</div>
+			<button
+				class="self-center"
+				on:click={() => {
+					show = false;
+				}}
+			>
+				<svg
+					xmlns="http://www.w3.org/2000/svg"
+					viewBox="0 0 20 20"
+					fill="currentColor"
+					class="w-5 h-5"
+				>
+					<path
+						d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+					/>
+				</svg>
+			</button>
+		</div>
+
+		<div class="flex flex-col md:flex-row w-full px-4 pb-4 md:space-x-4 dark:text-gray-200">
+			<div class=" flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
+				<form
+					class="flex flex-col w-full"
+					on:submit|preventDefault={() => {
+						submitHandler();
+					}}
+				>
+					<div class="px-1">
+						<div class="flex justify-center pb-3">
+							<input
+								bind:this={imageInputElement}
+								type="file"
+								hidden
+								accept="image/*"
+								on:change={(e) => {
+									const files = e.target.files ?? [];
+									let reader = new FileReader();
+									reader.onload = (event) => {
+										let originalImageUrl = `${event.target.result}`;
+
+										const img = new Image();
+										img.src = originalImageUrl;
+
+										img.onload = function () {
+											const canvas = document.createElement('canvas');
+											const ctx = canvas.getContext('2d');
+
+											// Calculate the aspect ratio of the image
+											const aspectRatio = img.width / img.height;
+
+											// Calculate the new width and height to fit within 250x250
+											let newWidth, newHeight;
+											if (aspectRatio > 1) {
+												newWidth = 250 * aspectRatio;
+												newHeight = 250;
+											} else {
+												newWidth = 250;
+												newHeight = 250 / aspectRatio;
+											}
+
+											// Set the canvas size
+											canvas.width = 250;
+											canvas.height = 250;
+
+											// Calculate the position to center the image
+											const offsetX = (250 - newWidth) / 2;
+											const offsetY = (250 - newHeight) / 2;
+
+											// Draw the image on the canvas
+											ctx.drawImage(img, offsetX, offsetY, newWidth, newHeight);
+
+											// Get the base64 representation of the compressed image
+											const compressedSrc = canvas.toDataURL('image/jpeg');
+
+											// Display the compressed image
+											profileImageUrl = compressedSrc;
+
+											e.target.files = null;
+										};
+									};
+
+									if (
+										files.length > 0 &&
+										['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(
+											files[0]['type']
+										)
+									) {
+										reader.readAsDataURL(files[0]);
+									}
+								}}
+							/>
+
+							<button
+								class="relative rounded-full w-fit h-fit shrink-0"
+								type="button"
+								on:click={() => {
+									imageInputElement.click();
+								}}
+							>
+								<img
+									src={profileImageUrl}
+									class="size-16 rounded-full object-cover shrink-0"
+									alt="Profile"
+								/>
+
+								<div
+									class="absolute flex justify-center rounded-full bottom-0 left-0 right-0 top-0 h-full w-full overflow-hidden bg-gray-700 bg-fixed opacity-0 transition duration-300 ease-in-out hover:opacity-50"
+								>
+									<div class="my-auto text-white">
+										<PencilSolid className="size-4" />
+									</div>
+								</div>
+							</button>
+						</div>
+						<div class="flex gap-2">
+							<div class="flex flex-col w-full">
+								<div class=" mb-0.5 text-xs text-gray-500">{$i18n.t('Name')}</div>
+
+								<div class="flex-1">
+									<input
+										class="w-full text-sm bg-transparent placeholder:text-gray-300 dark:placeholder:text-gray-700 outline-none"
+										type="text"
+										bind:value={name}
+										placeholder={$i18n.t('Model Name')}
+										autocomplete="off"
+										required
+									/>
+								</div>
+							</div>
+
+							<div class="flex flex-col w-full">
+								<div class=" mb-0.5 text-xs text-gray-500">{$i18n.t('ID')}</div>
+
+								<div class="flex-1">
+									<input
+										class="w-full text-sm bg-transparent placeholder:text-gray-300 dark:placeholder:text-gray-700 outline-none"
+										type="text"
+										bind:value={id}
+										placeholder={$i18n.t('Model ID')}
+										autocomplete="off"
+										required
+										disabled={edit}
+									/>
+								</div>
+							</div>
+						</div>
+
+						<div class="flex flex-col w-full mt-2">
+							<div class=" mb-1 text-xs text-gray-500">{$i18n.t('Description')}</div>
+
+							<div class="flex-1">
+								<input
+									class="w-full text-sm bg-transparent placeholder:text-gray-300 dark:placeholder:text-gray-700 outline-none"
+									type="text"
+									bind:value={description}
+									placeholder={$i18n.t('Enter description')}
+									autocomplete="off"
+									required
+								/>
+							</div>
+						</div>
+
+						<hr class=" border-gray-100 dark:border-gray-700/10 my-2.5 w-full" />
+
+						<div class="flex flex-col w-full">
+							<div class=" mb-1 text-xs text-gray-500">{$i18n.t('Models')}</div>
+
+							{#if modelIds.length > 0}
+								<div class="flex flex-col">
+									{#each modelIds as modelId, modelIdx}
+										<div class=" flex gap-2 w-full justify-between items-center">
+											<div class=" text-sm flex-1 py-1 rounded-lg">
+												{$models.find((model) => model.id === modelId)?.name}
+											</div>
+											<div class="flex-shrink-0">
+												<button
+													type="button"
+													on:click={() => {
+														modelIds = modelIds.filter((_, idx) => idx !== modelIdx);
+													}}
+												>
+													<Minus strokeWidth="2" className="size-3.5" />
+												</button>
+											</div>
+										</div>
+									{/each}
+								</div>
+							{:else}
+								<div class="text-gray-500 text-xs text-center py-2">
+									{$i18n.t('Leave empty to include all models or select specific models')}
+								</div>
+							{/if}
+						</div>
+
+						<hr class=" border-gray-100 dark:border-gray-700/10 my-2.5 w-full" />
+
+						<div class="flex items-center">
+							<select
+								class="w-full py-1 text-sm rounded-lg bg-transparent {selectedModelId
+									? ''
+									: 'text-gray-500'} placeholder:text-gray-300 dark:placeholder:text-gray-700 outline-none"
+								bind:value={selectedModelId}
+							>
+								<option value="">{$i18n.t('Select a model')}</option>
+								{#each $models.filter((m) => m?.owned_by !== 'arena') as model}
+									<option value={model.id}>{model.name}</option>
+								{/each}
+							</select>
+
+							<div>
+								<button
+									type="button"
+									on:click={() => {
+										addModelHandler();
+									}}
+								>
+									<Plus className="size-3.5" strokeWidth="2" />
+								</button>
+							</div>
+						</div>
+					</div>
+
+					<div class="flex justify-end pt-3 text-sm font-medium gap-1.5">
+						{#if edit}
+							<button
+								class="px-3.5 py-1.5 text-sm font-medium dark:bg-black dark:hover:bg-gray-900 dark:text-white bg-white text-black hover:bg-gray-100 transition rounded-full flex flex-row space-x-1 items-center"
+								type="button"
+								on:click={() => {
+									dispatch('delete', model);
+									show = false;
+								}}
+							>
+								{$i18n.t('Delete')}
+							</button>
+						{/if}
+
+						<button
+							class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full flex flex-row space-x-1 items-center {loading
+								? ' cursor-not-allowed'
+								: ''}"
+							type="submit"
+							disabled={loading}
+						>
+							{$i18n.t('Save')}
+
+							{#if loading}
+								<div class="ml-2 self-center">
+									<svg
+										class=" w-4 h-4"
+										viewBox="0 0 24 24"
+										fill="currentColor"
+										xmlns="http://www.w3.org/2000/svg"
+										><style>
+											.spinner_ajPY {
+												transform-origin: center;
+												animation: spinner_AtaB 0.75s infinite linear;
+											}
+											@keyframes spinner_AtaB {
+												100% {
+													transform: rotate(360deg);
+												}
+											}
+										</style><path
+											d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
+											opacity=".25"
+										/><path
+											d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
+											class="spinner_ajPY"
+										/></svg
+									>
+								</div>
+							{/if}
+						</button>
+					</div>
+				</form>
+			</div>
+		</div>
+	</div>
+</Modal>

src/lib/components/admin/Settings/Interface.svelte

@@ -62,7 +62,7 @@
 >
 	<div class="  overflow-y-scroll scrollbar-hidden h-full pr-1.5">
 		<div>
-			<div class=" mb-2.5 text-sm font-medium flex">
+			<div class=" mb-2.5 text-sm font-medium flex items-center">
 				<div class=" mr-1">{$i18n.t('Set Task Model')}</div>
 				<Tooltip
 					content={$i18n.t(
@@ -75,7 +75,7 @@
 						viewBox="0 0 24 24"
 						stroke-width="1.5"
 						stroke="currentColor"
-						class="w-5 h-5"
+						class="size-3.5"
 					>
 						<path
 							stroke-linecap="round"

src/lib/components/chat/Chat.svelte

@@ -1009,10 +1009,10 @@
 					}
 
 					let _response = null;
-					if (model?.owned_by === 'openai') {
-						_response = await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
-					} else if (model) {
+					if (model?.owned_by === 'ollama') {
 						_response = await sendPromptOllama(model, prompt, responseMessageId, _chatId);
+					} else if (model) {
+						_response = await sendPromptOpenAI(model, prompt, responseMessageId, _chatId);
 					}
 					_responses.push(_response);
 

src/lib/components/chat/ChatPlaceholder.svelte

@@ -82,8 +82,8 @@
 		>
 			<div>
 				<div class=" capitalize line-clamp-1" in:fade={{ duration: 200 }}>
-					{#if models[selectedModelIdx]?.info}
-						{models[selectedModelIdx]?.info?.name}
+					{#if models[selectedModelIdx]?.name}
+						{models[selectedModelIdx]?.name}
 					{:else}
 						{$i18n.t('Hello, {{name}}', { name: $user.name })}
 					{/if}

src/lib/components/chat/Controls/Controls.svelte

@@ -56,7 +56,7 @@
 				</div>
 			</Collapsible>
 
-			<hr class="my-2 border-gray-50 dark:border-gray-800" />
+			<hr class="my-2 border-gray-50 dark:border-gray-700/10" />
 		{/if}
 
 		<Collapsible title={$i18n.t('Valves')} buttonClassName="w-full">
@@ -65,7 +65,7 @@
 			</div>
 		</Collapsible>
 
-		<hr class="my-2 border-gray-50 dark:border-gray-800" />
+		<hr class="my-2 border-gray-50 dark:border-gray-700/10" />
 
 		<Collapsible title={$i18n.t('System Prompt')} open={true} buttonClassName="w-full">
 			<div class="" slot="content">
@@ -78,7 +78,7 @@
 			</div>
 		</Collapsible>
 
-		<hr class="my-2 border-gray-50 dark:border-gray-800" />
+		<hr class="my-2 border-gray-50 dark:border-gray-700/10" />
 
 		<Collapsible title={$i18n.t('Advanced Params')} open={true} buttonClassName="w-full">
 			<div class="text-sm mt-1.5" slot="content">

src/lib/components/chat/Placeholder.svelte

@@ -134,8 +134,8 @@
 					</div>
 
 					<div class=" capitalize text-3xl sm:text-4xl line-clamp-1" in:fade={{ duration: 100 }}>
-						{#if models[selectedModelIdx]?.info}
-							{models[selectedModelIdx]?.info?.name}
+						{#if models[selectedModelIdx]?.name}
+							{models[selectedModelIdx]?.name}
 						{:else}
 							{$i18n.t('Hello, {{name}}', { name: $user.name })}
 						{/if}

src/lib/components/chat/Settings/Account.svelte

@@ -93,23 +93,23 @@
 						// Calculate the aspect ratio of the image
 						const aspectRatio = img.width / img.height;
 
-						// Calculate the new width and height to fit within 100x100
+						// Calculate the new width and height to fit within 250x250
 						let newWidth, newHeight;
 						if (aspectRatio > 1) {
-							newWidth = 100 * aspectRatio;
-							newHeight = 100;
+							newWidth = 250 * aspectRatio;
+							newHeight = 250;
 						} else {
-							newWidth = 100;
-							newHeight = 100 / aspectRatio;
+							newWidth = 250;
+							newHeight = 250 / aspectRatio;
 						}
 
 						// Set the canvas size
-						canvas.width = 100;
-						canvas.height = 100;
+						canvas.width = 250;
+						canvas.height = 250;
 
 						// Calculate the position to center the image
-						const offsetX = (100 - newWidth) / 2;
-						const offsetY = (100 - newHeight) / 2;
+						const offsetX = (250 - newWidth) / 2;
+						const offsetY = (250 - newHeight) / 2;
 
 						// Draw the image on the canvas
 						ctx.drawImage(img, offsetX, offsetY, newWidth, newHeight);

src/lib/components/common/Switch.svelte

@@ -12,7 +12,7 @@
 		await tick();
 		dispatch('change', e);
 	}}
-	class="flex h-5 min-h-5 w-9 shrink-0 cursor-pointer items-center rounded-full px-[3px] transition  {state
+	class="flex h-5 min-h-5 w-9 shrink-0 cursor-pointer items-center rounded-full px-[3px] mx-[1px] transition  {state
 		? ' bg-emerald-600'
 		: 'bg-gray-200 dark:bg-transparent'} outline outline-1 outline-gray-100 dark:outline-gray-800"
 >

src/lib/components/icons/ChartBar.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class={className}>
+	<path
+		d="M18.375 2.25c-1.035 0-1.875.84-1.875 1.875v15.75c0 1.035.84 1.875 1.875 1.875h.75c1.035 0 1.875-.84 1.875-1.875V4.125c0-1.036-.84-1.875-1.875-1.875h-.75ZM9.75 8.625c0-1.036.84-1.875 1.875-1.875h.75c1.036 0 1.875.84 1.875 1.875v11.25c0 1.035-.84 1.875-1.875 1.875h-.75a1.875 1.875 0 0 1-1.875-1.875V8.625ZM3 13.125c0-1.036.84-1.875 1.875-1.875h.75c1.036 0 1.875.84 1.875 1.875v6.75c0 1.035-.84 1.875-1.875 1.875h-.75A1.875 1.875 0 0 1 3 19.875v-6.75Z"
+	/>
+</svg>

src/lib/components/icons/DocumentChartBar.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class={className}>
+	<path
+		fill-rule="evenodd"
+		d="M5.625 1.5H9a3.75 3.75 0 0 1 3.75 3.75v1.875c0 1.036.84 1.875 1.875 1.875H16.5a3.75 3.75 0 0 1 3.75 3.75v7.875c0 1.035-.84 1.875-1.875 1.875H5.625a1.875 1.875 0 0 1-1.875-1.875V3.375c0-1.036.84-1.875 1.875-1.875ZM9.75 17.25a.75.75 0 0 0-1.5 0V18a.75.75 0 0 0 1.5 0v-.75Zm2.25-3a.75.75 0 0 1 .75.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 .75-.75Zm3.75-1.5a.75.75 0 0 0-1.5 0V18a.75.75 0 0 0 1.5 0v-5.25Z"
+		clip-rule="evenodd"
+	/>
+	<path
+		d="M14.25 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 16.5 7.5h-1.875a.375.375 0 0 1-.375-.375V5.25Z"
+	/>
+</svg>

src/lib/components/icons/Minus.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+	export let className = 'w-4 h-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	fill="none"
+	viewBox="0 0 24 24"
+	stroke-width={strokeWidth}
+	stroke="currentColor"
+	class={className}
+>
+	<path stroke-linecap="round" stroke-linejoin="round" d="M5 12h14" />
+</svg>

src/lib/components/icons/PencilSolid.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+	export let className = 'w-4 h-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class={className}>
+	<path
+		d="M21.731 2.269a2.625 2.625 0 0 0-3.712 0l-1.157 1.157 3.712 3.712 1.157-1.157a2.625 2.625 0 0 0 0-3.712ZM19.513 8.199l-3.712-3.712-12.15 12.15a5.25 5.25 0 0 0-1.32 2.214l-.8 2.685a.75.75 0 0 0 .933.933l2.685-.8a5.25 5.25 0 0 0 2.214-1.32L19.513 8.2Z"
+	/>
+</svg>

src/lib/components/workspace/Models.svelte

@@ -243,7 +243,7 @@
 
 	onMount(async () => {
 		// Legacy code to sync localModelfiles with models
-		_models = $models;
+		_models = $models.filter((m) => m?.owned_by !== 'arena');
 		localModelfiles = JSON.parse(localStorage.getItem('modelfiles') ?? '[]');
 
 		if (localModelfiles) {

src/lib/stores/index.ts

@@ -58,7 +58,7 @@ type BaseModel = {
 	id: string;
 	name: string;
 	info?: ModelConfig;
-	owned_by: 'ollama' | 'openai';
+	owned_by: 'ollama' | 'openai' | 'arena';
 };
 
 export interface OpenAIModel extends BaseModel {

src/routes/(app)/admin/+layout.svelte

@@ -61,6 +61,15 @@
 							href="/admin">{$i18n.t('Dashboard')}</a
 						>
 
+						<a
+							class="min-w-fit rounded-full p-1.5 px-3 {$page.url.pathname.includes(
+								'/admin/evaluations'
+							)
+								? 'bg-gray-50 dark:bg-gray-850'
+								: ''} transition"
+							href="/admin/evaluations">{$i18n.t('Evaluations')}</a
+						>
+
 						<a
 							class="min-w-fit rounded-full p-1.5 px-3 {$page.url.pathname.includes(
 								'/admin/settings'

src/routes/(app)/admin/evaluations/+page.svelte

@@ -0,0 +1,5 @@
+<script>
+	import Evaluations from '$lib/components/admin/Evaluations.svelte';
+</script>
+
+<Evaluations />

src/routes/(app)/workspace/models/create/+page.svelte

@@ -184,7 +184,7 @@
 
 		if (model.info.base_model_id) {
 			const base_model = $models
-				.filter((m) => !m?.preset)
+				.filter((m) => !m?.preset && m?.owned_by !== 'arena')
 				.find((m) =>
 					[model.info.base_model_id, `${model.info.base_model_id}:latest`].includes(m.id)
 				);
@@ -451,7 +451,7 @@
 					required
 				>
 					<option value={null} class=" text-gray-900">{$i18n.t('Select a base model')}</option>
-					{#each $models.filter((m) => !m?.preset) as model}
+					{#each $models.filter((m) => !m?.preset && m?.owned_by !== 'arena') as model}
 						<option value={model.id} class=" text-gray-900">{model.name}</option>
 					{/each}
 				</select>

src/routes/(app)/workspace/models/edit/+page.svelte

@@ -139,7 +139,7 @@
 		const _id = $page.url.searchParams.get('id');
 
 		if (_id) {
-			model = $models.find((m) => m.id === _id);
+			model = $models.find((m) => m.id === _id && m?.owned_by !== 'arena');
 			if (model) {
 				id = model.id;
 				name = model.name;
@@ -395,7 +395,7 @@
 							required
 						>
 							<option value={null} class=" text-gray-900">{$i18n.t('Select a base model')}</option>
-							{#each $models.filter((m) => m.id !== model.id && !m?.preset) as model}
+							{#each $models.filter((m) => m.id !== model.id && !m?.preset && m?.owned_by !== 'arena') as model}
 								<option value={model.id} class=" text-gray-900">{model.name}</option>
 							{/each}
 						</select>