main.py 77 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396
  1. import base64
  2. import inspect
  3. import json
  4. import logging
  5. import mimetypes
  6. import os
  7. import shutil
  8. import sys
  9. import time
  10. import uuid
  11. import asyncio
  12. from contextlib import asynccontextmanager
  13. from typing import Optional
  14. import aiohttp
  15. import requests
  16. from open_webui.apps.audio.main import app as audio_app
  17. from open_webui.apps.images.main import app as images_app
  18. from open_webui.apps.ollama.main import app as ollama_app
  19. from open_webui.apps.ollama.main import (
  20. GenerateChatCompletionForm,
  21. generate_chat_completion as generate_ollama_chat_completion,
  22. generate_openai_chat_completion as generate_ollama_openai_chat_completion,
  23. )
  24. from open_webui.apps.ollama.main import get_all_models as get_ollama_models
  25. from open_webui.apps.openai.main import app as openai_app
  26. from open_webui.apps.openai.main import (
  27. generate_chat_completion as generate_openai_chat_completion,
  28. )
  29. from open_webui.apps.openai.main import get_all_models as get_openai_models
  30. from open_webui.apps.rag.main import app as rag_app
  31. from open_webui.apps.rag.utils import get_rag_context, rag_template
  32. from open_webui.apps.socket.main import app as socket_app, periodic_usage_pool_cleanup
  33. from open_webui.apps.socket.main import get_event_call, get_event_emitter
  34. from open_webui.apps.webui.internal.db import Session
  35. from open_webui.apps.webui.main import app as webui_app
  36. from open_webui.apps.webui.main import (
  37. generate_function_chat_completion,
  38. get_pipe_models,
  39. )
  40. from open_webui.apps.webui.models.auths import Auths
  41. from open_webui.apps.webui.models.functions import Functions
  42. from open_webui.apps.webui.models.models import Models
  43. from open_webui.apps.webui.models.users import UserModel, Users
  44. from open_webui.apps.webui.utils import load_function_module_by_id
  45. from authlib.integrations.starlette_client import OAuth
  46. from authlib.oidc.core import UserInfo
  47. from open_webui.config import (
  48. CACHE_DIR,
  49. CORS_ALLOW_ORIGIN,
  50. DEFAULT_LOCALE,
  51. ENABLE_ADMIN_CHAT_ACCESS,
  52. ENABLE_ADMIN_EXPORT,
  53. ENABLE_MODEL_FILTER,
  54. ENABLE_OAUTH_SIGNUP,
  55. ENABLE_OLLAMA_API,
  56. ENABLE_OPENAI_API,
  57. ENV,
  58. FRONTEND_BUILD_DIR,
  59. MODEL_FILTER_LIST,
  60. OAUTH_MERGE_ACCOUNTS_BY_EMAIL,
  61. OAUTH_PROVIDERS,
  62. ENABLE_SEARCH_QUERY,
  63. SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE,
  64. STATIC_DIR,
  65. TASK_MODEL,
  66. TASK_MODEL_EXTERNAL,
  67. TITLE_GENERATION_PROMPT_TEMPLATE,
  68. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  69. WEBHOOK_URL,
  70. WEBUI_AUTH,
  71. WEBUI_NAME,
  72. AppConfig,
  73. run_migrations,
  74. )
  75. from open_webui.constants import ERROR_MESSAGES, TASKS, WEBHOOK_MESSAGES
  76. from open_webui.env import (
  77. CHANGELOG,
  78. GLOBAL_LOG_LEVEL,
  79. SAFE_MODE,
  80. SRC_LOG_LEVELS,
  81. VERSION,
  82. WEBUI_BUILD_HASH,
  83. WEBUI_SECRET_KEY,
  84. WEBUI_SESSION_COOKIE_SAME_SITE,
  85. WEBUI_SESSION_COOKIE_SECURE,
  86. WEBUI_URL,
  87. )
  88. from fastapi import (
  89. Depends,
  90. FastAPI,
  91. File,
  92. Form,
  93. HTTPException,
  94. Request,
  95. UploadFile,
  96. status,
  97. )
  98. from fastapi.middleware.cors import CORSMiddleware
  99. from fastapi.responses import JSONResponse
  100. from fastapi.staticfiles import StaticFiles
  101. from pydantic import BaseModel
  102. from sqlalchemy import text
  103. from starlette.exceptions import HTTPException as StarletteHTTPException
  104. from starlette.middleware.base import BaseHTTPMiddleware
  105. from starlette.middleware.sessions import SessionMiddleware
  106. from starlette.responses import RedirectResponse, Response, StreamingResponse
  107. from open_webui.utils.security_headers import SecurityHeadersMiddleware
  108. from open_webui.utils.misc import (
  109. add_or_update_system_message,
  110. get_last_user_message,
  111. parse_duration,
  112. prepend_to_first_user_message_content,
  113. )
  114. from open_webui.utils.task import (
  115. moa_response_generation_template,
  116. search_query_generation_template,
  117. title_generation_template,
  118. tools_function_calling_generation_template,
  119. )
  120. from open_webui.utils.tools import get_tools
  121. from open_webui.utils.utils import (
  122. create_token,
  123. decode_token,
  124. get_admin_user,
  125. get_current_user,
  126. get_http_authorization_cred,
  127. get_password_hash,
  128. get_verified_user,
  129. )
  130. from open_webui.utils.webhook import post_webhook
  131. from open_webui.utils.payload import convert_payload_openai_to_ollama
  132. from open_webui.utils.response import (
  133. convert_response_ollama_to_openai,
  134. convert_streaming_response_ollama_to_openai,
  135. )
  136. if SAFE_MODE:
  137. print("SAFE MODE ENABLED")
  138. Functions.deactivate_all_functions()
  139. logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
  140. log = logging.getLogger(__name__)
  141. log.setLevel(SRC_LOG_LEVELS["MAIN"])
  142. class SPAStaticFiles(StaticFiles):
  143. async def get_response(self, path: str, scope):
  144. try:
  145. return await super().get_response(path, scope)
  146. except (HTTPException, StarletteHTTPException) as ex:
  147. if ex.status_code == 404:
  148. return await super().get_response("index.html", scope)
  149. else:
  150. raise ex
  151. print(
  152. rf"""
  153. ___ __ __ _ _ _ ___
  154. / _ \ _ __ ___ _ __ \ \ / /__| |__ | | | |_ _|
  155. | | | | '_ \ / _ \ '_ \ \ \ /\ / / _ \ '_ \| | | || |
  156. | |_| | |_) | __/ | | | \ V V / __/ |_) | |_| || |
  157. \___/| .__/ \___|_| |_| \_/\_/ \___|_.__/ \___/|___|
  158. |_|
  159. v{VERSION} - building the best open-source AI user interface.
  160. {f"Commit: {WEBUI_BUILD_HASH}" if WEBUI_BUILD_HASH != "dev-build" else ""}
  161. https://github.com/open-webui/open-webui
  162. """
  163. )
  164. @asynccontextmanager
  165. async def lifespan(app: FastAPI):
  166. run_migrations()
  167. asyncio.create_task(periodic_usage_pool_cleanup())
  168. yield
  169. app = FastAPI(
  170. docs_url="/docs" if ENV == "dev" else None, redoc_url=None, lifespan=lifespan
  171. )
  172. app.state.config = AppConfig()
  173. app.state.config.ENABLE_OPENAI_API = ENABLE_OPENAI_API
  174. app.state.config.ENABLE_OLLAMA_API = ENABLE_OLLAMA_API
  175. app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER
  176. app.state.config.MODEL_FILTER_LIST = MODEL_FILTER_LIST
  177. app.state.config.WEBHOOK_URL = WEBHOOK_URL
  178. app.state.config.TASK_MODEL = TASK_MODEL
  179. app.state.config.TASK_MODEL_EXTERNAL = TASK_MODEL_EXTERNAL
  180. app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE = TITLE_GENERATION_PROMPT_TEMPLATE
  181. app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE = (
  182. SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE
  183. )
  184. app.state.config.ENABLE_SEARCH_QUERY = ENABLE_SEARCH_QUERY
  185. app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE = (
  186. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  187. )
  188. app.state.MODELS = {}
  189. ##################################
  190. #
  191. # ChatCompletion Middleware
  192. #
  193. ##################################
  194. def get_task_model_id(default_model_id):
  195. # Set the task model
  196. task_model_id = default_model_id
  197. # Check if the user has a custom task model and use that model
  198. if app.state.MODELS[task_model_id]["owned_by"] == "ollama":
  199. if (
  200. app.state.config.TASK_MODEL
  201. and app.state.config.TASK_MODEL in app.state.MODELS
  202. ):
  203. task_model_id = app.state.config.TASK_MODEL
  204. else:
  205. if (
  206. app.state.config.TASK_MODEL_EXTERNAL
  207. and app.state.config.TASK_MODEL_EXTERNAL in app.state.MODELS
  208. ):
  209. task_model_id = app.state.config.TASK_MODEL_EXTERNAL
  210. return task_model_id
  211. def get_filter_function_ids(model):
  212. def get_priority(function_id):
  213. function = Functions.get_function_by_id(function_id)
  214. if function is not None and hasattr(function, "valves"):
  215. # TODO: Fix FunctionModel
  216. return (function.valves if function.valves else {}).get("priority", 0)
  217. return 0
  218. filter_ids = [function.id for function in Functions.get_global_filter_functions()]
  219. if "info" in model and "meta" in model["info"]:
  220. filter_ids.extend(model["info"]["meta"].get("filterIds", []))
  221. filter_ids = list(set(filter_ids))
  222. enabled_filter_ids = [
  223. function.id
  224. for function in Functions.get_functions_by_type("filter", active_only=True)
  225. ]
  226. filter_ids = [
  227. filter_id for filter_id in filter_ids if filter_id in enabled_filter_ids
  228. ]
  229. filter_ids.sort(key=get_priority)
  230. return filter_ids
  231. async def chat_completion_filter_functions_handler(body, model, extra_params):
  232. skip_files = None
  233. filter_ids = get_filter_function_ids(model)
  234. for filter_id in filter_ids:
  235. filter = Functions.get_function_by_id(filter_id)
  236. if not filter:
  237. continue
  238. if filter_id in webui_app.state.FUNCTIONS:
  239. function_module = webui_app.state.FUNCTIONS[filter_id]
  240. else:
  241. function_module, _, _ = load_function_module_by_id(filter_id)
  242. webui_app.state.FUNCTIONS[filter_id] = function_module
  243. # Check if the function has a file_handler variable
  244. if hasattr(function_module, "file_handler"):
  245. skip_files = function_module.file_handler
  246. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  247. valves = Functions.get_function_valves_by_id(filter_id)
  248. function_module.valves = function_module.Valves(
  249. **(valves if valves else {})
  250. )
  251. if not hasattr(function_module, "inlet"):
  252. continue
  253. try:
  254. inlet = function_module.inlet
  255. # Get the signature of the function
  256. sig = inspect.signature(inlet)
  257. params = {"body": body} | {
  258. k: v
  259. for k, v in {
  260. **extra_params,
  261. "__model__": model,
  262. "__id__": filter_id,
  263. }.items()
  264. if k in sig.parameters
  265. }
  266. if "__user__" in params and hasattr(function_module, "UserValves"):
  267. try:
  268. params["__user__"]["valves"] = function_module.UserValves(
  269. **Functions.get_user_valves_by_id_and_user_id(
  270. filter_id, params["__user__"]["id"]
  271. )
  272. )
  273. except Exception as e:
  274. print(e)
  275. if inspect.iscoroutinefunction(inlet):
  276. body = await inlet(**params)
  277. else:
  278. body = inlet(**params)
  279. except Exception as e:
  280. print(f"Error: {e}")
  281. raise e
  282. if skip_files and "files" in body.get("metadata", {}):
  283. del body["metadata"]["files"]
  284. return body, {}
  285. def get_tools_function_calling_payload(messages, task_model_id, content):
  286. user_message = get_last_user_message(messages)
  287. history = "\n".join(
  288. f"{message['role'].upper()}: \"\"\"{message['content']}\"\"\""
  289. for message in messages[::-1][:4]
  290. )
  291. prompt = f"History:\n{history}\nQuery: {user_message}"
  292. return {
  293. "model": task_model_id,
  294. "messages": [
  295. {"role": "system", "content": content},
  296. {"role": "user", "content": f"Query: {prompt}"},
  297. ],
  298. "stream": False,
  299. "metadata": {"task": str(TASKS.FUNCTION_CALLING)},
  300. }
  301. async def get_content_from_response(response) -> Optional[str]:
  302. content = None
  303. if hasattr(response, "body_iterator"):
  304. async for chunk in response.body_iterator:
  305. data = json.loads(chunk.decode("utf-8"))
  306. content = data["choices"][0]["message"]["content"]
  307. # Cleanup any remaining background tasks if necessary
  308. if response.background is not None:
  309. await response.background()
  310. else:
  311. content = response["choices"][0]["message"]["content"]
  312. return content
  313. async def chat_completion_tools_handler(
  314. body: dict, user: UserModel, extra_params: dict
  315. ) -> tuple[dict, dict]:
  316. # If tool_ids field is present, call the functions
  317. metadata = body.get("metadata", {})
  318. tool_ids = metadata.get("tool_ids", None)
  319. log.debug(f"{tool_ids=}")
  320. if not tool_ids:
  321. return body, {}
  322. skip_files = False
  323. contexts = []
  324. citations = []
  325. task_model_id = get_task_model_id(body["model"])
  326. tools = get_tools(
  327. webui_app,
  328. tool_ids,
  329. user,
  330. {
  331. **extra_params,
  332. "__model__": app.state.MODELS[task_model_id],
  333. "__messages__": body["messages"],
  334. "__files__": metadata.get("files", []),
  335. },
  336. )
  337. log.info(f"{tools=}")
  338. specs = [tool["spec"] for tool in tools.values()]
  339. tools_specs = json.dumps(specs)
  340. if app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE != "":
  341. template = app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  342. else:
  343. template = """Available Tools: {{TOOLS}}\nReturn an empty string if no tools match the query. If a function tool matches, construct and return a JSON object in the format {\"name\": \"functionName\", \"parameters\": {\"requiredFunctionParamKey\": \"requiredFunctionParamValue\"}} using the appropriate tool and its parameters. Only return the object and limit the response to the JSON object without additional text."""
  344. tools_function_calling_prompt = tools_function_calling_generation_template(
  345. template, tools_specs
  346. )
  347. log.info(f"{tools_function_calling_prompt=}")
  348. payload = get_tools_function_calling_payload(
  349. body["messages"], task_model_id, tools_function_calling_prompt
  350. )
  351. try:
  352. payload = filter_pipeline(payload, user)
  353. except Exception as e:
  354. raise e
  355. try:
  356. response = await generate_chat_completions(form_data=payload, user=user)
  357. log.debug(f"{response=}")
  358. content = await get_content_from_response(response)
  359. log.debug(f"{content=}")
  360. if not content:
  361. return body, {}
  362. result = json.loads(content)
  363. tool_function_name = result.get("name", None)
  364. if tool_function_name not in tools:
  365. return body, {}
  366. tool_function_params = result.get("parameters", {})
  367. try:
  368. tool_output = await tools[tool_function_name]["callable"](
  369. **tool_function_params
  370. )
  371. except Exception as e:
  372. tool_output = str(e)
  373. if tools[tool_function_name]["citation"]:
  374. citations.append(
  375. {
  376. "source": {
  377. "name": f"TOOL:{tools[tool_function_name]['toolkit_id']}/{tool_function_name}"
  378. },
  379. "document": [tool_output],
  380. "metadata": [{"source": tool_function_name}],
  381. }
  382. )
  383. if tools[tool_function_name]["file_handler"]:
  384. skip_files = True
  385. if isinstance(tool_output, str):
  386. contexts.append(tool_output)
  387. except Exception as e:
  388. log.exception(f"Error: {e}")
  389. content = None
  390. log.debug(f"tool_contexts: {contexts}")
  391. if skip_files and "files" in body.get("metadata", {}):
  392. del body["metadata"]["files"]
  393. return body, {"contexts": contexts, "citations": citations}
  394. async def chat_completion_files_handler(body) -> tuple[dict, dict[str, list]]:
  395. contexts = []
  396. citations = []
  397. if files := body.get("metadata", {}).get("files", None):
  398. contexts, citations = get_rag_context(
  399. files=files,
  400. messages=body["messages"],
  401. embedding_function=rag_app.state.EMBEDDING_FUNCTION,
  402. k=rag_app.state.config.TOP_K,
  403. reranking_function=rag_app.state.sentence_transformer_rf,
  404. r=rag_app.state.config.RELEVANCE_THRESHOLD,
  405. hybrid_search=rag_app.state.config.ENABLE_RAG_HYBRID_SEARCH,
  406. )
  407. log.debug(f"rag_contexts: {contexts}, citations: {citations}")
  408. return body, {"contexts": contexts, "citations": citations}
  409. def is_chat_completion_request(request):
  410. return request.method == "POST" and any(
  411. endpoint in request.url.path
  412. for endpoint in ["/ollama/api/chat", "/chat/completions"]
  413. )
  414. async def get_body_and_model_and_user(request):
  415. # Read the original request body
  416. body = await request.body()
  417. body_str = body.decode("utf-8")
  418. body = json.loads(body_str) if body_str else {}
  419. model_id = body["model"]
  420. if model_id not in app.state.MODELS:
  421. raise Exception("Model not found")
  422. model = app.state.MODELS[model_id]
  423. user = get_current_user(
  424. request,
  425. get_http_authorization_cred(request.headers.get("Authorization")),
  426. )
  427. return body, model, user
  428. class ChatCompletionMiddleware(BaseHTTPMiddleware):
  429. async def dispatch(self, request: Request, call_next):
  430. if not is_chat_completion_request(request):
  431. return await call_next(request)
  432. log.debug(f"request.url.path: {request.url.path}")
  433. try:
  434. body, model, user = await get_body_and_model_and_user(request)
  435. except Exception as e:
  436. return JSONResponse(
  437. status_code=status.HTTP_400_BAD_REQUEST,
  438. content={"detail": str(e)},
  439. )
  440. metadata = {
  441. "chat_id": body.pop("chat_id", None),
  442. "message_id": body.pop("id", None),
  443. "session_id": body.pop("session_id", None),
  444. "tool_ids": body.get("tool_ids", None),
  445. "files": body.get("files", None),
  446. }
  447. body["metadata"] = metadata
  448. extra_params = {
  449. "__event_emitter__": get_event_emitter(metadata),
  450. "__event_call__": get_event_call(metadata),
  451. "__user__": {
  452. "id": user.id,
  453. "email": user.email,
  454. "name": user.name,
  455. "role": user.role,
  456. },
  457. }
  458. # Initialize data_items to store additional data to be sent to the client
  459. # Initalize contexts and citation
  460. data_items = []
  461. contexts = []
  462. citations = []
  463. try:
  464. body, flags = await chat_completion_filter_functions_handler(
  465. body, model, extra_params
  466. )
  467. except Exception as e:
  468. return JSONResponse(
  469. status_code=status.HTTP_400_BAD_REQUEST,
  470. content={"detail": str(e)},
  471. )
  472. metadata = {
  473. **metadata,
  474. "tool_ids": body.pop("tool_ids", None),
  475. "files": body.pop("files", None),
  476. }
  477. body["metadata"] = metadata
  478. try:
  479. body, flags = await chat_completion_tools_handler(body, user, extra_params)
  480. contexts.extend(flags.get("contexts", []))
  481. citations.extend(flags.get("citations", []))
  482. except Exception as e:
  483. log.exception(e)
  484. try:
  485. body, flags = await chat_completion_files_handler(body)
  486. contexts.extend(flags.get("contexts", []))
  487. citations.extend(flags.get("citations", []))
  488. except Exception as e:
  489. log.exception(e)
  490. # If context is not empty, insert it into the messages
  491. if len(contexts) > 0:
  492. context_string = "/n".join(contexts).strip()
  493. prompt = get_last_user_message(body["messages"])
  494. if prompt is None:
  495. raise Exception("No user message found")
  496. if (
  497. rag_app.state.config.RELEVANCE_THRESHOLD == 0
  498. and context_string.strip() == ""
  499. ):
  500. log.debug(
  501. f"With a 0 relevancy threshold for RAG, the context cannot be empty"
  502. )
  503. # Workaround for Ollama 2.0+ system prompt issue
  504. # TODO: replace with add_or_update_system_message
  505. if model["owned_by"] == "ollama":
  506. body["messages"] = prepend_to_first_user_message_content(
  507. rag_template(
  508. rag_app.state.config.RAG_TEMPLATE, context_string, prompt
  509. ),
  510. body["messages"],
  511. )
  512. else:
  513. body["messages"] = add_or_update_system_message(
  514. rag_template(
  515. rag_app.state.config.RAG_TEMPLATE, context_string, prompt
  516. ),
  517. body["messages"],
  518. )
  519. # If there are citations, add them to the data_items
  520. if len(citations) > 0:
  521. data_items.append({"citations": citations})
  522. modified_body_bytes = json.dumps(body).encode("utf-8")
  523. # Replace the request body with the modified one
  524. request._body = modified_body_bytes
  525. # Set custom header to ensure content-length matches new body length
  526. request.headers.__dict__["_list"] = [
  527. (b"content-length", str(len(modified_body_bytes)).encode("utf-8")),
  528. *[(k, v) for k, v in request.headers.raw if k.lower() != b"content-length"],
  529. ]
  530. response = await call_next(request)
  531. if not isinstance(response, StreamingResponse):
  532. return response
  533. content_type = response.headers["Content-Type"]
  534. is_openai = "text/event-stream" in content_type
  535. is_ollama = "application/x-ndjson" in content_type
  536. if not is_openai and not is_ollama:
  537. return response
  538. def wrap_item(item):
  539. return f"data: {item}\n\n" if is_openai else f"{item}\n"
  540. async def stream_wrapper(original_generator, data_items):
  541. for item in data_items:
  542. yield wrap_item(json.dumps(item))
  543. async for data in original_generator:
  544. yield data
  545. return StreamingResponse(
  546. stream_wrapper(response.body_iterator, data_items),
  547. headers=dict(response.headers),
  548. )
  549. async def _receive(self, body: bytes):
  550. return {"type": "http.request", "body": body, "more_body": False}
  551. app.add_middleware(ChatCompletionMiddleware)
  552. ##################################
  553. #
  554. # Pipeline Middleware
  555. #
  556. ##################################
  557. def get_sorted_filters(model_id):
  558. filters = [
  559. model
  560. for model in app.state.MODELS.values()
  561. if "pipeline" in model
  562. and "type" in model["pipeline"]
  563. and model["pipeline"]["type"] == "filter"
  564. and (
  565. model["pipeline"]["pipelines"] == ["*"]
  566. or any(
  567. model_id == target_model_id
  568. for target_model_id in model["pipeline"]["pipelines"]
  569. )
  570. )
  571. ]
  572. sorted_filters = sorted(filters, key=lambda x: x["pipeline"]["priority"])
  573. return sorted_filters
  574. def filter_pipeline(payload, user):
  575. user = {"id": user.id, "email": user.email, "name": user.name, "role": user.role}
  576. model_id = payload["model"]
  577. sorted_filters = get_sorted_filters(model_id)
  578. model = app.state.MODELS[model_id]
  579. if "pipeline" in model:
  580. sorted_filters.append(model)
  581. for filter in sorted_filters:
  582. r = None
  583. try:
  584. urlIdx = filter["urlIdx"]
  585. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  586. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  587. if key == "":
  588. continue
  589. headers = {"Authorization": f"Bearer {key}"}
  590. r = requests.post(
  591. f"{url}/{filter['id']}/filter/inlet",
  592. headers=headers,
  593. json={
  594. "user": user,
  595. "body": payload,
  596. },
  597. )
  598. r.raise_for_status()
  599. payload = r.json()
  600. except Exception as e:
  601. # Handle connection error here
  602. print(f"Connection error: {e}")
  603. if r is not None:
  604. res = r.json()
  605. if "detail" in res:
  606. raise Exception(r.status_code, res["detail"])
  607. return payload
  608. class PipelineMiddleware(BaseHTTPMiddleware):
  609. async def dispatch(self, request: Request, call_next):
  610. if not is_chat_completion_request(request):
  611. return await call_next(request)
  612. log.debug(f"request.url.path: {request.url.path}")
  613. # Read the original request body
  614. body = await request.body()
  615. # Decode body to string
  616. body_str = body.decode("utf-8")
  617. # Parse string to JSON
  618. data = json.loads(body_str) if body_str else {}
  619. user = get_current_user(
  620. request,
  621. get_http_authorization_cred(request.headers["Authorization"]),
  622. )
  623. try:
  624. data = filter_pipeline(data, user)
  625. except Exception as e:
  626. if len(e.args) > 1:
  627. return JSONResponse(
  628. status_code=e.args[0],
  629. content={"detail": e.args[1]},
  630. )
  631. else:
  632. return JSONResponse(
  633. status_code=status.HTTP_400_BAD_REQUEST,
  634. content={"detail": str(e)},
  635. )
  636. modified_body_bytes = json.dumps(data).encode("utf-8")
  637. # Replace the request body with the modified one
  638. request._body = modified_body_bytes
  639. # Set custom header to ensure content-length matches new body length
  640. request.headers.__dict__["_list"] = [
  641. (b"content-length", str(len(modified_body_bytes)).encode("utf-8")),
  642. *[(k, v) for k, v in request.headers.raw if k.lower() != b"content-length"],
  643. ]
  644. response = await call_next(request)
  645. return response
  646. async def _receive(self, body: bytes):
  647. return {"type": "http.request", "body": body, "more_body": False}
  648. app.add_middleware(PipelineMiddleware)
  649. app.add_middleware(
  650. CORSMiddleware,
  651. allow_origins=CORS_ALLOW_ORIGIN,
  652. allow_credentials=True,
  653. allow_methods=["*"],
  654. allow_headers=["*"],
  655. )
  656. app.add_middleware(SecurityHeadersMiddleware)
  657. @app.middleware("http")
  658. async def commit_session_after_request(request: Request, call_next):
  659. response = await call_next(request)
  660. log.debug("Commit session after request")
  661. Session.commit()
  662. return response
  663. @app.middleware("http")
  664. async def check_url(request: Request, call_next):
  665. if len(app.state.MODELS) == 0:
  666. await get_all_models()
  667. else:
  668. pass
  669. start_time = int(time.time())
  670. response = await call_next(request)
  671. process_time = int(time.time()) - start_time
  672. response.headers["X-Process-Time"] = str(process_time)
  673. return response
  674. @app.middleware("http")
  675. async def update_embedding_function(request: Request, call_next):
  676. response = await call_next(request)
  677. if "/embedding/update" in request.url.path:
  678. webui_app.state.EMBEDDING_FUNCTION = rag_app.state.EMBEDDING_FUNCTION
  679. return response
  680. @app.middleware("http")
  681. async def inspect_websocket(request: Request, call_next):
  682. if (
  683. "/ws/socket.io" in request.url.path
  684. and request.query_params.get("transport") == "websocket"
  685. ):
  686. upgrade = (request.headers.get("Upgrade") or "").lower()
  687. connection = (request.headers.get("Connection") or "").lower().split(",")
  688. # Check that there's the correct headers for an upgrade, else reject the connection
  689. # This is to work around this upstream issue: https://github.com/miguelgrinberg/python-engineio/issues/367
  690. if upgrade != "websocket" or "upgrade" not in connection:
  691. return JSONResponse(
  692. status_code=status.HTTP_400_BAD_REQUEST,
  693. content={"detail": "Invalid WebSocket upgrade request"},
  694. )
  695. return await call_next(request)
  696. app.mount("/ws", socket_app)
  697. app.mount("/ollama", ollama_app)
  698. app.mount("/openai", openai_app)
  699. app.mount("/images/api/v1", images_app)
  700. app.mount("/audio/api/v1", audio_app)
  701. app.mount("/rag/api/v1", rag_app)
  702. app.mount("/api/v1", webui_app)
  703. webui_app.state.EMBEDDING_FUNCTION = rag_app.state.EMBEDDING_FUNCTION
  704. async def get_all_models():
  705. # TODO: Optimize this function
  706. pipe_models = []
  707. openai_models = []
  708. ollama_models = []
  709. pipe_models = await get_pipe_models()
  710. if app.state.config.ENABLE_OPENAI_API:
  711. openai_models = await get_openai_models()
  712. openai_models = openai_models["data"]
  713. if app.state.config.ENABLE_OLLAMA_API:
  714. ollama_models = await get_ollama_models()
  715. ollama_models = [
  716. {
  717. "id": model["model"],
  718. "name": model["name"],
  719. "object": "model",
  720. "created": int(time.time()),
  721. "owned_by": "ollama",
  722. "ollama": model,
  723. }
  724. for model in ollama_models["models"]
  725. ]
  726. models = pipe_models + openai_models + ollama_models
  727. global_action_ids = [
  728. function.id for function in Functions.get_global_action_functions()
  729. ]
  730. enabled_action_ids = [
  731. function.id
  732. for function in Functions.get_functions_by_type("action", active_only=True)
  733. ]
  734. custom_models = Models.get_all_models()
  735. for custom_model in custom_models:
  736. if custom_model.base_model_id is None:
  737. for model in models:
  738. if (
  739. custom_model.id == model["id"]
  740. or custom_model.id == model["id"].split(":")[0]
  741. ):
  742. model["name"] = custom_model.name
  743. model["info"] = custom_model.model_dump()
  744. action_ids = []
  745. if "info" in model and "meta" in model["info"]:
  746. action_ids.extend(model["info"]["meta"].get("actionIds", []))
  747. model["action_ids"] = action_ids
  748. else:
  749. owned_by = "openai"
  750. pipe = None
  751. action_ids = []
  752. for model in models:
  753. if (
  754. custom_model.base_model_id == model["id"]
  755. or custom_model.base_model_id == model["id"].split(":")[0]
  756. ):
  757. owned_by = model["owned_by"]
  758. if "pipe" in model:
  759. pipe = model["pipe"]
  760. if "info" in model and "meta" in model["info"]:
  761. action_ids.extend(model["info"]["meta"].get("actionIds", []))
  762. break
  763. models.append(
  764. {
  765. "id": custom_model.id,
  766. "name": custom_model.name,
  767. "object": "model",
  768. "created": custom_model.created_at,
  769. "owned_by": owned_by,
  770. "info": custom_model.model_dump(),
  771. "preset": True,
  772. **({"pipe": pipe} if pipe is not None else {}),
  773. "action_ids": action_ids,
  774. }
  775. )
  776. for model in models:
  777. action_ids = []
  778. if "action_ids" in model:
  779. action_ids = model["action_ids"]
  780. del model["action_ids"]
  781. action_ids = action_ids + global_action_ids
  782. action_ids = list(set(action_ids))
  783. action_ids = [
  784. action_id for action_id in action_ids if action_id in enabled_action_ids
  785. ]
  786. model["actions"] = []
  787. for action_id in action_ids:
  788. action = Functions.get_function_by_id(action_id)
  789. if action is None:
  790. raise Exception(f"Action not found: {action_id}")
  791. if action_id in webui_app.state.FUNCTIONS:
  792. function_module = webui_app.state.FUNCTIONS[action_id]
  793. else:
  794. function_module, _, _ = load_function_module_by_id(action_id)
  795. webui_app.state.FUNCTIONS[action_id] = function_module
  796. __webui__ = False
  797. if hasattr(function_module, "__webui__"):
  798. __webui__ = function_module.__webui__
  799. if hasattr(function_module, "actions"):
  800. actions = function_module.actions
  801. model["actions"].extend(
  802. [
  803. {
  804. "id": f"{action_id}.{_action['id']}",
  805. "name": _action.get(
  806. "name", f"{action.name} ({_action['id']})"
  807. ),
  808. "description": action.meta.description,
  809. "icon_url": _action.get(
  810. "icon_url", action.meta.manifest.get("icon_url", None)
  811. ),
  812. **({"__webui__": __webui__} if __webui__ else {}),
  813. }
  814. for _action in actions
  815. ]
  816. )
  817. else:
  818. model["actions"].append(
  819. {
  820. "id": action_id,
  821. "name": action.name,
  822. "description": action.meta.description,
  823. "icon_url": action.meta.manifest.get("icon_url", None),
  824. **({"__webui__": __webui__} if __webui__ else {}),
  825. }
  826. )
  827. app.state.MODELS = {model["id"]: model for model in models}
  828. webui_app.state.MODELS = app.state.MODELS
  829. return models
  830. @app.get("/api/models")
  831. async def get_models(user=Depends(get_verified_user)):
  832. models = await get_all_models()
  833. # Filter out filter pipelines
  834. models = [
  835. model
  836. for model in models
  837. if "pipeline" not in model or model["pipeline"].get("type", None) != "filter"
  838. ]
  839. if app.state.config.ENABLE_MODEL_FILTER:
  840. if user.role == "user":
  841. models = list(
  842. filter(
  843. lambda model: model["id"] in app.state.config.MODEL_FILTER_LIST,
  844. models,
  845. )
  846. )
  847. return {"data": models}
  848. return {"data": models}
  849. @app.post("/api/chat/completions")
  850. async def generate_chat_completions(form_data: dict, user=Depends(get_verified_user)):
  851. model_id = form_data["model"]
  852. if model_id not in app.state.MODELS:
  853. raise HTTPException(
  854. status_code=status.HTTP_404_NOT_FOUND,
  855. detail="Model not found",
  856. )
  857. if app.state.config.ENABLE_MODEL_FILTER:
  858. if user.role == "user" and model_id not in app.state.config.MODEL_FILTER_LIST:
  859. raise HTTPException(
  860. status_code=status.HTTP_403_FORBIDDEN,
  861. detail="Model not found",
  862. )
  863. model = app.state.MODELS[model_id]
  864. if model.get("pipe"):
  865. return await generate_function_chat_completion(form_data, user=user)
  866. if model["owned_by"] == "ollama":
  867. # Using /ollama/api/chat endpoint
  868. form_data = convert_payload_openai_to_ollama(form_data)
  869. form_data = GenerateChatCompletionForm(**form_data)
  870. response = await generate_ollama_chat_completion(form_data=form_data, user=user)
  871. if form_data.stream:
  872. response.headers["content-type"] = "text/event-stream"
  873. return StreamingResponse(
  874. convert_streaming_response_ollama_to_openai(response),
  875. headers=dict(response.headers),
  876. )
  877. else:
  878. return convert_response_ollama_to_openai(response)
  879. else:
  880. return await generate_openai_chat_completion(form_data, user=user)
  881. @app.post("/api/chat/completed")
  882. async def chat_completed(form_data: dict, user=Depends(get_verified_user)):
  883. data = form_data
  884. model_id = data["model"]
  885. if model_id not in app.state.MODELS:
  886. raise HTTPException(
  887. status_code=status.HTTP_404_NOT_FOUND,
  888. detail="Model not found",
  889. )
  890. model = app.state.MODELS[model_id]
  891. sorted_filters = get_sorted_filters(model_id)
  892. if "pipeline" in model:
  893. sorted_filters = [model] + sorted_filters
  894. for filter in sorted_filters:
  895. r = None
  896. try:
  897. urlIdx = filter["urlIdx"]
  898. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  899. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  900. if key != "":
  901. headers = {"Authorization": f"Bearer {key}"}
  902. r = requests.post(
  903. f"{url}/{filter['id']}/filter/outlet",
  904. headers=headers,
  905. json={
  906. "user": {
  907. "id": user.id,
  908. "name": user.name,
  909. "email": user.email,
  910. "role": user.role,
  911. },
  912. "body": data,
  913. },
  914. )
  915. r.raise_for_status()
  916. data = r.json()
  917. except Exception as e:
  918. # Handle connection error here
  919. print(f"Connection error: {e}")
  920. if r is not None:
  921. try:
  922. res = r.json()
  923. if "detail" in res:
  924. return JSONResponse(
  925. status_code=r.status_code,
  926. content=res,
  927. )
  928. except Exception:
  929. pass
  930. else:
  931. pass
  932. __event_emitter__ = get_event_emitter(
  933. {
  934. "chat_id": data["chat_id"],
  935. "message_id": data["id"],
  936. "session_id": data["session_id"],
  937. }
  938. )
  939. __event_call__ = get_event_call(
  940. {
  941. "chat_id": data["chat_id"],
  942. "message_id": data["id"],
  943. "session_id": data["session_id"],
  944. }
  945. )
  946. def get_priority(function_id):
  947. function = Functions.get_function_by_id(function_id)
  948. if function is not None and hasattr(function, "valves"):
  949. # TODO: Fix FunctionModel to include vavles
  950. return (function.valves if function.valves else {}).get("priority", 0)
  951. return 0
  952. filter_ids = [function.id for function in Functions.get_global_filter_functions()]
  953. if "info" in model and "meta" in model["info"]:
  954. filter_ids.extend(model["info"]["meta"].get("filterIds", []))
  955. filter_ids = list(set(filter_ids))
  956. enabled_filter_ids = [
  957. function.id
  958. for function in Functions.get_functions_by_type("filter", active_only=True)
  959. ]
  960. filter_ids = [
  961. filter_id for filter_id in filter_ids if filter_id in enabled_filter_ids
  962. ]
  963. # Sort filter_ids by priority, using the get_priority function
  964. filter_ids.sort(key=get_priority)
  965. for filter_id in filter_ids:
  966. filter = Functions.get_function_by_id(filter_id)
  967. if not filter:
  968. continue
  969. if filter_id in webui_app.state.FUNCTIONS:
  970. function_module = webui_app.state.FUNCTIONS[filter_id]
  971. else:
  972. function_module, _, _ = load_function_module_by_id(filter_id)
  973. webui_app.state.FUNCTIONS[filter_id] = function_module
  974. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  975. valves = Functions.get_function_valves_by_id(filter_id)
  976. function_module.valves = function_module.Valves(
  977. **(valves if valves else {})
  978. )
  979. if not hasattr(function_module, "outlet"):
  980. continue
  981. try:
  982. outlet = function_module.outlet
  983. # Get the signature of the function
  984. sig = inspect.signature(outlet)
  985. params = {"body": data}
  986. # Extra parameters to be passed to the function
  987. extra_params = {
  988. "__model__": model,
  989. "__id__": filter_id,
  990. "__event_emitter__": __event_emitter__,
  991. "__event_call__": __event_call__,
  992. }
  993. # Add extra params in contained in function signature
  994. for key, value in extra_params.items():
  995. if key in sig.parameters:
  996. params[key] = value
  997. if "__user__" in sig.parameters:
  998. __user__ = {
  999. "id": user.id,
  1000. "email": user.email,
  1001. "name": user.name,
  1002. "role": user.role,
  1003. }
  1004. try:
  1005. if hasattr(function_module, "UserValves"):
  1006. __user__["valves"] = function_module.UserValves(
  1007. **Functions.get_user_valves_by_id_and_user_id(
  1008. filter_id, user.id
  1009. )
  1010. )
  1011. except Exception as e:
  1012. print(e)
  1013. params = {**params, "__user__": __user__}
  1014. if inspect.iscoroutinefunction(outlet):
  1015. data = await outlet(**params)
  1016. else:
  1017. data = outlet(**params)
  1018. except Exception as e:
  1019. print(f"Error: {e}")
  1020. return JSONResponse(
  1021. status_code=status.HTTP_400_BAD_REQUEST,
  1022. content={"detail": str(e)},
  1023. )
  1024. return data
  1025. @app.post("/api/chat/actions/{action_id}")
  1026. async def chat_action(action_id: str, form_data: dict, user=Depends(get_verified_user)):
  1027. if "." in action_id:
  1028. action_id, sub_action_id = action_id.split(".")
  1029. else:
  1030. sub_action_id = None
  1031. action = Functions.get_function_by_id(action_id)
  1032. if not action:
  1033. raise HTTPException(
  1034. status_code=status.HTTP_404_NOT_FOUND,
  1035. detail="Action not found",
  1036. )
  1037. data = form_data
  1038. model_id = data["model"]
  1039. if model_id not in app.state.MODELS:
  1040. raise HTTPException(
  1041. status_code=status.HTTP_404_NOT_FOUND,
  1042. detail="Model not found",
  1043. )
  1044. model = app.state.MODELS[model_id]
  1045. __event_emitter__ = get_event_emitter(
  1046. {
  1047. "chat_id": data["chat_id"],
  1048. "message_id": data["id"],
  1049. "session_id": data["session_id"],
  1050. }
  1051. )
  1052. __event_call__ = get_event_call(
  1053. {
  1054. "chat_id": data["chat_id"],
  1055. "message_id": data["id"],
  1056. "session_id": data["session_id"],
  1057. }
  1058. )
  1059. if action_id in webui_app.state.FUNCTIONS:
  1060. function_module = webui_app.state.FUNCTIONS[action_id]
  1061. else:
  1062. function_module, _, _ = load_function_module_by_id(action_id)
  1063. webui_app.state.FUNCTIONS[action_id] = function_module
  1064. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  1065. valves = Functions.get_function_valves_by_id(action_id)
  1066. function_module.valves = function_module.Valves(**(valves if valves else {}))
  1067. if hasattr(function_module, "action"):
  1068. try:
  1069. action = function_module.action
  1070. # Get the signature of the function
  1071. sig = inspect.signature(action)
  1072. params = {"body": data}
  1073. # Extra parameters to be passed to the function
  1074. extra_params = {
  1075. "__model__": model,
  1076. "__id__": sub_action_id if sub_action_id is not None else action_id,
  1077. "__event_emitter__": __event_emitter__,
  1078. "__event_call__": __event_call__,
  1079. }
  1080. # Add extra params in contained in function signature
  1081. for key, value in extra_params.items():
  1082. if key in sig.parameters:
  1083. params[key] = value
  1084. if "__user__" in sig.parameters:
  1085. __user__ = {
  1086. "id": user.id,
  1087. "email": user.email,
  1088. "name": user.name,
  1089. "role": user.role,
  1090. }
  1091. try:
  1092. if hasattr(function_module, "UserValves"):
  1093. __user__["valves"] = function_module.UserValves(
  1094. **Functions.get_user_valves_by_id_and_user_id(
  1095. action_id, user.id
  1096. )
  1097. )
  1098. except Exception as e:
  1099. print(e)
  1100. params = {**params, "__user__": __user__}
  1101. if inspect.iscoroutinefunction(action):
  1102. data = await action(**params)
  1103. else:
  1104. data = action(**params)
  1105. except Exception as e:
  1106. print(f"Error: {e}")
  1107. return JSONResponse(
  1108. status_code=status.HTTP_400_BAD_REQUEST,
  1109. content={"detail": str(e)},
  1110. )
  1111. return data
  1112. ##################################
  1113. #
  1114. # Task Endpoints
  1115. #
  1116. ##################################
  1117. # TODO: Refactor task API endpoints below into a separate file
  1118. @app.get("/api/task/config")
  1119. async def get_task_config(user=Depends(get_verified_user)):
  1120. return {
  1121. "TASK_MODEL": app.state.config.TASK_MODEL,
  1122. "TASK_MODEL_EXTERNAL": app.state.config.TASK_MODEL_EXTERNAL,
  1123. "TITLE_GENERATION_PROMPT_TEMPLATE": app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
  1124. "ENABLE_SEARCH_QUERY": app.state.config.ENABLE_SEARCH_QUERY,
  1125. "SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE": app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE,
  1126. "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  1127. }
  1128. class TaskConfigForm(BaseModel):
  1129. TASK_MODEL: Optional[str]
  1130. TASK_MODEL_EXTERNAL: Optional[str]
  1131. TITLE_GENERATION_PROMPT_TEMPLATE: str
  1132. SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE: str
  1133. ENABLE_SEARCH_QUERY: bool
  1134. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE: str
  1135. @app.post("/api/task/config/update")
  1136. async def update_task_config(form_data: TaskConfigForm, user=Depends(get_admin_user)):
  1137. app.state.config.TASK_MODEL = form_data.TASK_MODEL
  1138. app.state.config.TASK_MODEL_EXTERNAL = form_data.TASK_MODEL_EXTERNAL
  1139. app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE = (
  1140. form_data.TITLE_GENERATION_PROMPT_TEMPLATE
  1141. )
  1142. app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE = (
  1143. form_data.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE
  1144. )
  1145. app.state.config.ENABLE_SEARCH_QUERY = form_data.ENABLE_SEARCH_QUERY
  1146. app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE = (
  1147. form_data.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  1148. )
  1149. return {
  1150. "TASK_MODEL": app.state.config.TASK_MODEL,
  1151. "TASK_MODEL_EXTERNAL": app.state.config.TASK_MODEL_EXTERNAL,
  1152. "TITLE_GENERATION_PROMPT_TEMPLATE": app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
  1153. "SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE": app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE,
  1154. "ENABLE_SEARCH_QUERY": app.state.config.ENABLE_SEARCH_QUERY,
  1155. "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  1156. }
  1157. @app.post("/api/task/title/completions")
  1158. async def generate_title(form_data: dict, user=Depends(get_verified_user)):
  1159. print("generate_title")
  1160. model_id = form_data["model"]
  1161. if model_id not in app.state.MODELS:
  1162. raise HTTPException(
  1163. status_code=status.HTTP_404_NOT_FOUND,
  1164. detail="Model not found",
  1165. )
  1166. # Check if the user has a custom task model
  1167. # If the user has a custom task model, use that model
  1168. task_model_id = get_task_model_id(model_id)
  1169. print(task_model_id)
  1170. model = app.state.MODELS[task_model_id]
  1171. if app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE != "":
  1172. template = app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE
  1173. else:
  1174. template = """Create a concise, 3-5 word title with an emoji as a title for the prompt in the given language. Suitable Emojis for the summary can be used to enhance understanding but avoid quotation marks or special formatting. RESPOND ONLY WITH THE TITLE TEXT.
  1175. Examples of titles:
  1176. 📉 Stock Market Trends
  1177. 🍪 Perfect Chocolate Chip Recipe
  1178. Evolution of Music Streaming
  1179. Remote Work Productivity Tips
  1180. Artificial Intelligence in Healthcare
  1181. 🎮 Video Game Development Insights
  1182. Prompt: {{prompt:middletruncate:8000}}"""
  1183. content = title_generation_template(
  1184. template,
  1185. form_data["prompt"],
  1186. {
  1187. "name": user.name,
  1188. "location": user.info.get("location") if user.info else None,
  1189. },
  1190. )
  1191. payload = {
  1192. "model": task_model_id,
  1193. "messages": [{"role": "user", "content": content}],
  1194. "stream": False,
  1195. **(
  1196. {"max_tokens": 50}
  1197. if app.state.MODELS[task_model_id]["owned_by"] == "ollama"
  1198. else {
  1199. "max_completion_tokens": 50,
  1200. }
  1201. ),
  1202. "chat_id": form_data.get("chat_id", None),
  1203. "metadata": {"task": str(TASKS.TITLE_GENERATION)},
  1204. }
  1205. log.debug(payload)
  1206. # Handle pipeline filters
  1207. try:
  1208. payload = filter_pipeline(payload, user)
  1209. except Exception as e:
  1210. if len(e.args) > 1:
  1211. return JSONResponse(
  1212. status_code=e.args[0],
  1213. content={"detail": e.args[1]},
  1214. )
  1215. else:
  1216. return JSONResponse(
  1217. status_code=status.HTTP_400_BAD_REQUEST,
  1218. content={"detail": str(e)},
  1219. )
  1220. if "chat_id" in payload:
  1221. del payload["chat_id"]
  1222. return await generate_chat_completions(form_data=payload, user=user)
  1223. @app.post("/api/task/query/completions")
  1224. async def generate_search_query(form_data: dict, user=Depends(get_verified_user)):
  1225. print("generate_search_query")
  1226. if not app.state.config.ENABLE_SEARCH_QUERY:
  1227. raise HTTPException(
  1228. status_code=status.HTTP_400_BAD_REQUEST,
  1229. detail=f"Search query generation is disabled",
  1230. )
  1231. model_id = form_data["model"]
  1232. if model_id not in app.state.MODELS:
  1233. raise HTTPException(
  1234. status_code=status.HTTP_404_NOT_FOUND,
  1235. detail="Model not found",
  1236. )
  1237. # Check if the user has a custom task model
  1238. # If the user has a custom task model, use that model
  1239. task_model_id = get_task_model_id(model_id)
  1240. print(task_model_id)
  1241. model = app.state.MODELS[task_model_id]
  1242. if app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE != "":
  1243. template = app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE
  1244. else:
  1245. template = """Given the user's message and interaction history, decide if a web search is necessary. You must be concise and exclusively provide a search query if one is necessary. Refrain from verbose responses or any additional commentary. Prefer suggesting a search if uncertain to provide comprehensive or updated information. If a search isn't needed at all, respond with an empty string. Default to a search query when in doubt. Today's date is {{CURRENT_DATE}}.
  1246. User Message:
  1247. {{prompt:end:4000}}
  1248. Interaction History:
  1249. {{MESSAGES:END:6}}
  1250. Search Query:"""
  1251. content = search_query_generation_template(
  1252. template, form_data["messages"], {"name": user.name}
  1253. )
  1254. print("content", content)
  1255. payload = {
  1256. "model": task_model_id,
  1257. "messages": [{"role": "user", "content": content}],
  1258. "stream": False,
  1259. **(
  1260. {"max_tokens": 30}
  1261. if app.state.MODELS[task_model_id]["owned_by"] == "ollama"
  1262. else {
  1263. "max_completion_tokens": 30,
  1264. }
  1265. ),
  1266. "metadata": {"task": str(TASKS.QUERY_GENERATION)},
  1267. }
  1268. log.debug(payload)
  1269. # Handle pipeline filters
  1270. try:
  1271. payload = filter_pipeline(payload, user)
  1272. except Exception as e:
  1273. if len(e.args) > 1:
  1274. return JSONResponse(
  1275. status_code=e.args[0],
  1276. content={"detail": e.args[1]},
  1277. )
  1278. else:
  1279. return JSONResponse(
  1280. status_code=status.HTTP_400_BAD_REQUEST,
  1281. content={"detail": str(e)},
  1282. )
  1283. if "chat_id" in payload:
  1284. del payload["chat_id"]
  1285. return await generate_chat_completions(form_data=payload, user=user)
  1286. @app.post("/api/task/emoji/completions")
  1287. async def generate_emoji(form_data: dict, user=Depends(get_verified_user)):
  1288. print("generate_emoji")
  1289. model_id = form_data["model"]
  1290. if model_id not in app.state.MODELS:
  1291. raise HTTPException(
  1292. status_code=status.HTTP_404_NOT_FOUND,
  1293. detail="Model not found",
  1294. )
  1295. # Check if the user has a custom task model
  1296. # If the user has a custom task model, use that model
  1297. task_model_id = get_task_model_id(model_id)
  1298. print(task_model_id)
  1299. model = app.state.MODELS[task_model_id]
  1300. template = '''
  1301. Your task is to reflect the speaker's likely facial expression through a fitting emoji. Interpret emotions from the message and reflect their facial expression using fitting, diverse emojis (e.g., 😊, 😢, 😡, 😱).
  1302. Message: """{{prompt}}"""
  1303. '''
  1304. content = title_generation_template(
  1305. template,
  1306. form_data["prompt"],
  1307. {
  1308. "name": user.name,
  1309. "location": user.info.get("location") if user.info else None,
  1310. },
  1311. )
  1312. payload = {
  1313. "model": task_model_id,
  1314. "messages": [{"role": "user", "content": content}],
  1315. "stream": False,
  1316. **(
  1317. {"max_tokens": 4}
  1318. if app.state.MODELS[task_model_id]["owned_by"] == "ollama"
  1319. else {
  1320. "max_completion_tokens": 4,
  1321. }
  1322. ),
  1323. "chat_id": form_data.get("chat_id", None),
  1324. "metadata": {"task": str(TASKS.EMOJI_GENERATION)},
  1325. }
  1326. log.debug(payload)
  1327. # Handle pipeline filters
  1328. try:
  1329. payload = filter_pipeline(payload, user)
  1330. except Exception as e:
  1331. if len(e.args) > 1:
  1332. return JSONResponse(
  1333. status_code=e.args[0],
  1334. content={"detail": e.args[1]},
  1335. )
  1336. else:
  1337. return JSONResponse(
  1338. status_code=status.HTTP_400_BAD_REQUEST,
  1339. content={"detail": str(e)},
  1340. )
  1341. if "chat_id" in payload:
  1342. del payload["chat_id"]
  1343. return await generate_chat_completions(form_data=payload, user=user)
  1344. @app.post("/api/task/moa/completions")
  1345. async def generate_moa_response(form_data: dict, user=Depends(get_verified_user)):
  1346. print("generate_moa_response")
  1347. model_id = form_data["model"]
  1348. if model_id not in app.state.MODELS:
  1349. raise HTTPException(
  1350. status_code=status.HTTP_404_NOT_FOUND,
  1351. detail="Model not found",
  1352. )
  1353. # Check if the user has a custom task model
  1354. # If the user has a custom task model, use that model
  1355. task_model_id = get_task_model_id(model_id)
  1356. print(task_model_id)
  1357. model = app.state.MODELS[task_model_id]
  1358. template = """You have been provided with a set of responses from various models to the latest user query: "{{prompt}}"
  1359. Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability.
  1360. Responses from models: {{responses}}"""
  1361. content = moa_response_generation_template(
  1362. template,
  1363. form_data["prompt"],
  1364. form_data["responses"],
  1365. )
  1366. payload = {
  1367. "model": task_model_id,
  1368. "messages": [{"role": "user", "content": content}],
  1369. "stream": form_data.get("stream", False),
  1370. "chat_id": form_data.get("chat_id", None),
  1371. "metadata": {"task": str(TASKS.MOA_RESPONSE_GENERATION)},
  1372. }
  1373. log.debug(payload)
  1374. try:
  1375. payload = filter_pipeline(payload, user)
  1376. except Exception as e:
  1377. if len(e.args) > 1:
  1378. return JSONResponse(
  1379. status_code=e.args[0],
  1380. content={"detail": e.args[1]},
  1381. )
  1382. else:
  1383. return JSONResponse(
  1384. status_code=status.HTTP_400_BAD_REQUEST,
  1385. content={"detail": str(e)},
  1386. )
  1387. if "chat_id" in payload:
  1388. del payload["chat_id"]
  1389. return await generate_chat_completions(form_data=payload, user=user)
  1390. ##################################
  1391. #
  1392. # Pipelines Endpoints
  1393. #
  1394. ##################################
  1395. # TODO: Refactor pipelines API endpoints below into a separate file
  1396. @app.get("/api/pipelines/list")
  1397. async def get_pipelines_list(user=Depends(get_admin_user)):
  1398. responses = await get_openai_models(raw=True)
  1399. print(responses)
  1400. urlIdxs = [
  1401. idx
  1402. for idx, response in enumerate(responses)
  1403. if response is not None and "pipelines" in response
  1404. ]
  1405. return {
  1406. "data": [
  1407. {
  1408. "url": openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx],
  1409. "idx": urlIdx,
  1410. }
  1411. for urlIdx in urlIdxs
  1412. ]
  1413. }
  1414. @app.post("/api/pipelines/upload")
  1415. async def upload_pipeline(
  1416. urlIdx: int = Form(...), file: UploadFile = File(...), user=Depends(get_admin_user)
  1417. ):
  1418. print("upload_pipeline", urlIdx, file.filename)
  1419. # Check if the uploaded file is a python file
  1420. if not (file.filename and file.filename.endswith(".py")):
  1421. raise HTTPException(
  1422. status_code=status.HTTP_400_BAD_REQUEST,
  1423. detail="Only Python (.py) files are allowed.",
  1424. )
  1425. upload_folder = f"{CACHE_DIR}/pipelines"
  1426. os.makedirs(upload_folder, exist_ok=True)
  1427. file_path = os.path.join(upload_folder, file.filename)
  1428. r = None
  1429. try:
  1430. # Save the uploaded file
  1431. with open(file_path, "wb") as buffer:
  1432. shutil.copyfileobj(file.file, buffer)
  1433. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1434. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1435. headers = {"Authorization": f"Bearer {key}"}
  1436. with open(file_path, "rb") as f:
  1437. files = {"file": f}
  1438. r = requests.post(f"{url}/pipelines/upload", headers=headers, files=files)
  1439. r.raise_for_status()
  1440. data = r.json()
  1441. return {**data}
  1442. except Exception as e:
  1443. # Handle connection error here
  1444. print(f"Connection error: {e}")
  1445. detail = "Pipeline not found"
  1446. status_code = status.HTTP_404_NOT_FOUND
  1447. if r is not None:
  1448. status_code = r.status_code
  1449. try:
  1450. res = r.json()
  1451. if "detail" in res:
  1452. detail = res["detail"]
  1453. except Exception:
  1454. pass
  1455. raise HTTPException(
  1456. status_code=status_code,
  1457. detail=detail,
  1458. )
  1459. finally:
  1460. # Ensure the file is deleted after the upload is completed or on failure
  1461. if os.path.exists(file_path):
  1462. os.remove(file_path)
  1463. class AddPipelineForm(BaseModel):
  1464. url: str
  1465. urlIdx: int
  1466. @app.post("/api/pipelines/add")
  1467. async def add_pipeline(form_data: AddPipelineForm, user=Depends(get_admin_user)):
  1468. r = None
  1469. try:
  1470. urlIdx = form_data.urlIdx
  1471. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1472. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1473. headers = {"Authorization": f"Bearer {key}"}
  1474. r = requests.post(
  1475. f"{url}/pipelines/add", headers=headers, json={"url": form_data.url}
  1476. )
  1477. r.raise_for_status()
  1478. data = r.json()
  1479. return {**data}
  1480. except Exception as e:
  1481. # Handle connection error here
  1482. print(f"Connection error: {e}")
  1483. detail = "Pipeline not found"
  1484. if r is not None:
  1485. try:
  1486. res = r.json()
  1487. if "detail" in res:
  1488. detail = res["detail"]
  1489. except Exception:
  1490. pass
  1491. raise HTTPException(
  1492. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1493. detail=detail,
  1494. )
  1495. class DeletePipelineForm(BaseModel):
  1496. id: str
  1497. urlIdx: int
  1498. @app.delete("/api/pipelines/delete")
  1499. async def delete_pipeline(form_data: DeletePipelineForm, user=Depends(get_admin_user)):
  1500. r = None
  1501. try:
  1502. urlIdx = form_data.urlIdx
  1503. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1504. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1505. headers = {"Authorization": f"Bearer {key}"}
  1506. r = requests.delete(
  1507. f"{url}/pipelines/delete", headers=headers, json={"id": form_data.id}
  1508. )
  1509. r.raise_for_status()
  1510. data = r.json()
  1511. return {**data}
  1512. except Exception as e:
  1513. # Handle connection error here
  1514. print(f"Connection error: {e}")
  1515. detail = "Pipeline not found"
  1516. if r is not None:
  1517. try:
  1518. res = r.json()
  1519. if "detail" in res:
  1520. detail = res["detail"]
  1521. except Exception:
  1522. pass
  1523. raise HTTPException(
  1524. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1525. detail=detail,
  1526. )
  1527. @app.get("/api/pipelines")
  1528. async def get_pipelines(urlIdx: Optional[int] = None, user=Depends(get_admin_user)):
  1529. r = None
  1530. try:
  1531. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1532. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1533. headers = {"Authorization": f"Bearer {key}"}
  1534. r = requests.get(f"{url}/pipelines", headers=headers)
  1535. r.raise_for_status()
  1536. data = r.json()
  1537. return {**data}
  1538. except Exception as e:
  1539. # Handle connection error here
  1540. print(f"Connection error: {e}")
  1541. detail = "Pipeline not found"
  1542. if r is not None:
  1543. try:
  1544. res = r.json()
  1545. if "detail" in res:
  1546. detail = res["detail"]
  1547. except Exception:
  1548. pass
  1549. raise HTTPException(
  1550. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1551. detail=detail,
  1552. )
  1553. @app.get("/api/pipelines/{pipeline_id}/valves")
  1554. async def get_pipeline_valves(
  1555. urlIdx: Optional[int],
  1556. pipeline_id: str,
  1557. user=Depends(get_admin_user),
  1558. ):
  1559. r = None
  1560. try:
  1561. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1562. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1563. headers = {"Authorization": f"Bearer {key}"}
  1564. r = requests.get(f"{url}/{pipeline_id}/valves", headers=headers)
  1565. r.raise_for_status()
  1566. data = r.json()
  1567. return {**data}
  1568. except Exception as e:
  1569. # Handle connection error here
  1570. print(f"Connection error: {e}")
  1571. detail = "Pipeline not found"
  1572. if r is not None:
  1573. try:
  1574. res = r.json()
  1575. if "detail" in res:
  1576. detail = res["detail"]
  1577. except Exception:
  1578. pass
  1579. raise HTTPException(
  1580. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1581. detail=detail,
  1582. )
  1583. @app.get("/api/pipelines/{pipeline_id}/valves/spec")
  1584. async def get_pipeline_valves_spec(
  1585. urlIdx: Optional[int],
  1586. pipeline_id: str,
  1587. user=Depends(get_admin_user),
  1588. ):
  1589. r = None
  1590. try:
  1591. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1592. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1593. headers = {"Authorization": f"Bearer {key}"}
  1594. r = requests.get(f"{url}/{pipeline_id}/valves/spec", headers=headers)
  1595. r.raise_for_status()
  1596. data = r.json()
  1597. return {**data}
  1598. except Exception as e:
  1599. # Handle connection error here
  1600. print(f"Connection error: {e}")
  1601. detail = "Pipeline not found"
  1602. if r is not None:
  1603. try:
  1604. res = r.json()
  1605. if "detail" in res:
  1606. detail = res["detail"]
  1607. except Exception:
  1608. pass
  1609. raise HTTPException(
  1610. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1611. detail=detail,
  1612. )
  1613. @app.post("/api/pipelines/{pipeline_id}/valves/update")
  1614. async def update_pipeline_valves(
  1615. urlIdx: Optional[int],
  1616. pipeline_id: str,
  1617. form_data: dict,
  1618. user=Depends(get_admin_user),
  1619. ):
  1620. r = None
  1621. try:
  1622. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1623. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1624. headers = {"Authorization": f"Bearer {key}"}
  1625. r = requests.post(
  1626. f"{url}/{pipeline_id}/valves/update",
  1627. headers=headers,
  1628. json={**form_data},
  1629. )
  1630. r.raise_for_status()
  1631. data = r.json()
  1632. return {**data}
  1633. except Exception as e:
  1634. # Handle connection error here
  1635. print(f"Connection error: {e}")
  1636. detail = "Pipeline not found"
  1637. if r is not None:
  1638. try:
  1639. res = r.json()
  1640. if "detail" in res:
  1641. detail = res["detail"]
  1642. except Exception:
  1643. pass
  1644. raise HTTPException(
  1645. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1646. detail=detail,
  1647. )
  1648. ##################################
  1649. #
  1650. # Config Endpoints
  1651. #
  1652. ##################################
  1653. @app.get("/api/config")
  1654. async def get_app_config(request: Request):
  1655. user = None
  1656. if "token" in request.cookies:
  1657. token = request.cookies.get("token")
  1658. data = decode_token(token)
  1659. if data is not None and "id" in data:
  1660. user = Users.get_user_by_id(data["id"])
  1661. return {
  1662. "status": True,
  1663. "name": WEBUI_NAME,
  1664. "version": VERSION,
  1665. "default_locale": str(DEFAULT_LOCALE),
  1666. "oauth": {
  1667. "providers": {
  1668. name: config.get("name", name)
  1669. for name, config in OAUTH_PROVIDERS.items()
  1670. }
  1671. },
  1672. "features": {
  1673. "auth": WEBUI_AUTH,
  1674. "auth_trusted_header": bool(webui_app.state.AUTH_TRUSTED_EMAIL_HEADER),
  1675. "enable_signup": webui_app.state.config.ENABLE_SIGNUP,
  1676. "enable_login_form": webui_app.state.config.ENABLE_LOGIN_FORM,
  1677. **(
  1678. {
  1679. "enable_web_search": rag_app.state.config.ENABLE_RAG_WEB_SEARCH,
  1680. "enable_image_generation": images_app.state.config.ENABLED,
  1681. "enable_community_sharing": webui_app.state.config.ENABLE_COMMUNITY_SHARING,
  1682. "enable_message_rating": webui_app.state.config.ENABLE_MESSAGE_RATING,
  1683. "enable_admin_export": ENABLE_ADMIN_EXPORT,
  1684. "enable_admin_chat_access": ENABLE_ADMIN_CHAT_ACCESS,
  1685. }
  1686. if user is not None
  1687. else {}
  1688. ),
  1689. },
  1690. **(
  1691. {
  1692. "default_models": webui_app.state.config.DEFAULT_MODELS,
  1693. "default_prompt_suggestions": webui_app.state.config.DEFAULT_PROMPT_SUGGESTIONS,
  1694. "audio": {
  1695. "tts": {
  1696. "engine": audio_app.state.config.TTS_ENGINE,
  1697. "voice": audio_app.state.config.TTS_VOICE,
  1698. "split_on": audio_app.state.config.TTS_SPLIT_ON,
  1699. },
  1700. "stt": {
  1701. "engine": audio_app.state.config.STT_ENGINE,
  1702. },
  1703. },
  1704. "file": {
  1705. "max_size": rag_app.state.config.FILE_MAX_SIZE,
  1706. "max_count": rag_app.state.config.FILE_MAX_COUNT,
  1707. },
  1708. "permissions": {**webui_app.state.config.USER_PERMISSIONS},
  1709. }
  1710. if user is not None
  1711. else {}
  1712. ),
  1713. }
  1714. @app.get("/api/config/model/filter")
  1715. async def get_model_filter_config(user=Depends(get_admin_user)):
  1716. return {
  1717. "enabled": app.state.config.ENABLE_MODEL_FILTER,
  1718. "models": app.state.config.MODEL_FILTER_LIST,
  1719. }
  1720. class ModelFilterConfigForm(BaseModel):
  1721. enabled: bool
  1722. models: list[str]
  1723. @app.post("/api/config/model/filter")
  1724. async def update_model_filter_config(
  1725. form_data: ModelFilterConfigForm, user=Depends(get_admin_user)
  1726. ):
  1727. app.state.config.ENABLE_MODEL_FILTER = form_data.enabled
  1728. app.state.config.MODEL_FILTER_LIST = form_data.models
  1729. return {
  1730. "enabled": app.state.config.ENABLE_MODEL_FILTER,
  1731. "models": app.state.config.MODEL_FILTER_LIST,
  1732. }
  1733. # TODO: webhook endpoint should be under config endpoints
  1734. @app.get("/api/webhook")
  1735. async def get_webhook_url(user=Depends(get_admin_user)):
  1736. return {
  1737. "url": app.state.config.WEBHOOK_URL,
  1738. }
  1739. class UrlForm(BaseModel):
  1740. url: str
  1741. @app.post("/api/webhook")
  1742. async def update_webhook_url(form_data: UrlForm, user=Depends(get_admin_user)):
  1743. app.state.config.WEBHOOK_URL = form_data.url
  1744. webui_app.state.WEBHOOK_URL = app.state.config.WEBHOOK_URL
  1745. return {"url": app.state.config.WEBHOOK_URL}
  1746. @app.get("/api/version")
  1747. async def get_app_version():
  1748. return {
  1749. "version": VERSION,
  1750. }
  1751. @app.get("/api/changelog")
  1752. async def get_app_changelog():
  1753. return {key: CHANGELOG[key] for idx, key in enumerate(CHANGELOG) if idx < 5}
  1754. @app.get("/api/version/updates")
  1755. async def get_app_latest_release_version():
  1756. try:
  1757. async with aiohttp.ClientSession(trust_env=True) as session:
  1758. async with session.get(
  1759. "https://api.github.com/repos/open-webui/open-webui/releases/latest"
  1760. ) as response:
  1761. response.raise_for_status()
  1762. data = await response.json()
  1763. latest_version = data["tag_name"]
  1764. return {"current": VERSION, "latest": latest_version[1:]}
  1765. except aiohttp.ClientError:
  1766. raise HTTPException(
  1767. status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
  1768. detail=ERROR_MESSAGES.RATE_LIMIT_EXCEEDED,
  1769. )
  1770. ############################
  1771. # OAuth Login & Callback
  1772. ############################
  1773. oauth = OAuth()
  1774. for provider_name, provider_config in OAUTH_PROVIDERS.items():
  1775. oauth.register(
  1776. name=provider_name,
  1777. client_id=provider_config["client_id"],
  1778. client_secret=provider_config["client_secret"],
  1779. server_metadata_url=provider_config["server_metadata_url"],
  1780. client_kwargs={
  1781. "scope": provider_config["scope"],
  1782. },
  1783. redirect_uri=provider_config["redirect_uri"],
  1784. )
  1785. # SessionMiddleware is used by authlib for oauth
  1786. if len(OAUTH_PROVIDERS) > 0:
  1787. app.add_middleware(
  1788. SessionMiddleware,
  1789. secret_key=WEBUI_SECRET_KEY,
  1790. session_cookie="oui-session",
  1791. same_site=WEBUI_SESSION_COOKIE_SAME_SITE,
  1792. https_only=WEBUI_SESSION_COOKIE_SECURE,
  1793. )
  1794. @app.get("/oauth/{provider}/login")
  1795. async def oauth_login(provider: str, request: Request):
  1796. if provider not in OAUTH_PROVIDERS:
  1797. raise HTTPException(404)
  1798. # If the provider has a custom redirect URL, use that, otherwise automatically generate one
  1799. redirect_uri = OAUTH_PROVIDERS[provider].get("redirect_uri") or request.url_for(
  1800. "oauth_callback", provider=provider
  1801. )
  1802. client = oauth.create_client(provider)
  1803. if client is None:
  1804. raise HTTPException(404)
  1805. return await client.authorize_redirect(request, redirect_uri)
  1806. # OAuth login logic is as follows:
  1807. # 1. Attempt to find a user with matching subject ID, tied to the provider
  1808. # 2. If OAUTH_MERGE_ACCOUNTS_BY_EMAIL is true, find a user with the email address provided via OAuth
  1809. # - This is considered insecure in general, as OAuth providers do not always verify email addresses
  1810. # 3. If there is no user, and ENABLE_OAUTH_SIGNUP is true, create a user
  1811. # - Email addresses are considered unique, so we fail registration if the email address is alreayd taken
  1812. @app.get("/oauth/{provider}/callback")
  1813. async def oauth_callback(provider: str, request: Request, response: Response):
  1814. if provider not in OAUTH_PROVIDERS:
  1815. raise HTTPException(404)
  1816. client = oauth.create_client(provider)
  1817. try:
  1818. token = await client.authorize_access_token(request)
  1819. except Exception as e:
  1820. log.warning(f"OAuth callback error: {e}")
  1821. raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
  1822. user_data: UserInfo = token["userinfo"]
  1823. sub = user_data.get("sub")
  1824. if not sub:
  1825. log.warning(f"OAuth callback failed, sub is missing: {user_data}")
  1826. raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
  1827. provider_sub = f"{provider}@{sub}"
  1828. email_claim = webui_app.state.config.OAUTH_EMAIL_CLAIM
  1829. email = user_data.get(email_claim, "").lower()
  1830. # We currently mandate that email addresses are provided
  1831. if not email:
  1832. log.warning(f"OAuth callback failed, email is missing: {user_data}")
  1833. raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
  1834. # Check if the user exists
  1835. user = Users.get_user_by_oauth_sub(provider_sub)
  1836. if not user:
  1837. # If the user does not exist, check if merging is enabled
  1838. if OAUTH_MERGE_ACCOUNTS_BY_EMAIL.value:
  1839. # Check if the user exists by email
  1840. user = Users.get_user_by_email(email)
  1841. if user:
  1842. # Update the user with the new oauth sub
  1843. Users.update_user_oauth_sub_by_id(user.id, provider_sub)
  1844. if not user:
  1845. # If the user does not exist, check if signups are enabled
  1846. if ENABLE_OAUTH_SIGNUP.value:
  1847. # Check if an existing user with the same email already exists
  1848. existing_user = Users.get_user_by_email(user_data.get("email", "").lower())
  1849. if existing_user:
  1850. raise HTTPException(400, detail=ERROR_MESSAGES.EMAIL_TAKEN)
  1851. picture_claim = webui_app.state.config.OAUTH_PICTURE_CLAIM
  1852. picture_url = user_data.get(picture_claim, "")
  1853. if picture_url:
  1854. # Download the profile image into a base64 string
  1855. try:
  1856. async with aiohttp.ClientSession() as session:
  1857. async with session.get(picture_url) as resp:
  1858. picture = await resp.read()
  1859. base64_encoded_picture = base64.b64encode(picture).decode(
  1860. "utf-8"
  1861. )
  1862. guessed_mime_type = mimetypes.guess_type(picture_url)[0]
  1863. if guessed_mime_type is None:
  1864. # assume JPG, browsers are tolerant enough of image formats
  1865. guessed_mime_type = "image/jpeg"
  1866. picture_url = f"data:{guessed_mime_type};base64,{base64_encoded_picture}"
  1867. except Exception as e:
  1868. log.error(f"Error downloading profile image '{picture_url}': {e}")
  1869. picture_url = ""
  1870. if not picture_url:
  1871. picture_url = "/user.png"
  1872. username_claim = webui_app.state.config.OAUTH_USERNAME_CLAIM
  1873. role = (
  1874. "admin"
  1875. if Users.get_num_users() == 0
  1876. else webui_app.state.config.DEFAULT_USER_ROLE
  1877. )
  1878. user = Auths.insert_new_auth(
  1879. email=email,
  1880. password=get_password_hash(
  1881. str(uuid.uuid4())
  1882. ), # Random password, not used
  1883. name=user_data.get(username_claim, "User"),
  1884. profile_image_url=picture_url,
  1885. role=role,
  1886. oauth_sub=provider_sub,
  1887. )
  1888. if webui_app.state.config.WEBHOOK_URL:
  1889. post_webhook(
  1890. webui_app.state.config.WEBHOOK_URL,
  1891. WEBHOOK_MESSAGES.USER_SIGNUP(user.name),
  1892. {
  1893. "action": "signup",
  1894. "message": WEBHOOK_MESSAGES.USER_SIGNUP(user.name),
  1895. "user": user.model_dump_json(exclude_none=True),
  1896. },
  1897. )
  1898. else:
  1899. raise HTTPException(
  1900. status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.ACCESS_PROHIBITED
  1901. )
  1902. jwt_token = create_token(
  1903. data={"id": user.id},
  1904. expires_delta=parse_duration(webui_app.state.config.JWT_EXPIRES_IN),
  1905. )
  1906. # Set the cookie token
  1907. response.set_cookie(
  1908. key="token",
  1909. value=jwt_token,
  1910. httponly=True, # Ensures the cookie is not accessible via JavaScript
  1911. )
  1912. # Redirect back to the frontend with the JWT token
  1913. redirect_url = f"{request.base_url}auth#token={jwt_token}"
  1914. return RedirectResponse(url=redirect_url)
  1915. @app.get("/manifest.json")
  1916. async def get_manifest_json():
  1917. return {
  1918. "name": WEBUI_NAME,
  1919. "short_name": WEBUI_NAME,
  1920. "description": "Open WebUI is an open, extensible, user-friendly interface for AI that adapts to your workflow.",
  1921. "start_url": "/",
  1922. "display": "standalone",
  1923. "background_color": "#343541",
  1924. "orientation": "any",
  1925. "icons": [
  1926. {
  1927. "src": "/static/logo.png",
  1928. "type": "image/png",
  1929. "sizes": "500x500",
  1930. "purpose": "any",
  1931. },
  1932. {
  1933. "src": "/static/logo.png",
  1934. "type": "image/png",
  1935. "sizes": "500x500",
  1936. "purpose": "maskable",
  1937. },
  1938. ],
  1939. }
  1940. @app.get("/opensearch.xml")
  1941. async def get_opensearch_xml():
  1942. xml_content = rf"""
  1943. <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" xmlns:moz="http://www.mozilla.org/2006/browser/search/">
  1944. <ShortName>{WEBUI_NAME}</ShortName>
  1945. <Description>Search {WEBUI_NAME}</Description>
  1946. <InputEncoding>UTF-8</InputEncoding>
  1947. <Image width="16" height="16" type="image/x-icon">{WEBUI_URL}/static/favicon.png</Image>
  1948. <Url type="text/html" method="get" template="{WEBUI_URL}/?q={"{searchTerms}"}"/>
  1949. <moz:SearchForm>{WEBUI_URL}</moz:SearchForm>
  1950. </OpenSearchDescription>
  1951. """
  1952. return Response(content=xml_content, media_type="application/xml")
  1953. @app.get("/health")
  1954. async def healthcheck():
  1955. return {"status": True}
  1956. @app.get("/health/db")
  1957. async def healthcheck_with_db():
  1958. Session.execute(text("SELECT 1;")).all()
  1959. return {"status": True}
  1960. app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
  1961. app.mount("/cache", StaticFiles(directory=CACHE_DIR), name="cache")
  1962. if os.path.exists(FRONTEND_BUILD_DIR):
  1963. mimetypes.add_type("text/javascript", ".js")
  1964. app.mount(
  1965. "/",
  1966. SPAStaticFiles(directory=FRONTEND_BUILD_DIR, html=True),
  1967. name="spa-static-files",
  1968. )
  1969. else:
  1970. log.warning(
  1971. f"Frontend build directory not found at '{FRONTEND_BUILD_DIR}'. Serving API only."
  1972. )