main.py 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637
  1. import asyncio
  2. import inspect
  3. import json
  4. import logging
  5. import mimetypes
  6. import os
  7. import shutil
  8. import sys
  9. import time
  10. import random
  11. from contextlib import asynccontextmanager
  12. from typing import Optional
  13. from aiocache import cached
  14. import aiohttp
  15. import requests
  16. from fastapi import (
  17. Depends,
  18. FastAPI,
  19. File,
  20. Form,
  21. HTTPException,
  22. Request,
  23. UploadFile,
  24. status,
  25. )
  26. from fastapi.middleware.cors import CORSMiddleware
  27. from fastapi.responses import JSONResponse, RedirectResponse
  28. from fastapi.staticfiles import StaticFiles
  29. from pydantic import BaseModel
  30. from sqlalchemy import text
  31. from starlette.exceptions import HTTPException as StarletteHTTPException
  32. from starlette.middleware.base import BaseHTTPMiddleware
  33. from starlette.middleware.sessions import SessionMiddleware
  34. from starlette.responses import Response, StreamingResponse
  35. from open_webui.apps.audio.main import app as audio_app
  36. from open_webui.apps.images.main import app as images_app
  37. from open_webui.apps.ollama.main import (
  38. app as ollama_app,
  39. get_all_models as get_ollama_models,
  40. generate_chat_completion as generate_ollama_chat_completion,
  41. GenerateChatCompletionForm,
  42. )
  43. from open_webui.apps.openai.main import (
  44. app as openai_app,
  45. generate_chat_completion as generate_openai_chat_completion,
  46. get_all_models as get_openai_models,
  47. get_all_models_responses as get_openai_models_responses,
  48. )
  49. from open_webui.apps.retrieval.main import app as retrieval_app
  50. from open_webui.apps.retrieval.utils import get_rag_context, rag_template
  51. from open_webui.apps.socket.main import (
  52. app as socket_app,
  53. periodic_usage_pool_cleanup,
  54. get_event_call,
  55. get_event_emitter,
  56. )
  57. from open_webui.apps.webui.internal.db import Session
  58. from open_webui.apps.webui.main import (
  59. app as webui_app,
  60. generate_function_chat_completion,
  61. get_all_models as get_open_webui_models,
  62. )
  63. from open_webui.apps.webui.models.functions import Functions
  64. from open_webui.apps.webui.models.models import Models
  65. from open_webui.apps.webui.models.users import UserModel, Users
  66. from open_webui.apps.webui.utils import load_function_module_by_id
  67. from open_webui.config import (
  68. CACHE_DIR,
  69. CORS_ALLOW_ORIGIN,
  70. DEFAULT_LOCALE,
  71. ENABLE_ADMIN_CHAT_ACCESS,
  72. ENABLE_ADMIN_EXPORT,
  73. ENABLE_OLLAMA_API,
  74. ENABLE_OPENAI_API,
  75. ENABLE_TAGS_GENERATION,
  76. ENV,
  77. FRONTEND_BUILD_DIR,
  78. OAUTH_PROVIDERS,
  79. STATIC_DIR,
  80. TASK_MODEL,
  81. TASK_MODEL_EXTERNAL,
  82. ENABLE_SEARCH_QUERY_GENERATION,
  83. ENABLE_RETRIEVAL_QUERY_GENERATION,
  84. QUERY_GENERATION_PROMPT_TEMPLATE,
  85. DEFAULT_QUERY_GENERATION_PROMPT_TEMPLATE,
  86. TITLE_GENERATION_PROMPT_TEMPLATE,
  87. TAGS_GENERATION_PROMPT_TEMPLATE,
  88. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  89. WEBHOOK_URL,
  90. WEBUI_AUTH,
  91. WEBUI_NAME,
  92. AppConfig,
  93. reset_config,
  94. )
  95. from open_webui.constants import TASKS
  96. from open_webui.env import (
  97. CHANGELOG,
  98. GLOBAL_LOG_LEVEL,
  99. SAFE_MODE,
  100. SRC_LOG_LEVELS,
  101. VERSION,
  102. WEBUI_BUILD_HASH,
  103. WEBUI_SECRET_KEY,
  104. WEBUI_SESSION_COOKIE_SAME_SITE,
  105. WEBUI_SESSION_COOKIE_SECURE,
  106. WEBUI_URL,
  107. RESET_CONFIG_ON_START,
  108. OFFLINE_MODE,
  109. )
  110. from open_webui.utils.misc import (
  111. add_or_update_system_message,
  112. get_last_user_message,
  113. prepend_to_first_user_message_content,
  114. )
  115. from open_webui.utils.oauth import oauth_manager
  116. from open_webui.utils.payload import convert_payload_openai_to_ollama
  117. from open_webui.utils.response import (
  118. convert_response_ollama_to_openai,
  119. convert_streaming_response_ollama_to_openai,
  120. )
  121. from open_webui.utils.security_headers import SecurityHeadersMiddleware
  122. from open_webui.utils.task import (
  123. moa_response_generation_template,
  124. tags_generation_template,
  125. query_generation_template,
  126. emoji_generation_template,
  127. title_generation_template,
  128. tools_function_calling_generation_template,
  129. )
  130. from open_webui.utils.tools import get_tools
  131. from open_webui.utils.utils import (
  132. decode_token,
  133. get_admin_user,
  134. get_current_user,
  135. get_http_authorization_cred,
  136. get_verified_user,
  137. )
  138. from open_webui.utils.access_control import has_access
  139. if SAFE_MODE:
  140. print("SAFE MODE ENABLED")
  141. Functions.deactivate_all_functions()
  142. logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL)
  143. log = logging.getLogger(__name__)
  144. log.setLevel(SRC_LOG_LEVELS["MAIN"])
  145. class SPAStaticFiles(StaticFiles):
  146. async def get_response(self, path: str, scope):
  147. try:
  148. return await super().get_response(path, scope)
  149. except (HTTPException, StarletteHTTPException) as ex:
  150. if ex.status_code == 404:
  151. return await super().get_response("index.html", scope)
  152. else:
  153. raise ex
  154. print(
  155. rf"""
  156. ___ __ __ _ _ _ ___
  157. / _ \ _ __ ___ _ __ \ \ / /__| |__ | | | |_ _|
  158. | | | | '_ \ / _ \ '_ \ \ \ /\ / / _ \ '_ \| | | || |
  159. | |_| | |_) | __/ | | | \ V V / __/ |_) | |_| || |
  160. \___/| .__/ \___|_| |_| \_/\_/ \___|_.__/ \___/|___|
  161. |_|
  162. v{VERSION} - building the best open-source AI user interface.
  163. {f"Commit: {WEBUI_BUILD_HASH}" if WEBUI_BUILD_HASH != "dev-build" else ""}
  164. https://github.com/open-webui/open-webui
  165. """
  166. )
  167. @asynccontextmanager
  168. async def lifespan(app: FastAPI):
  169. if RESET_CONFIG_ON_START:
  170. reset_config()
  171. asyncio.create_task(periodic_usage_pool_cleanup())
  172. yield
  173. app = FastAPI(
  174. docs_url="/docs" if ENV == "dev" else None,
  175. openapi_url="/openapi.json" if ENV == "dev" else None,
  176. redoc_url=None,
  177. lifespan=lifespan,
  178. )
  179. app.state.config = AppConfig()
  180. app.state.config.ENABLE_OPENAI_API = ENABLE_OPENAI_API
  181. app.state.config.ENABLE_OLLAMA_API = ENABLE_OLLAMA_API
  182. app.state.config.WEBHOOK_URL = WEBHOOK_URL
  183. app.state.config.TASK_MODEL = TASK_MODEL
  184. app.state.config.TASK_MODEL_EXTERNAL = TASK_MODEL_EXTERNAL
  185. app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE = TITLE_GENERATION_PROMPT_TEMPLATE
  186. app.state.config.ENABLE_TAGS_GENERATION = ENABLE_TAGS_GENERATION
  187. app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE = TAGS_GENERATION_PROMPT_TEMPLATE
  188. app.state.config.ENABLE_SEARCH_QUERY_GENERATION = ENABLE_SEARCH_QUERY_GENERATION
  189. app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION = ENABLE_RETRIEVAL_QUERY_GENERATION
  190. app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE = QUERY_GENERATION_PROMPT_TEMPLATE
  191. app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE = (
  192. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  193. )
  194. ##################################
  195. #
  196. # ChatCompletion Middleware
  197. #
  198. ##################################
  199. def get_filter_function_ids(model):
  200. def get_priority(function_id):
  201. function = Functions.get_function_by_id(function_id)
  202. if function is not None and hasattr(function, "valves"):
  203. # TODO: Fix FunctionModel
  204. return (function.valves if function.valves else {}).get("priority", 0)
  205. return 0
  206. filter_ids = [function.id for function in Functions.get_global_filter_functions()]
  207. if "info" in model and "meta" in model["info"]:
  208. filter_ids.extend(model["info"]["meta"].get("filterIds", []))
  209. filter_ids = list(set(filter_ids))
  210. enabled_filter_ids = [
  211. function.id
  212. for function in Functions.get_functions_by_type("filter", active_only=True)
  213. ]
  214. filter_ids = [
  215. filter_id for filter_id in filter_ids if filter_id in enabled_filter_ids
  216. ]
  217. filter_ids.sort(key=get_priority)
  218. return filter_ids
  219. async def chat_completion_filter_functions_handler(body, model, extra_params):
  220. skip_files = None
  221. filter_ids = get_filter_function_ids(model)
  222. for filter_id in filter_ids:
  223. filter = Functions.get_function_by_id(filter_id)
  224. if not filter:
  225. continue
  226. if filter_id in webui_app.state.FUNCTIONS:
  227. function_module = webui_app.state.FUNCTIONS[filter_id]
  228. else:
  229. function_module, _, _ = load_function_module_by_id(filter_id)
  230. webui_app.state.FUNCTIONS[filter_id] = function_module
  231. # Check if the function has a file_handler variable
  232. if hasattr(function_module, "file_handler"):
  233. skip_files = function_module.file_handler
  234. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  235. valves = Functions.get_function_valves_by_id(filter_id)
  236. function_module.valves = function_module.Valves(
  237. **(valves if valves else {})
  238. )
  239. if not hasattr(function_module, "inlet"):
  240. continue
  241. try:
  242. inlet = function_module.inlet
  243. # Get the signature of the function
  244. sig = inspect.signature(inlet)
  245. params = {"body": body} | {
  246. k: v
  247. for k, v in {
  248. **extra_params,
  249. "__model__": model,
  250. "__id__": filter_id,
  251. }.items()
  252. if k in sig.parameters
  253. }
  254. if "__user__" in params and hasattr(function_module, "UserValves"):
  255. try:
  256. params["__user__"]["valves"] = function_module.UserValves(
  257. **Functions.get_user_valves_by_id_and_user_id(
  258. filter_id, params["__user__"]["id"]
  259. )
  260. )
  261. except Exception as e:
  262. print(e)
  263. if inspect.iscoroutinefunction(inlet):
  264. body = await inlet(**params)
  265. else:
  266. body = inlet(**params)
  267. except Exception as e:
  268. print(f"Error: {e}")
  269. raise e
  270. if skip_files and "files" in body.get("metadata", {}):
  271. del body["metadata"]["files"]
  272. return body, {}
  273. def get_tools_function_calling_payload(messages, task_model_id, content):
  274. user_message = get_last_user_message(messages)
  275. history = "\n".join(
  276. f"{message['role'].upper()}: \"\"\"{message['content']}\"\"\""
  277. for message in messages[::-1][:4]
  278. )
  279. prompt = f"History:\n{history}\nQuery: {user_message}"
  280. return {
  281. "model": task_model_id,
  282. "messages": [
  283. {"role": "system", "content": content},
  284. {"role": "user", "content": f"Query: {prompt}"},
  285. ],
  286. "stream": False,
  287. "metadata": {"task": str(TASKS.FUNCTION_CALLING)},
  288. }
  289. async def get_content_from_response(response) -> Optional[str]:
  290. content = None
  291. if hasattr(response, "body_iterator"):
  292. async for chunk in response.body_iterator:
  293. data = json.loads(chunk.decode("utf-8"))
  294. content = data["choices"][0]["message"]["content"]
  295. # Cleanup any remaining background tasks if necessary
  296. if response.background is not None:
  297. await response.background()
  298. else:
  299. content = response["choices"][0]["message"]["content"]
  300. return content
  301. def get_task_model_id(
  302. default_model_id: str, task_model: str, task_model_external: str, models
  303. ) -> str:
  304. # Set the task model
  305. task_model_id = default_model_id
  306. # Check if the user has a custom task model and use that model
  307. if models[task_model_id]["owned_by"] == "ollama":
  308. if task_model and task_model in models:
  309. task_model_id = task_model
  310. else:
  311. if task_model_external and task_model_external in models:
  312. task_model_id = task_model_external
  313. return task_model_id
  314. async def chat_completion_tools_handler(
  315. body: dict, user: UserModel, models, extra_params: dict
  316. ) -> tuple[dict, dict]:
  317. # If tool_ids field is present, call the functions
  318. metadata = body.get("metadata", {})
  319. tool_ids = metadata.get("tool_ids", None)
  320. log.debug(f"{tool_ids=}")
  321. if not tool_ids:
  322. return body, {}
  323. skip_files = False
  324. contexts = []
  325. citations = []
  326. task_model_id = get_task_model_id(
  327. body["model"],
  328. app.state.config.TASK_MODEL,
  329. app.state.config.TASK_MODEL_EXTERNAL,
  330. models,
  331. )
  332. tools = get_tools(
  333. webui_app,
  334. tool_ids,
  335. user,
  336. {
  337. **extra_params,
  338. "__model__": models[task_model_id],
  339. "__messages__": body["messages"],
  340. "__files__": metadata.get("files", []),
  341. },
  342. )
  343. log.info(f"{tools=}")
  344. specs = [tool["spec"] for tool in tools.values()]
  345. tools_specs = json.dumps(specs)
  346. if app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE != "":
  347. template = app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  348. else:
  349. template = """Available Tools: {{TOOLS}}\nReturn an empty string if no tools match the query. If a function tool matches, construct and return a JSON object in the format {\"name\": \"functionName\", \"parameters\": {\"requiredFunctionParamKey\": \"requiredFunctionParamValue\"}} using the appropriate tool and its parameters. Only return the object and limit the response to the JSON object without additional text."""
  350. tools_function_calling_prompt = tools_function_calling_generation_template(
  351. template, tools_specs
  352. )
  353. log.info(f"{tools_function_calling_prompt=}")
  354. payload = get_tools_function_calling_payload(
  355. body["messages"], task_model_id, tools_function_calling_prompt
  356. )
  357. try:
  358. payload = filter_pipeline(payload, user, models)
  359. except Exception as e:
  360. raise e
  361. try:
  362. response = await generate_chat_completions(form_data=payload, user=user)
  363. log.debug(f"{response=}")
  364. content = await get_content_from_response(response)
  365. log.debug(f"{content=}")
  366. if not content:
  367. return body, {}
  368. try:
  369. content = content[content.find("{") : content.rfind("}") + 1]
  370. if not content:
  371. raise Exception("No JSON object found in the response")
  372. result = json.loads(content)
  373. tool_function_name = result.get("name", None)
  374. if tool_function_name not in tools:
  375. return body, {}
  376. tool_function_params = result.get("parameters", {})
  377. try:
  378. required_params = (
  379. tools[tool_function_name]
  380. .get("spec", {})
  381. .get("parameters", {})
  382. .get("required", [])
  383. )
  384. tool_function = tools[tool_function_name]["callable"]
  385. tool_function_params = {
  386. k: v
  387. for k, v in tool_function_params.items()
  388. if k in required_params
  389. }
  390. tool_output = await tool_function(**tool_function_params)
  391. except Exception as e:
  392. tool_output = str(e)
  393. if tools[tool_function_name]["citation"]:
  394. citations.append(
  395. {
  396. "source": {
  397. "name": f"TOOL:{tools[tool_function_name]['toolkit_id']}/{tool_function_name}"
  398. },
  399. "document": [tool_output],
  400. "metadata": [{"source": tool_function_name}],
  401. }
  402. )
  403. if tools[tool_function_name]["file_handler"]:
  404. skip_files = True
  405. if isinstance(tool_output, str):
  406. contexts.append(tool_output)
  407. except Exception as e:
  408. log.exception(f"Error: {e}")
  409. content = None
  410. except Exception as e:
  411. log.exception(f"Error: {e}")
  412. content = None
  413. log.debug(f"tool_contexts: {contexts}")
  414. if skip_files and "files" in body.get("metadata", {}):
  415. del body["metadata"]["files"]
  416. return body, {"contexts": contexts, "citations": citations}
  417. async def chat_completion_files_handler(
  418. body: dict, user: UserModel
  419. ) -> tuple[dict, dict[str, list]]:
  420. contexts = []
  421. citations = []
  422. try:
  423. queries_response = await generate_queries(
  424. {
  425. "model": body["model"],
  426. "messages": body["messages"],
  427. "type": "retrieval",
  428. },
  429. user,
  430. )
  431. queries_response = queries_response["choices"][0]["message"]["content"]
  432. try:
  433. queries_response = json.loads(queries_response)
  434. except Exception as e:
  435. queries_response = {"queries": []}
  436. queries = queries_response.get("queries", [])
  437. except Exception as e:
  438. queries = []
  439. if len(queries) == 0:
  440. queries = [get_last_user_message(body["messages"])]
  441. print(f"{queries=}")
  442. if files := body.get("metadata", {}).get("files", None):
  443. contexts, citations = get_rag_context(
  444. files=files,
  445. queries=queries,
  446. embedding_function=retrieval_app.state.EMBEDDING_FUNCTION,
  447. k=retrieval_app.state.config.TOP_K,
  448. reranking_function=retrieval_app.state.sentence_transformer_rf,
  449. r=retrieval_app.state.config.RELEVANCE_THRESHOLD,
  450. hybrid_search=retrieval_app.state.config.ENABLE_RAG_HYBRID_SEARCH,
  451. )
  452. log.debug(f"rag_contexts: {contexts}, citations: {citations}")
  453. return body, {"contexts": contexts, "citations": citations}
  454. def is_chat_completion_request(request):
  455. return request.method == "POST" and any(
  456. endpoint in request.url.path
  457. for endpoint in ["/ollama/api/chat", "/chat/completions"]
  458. )
  459. async def get_body_and_model_and_user(request, models):
  460. # Read the original request body
  461. body = await request.body()
  462. body_str = body.decode("utf-8")
  463. body = json.loads(body_str) if body_str else {}
  464. model_id = body["model"]
  465. if model_id not in models:
  466. raise Exception("Model not found")
  467. model = models[model_id]
  468. user = get_current_user(
  469. request,
  470. get_http_authorization_cred(request.headers.get("Authorization")),
  471. )
  472. return body, model, user
  473. class ChatCompletionMiddleware(BaseHTTPMiddleware):
  474. async def dispatch(self, request: Request, call_next):
  475. if not is_chat_completion_request(request):
  476. return await call_next(request)
  477. log.debug(f"request.url.path: {request.url.path}")
  478. model_list = await get_all_models()
  479. models = {model["id"]: model for model in model_list}
  480. try:
  481. body, model, user = await get_body_and_model_and_user(request, models)
  482. except Exception as e:
  483. return JSONResponse(
  484. status_code=status.HTTP_400_BAD_REQUEST,
  485. content={"detail": str(e)},
  486. )
  487. model_info = Models.get_model_by_id(model["id"])
  488. if user.role == "user":
  489. if model.get("arena"):
  490. if not has_access(
  491. user.id,
  492. type="read",
  493. access_control=model.get("info", {})
  494. .get("meta", {})
  495. .get("access_control", {}),
  496. ):
  497. raise HTTPException(
  498. status_code=403,
  499. detail="Model not found",
  500. )
  501. else:
  502. if not model_info:
  503. return JSONResponse(
  504. status_code=status.HTTP_404_NOT_FOUND,
  505. content={"detail": "Model not found"},
  506. )
  507. elif not (
  508. user.id == model_info.user_id
  509. or has_access(
  510. user.id, type="read", access_control=model_info.access_control
  511. )
  512. ):
  513. return JSONResponse(
  514. status_code=status.HTTP_403_FORBIDDEN,
  515. content={"detail": "User does not have access to the model"},
  516. )
  517. metadata = {
  518. "chat_id": body.pop("chat_id", None),
  519. "message_id": body.pop("id", None),
  520. "session_id": body.pop("session_id", None),
  521. "tool_ids": body.get("tool_ids", None),
  522. "files": body.get("files", None),
  523. }
  524. body["metadata"] = metadata
  525. extra_params = {
  526. "__event_emitter__": get_event_emitter(metadata),
  527. "__event_call__": get_event_call(metadata),
  528. "__user__": {
  529. "id": user.id,
  530. "email": user.email,
  531. "name": user.name,
  532. "role": user.role,
  533. },
  534. "__metadata__": metadata,
  535. }
  536. # Initialize data_items to store additional data to be sent to the client
  537. # Initialize contexts and citation
  538. data_items = []
  539. contexts = []
  540. citations = []
  541. try:
  542. body, flags = await chat_completion_filter_functions_handler(
  543. body, model, extra_params
  544. )
  545. except Exception as e:
  546. return JSONResponse(
  547. status_code=status.HTTP_400_BAD_REQUEST,
  548. content={"detail": str(e)},
  549. )
  550. tool_ids = body.pop("tool_ids", None)
  551. files = body.pop("files", None)
  552. metadata = {
  553. **metadata,
  554. "tool_ids": tool_ids,
  555. "files": files,
  556. }
  557. body["metadata"] = metadata
  558. try:
  559. body, flags = await chat_completion_tools_handler(
  560. body, user, models, extra_params
  561. )
  562. contexts.extend(flags.get("contexts", []))
  563. citations.extend(flags.get("citations", []))
  564. except Exception as e:
  565. log.exception(e)
  566. try:
  567. body, flags = await chat_completion_files_handler(body, user)
  568. contexts.extend(flags.get("contexts", []))
  569. citations.extend(flags.get("citations", []))
  570. except Exception as e:
  571. log.exception(e)
  572. # If context is not empty, insert it into the messages
  573. if len(contexts) > 0:
  574. context_string = ""
  575. for context_idx, context in enumerate(contexts):
  576. print(context)
  577. source_id = citations[context_idx].get("source", {}).get("name", "")
  578. context_string += f"<source><source_id>{source_id}</source_id><source_context>{context}</source_context></source>\n"
  579. context_string = context_string.strip()
  580. prompt = get_last_user_message(body["messages"])
  581. if prompt is None:
  582. raise Exception("No user message found")
  583. if (
  584. retrieval_app.state.config.RELEVANCE_THRESHOLD == 0
  585. and context_string.strip() == ""
  586. ):
  587. log.debug(
  588. f"With a 0 relevancy threshold for RAG, the context cannot be empty"
  589. )
  590. # Workaround for Ollama 2.0+ system prompt issue
  591. # TODO: replace with add_or_update_system_message
  592. if model["owned_by"] == "ollama":
  593. body["messages"] = prepend_to_first_user_message_content(
  594. rag_template(
  595. retrieval_app.state.config.RAG_TEMPLATE, context_string, prompt
  596. ),
  597. body["messages"],
  598. )
  599. else:
  600. body["messages"] = add_or_update_system_message(
  601. rag_template(
  602. retrieval_app.state.config.RAG_TEMPLATE, context_string, prompt
  603. ),
  604. body["messages"],
  605. )
  606. # If there are citations, add them to the data_items
  607. if len(citations) > 0:
  608. data_items.append({"citations": citations})
  609. modified_body_bytes = json.dumps(body).encode("utf-8")
  610. # Replace the request body with the modified one
  611. request._body = modified_body_bytes
  612. # Set custom header to ensure content-length matches new body length
  613. request.headers.__dict__["_list"] = [
  614. (b"content-length", str(len(modified_body_bytes)).encode("utf-8")),
  615. *[(k, v) for k, v in request.headers.raw if k.lower() != b"content-length"],
  616. ]
  617. response = await call_next(request)
  618. if not isinstance(response, StreamingResponse):
  619. return response
  620. content_type = response.headers["Content-Type"]
  621. is_openai = "text/event-stream" in content_type
  622. is_ollama = "application/x-ndjson" in content_type
  623. if not is_openai and not is_ollama:
  624. return response
  625. def wrap_item(item):
  626. return f"data: {item}\n\n" if is_openai else f"{item}\n"
  627. async def stream_wrapper(original_generator, data_items):
  628. for item in data_items:
  629. yield wrap_item(json.dumps(item))
  630. async for data in original_generator:
  631. yield data
  632. return StreamingResponse(
  633. stream_wrapper(response.body_iterator, data_items),
  634. headers=dict(response.headers),
  635. )
  636. async def _receive(self, body: bytes):
  637. return {"type": "http.request", "body": body, "more_body": False}
  638. app.add_middleware(ChatCompletionMiddleware)
  639. ##################################
  640. #
  641. # Pipeline Middleware
  642. #
  643. ##################################
  644. def get_sorted_filters(model_id, models):
  645. filters = [
  646. model
  647. for model in models.values()
  648. if "pipeline" in model
  649. and "type" in model["pipeline"]
  650. and model["pipeline"]["type"] == "filter"
  651. and (
  652. model["pipeline"]["pipelines"] == ["*"]
  653. or any(
  654. model_id == target_model_id
  655. for target_model_id in model["pipeline"]["pipelines"]
  656. )
  657. )
  658. ]
  659. sorted_filters = sorted(filters, key=lambda x: x["pipeline"]["priority"])
  660. return sorted_filters
  661. def filter_pipeline(payload, user, models):
  662. user = {"id": user.id, "email": user.email, "name": user.name, "role": user.role}
  663. model_id = payload["model"]
  664. sorted_filters = get_sorted_filters(model_id, models)
  665. model = models[model_id]
  666. if "pipeline" in model:
  667. sorted_filters.append(model)
  668. for filter in sorted_filters:
  669. r = None
  670. try:
  671. urlIdx = filter["urlIdx"]
  672. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  673. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  674. if key == "":
  675. continue
  676. headers = {"Authorization": f"Bearer {key}"}
  677. r = requests.post(
  678. f"{url}/{filter['id']}/filter/inlet",
  679. headers=headers,
  680. json={
  681. "user": user,
  682. "body": payload,
  683. },
  684. )
  685. r.raise_for_status()
  686. payload = r.json()
  687. except Exception as e:
  688. # Handle connection error here
  689. print(f"Connection error: {e}")
  690. if r is not None:
  691. res = r.json()
  692. if "detail" in res:
  693. raise Exception(r.status_code, res["detail"])
  694. return payload
  695. class PipelineMiddleware(BaseHTTPMiddleware):
  696. async def dispatch(self, request: Request, call_next):
  697. if not is_chat_completion_request(request):
  698. return await call_next(request)
  699. log.debug(f"request.url.path: {request.url.path}")
  700. # Read the original request body
  701. body = await request.body()
  702. # Decode body to string
  703. body_str = body.decode("utf-8")
  704. # Parse string to JSON
  705. data = json.loads(body_str) if body_str else {}
  706. try:
  707. user = get_current_user(
  708. request,
  709. get_http_authorization_cred(request.headers["Authorization"]),
  710. )
  711. except KeyError as e:
  712. if len(e.args) > 1:
  713. return JSONResponse(
  714. status_code=e.args[0],
  715. content={"detail": e.args[1]},
  716. )
  717. else:
  718. return JSONResponse(
  719. status_code=status.HTTP_401_UNAUTHORIZED,
  720. content={"detail": "Not authenticated"},
  721. )
  722. except HTTPException as e:
  723. return JSONResponse(
  724. status_code=e.status_code,
  725. content={"detail": e.detail},
  726. )
  727. model_list = await get_all_models()
  728. models = {model["id"]: model for model in model_list}
  729. try:
  730. data = filter_pipeline(data, user, models)
  731. except Exception as e:
  732. if len(e.args) > 1:
  733. return JSONResponse(
  734. status_code=e.args[0],
  735. content={"detail": e.args[1]},
  736. )
  737. else:
  738. return JSONResponse(
  739. status_code=status.HTTP_400_BAD_REQUEST,
  740. content={"detail": str(e)},
  741. )
  742. modified_body_bytes = json.dumps(data).encode("utf-8")
  743. # Replace the request body with the modified one
  744. request._body = modified_body_bytes
  745. # Set custom header to ensure content-length matches new body length
  746. request.headers.__dict__["_list"] = [
  747. (b"content-length", str(len(modified_body_bytes)).encode("utf-8")),
  748. *[(k, v) for k, v in request.headers.raw if k.lower() != b"content-length"],
  749. ]
  750. response = await call_next(request)
  751. return response
  752. async def _receive(self, body: bytes):
  753. return {"type": "http.request", "body": body, "more_body": False}
  754. app.add_middleware(PipelineMiddleware)
  755. from urllib.parse import urlencode, parse_qs, urlparse
  756. class RedirectMiddleware(BaseHTTPMiddleware):
  757. async def dispatch(self, request: Request, call_next):
  758. # Check if the request is a GET request
  759. if request.method == "GET":
  760. path = request.url.path
  761. query_params = dict(parse_qs(urlparse(str(request.url)).query))
  762. # Check for the specific watch path and the presence of 'v' parameter
  763. if path.endswith("/watch") and "v" in query_params:
  764. video_id = query_params["v"][0] # Extract the first 'v' parameter
  765. encoded_video_id = urlencode({"youtube": video_id})
  766. redirect_url = f"/?{encoded_video_id}"
  767. return RedirectResponse(url=redirect_url)
  768. # Proceed with the normal flow of other requests
  769. response = await call_next(request)
  770. return response
  771. # Add the middleware to the app
  772. app.add_middleware(RedirectMiddleware)
  773. app.add_middleware(
  774. CORSMiddleware,
  775. allow_origins=CORS_ALLOW_ORIGIN,
  776. allow_credentials=True,
  777. allow_methods=["*"],
  778. allow_headers=["*"],
  779. )
  780. app.add_middleware(SecurityHeadersMiddleware)
  781. @app.middleware("http")
  782. async def commit_session_after_request(request: Request, call_next):
  783. response = await call_next(request)
  784. log.debug("Commit session after request")
  785. Session.commit()
  786. return response
  787. @app.middleware("http")
  788. async def check_url(request: Request, call_next):
  789. start_time = int(time.time())
  790. request.state.enable_api_key = webui_app.state.config.ENABLE_API_KEY
  791. response = await call_next(request)
  792. process_time = int(time.time()) - start_time
  793. response.headers["X-Process-Time"] = str(process_time)
  794. return response
  795. @app.middleware("http")
  796. async def update_embedding_function(request: Request, call_next):
  797. response = await call_next(request)
  798. if "/embedding/update" in request.url.path:
  799. webui_app.state.EMBEDDING_FUNCTION = retrieval_app.state.EMBEDDING_FUNCTION
  800. return response
  801. @app.middleware("http")
  802. async def inspect_websocket(request: Request, call_next):
  803. if (
  804. "/ws/socket.io" in request.url.path
  805. and request.query_params.get("transport") == "websocket"
  806. ):
  807. upgrade = (request.headers.get("Upgrade") or "").lower()
  808. connection = (request.headers.get("Connection") or "").lower().split(",")
  809. # Check that there's the correct headers for an upgrade, else reject the connection
  810. # This is to work around this upstream issue: https://github.com/miguelgrinberg/python-engineio/issues/367
  811. if upgrade != "websocket" or "upgrade" not in connection:
  812. return JSONResponse(
  813. status_code=status.HTTP_400_BAD_REQUEST,
  814. content={"detail": "Invalid WebSocket upgrade request"},
  815. )
  816. return await call_next(request)
  817. app.mount("/ws", socket_app)
  818. app.mount("/ollama", ollama_app)
  819. app.mount("/openai", openai_app)
  820. app.mount("/images/api/v1", images_app)
  821. app.mount("/audio/api/v1", audio_app)
  822. app.mount("/retrieval/api/v1", retrieval_app)
  823. app.mount("/api/v1", webui_app)
  824. webui_app.state.EMBEDDING_FUNCTION = retrieval_app.state.EMBEDDING_FUNCTION
  825. async def get_all_base_models():
  826. open_webui_models = []
  827. openai_models = []
  828. ollama_models = []
  829. if app.state.config.ENABLE_OPENAI_API:
  830. openai_models = await get_openai_models()
  831. openai_models = openai_models["data"]
  832. if app.state.config.ENABLE_OLLAMA_API:
  833. ollama_models = await get_ollama_models()
  834. ollama_models = [
  835. {
  836. "id": model["model"],
  837. "name": model["name"],
  838. "object": "model",
  839. "created": int(time.time()),
  840. "owned_by": "ollama",
  841. "ollama": model,
  842. }
  843. for model in ollama_models["models"]
  844. ]
  845. open_webui_models = await get_open_webui_models()
  846. models = open_webui_models + openai_models + ollama_models
  847. return models
  848. @cached(ttl=3)
  849. async def get_all_models():
  850. models = await get_all_base_models()
  851. # If there are no models, return an empty list
  852. if len([model for model in models if not model.get("arena", False)]) == 0:
  853. return []
  854. global_action_ids = [
  855. function.id for function in Functions.get_global_action_functions()
  856. ]
  857. enabled_action_ids = [
  858. function.id
  859. for function in Functions.get_functions_by_type("action", active_only=True)
  860. ]
  861. custom_models = Models.get_all_models()
  862. for custom_model in custom_models:
  863. if custom_model.base_model_id is None:
  864. for model in models:
  865. if (
  866. custom_model.id == model["id"]
  867. or custom_model.id == model["id"].split(":")[0]
  868. ):
  869. if custom_model.is_active:
  870. model["name"] = custom_model.name
  871. model["info"] = custom_model.model_dump()
  872. action_ids = []
  873. if "info" in model and "meta" in model["info"]:
  874. action_ids.extend(
  875. model["info"]["meta"].get("actionIds", [])
  876. )
  877. model["action_ids"] = action_ids
  878. else:
  879. models.remove(model)
  880. elif custom_model.is_active and (
  881. custom_model.id not in [model["id"] for model in models]
  882. ):
  883. owned_by = "openai"
  884. pipe = None
  885. action_ids = []
  886. for model in models:
  887. if (
  888. custom_model.base_model_id == model["id"]
  889. or custom_model.base_model_id == model["id"].split(":")[0]
  890. ):
  891. owned_by = model["owned_by"]
  892. if "pipe" in model:
  893. pipe = model["pipe"]
  894. break
  895. if custom_model.meta:
  896. meta = custom_model.meta.model_dump()
  897. if "actionIds" in meta:
  898. action_ids.extend(meta["actionIds"])
  899. models.append(
  900. {
  901. "id": f"{custom_model.id}",
  902. "name": custom_model.name,
  903. "object": "model",
  904. "created": custom_model.created_at,
  905. "owned_by": owned_by,
  906. "info": custom_model.model_dump(),
  907. "preset": True,
  908. **({"pipe": pipe} if pipe is not None else {}),
  909. "action_ids": action_ids,
  910. }
  911. )
  912. # Process action_ids to get the actions
  913. def get_action_items_from_module(function, module):
  914. actions = []
  915. if hasattr(module, "actions"):
  916. actions = module.actions
  917. return [
  918. {
  919. "id": f"{function.id}.{action['id']}",
  920. "name": action.get("name", f"{function.name} ({action['id']})"),
  921. "description": function.meta.description,
  922. "icon_url": action.get(
  923. "icon_url", function.meta.manifest.get("icon_url", None)
  924. ),
  925. }
  926. for action in actions
  927. ]
  928. else:
  929. return [
  930. {
  931. "id": function.id,
  932. "name": function.name,
  933. "description": function.meta.description,
  934. "icon_url": function.meta.manifest.get("icon_url", None),
  935. }
  936. ]
  937. def get_function_module_by_id(function_id):
  938. if function_id in webui_app.state.FUNCTIONS:
  939. function_module = webui_app.state.FUNCTIONS[function_id]
  940. else:
  941. function_module, _, _ = load_function_module_by_id(function_id)
  942. webui_app.state.FUNCTIONS[function_id] = function_module
  943. for model in models:
  944. action_ids = [
  945. action_id
  946. for action_id in list(set(model.pop("action_ids", []) + global_action_ids))
  947. if action_id in enabled_action_ids
  948. ]
  949. model["actions"] = []
  950. for action_id in action_ids:
  951. action_function = Functions.get_function_by_id(action_id)
  952. if action_function is None:
  953. raise Exception(f"Action not found: {action_id}")
  954. function_module = get_function_module_by_id(action_id)
  955. model["actions"].extend(
  956. get_action_items_from_module(action_function, function_module)
  957. )
  958. return models
  959. @app.get("/api/models")
  960. async def get_models(user=Depends(get_verified_user)):
  961. models = await get_all_models()
  962. # Filter out filter pipelines
  963. models = [
  964. model
  965. for model in models
  966. if "pipeline" not in model or model["pipeline"].get("type", None) != "filter"
  967. ]
  968. # Filter out models that the user does not have access to
  969. if user.role == "user":
  970. filtered_models = []
  971. for model in models:
  972. if model.get("arena"):
  973. if has_access(
  974. user.id,
  975. type="read",
  976. access_control=model.get("info", {})
  977. .get("meta", {})
  978. .get("access_control", {}),
  979. ):
  980. filtered_models.append(model)
  981. continue
  982. model_info = Models.get_model_by_id(model["id"])
  983. if model_info:
  984. if user.id == model_info.user_id or has_access(
  985. user.id, type="read", access_control=model_info.access_control
  986. ):
  987. filtered_models.append(model)
  988. models = filtered_models
  989. return {"data": models}
  990. @app.get("/api/models/base")
  991. async def get_base_models(user=Depends(get_admin_user)):
  992. models = await get_all_base_models()
  993. # Filter out arena models
  994. models = [model for model in models if not model.get("arena", False)]
  995. return {"data": models}
  996. @app.post("/api/chat/completions")
  997. async def generate_chat_completions(
  998. form_data: dict, user=Depends(get_verified_user), bypass_filter: bool = False
  999. ):
  1000. model_list = await get_all_models()
  1001. models = {model["id"]: model for model in model_list}
  1002. model_id = form_data["model"]
  1003. if model_id not in models:
  1004. raise HTTPException(
  1005. status_code=status.HTTP_404_NOT_FOUND,
  1006. detail="Model not found",
  1007. )
  1008. model = models[model_id]
  1009. # Check if user has access to the model
  1010. if not bypass_filter and user.role == "user":
  1011. if model.get("arena"):
  1012. if not has_access(
  1013. user.id,
  1014. type="read",
  1015. access_control=model.get("info", {})
  1016. .get("meta", {})
  1017. .get("access_control", {}),
  1018. ):
  1019. raise HTTPException(
  1020. status_code=403,
  1021. detail="Model not found",
  1022. )
  1023. else:
  1024. model_info = Models.get_model_by_id(model_id)
  1025. if not model_info:
  1026. raise HTTPException(
  1027. status_code=404,
  1028. detail="Model not found",
  1029. )
  1030. elif not (
  1031. user.id == model_info.user_id
  1032. or has_access(
  1033. user.id, type="read", access_control=model_info.access_control
  1034. )
  1035. ):
  1036. raise HTTPException(
  1037. status_code=403,
  1038. detail="Model not found",
  1039. )
  1040. if model["owned_by"] == "arena":
  1041. model_ids = model.get("info", {}).get("meta", {}).get("model_ids")
  1042. filter_mode = model.get("info", {}).get("meta", {}).get("filter_mode")
  1043. if model_ids and filter_mode == "exclude":
  1044. model_ids = [
  1045. model["id"]
  1046. for model in await get_all_models()
  1047. if model.get("owned_by") != "arena" and model["id"] not in model_ids
  1048. ]
  1049. selected_model_id = None
  1050. if isinstance(model_ids, list) and model_ids:
  1051. selected_model_id = random.choice(model_ids)
  1052. else:
  1053. model_ids = [
  1054. model["id"]
  1055. for model in await get_all_models()
  1056. if model.get("owned_by") != "arena"
  1057. ]
  1058. selected_model_id = random.choice(model_ids)
  1059. form_data["model"] = selected_model_id
  1060. if form_data.get("stream") == True:
  1061. async def stream_wrapper(stream):
  1062. yield f"data: {json.dumps({'selected_model_id': selected_model_id})}\n\n"
  1063. async for chunk in stream:
  1064. yield chunk
  1065. response = await generate_chat_completions(
  1066. form_data, user, bypass_filter=True
  1067. )
  1068. return StreamingResponse(
  1069. stream_wrapper(response.body_iterator), media_type="text/event-stream"
  1070. )
  1071. else:
  1072. return {
  1073. **(
  1074. await generate_chat_completions(form_data, user, bypass_filter=True)
  1075. ),
  1076. "selected_model_id": selected_model_id,
  1077. }
  1078. if model.get("pipe"):
  1079. # Below does not require bypass_filter because this is the only route the uses this function and it is already bypassing the filter
  1080. return await generate_function_chat_completion(
  1081. form_data, user=user, models=models
  1082. )
  1083. if model["owned_by"] == "ollama":
  1084. # Using /ollama/api/chat endpoint
  1085. form_data = convert_payload_openai_to_ollama(form_data)
  1086. form_data = GenerateChatCompletionForm(**form_data)
  1087. response = await generate_ollama_chat_completion(
  1088. form_data=form_data, user=user, bypass_filter=bypass_filter
  1089. )
  1090. if form_data.stream:
  1091. response.headers["content-type"] = "text/event-stream"
  1092. return StreamingResponse(
  1093. convert_streaming_response_ollama_to_openai(response),
  1094. headers=dict(response.headers),
  1095. )
  1096. else:
  1097. return convert_response_ollama_to_openai(response)
  1098. else:
  1099. return await generate_openai_chat_completion(
  1100. form_data, user=user, bypass_filter=bypass_filter
  1101. )
  1102. @app.post("/api/chat/completed")
  1103. async def chat_completed(form_data: dict, user=Depends(get_verified_user)):
  1104. model_list = await get_all_models()
  1105. models = {model["id"]: model for model in model_list}
  1106. data = form_data
  1107. model_id = data["model"]
  1108. if model_id not in models:
  1109. raise HTTPException(
  1110. status_code=status.HTTP_404_NOT_FOUND,
  1111. detail="Model not found",
  1112. )
  1113. model = models[model_id]
  1114. sorted_filters = get_sorted_filters(model_id, models)
  1115. if "pipeline" in model:
  1116. sorted_filters = [model] + sorted_filters
  1117. for filter in sorted_filters:
  1118. r = None
  1119. try:
  1120. urlIdx = filter["urlIdx"]
  1121. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1122. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1123. if key != "":
  1124. headers = {"Authorization": f"Bearer {key}"}
  1125. r = requests.post(
  1126. f"{url}/{filter['id']}/filter/outlet",
  1127. headers=headers,
  1128. json={
  1129. "user": {
  1130. "id": user.id,
  1131. "name": user.name,
  1132. "email": user.email,
  1133. "role": user.role,
  1134. },
  1135. "body": data,
  1136. },
  1137. )
  1138. r.raise_for_status()
  1139. data = r.json()
  1140. except Exception as e:
  1141. # Handle connection error here
  1142. print(f"Connection error: {e}")
  1143. if r is not None:
  1144. try:
  1145. res = r.json()
  1146. if "detail" in res:
  1147. return JSONResponse(
  1148. status_code=r.status_code,
  1149. content=res,
  1150. )
  1151. except Exception:
  1152. pass
  1153. else:
  1154. pass
  1155. __event_emitter__ = get_event_emitter(
  1156. {
  1157. "chat_id": data["chat_id"],
  1158. "message_id": data["id"],
  1159. "session_id": data["session_id"],
  1160. }
  1161. )
  1162. __event_call__ = get_event_call(
  1163. {
  1164. "chat_id": data["chat_id"],
  1165. "message_id": data["id"],
  1166. "session_id": data["session_id"],
  1167. }
  1168. )
  1169. def get_priority(function_id):
  1170. function = Functions.get_function_by_id(function_id)
  1171. if function is not None and hasattr(function, "valves"):
  1172. # TODO: Fix FunctionModel to include vavles
  1173. return (function.valves if function.valves else {}).get("priority", 0)
  1174. return 0
  1175. filter_ids = [function.id for function in Functions.get_global_filter_functions()]
  1176. if "info" in model and "meta" in model["info"]:
  1177. filter_ids.extend(model["info"]["meta"].get("filterIds", []))
  1178. filter_ids = list(set(filter_ids))
  1179. enabled_filter_ids = [
  1180. function.id
  1181. for function in Functions.get_functions_by_type("filter", active_only=True)
  1182. ]
  1183. filter_ids = [
  1184. filter_id for filter_id in filter_ids if filter_id in enabled_filter_ids
  1185. ]
  1186. # Sort filter_ids by priority, using the get_priority function
  1187. filter_ids.sort(key=get_priority)
  1188. for filter_id in filter_ids:
  1189. filter = Functions.get_function_by_id(filter_id)
  1190. if not filter:
  1191. continue
  1192. if filter_id in webui_app.state.FUNCTIONS:
  1193. function_module = webui_app.state.FUNCTIONS[filter_id]
  1194. else:
  1195. function_module, _, _ = load_function_module_by_id(filter_id)
  1196. webui_app.state.FUNCTIONS[filter_id] = function_module
  1197. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  1198. valves = Functions.get_function_valves_by_id(filter_id)
  1199. function_module.valves = function_module.Valves(
  1200. **(valves if valves else {})
  1201. )
  1202. if not hasattr(function_module, "outlet"):
  1203. continue
  1204. try:
  1205. outlet = function_module.outlet
  1206. # Get the signature of the function
  1207. sig = inspect.signature(outlet)
  1208. params = {"body": data}
  1209. # Extra parameters to be passed to the function
  1210. extra_params = {
  1211. "__model__": model,
  1212. "__id__": filter_id,
  1213. "__event_emitter__": __event_emitter__,
  1214. "__event_call__": __event_call__,
  1215. }
  1216. # Add extra params in contained in function signature
  1217. for key, value in extra_params.items():
  1218. if key in sig.parameters:
  1219. params[key] = value
  1220. if "__user__" in sig.parameters:
  1221. __user__ = {
  1222. "id": user.id,
  1223. "email": user.email,
  1224. "name": user.name,
  1225. "role": user.role,
  1226. }
  1227. try:
  1228. if hasattr(function_module, "UserValves"):
  1229. __user__["valves"] = function_module.UserValves(
  1230. **Functions.get_user_valves_by_id_and_user_id(
  1231. filter_id, user.id
  1232. )
  1233. )
  1234. except Exception as e:
  1235. print(e)
  1236. params = {**params, "__user__": __user__}
  1237. if inspect.iscoroutinefunction(outlet):
  1238. data = await outlet(**params)
  1239. else:
  1240. data = outlet(**params)
  1241. except Exception as e:
  1242. print(f"Error: {e}")
  1243. return JSONResponse(
  1244. status_code=status.HTTP_400_BAD_REQUEST,
  1245. content={"detail": str(e)},
  1246. )
  1247. return data
  1248. @app.post("/api/chat/actions/{action_id}")
  1249. async def chat_action(action_id: str, form_data: dict, user=Depends(get_verified_user)):
  1250. if "." in action_id:
  1251. action_id, sub_action_id = action_id.split(".")
  1252. else:
  1253. sub_action_id = None
  1254. action = Functions.get_function_by_id(action_id)
  1255. if not action:
  1256. raise HTTPException(
  1257. status_code=status.HTTP_404_NOT_FOUND,
  1258. detail="Action not found",
  1259. )
  1260. model_list = await get_all_models()
  1261. models = {model["id"]: model for model in model_list}
  1262. data = form_data
  1263. model_id = data["model"]
  1264. if model_id not in models:
  1265. raise HTTPException(
  1266. status_code=status.HTTP_404_NOT_FOUND,
  1267. detail="Model not found",
  1268. )
  1269. model = models[model_id]
  1270. __event_emitter__ = get_event_emitter(
  1271. {
  1272. "chat_id": data["chat_id"],
  1273. "message_id": data["id"],
  1274. "session_id": data["session_id"],
  1275. }
  1276. )
  1277. __event_call__ = get_event_call(
  1278. {
  1279. "chat_id": data["chat_id"],
  1280. "message_id": data["id"],
  1281. "session_id": data["session_id"],
  1282. }
  1283. )
  1284. if action_id in webui_app.state.FUNCTIONS:
  1285. function_module = webui_app.state.FUNCTIONS[action_id]
  1286. else:
  1287. function_module, _, _ = load_function_module_by_id(action_id)
  1288. webui_app.state.FUNCTIONS[action_id] = function_module
  1289. if hasattr(function_module, "valves") and hasattr(function_module, "Valves"):
  1290. valves = Functions.get_function_valves_by_id(action_id)
  1291. function_module.valves = function_module.Valves(**(valves if valves else {}))
  1292. if hasattr(function_module, "action"):
  1293. try:
  1294. action = function_module.action
  1295. # Get the signature of the function
  1296. sig = inspect.signature(action)
  1297. params = {"body": data}
  1298. # Extra parameters to be passed to the function
  1299. extra_params = {
  1300. "__model__": model,
  1301. "__id__": sub_action_id if sub_action_id is not None else action_id,
  1302. "__event_emitter__": __event_emitter__,
  1303. "__event_call__": __event_call__,
  1304. }
  1305. # Add extra params in contained in function signature
  1306. for key, value in extra_params.items():
  1307. if key in sig.parameters:
  1308. params[key] = value
  1309. if "__user__" in sig.parameters:
  1310. __user__ = {
  1311. "id": user.id,
  1312. "email": user.email,
  1313. "name": user.name,
  1314. "role": user.role,
  1315. }
  1316. try:
  1317. if hasattr(function_module, "UserValves"):
  1318. __user__["valves"] = function_module.UserValves(
  1319. **Functions.get_user_valves_by_id_and_user_id(
  1320. action_id, user.id
  1321. )
  1322. )
  1323. except Exception as e:
  1324. print(e)
  1325. params = {**params, "__user__": __user__}
  1326. if inspect.iscoroutinefunction(action):
  1327. data = await action(**params)
  1328. else:
  1329. data = action(**params)
  1330. except Exception as e:
  1331. print(f"Error: {e}")
  1332. return JSONResponse(
  1333. status_code=status.HTTP_400_BAD_REQUEST,
  1334. content={"detail": str(e)},
  1335. )
  1336. return data
  1337. ##################################
  1338. #
  1339. # Task Endpoints
  1340. #
  1341. ##################################
  1342. # TODO: Refactor task API endpoints below into a separate file
  1343. @app.get("/api/task/config")
  1344. async def get_task_config(user=Depends(get_verified_user)):
  1345. return {
  1346. "TASK_MODEL": app.state.config.TASK_MODEL,
  1347. "TASK_MODEL_EXTERNAL": app.state.config.TASK_MODEL_EXTERNAL,
  1348. "TITLE_GENERATION_PROMPT_TEMPLATE": app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
  1349. "TAGS_GENERATION_PROMPT_TEMPLATE": app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE,
  1350. "ENABLE_TAGS_GENERATION": app.state.config.ENABLE_TAGS_GENERATION,
  1351. "ENABLE_SEARCH_QUERY_GENERATION": app.state.config.ENABLE_SEARCH_QUERY_GENERATION,
  1352. "ENABLE_RETRIEVAL_QUERY_GENERATION": app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION,
  1353. "QUERY_GENERATION_PROMPT_TEMPLATE": app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE,
  1354. "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  1355. }
  1356. class TaskConfigForm(BaseModel):
  1357. TASK_MODEL: Optional[str]
  1358. TASK_MODEL_EXTERNAL: Optional[str]
  1359. TITLE_GENERATION_PROMPT_TEMPLATE: str
  1360. TAGS_GENERATION_PROMPT_TEMPLATE: str
  1361. ENABLE_TAGS_GENERATION: bool
  1362. ENABLE_SEARCH_QUERY_GENERATION: bool
  1363. ENABLE_RETRIEVAL_QUERY_GENERATION: bool
  1364. QUERY_GENERATION_PROMPT_TEMPLATE: str
  1365. TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE: str
  1366. @app.post("/api/task/config/update")
  1367. async def update_task_config(form_data: TaskConfigForm, user=Depends(get_admin_user)):
  1368. app.state.config.TASK_MODEL = form_data.TASK_MODEL
  1369. app.state.config.TASK_MODEL_EXTERNAL = form_data.TASK_MODEL_EXTERNAL
  1370. app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE = (
  1371. form_data.TITLE_GENERATION_PROMPT_TEMPLATE
  1372. )
  1373. app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE = (
  1374. form_data.TAGS_GENERATION_PROMPT_TEMPLATE
  1375. )
  1376. app.state.config.ENABLE_TAGS_GENERATION = form_data.ENABLE_TAGS_GENERATION
  1377. app.state.config.ENABLE_SEARCH_QUERY_GENERATION = (
  1378. form_data.ENABLE_SEARCH_QUERY_GENERATION
  1379. )
  1380. app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION = (
  1381. form_data.ENABLE_RETRIEVAL_QUERY_GENERATION
  1382. )
  1383. app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE = (
  1384. form_data.QUERY_GENERATION_PROMPT_TEMPLATE
  1385. )
  1386. app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE = (
  1387. form_data.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
  1388. )
  1389. return {
  1390. "TASK_MODEL": app.state.config.TASK_MODEL,
  1391. "TASK_MODEL_EXTERNAL": app.state.config.TASK_MODEL_EXTERNAL,
  1392. "TITLE_GENERATION_PROMPT_TEMPLATE": app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
  1393. "TAGS_GENERATION_PROMPT_TEMPLATE": app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE,
  1394. "ENABLE_TAGS_GENERATION": app.state.config.ENABLE_TAGS_GENERATION,
  1395. "ENABLE_SEARCH_QUERY_GENERATION": app.state.config.ENABLE_SEARCH_QUERY_GENERATION,
  1396. "ENABLE_RETRIEVAL_QUERY_GENERATION": app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION,
  1397. "QUERY_GENERATION_PROMPT_TEMPLATE": app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE,
  1398. "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
  1399. }
  1400. @app.post("/api/task/title/completions")
  1401. async def generate_title(form_data: dict, user=Depends(get_verified_user)):
  1402. print("generate_title")
  1403. model_list = await get_all_models()
  1404. models = {model["id"]: model for model in model_list}
  1405. model_id = form_data["model"]
  1406. if model_id not in models:
  1407. raise HTTPException(
  1408. status_code=status.HTTP_404_NOT_FOUND,
  1409. detail="Model not found",
  1410. )
  1411. # Check if the user has a custom task model
  1412. # If the user has a custom task model, use that model
  1413. task_model_id = get_task_model_id(
  1414. model_id,
  1415. app.state.config.TASK_MODEL,
  1416. app.state.config.TASK_MODEL_EXTERNAL,
  1417. models,
  1418. )
  1419. print(task_model_id)
  1420. model = models[task_model_id]
  1421. if app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE != "":
  1422. template = app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE
  1423. else:
  1424. template = """Create a concise, 3-5 word title with an emoji as a title for the chat history, in the given language. Suitable Emojis for the summary can be used to enhance understanding but avoid quotation marks or special formatting. RESPOND ONLY WITH THE TITLE TEXT.
  1425. Examples of titles:
  1426. 📉 Stock Market Trends
  1427. 🍪 Perfect Chocolate Chip Recipe
  1428. Evolution of Music Streaming
  1429. Remote Work Productivity Tips
  1430. Artificial Intelligence in Healthcare
  1431. 🎮 Video Game Development Insights
  1432. <chat_history>
  1433. {{MESSAGES:END:2}}
  1434. </chat_history>"""
  1435. content = title_generation_template(
  1436. template,
  1437. form_data["messages"],
  1438. {
  1439. "name": user.name,
  1440. "location": user.info.get("location") if user.info else None,
  1441. },
  1442. )
  1443. payload = {
  1444. "model": task_model_id,
  1445. "messages": [{"role": "user", "content": content}],
  1446. "stream": False,
  1447. **(
  1448. {"max_tokens": 50}
  1449. if models[task_model_id]["owned_by"] == "ollama"
  1450. else {
  1451. "max_completion_tokens": 50,
  1452. }
  1453. ),
  1454. "chat_id": form_data.get("chat_id", None),
  1455. "metadata": {"task": str(TASKS.TITLE_GENERATION), "task_body": form_data},
  1456. }
  1457. log.debug(payload)
  1458. # Handle pipeline filters
  1459. try:
  1460. payload = filter_pipeline(payload, user, models)
  1461. except Exception as e:
  1462. if len(e.args) > 1:
  1463. return JSONResponse(
  1464. status_code=e.args[0],
  1465. content={"detail": e.args[1]},
  1466. )
  1467. else:
  1468. return JSONResponse(
  1469. status_code=status.HTTP_400_BAD_REQUEST,
  1470. content={"detail": str(e)},
  1471. )
  1472. if "chat_id" in payload:
  1473. del payload["chat_id"]
  1474. return await generate_chat_completions(form_data=payload, user=user)
  1475. @app.post("/api/task/tags/completions")
  1476. async def generate_chat_tags(form_data: dict, user=Depends(get_verified_user)):
  1477. print("generate_chat_tags")
  1478. if not app.state.config.ENABLE_TAGS_GENERATION:
  1479. return JSONResponse(
  1480. status_code=status.HTTP_200_OK,
  1481. content={"detail": "Tags generation is disabled"},
  1482. )
  1483. model_list = await get_all_models()
  1484. models = {model["id"]: model for model in model_list}
  1485. model_id = form_data["model"]
  1486. if model_id not in models:
  1487. raise HTTPException(
  1488. status_code=status.HTTP_404_NOT_FOUND,
  1489. detail="Model not found",
  1490. )
  1491. # Check if the user has a custom task model
  1492. # If the user has a custom task model, use that model
  1493. task_model_id = get_task_model_id(
  1494. model_id,
  1495. app.state.config.TASK_MODEL,
  1496. app.state.config.TASK_MODEL_EXTERNAL,
  1497. models,
  1498. )
  1499. print(task_model_id)
  1500. if app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE != "":
  1501. template = app.state.config.TAGS_GENERATION_PROMPT_TEMPLATE
  1502. else:
  1503. template = """### Task:
  1504. Generate 1-3 broad tags categorizing the main themes of the chat history, along with 1-3 more specific subtopic tags.
  1505. ### Guidelines:
  1506. - Start with high-level domains (e.g. Science, Technology, Philosophy, Arts, Politics, Business, Health, Sports, Entertainment, Education)
  1507. - Consider including relevant subfields/subdomains if they are strongly represented throughout the conversation
  1508. - If content is too short (less than 3 messages) or too diverse, use only ["General"]
  1509. - Use the chat's primary language; default to English if multilingual
  1510. - Prioritize accuracy over specificity
  1511. ### Output:
  1512. JSON format: { "tags": ["tag1", "tag2", "tag3"] }
  1513. ### Chat History:
  1514. <chat_history>
  1515. {{MESSAGES:END:6}}
  1516. </chat_history>"""
  1517. content = tags_generation_template(
  1518. template, form_data["messages"], {"name": user.name}
  1519. )
  1520. payload = {
  1521. "model": task_model_id,
  1522. "messages": [{"role": "user", "content": content}],
  1523. "stream": False,
  1524. "metadata": {"task": str(TASKS.TAGS_GENERATION), "task_body": form_data},
  1525. }
  1526. log.debug(payload)
  1527. # Handle pipeline filters
  1528. try:
  1529. payload = filter_pipeline(payload, user, models)
  1530. except Exception as e:
  1531. if len(e.args) > 1:
  1532. return JSONResponse(
  1533. status_code=e.args[0],
  1534. content={"detail": e.args[1]},
  1535. )
  1536. else:
  1537. return JSONResponse(
  1538. status_code=status.HTTP_400_BAD_REQUEST,
  1539. content={"detail": str(e)},
  1540. )
  1541. if "chat_id" in payload:
  1542. del payload["chat_id"]
  1543. return await generate_chat_completions(form_data=payload, user=user)
  1544. @app.post("/api/task/queries/completions")
  1545. async def generate_queries(form_data: dict, user=Depends(get_verified_user)):
  1546. print("generate_queries")
  1547. type = form_data.get("type")
  1548. if type == "web_search":
  1549. if not app.state.config.ENABLE_SEARCH_QUERY_GENERATION:
  1550. raise HTTPException(
  1551. status_code=status.HTTP_400_BAD_REQUEST,
  1552. detail=f"Search query generation is disabled",
  1553. )
  1554. elif type == "retrieval":
  1555. if not app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION:
  1556. raise HTTPException(
  1557. status_code=status.HTTP_400_BAD_REQUEST,
  1558. detail=f"Query generation is disabled",
  1559. )
  1560. model_list = await get_all_models()
  1561. models = {model["id"]: model for model in model_list}
  1562. model_id = form_data["model"]
  1563. if model_id not in models:
  1564. raise HTTPException(
  1565. status_code=status.HTTP_404_NOT_FOUND,
  1566. detail="Model not found",
  1567. )
  1568. # Check if the user has a custom task model
  1569. # If the user has a custom task model, use that model
  1570. task_model_id = get_task_model_id(
  1571. model_id,
  1572. app.state.config.TASK_MODEL,
  1573. app.state.config.TASK_MODEL_EXTERNAL,
  1574. models,
  1575. )
  1576. print(task_model_id)
  1577. model = models[task_model_id]
  1578. if app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE != "":
  1579. template = app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE
  1580. else:
  1581. template = DEFAULT_QUERY_GENERATION_PROMPT_TEMPLATE
  1582. content = query_generation_template(
  1583. template, form_data["messages"], {"name": user.name}
  1584. )
  1585. payload = {
  1586. "model": task_model_id,
  1587. "messages": [{"role": "user", "content": content}],
  1588. "stream": False,
  1589. "metadata": {"task": str(TASKS.QUERY_GENERATION), "task_body": form_data},
  1590. }
  1591. log.debug(payload)
  1592. # Handle pipeline filters
  1593. try:
  1594. payload = filter_pipeline(payload, user, models)
  1595. except Exception as e:
  1596. if len(e.args) > 1:
  1597. return JSONResponse(
  1598. status_code=e.args[0],
  1599. content={"detail": e.args[1]},
  1600. )
  1601. else:
  1602. return JSONResponse(
  1603. status_code=status.HTTP_400_BAD_REQUEST,
  1604. content={"detail": str(e)},
  1605. )
  1606. if "chat_id" in payload:
  1607. del payload["chat_id"]
  1608. return await generate_chat_completions(form_data=payload, user=user)
  1609. @app.post("/api/task/emoji/completions")
  1610. async def generate_emoji(form_data: dict, user=Depends(get_verified_user)):
  1611. print("generate_emoji")
  1612. model_list = await get_all_models()
  1613. models = {model["id"]: model for model in model_list}
  1614. model_id = form_data["model"]
  1615. if model_id not in models:
  1616. raise HTTPException(
  1617. status_code=status.HTTP_404_NOT_FOUND,
  1618. detail="Model not found",
  1619. )
  1620. # Check if the user has a custom task model
  1621. # If the user has a custom task model, use that model
  1622. task_model_id = get_task_model_id(
  1623. model_id,
  1624. app.state.config.TASK_MODEL,
  1625. app.state.config.TASK_MODEL_EXTERNAL,
  1626. models,
  1627. )
  1628. print(task_model_id)
  1629. model = models[task_model_id]
  1630. template = '''
  1631. Your task is to reflect the speaker's likely facial expression through a fitting emoji. Interpret emotions from the message and reflect their facial expression using fitting, diverse emojis (e.g., 😊, 😢, 😡, 😱).
  1632. Message: """{{prompt}}"""
  1633. '''
  1634. content = emoji_generation_template(
  1635. template,
  1636. form_data["prompt"],
  1637. {
  1638. "name": user.name,
  1639. "location": user.info.get("location") if user.info else None,
  1640. },
  1641. )
  1642. payload = {
  1643. "model": task_model_id,
  1644. "messages": [{"role": "user", "content": content}],
  1645. "stream": False,
  1646. **(
  1647. {"max_tokens": 4}
  1648. if models[task_model_id]["owned_by"] == "ollama"
  1649. else {
  1650. "max_completion_tokens": 4,
  1651. }
  1652. ),
  1653. "chat_id": form_data.get("chat_id", None),
  1654. "metadata": {"task": str(TASKS.EMOJI_GENERATION), "task_body": form_data},
  1655. }
  1656. log.debug(payload)
  1657. # Handle pipeline filters
  1658. try:
  1659. payload = filter_pipeline(payload, user, models)
  1660. except Exception as e:
  1661. if len(e.args) > 1:
  1662. return JSONResponse(
  1663. status_code=e.args[0],
  1664. content={"detail": e.args[1]},
  1665. )
  1666. else:
  1667. return JSONResponse(
  1668. status_code=status.HTTP_400_BAD_REQUEST,
  1669. content={"detail": str(e)},
  1670. )
  1671. if "chat_id" in payload:
  1672. del payload["chat_id"]
  1673. return await generate_chat_completions(form_data=payload, user=user)
  1674. @app.post("/api/task/moa/completions")
  1675. async def generate_moa_response(form_data: dict, user=Depends(get_verified_user)):
  1676. print("generate_moa_response")
  1677. model_list = await get_all_models()
  1678. models = {model["id"]: model for model in model_list}
  1679. model_id = form_data["model"]
  1680. if model_id not in models:
  1681. raise HTTPException(
  1682. status_code=status.HTTP_404_NOT_FOUND,
  1683. detail="Model not found",
  1684. )
  1685. # Check if the user has a custom task model
  1686. # If the user has a custom task model, use that model
  1687. task_model_id = get_task_model_id(
  1688. model_id,
  1689. app.state.config.TASK_MODEL,
  1690. app.state.config.TASK_MODEL_EXTERNAL,
  1691. models,
  1692. )
  1693. print(task_model_id)
  1694. model = models[task_model_id]
  1695. template = """You have been provided with a set of responses from various models to the latest user query: "{{prompt}}"
  1696. Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability.
  1697. Responses from models: {{responses}}"""
  1698. content = moa_response_generation_template(
  1699. template,
  1700. form_data["prompt"],
  1701. form_data["responses"],
  1702. )
  1703. payload = {
  1704. "model": task_model_id,
  1705. "messages": [{"role": "user", "content": content}],
  1706. "stream": form_data.get("stream", False),
  1707. "chat_id": form_data.get("chat_id", None),
  1708. "metadata": {
  1709. "task": str(TASKS.MOA_RESPONSE_GENERATION),
  1710. "task_body": form_data,
  1711. },
  1712. }
  1713. log.debug(payload)
  1714. try:
  1715. payload = filter_pipeline(payload, user, models)
  1716. except Exception as e:
  1717. if len(e.args) > 1:
  1718. return JSONResponse(
  1719. status_code=e.args[0],
  1720. content={"detail": e.args[1]},
  1721. )
  1722. else:
  1723. return JSONResponse(
  1724. status_code=status.HTTP_400_BAD_REQUEST,
  1725. content={"detail": str(e)},
  1726. )
  1727. if "chat_id" in payload:
  1728. del payload["chat_id"]
  1729. return await generate_chat_completions(form_data=payload, user=user)
  1730. ##################################
  1731. #
  1732. # Pipelines Endpoints
  1733. #
  1734. ##################################
  1735. # TODO: Refactor pipelines API endpoints below into a separate file
  1736. @app.get("/api/pipelines/list")
  1737. async def get_pipelines_list(user=Depends(get_admin_user)):
  1738. responses = await get_openai_models_responses()
  1739. print(responses)
  1740. urlIdxs = [
  1741. idx
  1742. for idx, response in enumerate(responses)
  1743. if response is not None and "pipelines" in response
  1744. ]
  1745. return {
  1746. "data": [
  1747. {
  1748. "url": openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx],
  1749. "idx": urlIdx,
  1750. }
  1751. for urlIdx in urlIdxs
  1752. ]
  1753. }
  1754. @app.post("/api/pipelines/upload")
  1755. async def upload_pipeline(
  1756. urlIdx: int = Form(...), file: UploadFile = File(...), user=Depends(get_admin_user)
  1757. ):
  1758. print("upload_pipeline", urlIdx, file.filename)
  1759. # Check if the uploaded file is a python file
  1760. if not (file.filename and file.filename.endswith(".py")):
  1761. raise HTTPException(
  1762. status_code=status.HTTP_400_BAD_REQUEST,
  1763. detail="Only Python (.py) files are allowed.",
  1764. )
  1765. upload_folder = f"{CACHE_DIR}/pipelines"
  1766. os.makedirs(upload_folder, exist_ok=True)
  1767. file_path = os.path.join(upload_folder, file.filename)
  1768. r = None
  1769. try:
  1770. # Save the uploaded file
  1771. with open(file_path, "wb") as buffer:
  1772. shutil.copyfileobj(file.file, buffer)
  1773. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1774. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1775. headers = {"Authorization": f"Bearer {key}"}
  1776. with open(file_path, "rb") as f:
  1777. files = {"file": f}
  1778. r = requests.post(f"{url}/pipelines/upload", headers=headers, files=files)
  1779. r.raise_for_status()
  1780. data = r.json()
  1781. return {**data}
  1782. except Exception as e:
  1783. # Handle connection error here
  1784. print(f"Connection error: {e}")
  1785. detail = "Pipeline not found"
  1786. status_code = status.HTTP_404_NOT_FOUND
  1787. if r is not None:
  1788. status_code = r.status_code
  1789. try:
  1790. res = r.json()
  1791. if "detail" in res:
  1792. detail = res["detail"]
  1793. except Exception:
  1794. pass
  1795. raise HTTPException(
  1796. status_code=status_code,
  1797. detail=detail,
  1798. )
  1799. finally:
  1800. # Ensure the file is deleted after the upload is completed or on failure
  1801. if os.path.exists(file_path):
  1802. os.remove(file_path)
  1803. class AddPipelineForm(BaseModel):
  1804. url: str
  1805. urlIdx: int
  1806. @app.post("/api/pipelines/add")
  1807. async def add_pipeline(form_data: AddPipelineForm, user=Depends(get_admin_user)):
  1808. r = None
  1809. try:
  1810. urlIdx = form_data.urlIdx
  1811. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1812. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1813. headers = {"Authorization": f"Bearer {key}"}
  1814. r = requests.post(
  1815. f"{url}/pipelines/add", headers=headers, json={"url": form_data.url}
  1816. )
  1817. r.raise_for_status()
  1818. data = r.json()
  1819. return {**data}
  1820. except Exception as e:
  1821. # Handle connection error here
  1822. print(f"Connection error: {e}")
  1823. detail = "Pipeline not found"
  1824. if r is not None:
  1825. try:
  1826. res = r.json()
  1827. if "detail" in res:
  1828. detail = res["detail"]
  1829. except Exception:
  1830. pass
  1831. raise HTTPException(
  1832. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1833. detail=detail,
  1834. )
  1835. class DeletePipelineForm(BaseModel):
  1836. id: str
  1837. urlIdx: int
  1838. @app.delete("/api/pipelines/delete")
  1839. async def delete_pipeline(form_data: DeletePipelineForm, user=Depends(get_admin_user)):
  1840. r = None
  1841. try:
  1842. urlIdx = form_data.urlIdx
  1843. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1844. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1845. headers = {"Authorization": f"Bearer {key}"}
  1846. r = requests.delete(
  1847. f"{url}/pipelines/delete", headers=headers, json={"id": form_data.id}
  1848. )
  1849. r.raise_for_status()
  1850. data = r.json()
  1851. return {**data}
  1852. except Exception as e:
  1853. # Handle connection error here
  1854. print(f"Connection error: {e}")
  1855. detail = "Pipeline not found"
  1856. if r is not None:
  1857. try:
  1858. res = r.json()
  1859. if "detail" in res:
  1860. detail = res["detail"]
  1861. except Exception:
  1862. pass
  1863. raise HTTPException(
  1864. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1865. detail=detail,
  1866. )
  1867. @app.get("/api/pipelines")
  1868. async def get_pipelines(urlIdx: Optional[int] = None, user=Depends(get_admin_user)):
  1869. r = None
  1870. try:
  1871. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1872. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1873. headers = {"Authorization": f"Bearer {key}"}
  1874. r = requests.get(f"{url}/pipelines", headers=headers)
  1875. r.raise_for_status()
  1876. data = r.json()
  1877. return {**data}
  1878. except Exception as e:
  1879. # Handle connection error here
  1880. print(f"Connection error: {e}")
  1881. detail = "Pipeline not found"
  1882. if r is not None:
  1883. try:
  1884. res = r.json()
  1885. if "detail" in res:
  1886. detail = res["detail"]
  1887. except Exception:
  1888. pass
  1889. raise HTTPException(
  1890. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1891. detail=detail,
  1892. )
  1893. @app.get("/api/pipelines/{pipeline_id}/valves")
  1894. async def get_pipeline_valves(
  1895. urlIdx: Optional[int],
  1896. pipeline_id: str,
  1897. user=Depends(get_admin_user),
  1898. ):
  1899. r = None
  1900. try:
  1901. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1902. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1903. headers = {"Authorization": f"Bearer {key}"}
  1904. r = requests.get(f"{url}/{pipeline_id}/valves", headers=headers)
  1905. r.raise_for_status()
  1906. data = r.json()
  1907. return {**data}
  1908. except Exception as e:
  1909. # Handle connection error here
  1910. print(f"Connection error: {e}")
  1911. detail = "Pipeline not found"
  1912. if r is not None:
  1913. try:
  1914. res = r.json()
  1915. if "detail" in res:
  1916. detail = res["detail"]
  1917. except Exception:
  1918. pass
  1919. raise HTTPException(
  1920. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1921. detail=detail,
  1922. )
  1923. @app.get("/api/pipelines/{pipeline_id}/valves/spec")
  1924. async def get_pipeline_valves_spec(
  1925. urlIdx: Optional[int],
  1926. pipeline_id: str,
  1927. user=Depends(get_admin_user),
  1928. ):
  1929. r = None
  1930. try:
  1931. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1932. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1933. headers = {"Authorization": f"Bearer {key}"}
  1934. r = requests.get(f"{url}/{pipeline_id}/valves/spec", headers=headers)
  1935. r.raise_for_status()
  1936. data = r.json()
  1937. return {**data}
  1938. except Exception as e:
  1939. # Handle connection error here
  1940. print(f"Connection error: {e}")
  1941. detail = "Pipeline not found"
  1942. if r is not None:
  1943. try:
  1944. res = r.json()
  1945. if "detail" in res:
  1946. detail = res["detail"]
  1947. except Exception:
  1948. pass
  1949. raise HTTPException(
  1950. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1951. detail=detail,
  1952. )
  1953. @app.post("/api/pipelines/{pipeline_id}/valves/update")
  1954. async def update_pipeline_valves(
  1955. urlIdx: Optional[int],
  1956. pipeline_id: str,
  1957. form_data: dict,
  1958. user=Depends(get_admin_user),
  1959. ):
  1960. r = None
  1961. try:
  1962. url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx]
  1963. key = openai_app.state.config.OPENAI_API_KEYS[urlIdx]
  1964. headers = {"Authorization": f"Bearer {key}"}
  1965. r = requests.post(
  1966. f"{url}/{pipeline_id}/valves/update",
  1967. headers=headers,
  1968. json={**form_data},
  1969. )
  1970. r.raise_for_status()
  1971. data = r.json()
  1972. return {**data}
  1973. except Exception as e:
  1974. # Handle connection error here
  1975. print(f"Connection error: {e}")
  1976. detail = "Pipeline not found"
  1977. if r is not None:
  1978. try:
  1979. res = r.json()
  1980. if "detail" in res:
  1981. detail = res["detail"]
  1982. except Exception:
  1983. pass
  1984. raise HTTPException(
  1985. status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND),
  1986. detail=detail,
  1987. )
  1988. ##################################
  1989. #
  1990. # Config Endpoints
  1991. #
  1992. ##################################
  1993. @app.get("/api/config")
  1994. async def get_app_config(request: Request):
  1995. user = None
  1996. if "token" in request.cookies:
  1997. token = request.cookies.get("token")
  1998. try:
  1999. data = decode_token(token)
  2000. except Exception as e:
  2001. log.debug(e)
  2002. raise HTTPException(
  2003. status_code=status.HTTP_401_UNAUTHORIZED,
  2004. detail="Invalid token",
  2005. )
  2006. if data is not None and "id" in data:
  2007. user = Users.get_user_by_id(data["id"])
  2008. onboarding = False
  2009. if user is None:
  2010. user_count = Users.get_num_users()
  2011. onboarding = user_count == 0
  2012. return {
  2013. **({"onboarding": True} if onboarding else {}),
  2014. "status": True,
  2015. "name": WEBUI_NAME,
  2016. "version": VERSION,
  2017. "default_locale": str(DEFAULT_LOCALE),
  2018. "oauth": {
  2019. "providers": {
  2020. name: config.get("name", name)
  2021. for name, config in OAUTH_PROVIDERS.items()
  2022. }
  2023. },
  2024. "features": {
  2025. "auth": WEBUI_AUTH,
  2026. "auth_trusted_header": bool(webui_app.state.AUTH_TRUSTED_EMAIL_HEADER),
  2027. "enable_ldap": webui_app.state.config.ENABLE_LDAP,
  2028. "enable_api_key": webui_app.state.config.ENABLE_API_KEY,
  2029. "enable_signup": webui_app.state.config.ENABLE_SIGNUP,
  2030. "enable_login_form": webui_app.state.config.ENABLE_LOGIN_FORM,
  2031. **(
  2032. {
  2033. "enable_web_search": retrieval_app.state.config.ENABLE_RAG_WEB_SEARCH,
  2034. "enable_image_generation": images_app.state.config.ENABLED,
  2035. "enable_community_sharing": webui_app.state.config.ENABLE_COMMUNITY_SHARING,
  2036. "enable_message_rating": webui_app.state.config.ENABLE_MESSAGE_RATING,
  2037. "enable_admin_export": ENABLE_ADMIN_EXPORT,
  2038. "enable_admin_chat_access": ENABLE_ADMIN_CHAT_ACCESS,
  2039. }
  2040. if user is not None
  2041. else {}
  2042. ),
  2043. },
  2044. **(
  2045. {
  2046. "default_models": webui_app.state.config.DEFAULT_MODELS,
  2047. "default_prompt_suggestions": webui_app.state.config.DEFAULT_PROMPT_SUGGESTIONS,
  2048. "audio": {
  2049. "tts": {
  2050. "engine": audio_app.state.config.TTS_ENGINE,
  2051. "voice": audio_app.state.config.TTS_VOICE,
  2052. "split_on": audio_app.state.config.TTS_SPLIT_ON,
  2053. },
  2054. "stt": {
  2055. "engine": audio_app.state.config.STT_ENGINE,
  2056. },
  2057. },
  2058. "file": {
  2059. "max_size": retrieval_app.state.config.FILE_MAX_SIZE,
  2060. "max_count": retrieval_app.state.config.FILE_MAX_COUNT,
  2061. },
  2062. "permissions": {**webui_app.state.config.USER_PERMISSIONS},
  2063. }
  2064. if user is not None
  2065. else {}
  2066. ),
  2067. }
  2068. # TODO: webhook endpoint should be under config endpoints
  2069. @app.get("/api/webhook")
  2070. async def get_webhook_url(user=Depends(get_admin_user)):
  2071. return {
  2072. "url": app.state.config.WEBHOOK_URL,
  2073. }
  2074. class UrlForm(BaseModel):
  2075. url: str
  2076. @app.post("/api/webhook")
  2077. async def update_webhook_url(form_data: UrlForm, user=Depends(get_admin_user)):
  2078. app.state.config.WEBHOOK_URL = form_data.url
  2079. webui_app.state.WEBHOOK_URL = app.state.config.WEBHOOK_URL
  2080. return {"url": app.state.config.WEBHOOK_URL}
  2081. @app.get("/api/version")
  2082. async def get_app_version():
  2083. return {
  2084. "version": VERSION,
  2085. }
  2086. @app.get("/api/changelog")
  2087. async def get_app_changelog():
  2088. return {key: CHANGELOG[key] for idx, key in enumerate(CHANGELOG) if idx < 5}
  2089. @app.get("/api/version/updates")
  2090. async def get_app_latest_release_version():
  2091. if OFFLINE_MODE:
  2092. log.debug(
  2093. f"Offline mode is enabled, returning current version as latest version"
  2094. )
  2095. return {"current": VERSION, "latest": VERSION}
  2096. try:
  2097. timeout = aiohttp.ClientTimeout(total=1)
  2098. async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
  2099. async with session.get(
  2100. "https://api.github.com/repos/open-webui/open-webui/releases/latest"
  2101. ) as response:
  2102. response.raise_for_status()
  2103. data = await response.json()
  2104. latest_version = data["tag_name"]
  2105. return {"current": VERSION, "latest": latest_version[1:]}
  2106. except Exception as e:
  2107. log.debug(e)
  2108. return {"current": VERSION, "latest": VERSION}
  2109. ############################
  2110. # OAuth Login & Callback
  2111. ############################
  2112. # SessionMiddleware is used by authlib for oauth
  2113. if len(OAUTH_PROVIDERS) > 0:
  2114. app.add_middleware(
  2115. SessionMiddleware,
  2116. secret_key=WEBUI_SECRET_KEY,
  2117. session_cookie="oui-session",
  2118. same_site=WEBUI_SESSION_COOKIE_SAME_SITE,
  2119. https_only=WEBUI_SESSION_COOKIE_SECURE,
  2120. )
  2121. @app.get("/oauth/{provider}/login")
  2122. async def oauth_login(provider: str, request: Request):
  2123. return await oauth_manager.handle_login(provider, request)
  2124. # OAuth login logic is as follows:
  2125. # 1. Attempt to find a user with matching subject ID, tied to the provider
  2126. # 2. If OAUTH_MERGE_ACCOUNTS_BY_EMAIL is true, find a user with the email address provided via OAuth
  2127. # - This is considered insecure in general, as OAuth providers do not always verify email addresses
  2128. # 3. If there is no user, and ENABLE_OAUTH_SIGNUP is true, create a user
  2129. # - Email addresses are considered unique, so we fail registration if the email address is already taken
  2130. @app.get("/oauth/{provider}/callback")
  2131. async def oauth_callback(provider: str, request: Request, response: Response):
  2132. return await oauth_manager.handle_callback(provider, request, response)
  2133. @app.get("/manifest.json")
  2134. async def get_manifest_json():
  2135. return {
  2136. "name": WEBUI_NAME,
  2137. "short_name": WEBUI_NAME,
  2138. "description": "Open WebUI is an open, extensible, user-friendly interface for AI that adapts to your workflow.",
  2139. "start_url": "/",
  2140. "display": "standalone",
  2141. "background_color": "#343541",
  2142. "orientation": "natural",
  2143. "icons": [
  2144. {
  2145. "src": "/static/logo.png",
  2146. "type": "image/png",
  2147. "sizes": "500x500",
  2148. "purpose": "any",
  2149. },
  2150. {
  2151. "src": "/static/logo.png",
  2152. "type": "image/png",
  2153. "sizes": "500x500",
  2154. "purpose": "maskable",
  2155. },
  2156. ],
  2157. }
  2158. @app.get("/opensearch.xml")
  2159. async def get_opensearch_xml():
  2160. xml_content = rf"""
  2161. <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" xmlns:moz="http://www.mozilla.org/2006/browser/search/">
  2162. <ShortName>{WEBUI_NAME}</ShortName>
  2163. <Description>Search {WEBUI_NAME}</Description>
  2164. <InputEncoding>UTF-8</InputEncoding>
  2165. <Image width="16" height="16" type="image/x-icon">{WEBUI_URL}/static/favicon.png</Image>
  2166. <Url type="text/html" method="get" template="{WEBUI_URL}/?q={"{searchTerms}"}"/>
  2167. <moz:SearchForm>{WEBUI_URL}</moz:SearchForm>
  2168. </OpenSearchDescription>
  2169. """
  2170. return Response(content=xml_content, media_type="application/xml")
  2171. @app.get("/health")
  2172. async def healthcheck():
  2173. return {"status": True}
  2174. @app.get("/health/db")
  2175. async def healthcheck_with_db():
  2176. Session.execute(text("SELECT 1;")).all()
  2177. return {"status": True}
  2178. app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
  2179. app.mount("/cache", StaticFiles(directory=CACHE_DIR), name="cache")
  2180. if os.path.exists(FRONTEND_BUILD_DIR):
  2181. mimetypes.add_type("text/javascript", ".js")
  2182. app.mount(
  2183. "/",
  2184. SPAStaticFiles(directory=FRONTEND_BUILD_DIR, html=True),
  2185. name="spa-static-files",
  2186. )
  2187. else:
  2188. log.warning(
  2189. f"Frontend build directory not found at '{FRONTEND_BUILD_DIR}'. Serving API only."
  2190. )