main.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703
  1. import hashlib
  2. import json
  3. import logging
  4. import os
  5. import uuid
  6. from functools import lru_cache
  7. from pathlib import Path
  8. from pydub import AudioSegment
  9. from pydub.silence import split_on_silence
  10. import aiohttp
  11. import aiofiles
  12. import requests
  13. from open_webui.config import (
  14. AUDIO_STT_ENGINE,
  15. AUDIO_STT_MODEL,
  16. AUDIO_STT_OPENAI_API_BASE_URL,
  17. AUDIO_STT_OPENAI_API_KEY,
  18. AUDIO_TTS_API_KEY,
  19. AUDIO_TTS_ENGINE,
  20. AUDIO_TTS_MODEL,
  21. AUDIO_TTS_OPENAI_API_BASE_URL,
  22. AUDIO_TTS_OPENAI_API_KEY,
  23. AUDIO_TTS_SPLIT_ON,
  24. AUDIO_TTS_VOICE,
  25. AUDIO_TTS_AZURE_SPEECH_REGION,
  26. AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  27. CACHE_DIR,
  28. CORS_ALLOW_ORIGIN,
  29. WHISPER_MODEL,
  30. WHISPER_MODEL_AUTO_UPDATE,
  31. WHISPER_MODEL_DIR,
  32. AppConfig,
  33. )
  34. from open_webui.constants import ERROR_MESSAGES
  35. from open_webui.env import (
  36. ENV,
  37. SRC_LOG_LEVELS,
  38. DEVICE_TYPE,
  39. ENABLE_FORWARD_USER_INFO_HEADERS,
  40. )
  41. from fastapi import Depends, FastAPI, File, HTTPException, Request, UploadFile, status
  42. from fastapi.middleware.cors import CORSMiddleware
  43. from fastapi.responses import FileResponse
  44. from pydantic import BaseModel
  45. from open_webui.utils.auth import get_admin_user, get_verified_user
  46. # Constants
  47. MAX_FILE_SIZE_MB = 25
  48. MAX_FILE_SIZE = MAX_FILE_SIZE_MB * 1024 * 1024 # Convert MB to bytes
  49. log = logging.getLogger(__name__)
  50. log.setLevel(SRC_LOG_LEVELS["AUDIO"])
  51. app = FastAPI(
  52. docs_url="/docs" if ENV == "dev" else None,
  53. openapi_url="/openapi.json" if ENV == "dev" else None,
  54. redoc_url=None,
  55. )
  56. app.add_middleware(
  57. CORSMiddleware,
  58. allow_origins=CORS_ALLOW_ORIGIN,
  59. allow_credentials=True,
  60. allow_methods=["*"],
  61. allow_headers=["*"],
  62. )
  63. app.state.config = AppConfig()
  64. app.state.config.STT_OPENAI_API_BASE_URL = AUDIO_STT_OPENAI_API_BASE_URL
  65. app.state.config.STT_OPENAI_API_KEY = AUDIO_STT_OPENAI_API_KEY
  66. app.state.config.STT_ENGINE = AUDIO_STT_ENGINE
  67. app.state.config.STT_MODEL = AUDIO_STT_MODEL
  68. app.state.config.WHISPER_MODEL = WHISPER_MODEL
  69. app.state.faster_whisper_model = None
  70. app.state.config.TTS_OPENAI_API_BASE_URL = AUDIO_TTS_OPENAI_API_BASE_URL
  71. app.state.config.TTS_OPENAI_API_KEY = AUDIO_TTS_OPENAI_API_KEY
  72. app.state.config.TTS_ENGINE = AUDIO_TTS_ENGINE
  73. app.state.config.TTS_MODEL = AUDIO_TTS_MODEL
  74. app.state.config.TTS_VOICE = AUDIO_TTS_VOICE
  75. app.state.config.TTS_API_KEY = AUDIO_TTS_API_KEY
  76. app.state.config.TTS_SPLIT_ON = AUDIO_TTS_SPLIT_ON
  77. app.state.speech_synthesiser = None
  78. app.state.speech_speaker_embeddings_dataset = None
  79. app.state.config.TTS_AZURE_SPEECH_REGION = AUDIO_TTS_AZURE_SPEECH_REGION
  80. app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT
  81. # setting device type for whisper model
  82. whisper_device_type = DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == "cuda" else "cpu"
  83. log.info(f"whisper_device_type: {whisper_device_type}")
  84. SPEECH_CACHE_DIR = Path(CACHE_DIR).joinpath("./audio/speech/")
  85. SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
  86. def set_faster_whisper_model(model: str, auto_update: bool = False):
  87. if model and app.state.config.STT_ENGINE == "":
  88. from faster_whisper import WhisperModel
  89. faster_whisper_kwargs = {
  90. "model_size_or_path": model,
  91. "device": whisper_device_type,
  92. "compute_type": "int8",
  93. "download_root": WHISPER_MODEL_DIR,
  94. "local_files_only": not auto_update,
  95. }
  96. try:
  97. app.state.faster_whisper_model = WhisperModel(**faster_whisper_kwargs)
  98. except Exception:
  99. log.warning(
  100. "WhisperModel initialization failed, attempting download with local_files_only=False"
  101. )
  102. faster_whisper_kwargs["local_files_only"] = False
  103. app.state.faster_whisper_model = WhisperModel(**faster_whisper_kwargs)
  104. else:
  105. app.state.faster_whisper_model = None
  106. class TTSConfigForm(BaseModel):
  107. OPENAI_API_BASE_URL: str
  108. OPENAI_API_KEY: str
  109. API_KEY: str
  110. ENGINE: str
  111. MODEL: str
  112. VOICE: str
  113. SPLIT_ON: str
  114. AZURE_SPEECH_REGION: str
  115. AZURE_SPEECH_OUTPUT_FORMAT: str
  116. class STTConfigForm(BaseModel):
  117. OPENAI_API_BASE_URL: str
  118. OPENAI_API_KEY: str
  119. ENGINE: str
  120. MODEL: str
  121. WHISPER_MODEL: str
  122. class AudioConfigUpdateForm(BaseModel):
  123. tts: TTSConfigForm
  124. stt: STTConfigForm
  125. from pydub import AudioSegment
  126. from pydub.utils import mediainfo
  127. def is_mp4_audio(file_path):
  128. """Check if the given file is an MP4 audio file."""
  129. if not os.path.isfile(file_path):
  130. print(f"File not found: {file_path}")
  131. return False
  132. info = mediainfo(file_path)
  133. if (
  134. info.get("codec_name") == "aac"
  135. and info.get("codec_type") == "audio"
  136. and info.get("codec_tag_string") == "mp4a"
  137. ):
  138. return True
  139. return False
  140. def convert_mp4_to_wav(file_path, output_path):
  141. """Convert MP4 audio file to WAV format."""
  142. audio = AudioSegment.from_file(file_path, format="mp4")
  143. audio.export(output_path, format="wav")
  144. print(f"Converted {file_path} to {output_path}")
  145. @app.get("/config")
  146. async def get_audio_config(user=Depends(get_admin_user)):
  147. return {
  148. "tts": {
  149. "OPENAI_API_BASE_URL": app.state.config.TTS_OPENAI_API_BASE_URL,
  150. "OPENAI_API_KEY": app.state.config.TTS_OPENAI_API_KEY,
  151. "API_KEY": app.state.config.TTS_API_KEY,
  152. "ENGINE": app.state.config.TTS_ENGINE,
  153. "MODEL": app.state.config.TTS_MODEL,
  154. "VOICE": app.state.config.TTS_VOICE,
  155. "SPLIT_ON": app.state.config.TTS_SPLIT_ON,
  156. "AZURE_SPEECH_REGION": app.state.config.TTS_AZURE_SPEECH_REGION,
  157. "AZURE_SPEECH_OUTPUT_FORMAT": app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  158. },
  159. "stt": {
  160. "OPENAI_API_BASE_URL": app.state.config.STT_OPENAI_API_BASE_URL,
  161. "OPENAI_API_KEY": app.state.config.STT_OPENAI_API_KEY,
  162. "ENGINE": app.state.config.STT_ENGINE,
  163. "MODEL": app.state.config.STT_MODEL,
  164. "WHISPER_MODEL": app.state.config.WHISPER_MODEL,
  165. },
  166. }
  167. @app.post("/config/update")
  168. async def update_audio_config(
  169. form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)
  170. ):
  171. app.state.config.TTS_OPENAI_API_BASE_URL = form_data.tts.OPENAI_API_BASE_URL
  172. app.state.config.TTS_OPENAI_API_KEY = form_data.tts.OPENAI_API_KEY
  173. app.state.config.TTS_API_KEY = form_data.tts.API_KEY
  174. app.state.config.TTS_ENGINE = form_data.tts.ENGINE
  175. app.state.config.TTS_MODEL = form_data.tts.MODEL
  176. app.state.config.TTS_VOICE = form_data.tts.VOICE
  177. app.state.config.TTS_SPLIT_ON = form_data.tts.SPLIT_ON
  178. app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
  179. app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = (
  180. form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
  181. )
  182. app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
  183. app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
  184. app.state.config.STT_ENGINE = form_data.stt.ENGINE
  185. app.state.config.STT_MODEL = form_data.stt.MODEL
  186. app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
  187. set_faster_whisper_model(form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE)
  188. return {
  189. "tts": {
  190. "OPENAI_API_BASE_URL": app.state.config.TTS_OPENAI_API_BASE_URL,
  191. "OPENAI_API_KEY": app.state.config.TTS_OPENAI_API_KEY,
  192. "API_KEY": app.state.config.TTS_API_KEY,
  193. "ENGINE": app.state.config.TTS_ENGINE,
  194. "MODEL": app.state.config.TTS_MODEL,
  195. "VOICE": app.state.config.TTS_VOICE,
  196. "SPLIT_ON": app.state.config.TTS_SPLIT_ON,
  197. "AZURE_SPEECH_REGION": app.state.config.TTS_AZURE_SPEECH_REGION,
  198. "AZURE_SPEECH_OUTPUT_FORMAT": app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  199. },
  200. "stt": {
  201. "OPENAI_API_BASE_URL": app.state.config.STT_OPENAI_API_BASE_URL,
  202. "OPENAI_API_KEY": app.state.config.STT_OPENAI_API_KEY,
  203. "ENGINE": app.state.config.STT_ENGINE,
  204. "MODEL": app.state.config.STT_MODEL,
  205. "WHISPER_MODEL": app.state.config.WHISPER_MODEL,
  206. },
  207. }
  208. def load_speech_pipeline():
  209. from transformers import pipeline
  210. from datasets import load_dataset
  211. if app.state.speech_synthesiser is None:
  212. app.state.speech_synthesiser = pipeline(
  213. "text-to-speech", "microsoft/speecht5_tts"
  214. )
  215. if app.state.speech_speaker_embeddings_dataset is None:
  216. app.state.speech_speaker_embeddings_dataset = load_dataset(
  217. "Matthijs/cmu-arctic-xvectors", split="validation"
  218. )
  219. @app.post("/speech")
  220. async def speech(request: Request, user=Depends(get_verified_user)):
  221. body = await request.body()
  222. name = hashlib.sha256(body).hexdigest()
  223. file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
  224. file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
  225. # Check if the file already exists in the cache
  226. if file_path.is_file():
  227. return FileResponse(file_path)
  228. if app.state.config.TTS_ENGINE == "openai":
  229. headers = {}
  230. headers["Authorization"] = f"Bearer {app.state.config.TTS_OPENAI_API_KEY}"
  231. headers["Content-Type"] = "application/json"
  232. if ENABLE_FORWARD_USER_INFO_HEADERS:
  233. headers["X-OpenWebUI-User-Name"] = user.name
  234. headers["X-OpenWebUI-User-Id"] = user.id
  235. headers["X-OpenWebUI-User-Email"] = user.email
  236. headers["X-OpenWebUI-User-Role"] = user.role
  237. try:
  238. body = body.decode("utf-8")
  239. body = json.loads(body)
  240. body["model"] = app.state.config.TTS_MODEL
  241. body = json.dumps(body).encode("utf-8")
  242. except Exception:
  243. pass
  244. try:
  245. async with aiohttp.ClientSession() as session:
  246. async with session.post(
  247. url=f"{app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech",
  248. data=body,
  249. headers=headers,
  250. ) as r:
  251. r.raise_for_status()
  252. async with aiofiles.open(file_path, "wb") as f:
  253. await f.write(await r.read())
  254. async with aiofiles.open(file_body_path, "w") as f:
  255. await f.write(json.dumps(json.loads(body.decode("utf-8"))))
  256. return FileResponse(file_path)
  257. except Exception as e:
  258. log.exception(e)
  259. error_detail = "Open WebUI: Server Connection Error"
  260. try:
  261. if r.status != 200:
  262. res = await r.json()
  263. if "error" in res:
  264. error_detail = f"External: {res['error']['message']}"
  265. except Exception:
  266. error_detail = f"External: {e}"
  267. raise HTTPException(
  268. status_code=getattr(r, "status", 500),
  269. detail=error_detail,
  270. )
  271. elif app.state.config.TTS_ENGINE == "elevenlabs":
  272. try:
  273. payload = json.loads(body.decode("utf-8"))
  274. except Exception as e:
  275. log.exception(e)
  276. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  277. voice_id = payload.get("voice", "")
  278. if voice_id not in get_available_voices():
  279. raise HTTPException(
  280. status_code=400,
  281. detail="Invalid voice id",
  282. )
  283. url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"
  284. headers = {
  285. "Accept": "audio/mpeg",
  286. "Content-Type": "application/json",
  287. "xi-api-key": app.state.config.TTS_API_KEY,
  288. }
  289. data = {
  290. "text": payload["input"],
  291. "model_id": app.state.config.TTS_MODEL,
  292. "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
  293. }
  294. try:
  295. async with aiohttp.ClientSession() as session:
  296. async with session.post(url, json=data, headers=headers) as r:
  297. r.raise_for_status()
  298. async with aiofiles.open(file_path, "wb") as f:
  299. await f.write(await r.read())
  300. async with aiofiles.open(file_body_path, "w") as f:
  301. await f.write(json.dumps(json.loads(body.decode("utf-8"))))
  302. return FileResponse(file_path)
  303. except Exception as e:
  304. log.exception(e)
  305. error_detail = "Open WebUI: Server Connection Error"
  306. try:
  307. if r.status != 200:
  308. res = await r.json()
  309. if "error" in res:
  310. error_detail = f"External: {res['error']['message']}"
  311. except Exception:
  312. error_detail = f"External: {e}"
  313. raise HTTPException(
  314. status_code=getattr(r, "status", 500),
  315. detail=error_detail,
  316. )
  317. elif app.state.config.TTS_ENGINE == "azure":
  318. try:
  319. payload = json.loads(body.decode("utf-8"))
  320. except Exception as e:
  321. log.exception(e)
  322. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  323. region = app.state.config.TTS_AZURE_SPEECH_REGION
  324. language = app.state.config.TTS_VOICE
  325. locale = "-".join(app.state.config.TTS_VOICE.split("-")[:1])
  326. output_format = app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT
  327. url = f"https://{region}.tts.speech.microsoft.com/cognitiveservices/v1"
  328. headers = {
  329. "Ocp-Apim-Subscription-Key": app.state.config.TTS_API_KEY,
  330. "Content-Type": "application/ssml+xml",
  331. "X-Microsoft-OutputFormat": output_format,
  332. }
  333. data = f"""<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{locale}">
  334. <voice name="{language}">{payload["input"]}</voice>
  335. </speak>"""
  336. try:
  337. async with aiohttp.ClientSession() as session:
  338. async with session.post(url, headers=headers, data=data) as response:
  339. if response.status == 200:
  340. async with aiofiles.open(file_path, "wb") as f:
  341. await f.write(await response.read())
  342. return FileResponse(file_path)
  343. else:
  344. error_msg = f"Error synthesizing speech - {response.reason}"
  345. log.error(error_msg)
  346. raise HTTPException(status_code=500, detail=error_msg)
  347. except Exception as e:
  348. log.exception(e)
  349. raise HTTPException(status_code=500, detail=str(e))
  350. elif app.state.config.TTS_ENGINE == "transformers":
  351. payload = None
  352. try:
  353. payload = json.loads(body.decode("utf-8"))
  354. except Exception as e:
  355. log.exception(e)
  356. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  357. import torch
  358. import soundfile as sf
  359. load_speech_pipeline()
  360. embeddings_dataset = app.state.speech_speaker_embeddings_dataset
  361. speaker_index = 6799
  362. try:
  363. speaker_index = embeddings_dataset["filename"].index(
  364. app.state.config.TTS_MODEL
  365. )
  366. except Exception:
  367. pass
  368. speaker_embedding = torch.tensor(
  369. embeddings_dataset[speaker_index]["xvector"]
  370. ).unsqueeze(0)
  371. speech = app.state.speech_synthesiser(
  372. payload["input"],
  373. forward_params={"speaker_embeddings": speaker_embedding},
  374. )
  375. sf.write(file_path, speech["audio"], samplerate=speech["sampling_rate"])
  376. with open(file_body_path, "w") as f:
  377. json.dump(json.loads(body.decode("utf-8")), f)
  378. return FileResponse(file_path)
  379. def transcribe(file_path):
  380. print("transcribe", file_path)
  381. filename = os.path.basename(file_path)
  382. file_dir = os.path.dirname(file_path)
  383. id = filename.split(".")[0]
  384. if app.state.config.STT_ENGINE == "":
  385. if app.state.faster_whisper_model is None:
  386. set_faster_whisper_model(app.state.config.WHISPER_MODEL)
  387. model = app.state.faster_whisper_model
  388. segments, info = model.transcribe(file_path, beam_size=5)
  389. log.info(
  390. "Detected language '%s' with probability %f"
  391. % (info.language, info.language_probability)
  392. )
  393. transcript = "".join([segment.text for segment in list(segments)])
  394. data = {"text": transcript.strip()}
  395. # save the transcript to a json file
  396. transcript_file = f"{file_dir}/{id}.json"
  397. with open(transcript_file, "w") as f:
  398. json.dump(data, f)
  399. log.debug(data)
  400. return data
  401. elif app.state.config.STT_ENGINE == "openai":
  402. if is_mp4_audio(file_path):
  403. print("is_mp4_audio")
  404. os.rename(file_path, file_path.replace(".wav", ".mp4"))
  405. # Convert MP4 audio file to WAV format
  406. convert_mp4_to_wav(file_path.replace(".wav", ".mp4"), file_path)
  407. headers = {"Authorization": f"Bearer {app.state.config.STT_OPENAI_API_KEY}"}
  408. files = {"file": (filename, open(file_path, "rb"))}
  409. data = {"model": app.state.config.STT_MODEL}
  410. log.debug(files, data)
  411. r = None
  412. try:
  413. r = requests.post(
  414. url=f"{app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions",
  415. headers=headers,
  416. files=files,
  417. data=data,
  418. )
  419. r.raise_for_status()
  420. data = r.json()
  421. # save the transcript to a json file
  422. transcript_file = f"{file_dir}/{id}.json"
  423. with open(transcript_file, "w") as f:
  424. json.dump(data, f)
  425. print(data)
  426. return data
  427. except Exception as e:
  428. log.exception(e)
  429. error_detail = "Open WebUI: Server Connection Error"
  430. if r is not None:
  431. try:
  432. res = r.json()
  433. if "error" in res:
  434. error_detail = f"External: {res['error']['message']}"
  435. except Exception:
  436. error_detail = f"External: {e}"
  437. raise Exception(error_detail)
  438. @app.post("/transcriptions")
  439. def transcription(
  440. file: UploadFile = File(...),
  441. user=Depends(get_verified_user),
  442. ):
  443. log.info(f"file.content_type: {file.content_type}")
  444. if file.content_type not in ["audio/mpeg", "audio/wav", "audio/ogg", "audio/x-m4a"]:
  445. raise HTTPException(
  446. status_code=status.HTTP_400_BAD_REQUEST,
  447. detail=ERROR_MESSAGES.FILE_NOT_SUPPORTED,
  448. )
  449. try:
  450. ext = file.filename.split(".")[-1]
  451. id = uuid.uuid4()
  452. filename = f"{id}.{ext}"
  453. contents = file.file.read()
  454. file_dir = f"{CACHE_DIR}/audio/transcriptions"
  455. os.makedirs(file_dir, exist_ok=True)
  456. file_path = f"{file_dir}/{filename}"
  457. with open(file_path, "wb") as f:
  458. f.write(contents)
  459. try:
  460. if os.path.getsize(file_path) > MAX_FILE_SIZE: # file is bigger than 25MB
  461. log.debug(f"File size is larger than {MAX_FILE_SIZE_MB}MB")
  462. audio = AudioSegment.from_file(file_path)
  463. audio = audio.set_frame_rate(16000).set_channels(1) # Compress audio
  464. compressed_path = f"{file_dir}/{id}_compressed.opus"
  465. audio.export(compressed_path, format="opus", bitrate="32k")
  466. log.debug(f"Compressed audio to {compressed_path}")
  467. file_path = compressed_path
  468. if (
  469. os.path.getsize(file_path) > MAX_FILE_SIZE
  470. ): # Still larger than 25MB after compression
  471. log.debug(
  472. f"Compressed file size is still larger than {MAX_FILE_SIZE_MB}MB: {os.path.getsize(file_path)}"
  473. )
  474. raise HTTPException(
  475. status_code=status.HTTP_400_BAD_REQUEST,
  476. detail=ERROR_MESSAGES.FILE_TOO_LARGE(
  477. size=f"{MAX_FILE_SIZE_MB}MB"
  478. ),
  479. )
  480. data = transcribe(file_path)
  481. else:
  482. data = transcribe(file_path)
  483. file_path = file_path.split("/")[-1]
  484. return {**data, "filename": file_path}
  485. except Exception as e:
  486. log.exception(e)
  487. raise HTTPException(
  488. status_code=status.HTTP_400_BAD_REQUEST,
  489. detail=ERROR_MESSAGES.DEFAULT(e),
  490. )
  491. except Exception as e:
  492. log.exception(e)
  493. raise HTTPException(
  494. status_code=status.HTTP_400_BAD_REQUEST,
  495. detail=ERROR_MESSAGES.DEFAULT(e),
  496. )
  497. def get_available_models() -> list[dict]:
  498. if app.state.config.TTS_ENGINE == "openai":
  499. return [{"id": "tts-1"}, {"id": "tts-1-hd"}]
  500. elif app.state.config.TTS_ENGINE == "elevenlabs":
  501. headers = {
  502. "xi-api-key": app.state.config.TTS_API_KEY,
  503. "Content-Type": "application/json",
  504. }
  505. try:
  506. response = requests.get(
  507. "https://api.elevenlabs.io/v1/models", headers=headers, timeout=5
  508. )
  509. response.raise_for_status()
  510. models = response.json()
  511. return [
  512. {"name": model["name"], "id": model["model_id"]} for model in models
  513. ]
  514. except requests.RequestException as e:
  515. log.error(f"Error fetching voices: {str(e)}")
  516. return []
  517. @app.get("/models")
  518. async def get_models(user=Depends(get_verified_user)):
  519. return {"models": get_available_models()}
  520. def get_available_voices() -> dict:
  521. """Returns {voice_id: voice_name} dict"""
  522. ret = {}
  523. if app.state.config.TTS_ENGINE == "openai":
  524. ret = {
  525. "alloy": "alloy",
  526. "echo": "echo",
  527. "fable": "fable",
  528. "onyx": "onyx",
  529. "nova": "nova",
  530. "shimmer": "shimmer",
  531. }
  532. elif app.state.config.TTS_ENGINE == "elevenlabs":
  533. try:
  534. ret = get_elevenlabs_voices()
  535. except Exception:
  536. # Avoided @lru_cache with exception
  537. pass
  538. elif app.state.config.TTS_ENGINE == "azure":
  539. try:
  540. region = app.state.config.TTS_AZURE_SPEECH_REGION
  541. url = f"https://{region}.tts.speech.microsoft.com/cognitiveservices/voices/list"
  542. headers = {"Ocp-Apim-Subscription-Key": app.state.config.TTS_API_KEY}
  543. response = requests.get(url, headers=headers)
  544. response.raise_for_status()
  545. voices = response.json()
  546. for voice in voices:
  547. ret[voice["ShortName"]] = (
  548. f"{voice['DisplayName']} ({voice['ShortName']})"
  549. )
  550. except requests.RequestException as e:
  551. log.error(f"Error fetching voices: {str(e)}")
  552. return ret
  553. @lru_cache
  554. def get_elevenlabs_voices() -> dict:
  555. """
  556. Note, set the following in your .env file to use Elevenlabs:
  557. AUDIO_TTS_ENGINE=elevenlabs
  558. AUDIO_TTS_API_KEY=sk_... # Your Elevenlabs API key
  559. AUDIO_TTS_VOICE=EXAVITQu4vr4xnSDxMaL # From https://api.elevenlabs.io/v1/voices
  560. AUDIO_TTS_MODEL=eleven_multilingual_v2
  561. """
  562. headers = {
  563. "xi-api-key": app.state.config.TTS_API_KEY,
  564. "Content-Type": "application/json",
  565. }
  566. try:
  567. # TODO: Add retries
  568. response = requests.get("https://api.elevenlabs.io/v1/voices", headers=headers)
  569. response.raise_for_status()
  570. voices_data = response.json()
  571. voices = {}
  572. for voice in voices_data.get("voices", []):
  573. voices[voice["voice_id"]] = voice["name"]
  574. except requests.RequestException as e:
  575. # Avoid @lru_cache with exception
  576. log.error(f"Error fetching voices: {str(e)}")
  577. raise RuntimeError(f"Error fetching voices: {str(e)}")
  578. return voices
  579. @app.get("/voices")
  580. async def get_voices(user=Depends(get_verified_user)):
  581. return {"voices": [{"id": k, "name": v} for k, v in get_available_voices().items()]}