audio.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785
  1. import hashlib
  2. import json
  3. import logging
  4. import os
  5. import uuid
  6. from functools import lru_cache
  7. from pathlib import Path
  8. from pydub import AudioSegment
  9. from pydub.silence import split_on_silence
  10. import aiohttp
  11. import aiofiles
  12. import requests
  13. import mimetypes
  14. from fastapi import (
  15. Depends,
  16. FastAPI,
  17. File,
  18. HTTPException,
  19. Request,
  20. UploadFile,
  21. status,
  22. APIRouter,
  23. )
  24. from fastapi.middleware.cors import CORSMiddleware
  25. from fastapi.responses import FileResponse
  26. from pydantic import BaseModel
  27. from open_webui.utils.auth import get_admin_user, get_verified_user
  28. from open_webui.config import (
  29. WHISPER_MODEL_AUTO_UPDATE,
  30. WHISPER_MODEL_DIR,
  31. CACHE_DIR,
  32. )
  33. from open_webui.constants import ERROR_MESSAGES
  34. from open_webui.env import (
  35. AIOHTTP_CLIENT_TIMEOUT,
  36. ENV,
  37. SRC_LOG_LEVELS,
  38. DEVICE_TYPE,
  39. ENABLE_FORWARD_USER_INFO_HEADERS,
  40. )
  41. router = APIRouter()
  42. # Constants
  43. MAX_FILE_SIZE_MB = 25
  44. MAX_FILE_SIZE = MAX_FILE_SIZE_MB * 1024 * 1024 # Convert MB to bytes
  45. log = logging.getLogger(__name__)
  46. log.setLevel(SRC_LOG_LEVELS["AUDIO"])
  47. SPEECH_CACHE_DIR = Path(CACHE_DIR).joinpath("./audio/speech/")
  48. SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
  49. ##########################################
  50. #
  51. # Utility functions
  52. #
  53. ##########################################
  54. from pydub import AudioSegment
  55. from pydub.utils import mediainfo
  56. def is_mp4_audio(file_path):
  57. """Check if the given file is an MP4 audio file."""
  58. if not os.path.isfile(file_path):
  59. print(f"File not found: {file_path}")
  60. return False
  61. info = mediainfo(file_path)
  62. if (
  63. info.get("codec_name") == "aac"
  64. and info.get("codec_type") == "audio"
  65. and info.get("codec_tag_string") == "mp4a"
  66. ):
  67. return True
  68. return False
  69. def convert_mp4_to_wav(file_path, output_path):
  70. """Convert MP4 audio file to WAV format."""
  71. audio = AudioSegment.from_file(file_path, format="mp4")
  72. audio.export(output_path, format="wav")
  73. print(f"Converted {file_path} to {output_path}")
  74. def set_faster_whisper_model(model: str, auto_update: bool = False):
  75. whisper_model = None
  76. if model:
  77. from faster_whisper import WhisperModel
  78. faster_whisper_kwargs = {
  79. "model_size_or_path": model,
  80. "device": DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == "cuda" else "cpu",
  81. "compute_type": "int8",
  82. "download_root": WHISPER_MODEL_DIR,
  83. "local_files_only": not auto_update,
  84. }
  85. try:
  86. whisper_model = WhisperModel(**faster_whisper_kwargs)
  87. except Exception:
  88. log.warning(
  89. "WhisperModel initialization failed, attempting download with local_files_only=False"
  90. )
  91. faster_whisper_kwargs["local_files_only"] = False
  92. whisper_model = WhisperModel(**faster_whisper_kwargs)
  93. return whisper_model
  94. ##########################################
  95. #
  96. # Audio API
  97. #
  98. ##########################################
  99. class TTSConfigForm(BaseModel):
  100. OPENAI_API_BASE_URL: str
  101. OPENAI_API_KEY: str
  102. API_KEY: str
  103. ENGINE: str
  104. MODEL: str
  105. VOICE: str
  106. SPLIT_ON: str
  107. AZURE_SPEECH_REGION: str
  108. AZURE_SPEECH_OUTPUT_FORMAT: str
  109. class STTConfigForm(BaseModel):
  110. OPENAI_API_BASE_URL: str
  111. OPENAI_API_KEY: str
  112. ENGINE: str
  113. MODEL: str
  114. WHISPER_MODEL: str
  115. DEEPGRAM_API_KEY: str
  116. class AudioConfigUpdateForm(BaseModel):
  117. tts: TTSConfigForm
  118. stt: STTConfigForm
  119. @router.get("/config")
  120. async def get_audio_config(request: Request, user=Depends(get_admin_user)):
  121. return {
  122. "tts": {
  123. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  124. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  125. "API_KEY": request.app.state.config.TTS_API_KEY,
  126. "ENGINE": request.app.state.config.TTS_ENGINE,
  127. "MODEL": request.app.state.config.TTS_MODEL,
  128. "VOICE": request.app.state.config.TTS_VOICE,
  129. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  130. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  131. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  132. },
  133. "stt": {
  134. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  135. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  136. "ENGINE": request.app.state.config.STT_ENGINE,
  137. "MODEL": request.app.state.config.STT_MODEL,
  138. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  139. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  140. },
  141. }
  142. @router.post("/config/update")
  143. async def update_audio_config(
  144. request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)
  145. ):
  146. request.app.state.config.TTS_OPENAI_API_BASE_URL = form_data.tts.OPENAI_API_BASE_URL
  147. request.app.state.config.TTS_OPENAI_API_KEY = form_data.tts.OPENAI_API_KEY
  148. request.app.state.config.TTS_API_KEY = form_data.tts.API_KEY
  149. request.app.state.config.TTS_ENGINE = form_data.tts.ENGINE
  150. request.app.state.config.TTS_MODEL = form_data.tts.MODEL
  151. request.app.state.config.TTS_VOICE = form_data.tts.VOICE
  152. request.app.state.config.TTS_SPLIT_ON = form_data.tts.SPLIT_ON
  153. request.app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
  154. request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = (
  155. form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
  156. )
  157. request.app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
  158. request.app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
  159. request.app.state.config.STT_ENGINE = form_data.stt.ENGINE
  160. request.app.state.config.STT_MODEL = form_data.stt.MODEL
  161. request.app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
  162. request.app.state.config.DEEPGRAM_API_KEY = form_data.stt.DEEPGRAM_API_KEY
  163. if request.app.state.config.STT_ENGINE == "":
  164. request.app.state.faster_whisper_model = set_faster_whisper_model(
  165. form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE
  166. )
  167. return {
  168. "tts": {
  169. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  170. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  171. "API_KEY": request.app.state.config.TTS_API_KEY,
  172. "ENGINE": request.app.state.config.TTS_ENGINE,
  173. "MODEL": request.app.state.config.TTS_MODEL,
  174. "VOICE": request.app.state.config.TTS_VOICE,
  175. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  176. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  177. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  178. },
  179. "stt": {
  180. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  181. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  182. "ENGINE": request.app.state.config.STT_ENGINE,
  183. "MODEL": request.app.state.config.STT_MODEL,
  184. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  185. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  186. },
  187. }
  188. def load_speech_pipeline(request):
  189. from transformers import pipeline
  190. from datasets import load_dataset
  191. if request.app.state.speech_synthesiser is None:
  192. request.app.state.speech_synthesiser = pipeline(
  193. "text-to-speech", "microsoft/speecht5_tts"
  194. )
  195. if request.app.state.speech_speaker_embeddings_dataset is None:
  196. request.app.state.speech_speaker_embeddings_dataset = load_dataset(
  197. "Matthijs/cmu-arctic-xvectors", split="validation"
  198. )
  199. @router.post("/speech")
  200. async def speech(request: Request, user=Depends(get_verified_user)):
  201. body = await request.body()
  202. name = hashlib.sha256(
  203. body
  204. + str(request.app.state.config.TTS_ENGINE).encode("utf-8")
  205. + str(request.app.state.config.TTS_MODEL).encode("utf-8")
  206. ).hexdigest()
  207. file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
  208. file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
  209. # Check if the file already exists in the cache
  210. if file_path.is_file():
  211. return FileResponse(file_path)
  212. payload = None
  213. try:
  214. payload = json.loads(body.decode("utf-8"))
  215. except Exception as e:
  216. log.exception(e)
  217. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  218. if request.app.state.config.TTS_ENGINE == "openai":
  219. payload["model"] = request.app.state.config.TTS_MODEL
  220. try:
  221. # print(payload)
  222. timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  223. async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
  224. async with session.post(
  225. url=f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech",
  226. json=payload,
  227. headers={
  228. "Content-Type": "application/json",
  229. "Authorization": f"Bearer {request.app.state.config.TTS_OPENAI_API_KEY}",
  230. **(
  231. {
  232. "X-OpenWebUI-User-Name": user.name,
  233. "X-OpenWebUI-User-Id": user.id,
  234. "X-OpenWebUI-User-Email": user.email,
  235. "X-OpenWebUI-User-Role": user.role,
  236. }
  237. if ENABLE_FORWARD_USER_INFO_HEADERS
  238. else {}
  239. ),
  240. },
  241. ) as r:
  242. r.raise_for_status()
  243. async with aiofiles.open(file_path, "wb") as f:
  244. await f.write(await r.read())
  245. async with aiofiles.open(file_body_path, "w") as f:
  246. await f.write(json.dumps(payload))
  247. return FileResponse(file_path)
  248. except Exception as e:
  249. log.exception(e)
  250. detail = None
  251. try:
  252. if r.status != 200:
  253. res = await r.json()
  254. if "error" in res:
  255. detail = f"External: {res['error'].get('message', '')}"
  256. except Exception:
  257. detail = f"External: {e}"
  258. raise HTTPException(
  259. status_code=getattr(r, "status", 500),
  260. detail=detail if detail else "Open WebUI: Server Connection Error",
  261. )
  262. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  263. voice_id = payload.get("voice", "")
  264. if voice_id not in get_available_voices(request):
  265. raise HTTPException(
  266. status_code=400,
  267. detail="Invalid voice id",
  268. )
  269. try:
  270. timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  271. async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
  272. async with session.post(
  273. f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
  274. json={
  275. "text": payload["input"],
  276. "model_id": request.app.state.config.TTS_MODEL,
  277. "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
  278. },
  279. headers={
  280. "Accept": "audio/mpeg",
  281. "Content-Type": "application/json",
  282. "xi-api-key": request.app.state.config.TTS_API_KEY,
  283. },
  284. ) as r:
  285. r.raise_for_status()
  286. async with aiofiles.open(file_path, "wb") as f:
  287. await f.write(await r.read())
  288. async with aiofiles.open(file_body_path, "w") as f:
  289. await f.write(json.dumps(payload))
  290. return FileResponse(file_path)
  291. except Exception as e:
  292. log.exception(e)
  293. detail = None
  294. try:
  295. if r.status != 200:
  296. res = await r.json()
  297. if "error" in res:
  298. detail = f"External: {res['error'].get('message', '')}"
  299. except Exception:
  300. detail = f"External: {e}"
  301. raise HTTPException(
  302. status_code=getattr(r, "status", 500),
  303. detail=detail if detail else "Open WebUI: Server Connection Error",
  304. )
  305. elif request.app.state.config.TTS_ENGINE == "azure":
  306. try:
  307. payload = json.loads(body.decode("utf-8"))
  308. except Exception as e:
  309. log.exception(e)
  310. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  311. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  312. language = request.app.state.config.TTS_VOICE
  313. locale = "-".join(request.app.state.config.TTS_VOICE.split("-")[:1])
  314. output_format = request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT
  315. try:
  316. data = f"""<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{locale}">
  317. <voice name="{language}">{payload["input"]}</voice>
  318. </speak>"""
  319. timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  320. async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
  321. async with session.post(
  322. f"https://{region}.tts.speech.microsoft.com/cognitiveservices/v1",
  323. headers={
  324. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY,
  325. "Content-Type": "application/ssml+xml",
  326. "X-Microsoft-OutputFormat": output_format,
  327. },
  328. data=data,
  329. ) as r:
  330. r.raise_for_status()
  331. async with aiofiles.open(file_path, "wb") as f:
  332. await f.write(await r.read())
  333. async with aiofiles.open(file_body_path, "w") as f:
  334. await f.write(json.dumps(payload))
  335. return FileResponse(file_path)
  336. except Exception as e:
  337. log.exception(e)
  338. detail = None
  339. try:
  340. if r.status != 200:
  341. res = await r.json()
  342. if "error" in res:
  343. detail = f"External: {res['error'].get('message', '')}"
  344. except Exception:
  345. detail = f"External: {e}"
  346. raise HTTPException(
  347. status_code=getattr(r, "status", 500),
  348. detail=detail if detail else "Open WebUI: Server Connection Error",
  349. )
  350. elif request.app.state.config.TTS_ENGINE == "transformers":
  351. payload = None
  352. try:
  353. payload = json.loads(body.decode("utf-8"))
  354. except Exception as e:
  355. log.exception(e)
  356. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  357. import torch
  358. import soundfile as sf
  359. load_speech_pipeline(request)
  360. embeddings_dataset = request.app.state.speech_speaker_embeddings_dataset
  361. speaker_index = 6799
  362. try:
  363. speaker_index = embeddings_dataset["filename"].index(
  364. request.app.state.config.TTS_MODEL
  365. )
  366. except Exception:
  367. pass
  368. speaker_embedding = torch.tensor(
  369. embeddings_dataset[speaker_index]["xvector"]
  370. ).unsqueeze(0)
  371. speech = request.app.state.speech_synthesiser(
  372. payload["input"],
  373. forward_params={"speaker_embeddings": speaker_embedding},
  374. )
  375. sf.write(file_path, speech["audio"], samplerate=speech["sampling_rate"])
  376. async with aiofiles.open(file_body_path, "w") as f:
  377. await f.write(json.dumps(payload))
  378. return FileResponse(file_path)
  379. def transcribe(request: Request, file_path):
  380. print("transcribe", file_path)
  381. filename = os.path.basename(file_path)
  382. file_dir = os.path.dirname(file_path)
  383. id = filename.split(".")[0]
  384. if request.app.state.config.STT_ENGINE == "":
  385. if request.app.state.faster_whisper_model is None:
  386. request.app.state.faster_whisper_model = set_faster_whisper_model(
  387. request.app.state.config.WHISPER_MODEL
  388. )
  389. model = request.app.state.faster_whisper_model
  390. segments, info = model.transcribe(file_path, beam_size=5)
  391. log.info(
  392. "Detected language '%s' with probability %f"
  393. % (info.language, info.language_probability)
  394. )
  395. transcript = "".join([segment.text for segment in list(segments)])
  396. data = {"text": transcript.strip()}
  397. # save the transcript to a json file
  398. transcript_file = f"{file_dir}/{id}.json"
  399. with open(transcript_file, "w") as f:
  400. json.dump(data, f)
  401. log.debug(data)
  402. return data
  403. elif request.app.state.config.STT_ENGINE == "openai":
  404. if is_mp4_audio(file_path):
  405. os.rename(file_path, file_path.replace(".wav", ".mp4"))
  406. # Convert MP4 audio file to WAV format
  407. convert_mp4_to_wav(file_path.replace(".wav", ".mp4"), file_path)
  408. r = None
  409. try:
  410. r = requests.post(
  411. url=f"{request.app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions",
  412. headers={
  413. "Authorization": f"Bearer {request.app.state.config.STT_OPENAI_API_KEY}"
  414. },
  415. files={"file": (filename, open(file_path, "rb"))},
  416. data={"model": request.app.state.config.STT_MODEL},
  417. )
  418. r.raise_for_status()
  419. data = r.json()
  420. # save the transcript to a json file
  421. transcript_file = f"{file_dir}/{id}.json"
  422. with open(transcript_file, "w") as f:
  423. json.dump(data, f)
  424. return data
  425. except Exception as e:
  426. log.exception(e)
  427. detail = None
  428. if r is not None:
  429. try:
  430. res = r.json()
  431. if "error" in res:
  432. detail = f"External: {res['error'].get('message', '')}"
  433. except Exception:
  434. detail = f"External: {e}"
  435. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  436. elif request.app.state.config.STT_ENGINE == "deepgram":
  437. try:
  438. # Determine the MIME type of the file
  439. mime, _ = mimetypes.guess_type(file_path)
  440. if not mime:
  441. mime = "audio/wav" # fallback to wav if undetectable
  442. # Read the audio file
  443. with open(file_path, "rb") as f:
  444. file_data = f.read()
  445. # Build headers and parameters
  446. headers = {
  447. "Authorization": f"Token {request.app.state.config.DEEPGRAM_API_KEY}",
  448. "Content-Type": mime,
  449. }
  450. # Add model if specified
  451. params = {}
  452. if request.app.state.config.STT_MODEL:
  453. params["model"] = request.app.state.config.STT_MODEL
  454. # Make request to Deepgram API
  455. r = requests.post(
  456. "https://api.deepgram.com/v1/listen",
  457. headers=headers,
  458. params=params,
  459. data=file_data,
  460. )
  461. r.raise_for_status()
  462. response_data = r.json()
  463. # Extract transcript from Deepgram response
  464. try:
  465. transcript = response_data["results"]["channels"][0]["alternatives"][
  466. 0
  467. ].get("transcript", "")
  468. except (KeyError, IndexError) as e:
  469. log.error(f"Malformed response from Deepgram: {str(e)}")
  470. raise Exception(
  471. "Failed to parse Deepgram response - unexpected response format"
  472. )
  473. data = {"text": transcript.strip()}
  474. # Save transcript
  475. transcript_file = f"{file_dir}/{id}.json"
  476. with open(transcript_file, "w") as f:
  477. json.dump(data, f)
  478. return data
  479. except Exception as e:
  480. log.exception(e)
  481. detail = None
  482. if r is not None:
  483. try:
  484. res = r.json()
  485. if "error" in res:
  486. detail = f"External: {res['error'].get('message', '')}"
  487. except Exception:
  488. detail = f"External: {e}"
  489. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  490. def compress_audio(file_path):
  491. if os.path.getsize(file_path) > MAX_FILE_SIZE:
  492. file_dir = os.path.dirname(file_path)
  493. audio = AudioSegment.from_file(file_path)
  494. audio = audio.set_frame_rate(16000).set_channels(1) # Compress audio
  495. compressed_path = f"{file_dir}/{id}_compressed.opus"
  496. audio.export(compressed_path, format="opus", bitrate="32k")
  497. log.debug(f"Compressed audio to {compressed_path}")
  498. if (
  499. os.path.getsize(compressed_path) > MAX_FILE_SIZE
  500. ): # Still larger than MAX_FILE_SIZE after compression
  501. raise Exception(ERROR_MESSAGES.FILE_TOO_LARGE(size=f"{MAX_FILE_SIZE_MB}MB"))
  502. return compressed_path
  503. else:
  504. return file_path
  505. @router.post("/transcriptions")
  506. def transcription(
  507. request: Request,
  508. file: UploadFile = File(...),
  509. user=Depends(get_verified_user),
  510. ):
  511. log.info(f"file.content_type: {file.content_type}")
  512. if file.content_type not in ["audio/mpeg", "audio/wav", "audio/ogg", "audio/x-m4a"]:
  513. raise HTTPException(
  514. status_code=status.HTTP_400_BAD_REQUEST,
  515. detail=ERROR_MESSAGES.FILE_NOT_SUPPORTED,
  516. )
  517. try:
  518. ext = file.filename.split(".")[-1]
  519. id = uuid.uuid4()
  520. filename = f"{id}.{ext}"
  521. contents = file.file.read()
  522. file_dir = f"{CACHE_DIR}/audio/transcriptions"
  523. os.makedirs(file_dir, exist_ok=True)
  524. file_path = f"{file_dir}/{filename}"
  525. with open(file_path, "wb") as f:
  526. f.write(contents)
  527. try:
  528. try:
  529. file_path = compress_audio(file_path)
  530. except Exception as e:
  531. log.exception(e)
  532. raise HTTPException(
  533. status_code=status.HTTP_400_BAD_REQUEST,
  534. detail=ERROR_MESSAGES.DEFAULT(e),
  535. )
  536. data = transcribe(request, file_path)
  537. file_path = file_path.split("/")[-1]
  538. return {**data, "filename": file_path}
  539. except Exception as e:
  540. log.exception(e)
  541. raise HTTPException(
  542. status_code=status.HTTP_400_BAD_REQUEST,
  543. detail=ERROR_MESSAGES.DEFAULT(e),
  544. )
  545. except Exception as e:
  546. log.exception(e)
  547. raise HTTPException(
  548. status_code=status.HTTP_400_BAD_REQUEST,
  549. detail=ERROR_MESSAGES.DEFAULT(e),
  550. )
  551. def get_available_models(request: Request) -> list[dict]:
  552. available_models = []
  553. if request.app.state.config.TTS_ENGINE == "openai":
  554. available_models = [{"id": "tts-1"}, {"id": "tts-1-hd"}]
  555. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  556. try:
  557. response = requests.get(
  558. "https://api.elevenlabs.io/v1/models",
  559. headers={
  560. "xi-api-key": request.app.state.config.TTS_API_KEY,
  561. "Content-Type": "application/json",
  562. },
  563. timeout=5,
  564. )
  565. response.raise_for_status()
  566. models = response.json()
  567. available_models = [
  568. {"name": model["name"], "id": model["model_id"]} for model in models
  569. ]
  570. except requests.RequestException as e:
  571. log.error(f"Error fetching voices: {str(e)}")
  572. return available_models
  573. @router.get("/models")
  574. async def get_models(request: Request, user=Depends(get_verified_user)):
  575. return {"models": get_available_models(request)}
  576. def get_available_voices(request) -> dict:
  577. """Returns {voice_id: voice_name} dict"""
  578. available_voices = {}
  579. if request.app.state.config.TTS_ENGINE == "openai":
  580. available_voices = {
  581. "alloy": "alloy",
  582. "echo": "echo",
  583. "fable": "fable",
  584. "onyx": "onyx",
  585. "nova": "nova",
  586. "shimmer": "shimmer",
  587. }
  588. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  589. try:
  590. available_voices = get_elevenlabs_voices(
  591. api_key=request.app.state.config.TTS_API_KEY
  592. )
  593. except Exception:
  594. # Avoided @lru_cache with exception
  595. pass
  596. elif request.app.state.config.TTS_ENGINE == "azure":
  597. try:
  598. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  599. url = f"https://{region}.tts.speech.microsoft.com/cognitiveservices/voices/list"
  600. headers = {
  601. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY
  602. }
  603. response = requests.get(url, headers=headers)
  604. response.raise_for_status()
  605. voices = response.json()
  606. for voice in voices:
  607. available_voices[voice["ShortName"]] = (
  608. f"{voice['DisplayName']} ({voice['ShortName']})"
  609. )
  610. except requests.RequestException as e:
  611. log.error(f"Error fetching voices: {str(e)}")
  612. return available_voices
  613. @lru_cache
  614. def get_elevenlabs_voices(api_key: str) -> dict:
  615. """
  616. Note, set the following in your .env file to use Elevenlabs:
  617. AUDIO_TTS_ENGINE=elevenlabs
  618. AUDIO_TTS_API_KEY=sk_... # Your Elevenlabs API key
  619. AUDIO_TTS_VOICE=EXAVITQu4vr4xnSDxMaL # From https://api.elevenlabs.io/v1/voices
  620. AUDIO_TTS_MODEL=eleven_multilingual_v2
  621. """
  622. try:
  623. # TODO: Add retries
  624. response = requests.get(
  625. "https://api.elevenlabs.io/v1/voices",
  626. headers={
  627. "xi-api-key": api_key,
  628. "Content-Type": "application/json",
  629. },
  630. )
  631. response.raise_for_status()
  632. voices_data = response.json()
  633. voices = {}
  634. for voice in voices_data.get("voices", []):
  635. voices[voice["voice_id"]] = voice["name"]
  636. except requests.RequestException as e:
  637. # Avoid @lru_cache with exception
  638. log.error(f"Error fetching voices: {str(e)}")
  639. raise RuntimeError(f"Error fetching voices: {str(e)}")
  640. return voices
  641. @router.get("/voices")
  642. async def get_voices(request: Request, user=Depends(get_verified_user)):
  643. return {
  644. "voices": [
  645. {"id": k, "name": v} for k, v in get_available_voices(request).items()
  646. ]
  647. }