audio.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777
  1. import hashlib
  2. import json
  3. import logging
  4. import os
  5. import uuid
  6. from functools import lru_cache
  7. from pathlib import Path
  8. from pydub import AudioSegment
  9. from pydub.silence import split_on_silence
  10. import aiohttp
  11. import aiofiles
  12. import requests
  13. import mimetypes
  14. from fastapi import (
  15. Depends,
  16. FastAPI,
  17. File,
  18. HTTPException,
  19. Request,
  20. UploadFile,
  21. status,
  22. APIRouter,
  23. )
  24. from fastapi.middleware.cors import CORSMiddleware
  25. from fastapi.responses import FileResponse
  26. from pydantic import BaseModel
  27. from open_webui.utils.auth import get_admin_user, get_verified_user
  28. from open_webui.config import (
  29. WHISPER_MODEL_AUTO_UPDATE,
  30. WHISPER_MODEL_DIR,
  31. CACHE_DIR,
  32. )
  33. from open_webui.constants import ERROR_MESSAGES
  34. from open_webui.env import (
  35. ENV,
  36. SRC_LOG_LEVELS,
  37. DEVICE_TYPE,
  38. ENABLE_FORWARD_USER_INFO_HEADERS,
  39. )
  40. router = APIRouter()
  41. # Constants
  42. MAX_FILE_SIZE_MB = 25
  43. MAX_FILE_SIZE = MAX_FILE_SIZE_MB * 1024 * 1024 # Convert MB to bytes
  44. log = logging.getLogger(__name__)
  45. log.setLevel(SRC_LOG_LEVELS["AUDIO"])
  46. SPEECH_CACHE_DIR = Path(CACHE_DIR).joinpath("./audio/speech/")
  47. SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
  48. ##########################################
  49. #
  50. # Utility functions
  51. #
  52. ##########################################
  53. from pydub import AudioSegment
  54. from pydub.utils import mediainfo
  55. def is_mp4_audio(file_path):
  56. """Check if the given file is an MP4 audio file."""
  57. if not os.path.isfile(file_path):
  58. print(f"File not found: {file_path}")
  59. return False
  60. info = mediainfo(file_path)
  61. if (
  62. info.get("codec_name") == "aac"
  63. and info.get("codec_type") == "audio"
  64. and info.get("codec_tag_string") == "mp4a"
  65. ):
  66. return True
  67. return False
  68. def convert_mp4_to_wav(file_path, output_path):
  69. """Convert MP4 audio file to WAV format."""
  70. audio = AudioSegment.from_file(file_path, format="mp4")
  71. audio.export(output_path, format="wav")
  72. print(f"Converted {file_path} to {output_path}")
  73. def set_faster_whisper_model(model: str, auto_update: bool = False):
  74. whisper_model = None
  75. if model:
  76. from faster_whisper import WhisperModel
  77. faster_whisper_kwargs = {
  78. "model_size_or_path": model,
  79. "device": DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == "cuda" else "cpu",
  80. "compute_type": "int8",
  81. "download_root": WHISPER_MODEL_DIR,
  82. "local_files_only": not auto_update,
  83. }
  84. try:
  85. whisper_model = WhisperModel(**faster_whisper_kwargs)
  86. except Exception:
  87. log.warning(
  88. "WhisperModel initialization failed, attempting download with local_files_only=False"
  89. )
  90. faster_whisper_kwargs["local_files_only"] = False
  91. whisper_model = WhisperModel(**faster_whisper_kwargs)
  92. return whisper_model
  93. ##########################################
  94. #
  95. # Audio API
  96. #
  97. ##########################################
  98. class TTSConfigForm(BaseModel):
  99. OPENAI_API_BASE_URL: str
  100. OPENAI_API_KEY: str
  101. API_KEY: str
  102. ENGINE: str
  103. MODEL: str
  104. VOICE: str
  105. SPLIT_ON: str
  106. AZURE_SPEECH_REGION: str
  107. AZURE_SPEECH_OUTPUT_FORMAT: str
  108. class STTConfigForm(BaseModel):
  109. OPENAI_API_BASE_URL: str
  110. OPENAI_API_KEY: str
  111. ENGINE: str
  112. MODEL: str
  113. WHISPER_MODEL: str
  114. DEEPGRAM_API_KEY: str
  115. class AudioConfigUpdateForm(BaseModel):
  116. tts: TTSConfigForm
  117. stt: STTConfigForm
  118. @router.get("/config")
  119. async def get_audio_config(request: Request, user=Depends(get_admin_user)):
  120. return {
  121. "tts": {
  122. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  123. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  124. "API_KEY": request.app.state.config.TTS_API_KEY,
  125. "ENGINE": request.app.state.config.TTS_ENGINE,
  126. "MODEL": request.app.state.config.TTS_MODEL,
  127. "VOICE": request.app.state.config.TTS_VOICE,
  128. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  129. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  130. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  131. },
  132. "stt": {
  133. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  134. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  135. "ENGINE": request.app.state.config.STT_ENGINE,
  136. "MODEL": request.app.state.config.STT_MODEL,
  137. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  138. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  139. },
  140. }
  141. @router.post("/config/update")
  142. async def update_audio_config(
  143. request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)
  144. ):
  145. request.app.state.config.TTS_OPENAI_API_BASE_URL = form_data.tts.OPENAI_API_BASE_URL
  146. request.app.state.config.TTS_OPENAI_API_KEY = form_data.tts.OPENAI_API_KEY
  147. request.app.state.config.TTS_API_KEY = form_data.tts.API_KEY
  148. request.app.state.config.TTS_ENGINE = form_data.tts.ENGINE
  149. request.app.state.config.TTS_MODEL = form_data.tts.MODEL
  150. request.app.state.config.TTS_VOICE = form_data.tts.VOICE
  151. request.app.state.config.TTS_SPLIT_ON = form_data.tts.SPLIT_ON
  152. request.app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
  153. request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = (
  154. form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
  155. )
  156. request.app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
  157. request.app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
  158. request.app.state.config.STT_ENGINE = form_data.stt.ENGINE
  159. request.app.state.config.STT_MODEL = form_data.stt.MODEL
  160. request.app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
  161. request.app.state.config.DEEPGRAM_API_KEY = form_data.stt.DEEPGRAM_API_KEY
  162. if request.app.state.config.STT_ENGINE == "":
  163. request.app.state.faster_whisper_model = set_faster_whisper_model(
  164. form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE
  165. )
  166. return {
  167. "tts": {
  168. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  169. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  170. "API_KEY": request.app.state.config.TTS_API_KEY,
  171. "ENGINE": request.app.state.config.TTS_ENGINE,
  172. "MODEL": request.app.state.config.TTS_MODEL,
  173. "VOICE": request.app.state.config.TTS_VOICE,
  174. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  175. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  176. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  177. },
  178. "stt": {
  179. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  180. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  181. "ENGINE": request.app.state.config.STT_ENGINE,
  182. "MODEL": request.app.state.config.STT_MODEL,
  183. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  184. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  185. },
  186. }
  187. def load_speech_pipeline(request):
  188. from transformers import pipeline
  189. from datasets import load_dataset
  190. if request.app.state.speech_synthesiser is None:
  191. request.app.state.speech_synthesiser = pipeline(
  192. "text-to-speech", "microsoft/speecht5_tts"
  193. )
  194. if request.app.state.speech_speaker_embeddings_dataset is None:
  195. request.app.state.speech_speaker_embeddings_dataset = load_dataset(
  196. "Matthijs/cmu-arctic-xvectors", split="validation"
  197. )
  198. @router.post("/speech")
  199. async def speech(request: Request, user=Depends(get_verified_user)):
  200. body = await request.body()
  201. name = hashlib.sha256(
  202. body
  203. + str(request.app.state.config.TTS_ENGINE).encode("utf-8")
  204. + str(request.app.state.config.TTS_MODEL).encode("utf-8")
  205. ).hexdigest()
  206. file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
  207. file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
  208. # Check if the file already exists in the cache
  209. if file_path.is_file():
  210. return FileResponse(file_path)
  211. payload = None
  212. try:
  213. payload = json.loads(body.decode("utf-8"))
  214. except Exception as e:
  215. log.exception(e)
  216. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  217. if request.app.state.config.TTS_ENGINE == "openai":
  218. payload["model"] = request.app.state.config.TTS_MODEL
  219. try:
  220. # print(payload)
  221. async with aiohttp.ClientSession() as session:
  222. async with session.post(
  223. url=f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech",
  224. json=payload,
  225. headers={
  226. "Content-Type": "application/json",
  227. "Authorization": f"Bearer {request.app.state.config.TTS_OPENAI_API_KEY}",
  228. **(
  229. {
  230. "X-OpenWebUI-User-Name": user.name,
  231. "X-OpenWebUI-User-Id": user.id,
  232. "X-OpenWebUI-User-Email": user.email,
  233. "X-OpenWebUI-User-Role": user.role,
  234. }
  235. if ENABLE_FORWARD_USER_INFO_HEADERS
  236. else {}
  237. ),
  238. },
  239. ) as r:
  240. r.raise_for_status()
  241. async with aiofiles.open(file_path, "wb") as f:
  242. await f.write(await r.read())
  243. async with aiofiles.open(file_body_path, "w") as f:
  244. await f.write(json.dumps(payload))
  245. return FileResponse(file_path)
  246. except Exception as e:
  247. log.exception(e)
  248. detail = None
  249. try:
  250. if r.status != 200:
  251. res = await r.json()
  252. if "error" in res:
  253. detail = f"External: {res['error'].get('message', '')}"
  254. except Exception:
  255. detail = f"External: {e}"
  256. raise HTTPException(
  257. status_code=getattr(r, "status", 500),
  258. detail=detail if detail else "Open WebUI: Server Connection Error",
  259. )
  260. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  261. voice_id = payload.get("voice", "")
  262. if voice_id not in get_available_voices(request):
  263. raise HTTPException(
  264. status_code=400,
  265. detail="Invalid voice id",
  266. )
  267. try:
  268. async with aiohttp.ClientSession() as session:
  269. async with session.post(
  270. f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
  271. json={
  272. "text": payload["input"],
  273. "model_id": request.app.state.config.TTS_MODEL,
  274. "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
  275. },
  276. headers={
  277. "Accept": "audio/mpeg",
  278. "Content-Type": "application/json",
  279. "xi-api-key": request.app.state.config.TTS_API_KEY,
  280. },
  281. ) as r:
  282. r.raise_for_status()
  283. async with aiofiles.open(file_path, "wb") as f:
  284. await f.write(await r.read())
  285. async with aiofiles.open(file_body_path, "w") as f:
  286. await f.write(json.dumps(payload))
  287. return FileResponse(file_path)
  288. except Exception as e:
  289. log.exception(e)
  290. detail = None
  291. try:
  292. if r.status != 200:
  293. res = await r.json()
  294. if "error" in res:
  295. detail = f"External: {res['error'].get('message', '')}"
  296. except Exception:
  297. detail = f"External: {e}"
  298. raise HTTPException(
  299. status_code=getattr(r, "status", 500),
  300. detail=detail if detail else "Open WebUI: Server Connection Error",
  301. )
  302. elif request.app.state.config.TTS_ENGINE == "azure":
  303. try:
  304. payload = json.loads(body.decode("utf-8"))
  305. except Exception as e:
  306. log.exception(e)
  307. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  308. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  309. language = request.app.state.config.TTS_VOICE
  310. locale = "-".join(request.app.state.config.TTS_VOICE.split("-")[:1])
  311. output_format = request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT
  312. try:
  313. data = f"""<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{locale}">
  314. <voice name="{language}">{payload["input"]}</voice>
  315. </speak>"""
  316. async with aiohttp.ClientSession() as session:
  317. async with session.post(
  318. f"https://{region}.tts.speech.microsoft.com/cognitiveservices/v1",
  319. headers={
  320. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY,
  321. "Content-Type": "application/ssml+xml",
  322. "X-Microsoft-OutputFormat": output_format,
  323. },
  324. data=data,
  325. ) as r:
  326. r.raise_for_status()
  327. async with aiofiles.open(file_path, "wb") as f:
  328. await f.write(await r.read())
  329. async with aiofiles.open(file_body_path, "w") as f:
  330. await f.write(json.dumps(payload))
  331. return FileResponse(file_path)
  332. except Exception as e:
  333. log.exception(e)
  334. detail = None
  335. try:
  336. if r.status != 200:
  337. res = await r.json()
  338. if "error" in res:
  339. detail = f"External: {res['error'].get('message', '')}"
  340. except Exception:
  341. detail = f"External: {e}"
  342. raise HTTPException(
  343. status_code=getattr(r, "status", 500),
  344. detail=detail if detail else "Open WebUI: Server Connection Error",
  345. )
  346. elif request.app.state.config.TTS_ENGINE == "transformers":
  347. payload = None
  348. try:
  349. payload = json.loads(body.decode("utf-8"))
  350. except Exception as e:
  351. log.exception(e)
  352. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  353. import torch
  354. import soundfile as sf
  355. load_speech_pipeline(request)
  356. embeddings_dataset = request.app.state.speech_speaker_embeddings_dataset
  357. speaker_index = 6799
  358. try:
  359. speaker_index = embeddings_dataset["filename"].index(
  360. request.app.state.config.TTS_MODEL
  361. )
  362. except Exception:
  363. pass
  364. speaker_embedding = torch.tensor(
  365. embeddings_dataset[speaker_index]["xvector"]
  366. ).unsqueeze(0)
  367. speech = request.app.state.speech_synthesiser(
  368. payload["input"],
  369. forward_params={"speaker_embeddings": speaker_embedding},
  370. )
  371. sf.write(file_path, speech["audio"], samplerate=speech["sampling_rate"])
  372. async with aiofiles.open(file_body_path, "w") as f:
  373. await f.write(json.dumps(payload))
  374. return FileResponse(file_path)
  375. def transcribe(request: Request, file_path):
  376. print("transcribe", file_path)
  377. filename = os.path.basename(file_path)
  378. file_dir = os.path.dirname(file_path)
  379. id = filename.split(".")[0]
  380. if request.app.state.config.STT_ENGINE == "":
  381. if request.app.state.faster_whisper_model is None:
  382. request.app.state.faster_whisper_model = set_faster_whisper_model(
  383. request.app.state.config.WHISPER_MODEL
  384. )
  385. model = request.app.state.faster_whisper_model
  386. segments, info = model.transcribe(file_path, beam_size=5)
  387. log.info(
  388. "Detected language '%s' with probability %f"
  389. % (info.language, info.language_probability)
  390. )
  391. transcript = "".join([segment.text for segment in list(segments)])
  392. data = {"text": transcript.strip()}
  393. # save the transcript to a json file
  394. transcript_file = f"{file_dir}/{id}.json"
  395. with open(transcript_file, "w") as f:
  396. json.dump(data, f)
  397. log.debug(data)
  398. return data
  399. elif request.app.state.config.STT_ENGINE == "openai":
  400. if is_mp4_audio(file_path):
  401. os.rename(file_path, file_path.replace(".wav", ".mp4"))
  402. # Convert MP4 audio file to WAV format
  403. convert_mp4_to_wav(file_path.replace(".wav", ".mp4"), file_path)
  404. r = None
  405. try:
  406. r = requests.post(
  407. url=f"{request.app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions",
  408. headers={
  409. "Authorization": f"Bearer {request.app.state.config.STT_OPENAI_API_KEY}"
  410. },
  411. files={"file": (filename, open(file_path, "rb"))},
  412. data={"model": request.app.state.config.STT_MODEL},
  413. )
  414. r.raise_for_status()
  415. data = r.json()
  416. # save the transcript to a json file
  417. transcript_file = f"{file_dir}/{id}.json"
  418. with open(transcript_file, "w") as f:
  419. json.dump(data, f)
  420. return data
  421. except Exception as e:
  422. log.exception(e)
  423. detail = None
  424. if r is not None:
  425. try:
  426. res = r.json()
  427. if "error" in res:
  428. detail = f"External: {res['error'].get('message', '')}"
  429. except Exception:
  430. detail = f"External: {e}"
  431. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  432. elif request.app.state.config.STT_ENGINE == "deepgram":
  433. try:
  434. # Determine the MIME type of the file
  435. mime, _ = mimetypes.guess_type(file_path)
  436. if not mime:
  437. mime = "audio/wav" # fallback to wav if undetectable
  438. # Read the audio file
  439. with open(file_path, "rb") as f:
  440. file_data = f.read()
  441. # Build headers and parameters
  442. headers = {
  443. "Authorization": f"Token {request.app.state.config.DEEPGRAM_API_KEY}",
  444. "Content-Type": mime,
  445. }
  446. # Add model if specified
  447. params = {}
  448. if request.app.state.config.STT_MODEL:
  449. params["model"] = request.app.state.config.STT_MODEL
  450. # Make request to Deepgram API
  451. r = requests.post(
  452. "https://api.deepgram.com/v1/listen",
  453. headers=headers,
  454. params=params,
  455. data=file_data,
  456. )
  457. r.raise_for_status()
  458. response_data = r.json()
  459. # Extract transcript from Deepgram response
  460. try:
  461. transcript = response_data["results"]["channels"][0]["alternatives"][0].get("transcript", "")
  462. except (KeyError, IndexError) as e:
  463. log.error(f"Malformed response from Deepgram: {str(e)}")
  464. raise Exception("Failed to parse Deepgram response - unexpected response format")
  465. data = {"text": transcript.strip()}
  466. # Save transcript
  467. transcript_file = f"{file_dir}/{id}.json"
  468. with open(transcript_file, "w") as f:
  469. json.dump(data, f)
  470. return data
  471. except Exception as e:
  472. log.exception(e)
  473. detail = None
  474. if r is not None:
  475. try:
  476. res = r.json()
  477. if "error" in res:
  478. detail = f"External: {res['error'].get('message', '')}"
  479. except Exception:
  480. detail = f"External: {e}"
  481. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  482. def compress_audio(file_path):
  483. if os.path.getsize(file_path) > MAX_FILE_SIZE:
  484. file_dir = os.path.dirname(file_path)
  485. audio = AudioSegment.from_file(file_path)
  486. audio = audio.set_frame_rate(16000).set_channels(1) # Compress audio
  487. compressed_path = f"{file_dir}/{id}_compressed.opus"
  488. audio.export(compressed_path, format="opus", bitrate="32k")
  489. log.debug(f"Compressed audio to {compressed_path}")
  490. if (
  491. os.path.getsize(compressed_path) > MAX_FILE_SIZE
  492. ): # Still larger than MAX_FILE_SIZE after compression
  493. raise Exception(ERROR_MESSAGES.FILE_TOO_LARGE(size=f"{MAX_FILE_SIZE_MB}MB"))
  494. return compressed_path
  495. else:
  496. return file_path
  497. @router.post("/transcriptions")
  498. def transcription(
  499. request: Request,
  500. file: UploadFile = File(...),
  501. user=Depends(get_verified_user),
  502. ):
  503. log.info(f"file.content_type: {file.content_type}")
  504. if file.content_type not in ["audio/mpeg", "audio/wav", "audio/ogg", "audio/x-m4a"]:
  505. raise HTTPException(
  506. status_code=status.HTTP_400_BAD_REQUEST,
  507. detail=ERROR_MESSAGES.FILE_NOT_SUPPORTED,
  508. )
  509. try:
  510. ext = file.filename.split(".")[-1]
  511. id = uuid.uuid4()
  512. filename = f"{id}.{ext}"
  513. contents = file.file.read()
  514. file_dir = f"{CACHE_DIR}/audio/transcriptions"
  515. os.makedirs(file_dir, exist_ok=True)
  516. file_path = f"{file_dir}/{filename}"
  517. with open(file_path, "wb") as f:
  518. f.write(contents)
  519. try:
  520. try:
  521. file_path = compress_audio(file_path)
  522. except Exception as e:
  523. log.exception(e)
  524. raise HTTPException(
  525. status_code=status.HTTP_400_BAD_REQUEST,
  526. detail=ERROR_MESSAGES.DEFAULT(e),
  527. )
  528. data = transcribe(request, file_path)
  529. file_path = file_path.split("/")[-1]
  530. return {**data, "filename": file_path}
  531. except Exception as e:
  532. log.exception(e)
  533. raise HTTPException(
  534. status_code=status.HTTP_400_BAD_REQUEST,
  535. detail=ERROR_MESSAGES.DEFAULT(e),
  536. )
  537. except Exception as e:
  538. log.exception(e)
  539. raise HTTPException(
  540. status_code=status.HTTP_400_BAD_REQUEST,
  541. detail=ERROR_MESSAGES.DEFAULT(e),
  542. )
  543. def get_available_models(request: Request) -> list[dict]:
  544. available_models = []
  545. if request.app.state.config.TTS_ENGINE == "openai":
  546. available_models = [{"id": "tts-1"}, {"id": "tts-1-hd"}]
  547. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  548. try:
  549. response = requests.get(
  550. "https://api.elevenlabs.io/v1/models",
  551. headers={
  552. "xi-api-key": request.app.state.config.TTS_API_KEY,
  553. "Content-Type": "application/json",
  554. },
  555. timeout=5,
  556. )
  557. response.raise_for_status()
  558. models = response.json()
  559. available_models = [
  560. {"name": model["name"], "id": model["model_id"]} for model in models
  561. ]
  562. except requests.RequestException as e:
  563. log.error(f"Error fetching voices: {str(e)}")
  564. return available_models
  565. @router.get("/models")
  566. async def get_models(request: Request, user=Depends(get_verified_user)):
  567. return {"models": get_available_models(request)}
  568. def get_available_voices(request) -> dict:
  569. """Returns {voice_id: voice_name} dict"""
  570. available_voices = {}
  571. if request.app.state.config.TTS_ENGINE == "openai":
  572. available_voices = {
  573. "alloy": "alloy",
  574. "echo": "echo",
  575. "fable": "fable",
  576. "onyx": "onyx",
  577. "nova": "nova",
  578. "shimmer": "shimmer",
  579. }
  580. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  581. try:
  582. available_voices = get_elevenlabs_voices(
  583. api_key=request.app.state.config.TTS_API_KEY
  584. )
  585. except Exception:
  586. # Avoided @lru_cache with exception
  587. pass
  588. elif request.app.state.config.TTS_ENGINE == "azure":
  589. try:
  590. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  591. url = f"https://{region}.tts.speech.microsoft.com/cognitiveservices/voices/list"
  592. headers = {
  593. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY
  594. }
  595. response = requests.get(url, headers=headers)
  596. response.raise_for_status()
  597. voices = response.json()
  598. for voice in voices:
  599. available_voices[voice["ShortName"]] = (
  600. f"{voice['DisplayName']} ({voice['ShortName']})"
  601. )
  602. except requests.RequestException as e:
  603. log.error(f"Error fetching voices: {str(e)}")
  604. return available_voices
  605. @lru_cache
  606. def get_elevenlabs_voices(api_key: str) -> dict:
  607. """
  608. Note, set the following in your .env file to use Elevenlabs:
  609. AUDIO_TTS_ENGINE=elevenlabs
  610. AUDIO_TTS_API_KEY=sk_... # Your Elevenlabs API key
  611. AUDIO_TTS_VOICE=EXAVITQu4vr4xnSDxMaL # From https://api.elevenlabs.io/v1/voices
  612. AUDIO_TTS_MODEL=eleven_multilingual_v2
  613. """
  614. try:
  615. # TODO: Add retries
  616. response = requests.get(
  617. "https://api.elevenlabs.io/v1/voices",
  618. headers={
  619. "xi-api-key": api_key,
  620. "Content-Type": "application/json",
  621. },
  622. )
  623. response.raise_for_status()
  624. voices_data = response.json()
  625. voices = {}
  626. for voice in voices_data.get("voices", []):
  627. voices[voice["voice_id"]] = voice["name"]
  628. except requests.RequestException as e:
  629. # Avoid @lru_cache with exception
  630. log.error(f"Error fetching voices: {str(e)}")
  631. raise RuntimeError(f"Error fetching voices: {str(e)}")
  632. return voices
  633. @router.get("/voices")
  634. async def get_voices(request: Request, user=Depends(get_verified_user)):
  635. return {
  636. "voices": [
  637. {"id": k, "name": v} for k, v in get_available_voices(request).items()
  638. ]
  639. }