index.ts 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072
  1. import { v4 as uuidv4 } from 'uuid';
  2. import sha256 from 'js-sha256';
  3. import dayjs from 'dayjs';
  4. import relativeTime from 'dayjs/plugin/relativeTime';
  5. import isToday from 'dayjs/plugin/isToday';
  6. import isYesterday from 'dayjs/plugin/isYesterday';
  7. import localizedFormat from 'dayjs/plugin/localizedFormat';
  8. dayjs.extend(relativeTime);
  9. dayjs.extend(isToday);
  10. dayjs.extend(isYesterday);
  11. dayjs.extend(localizedFormat);
  12. import { WEBUI_BASE_URL } from '$lib/constants';
  13. import { TTS_RESPONSE_SPLIT } from '$lib/types';
  14. //////////////////////////
  15. // Helper functions
  16. //////////////////////////
  17. export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
  18. function escapeRegExp(string: string): string {
  19. return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  20. }
  21. export const replaceTokens = (content, sourceIds, char, user) => {
  22. const tokens = [
  23. { regex: /{{char}}/gi, replacement: char },
  24. { regex: /{{user}}/gi, replacement: user },
  25. {
  26. regex: /{{VIDEO_FILE_ID_([a-f0-9-]+)}}/gi,
  27. replacement: (_, fileId) =>
  28. `<video src="${WEBUI_BASE_URL}/api/v1/files/${fileId}/content" controls></video>`
  29. },
  30. {
  31. regex: /{{HTML_FILE_ID_([a-f0-9-]+)}}/gi,
  32. replacement: (_, fileId) =>
  33. `<iframe src="${WEBUI_BASE_URL}/api/v1/files/${fileId}/content/html" width="100%" frameborder="0" onload="this.style.height=(this.contentWindow.document.body.scrollHeight+20)+'px';"></iframe>`
  34. }
  35. ];
  36. // Replace tokens outside code blocks only
  37. const processOutsideCodeBlocks = (text, replacementFn) => {
  38. return text
  39. .split(/(```[\s\S]*?```|`[\s\S]*?`)/)
  40. .map((segment) => {
  41. return segment.startsWith('```') || segment.startsWith('`')
  42. ? segment
  43. : replacementFn(segment);
  44. })
  45. .join('');
  46. };
  47. // Apply replacements
  48. content = processOutsideCodeBlocks(content, (segment) => {
  49. tokens.forEach(({ regex, replacement }) => {
  50. if (replacement !== undefined && replacement !== null) {
  51. segment = segment.replace(regex, replacement);
  52. }
  53. });
  54. if (Array.isArray(sourceIds)) {
  55. sourceIds.forEach((sourceId, idx) => {
  56. const regex = new RegExp(`\\[${idx}\\]`, 'g');
  57. segment = segment.replace(regex, `<source_id data="${idx}" title="${sourceId}" />`);
  58. });
  59. }
  60. return segment;
  61. });
  62. return content;
  63. };
  64. export const sanitizeResponseContent = (content: string) => {
  65. return content
  66. .replace(/<\|[a-z]*$/, '')
  67. .replace(/<\|[a-z]+\|$/, '')
  68. .replace(/<$/, '')
  69. .replaceAll(/<\|[a-z]+\|>/g, ' ')
  70. .replaceAll('<', '&lt;')
  71. .replaceAll('>', '&gt;')
  72. .trim();
  73. };
  74. export const processResponseContent = (content: string) => {
  75. return content.trim();
  76. };
  77. export function unescapeHtml(html: string) {
  78. const doc = new DOMParser().parseFromString(html, 'text/html');
  79. return doc.documentElement.textContent;
  80. }
  81. export const capitalizeFirstLetter = (string) => {
  82. return string.charAt(0).toUpperCase() + string.slice(1);
  83. };
  84. export const splitStream = (splitOn) => {
  85. let buffer = '';
  86. return new TransformStream({
  87. transform(chunk, controller) {
  88. buffer += chunk;
  89. const parts = buffer.split(splitOn);
  90. parts.slice(0, -1).forEach((part) => controller.enqueue(part));
  91. buffer = parts[parts.length - 1];
  92. },
  93. flush(controller) {
  94. if (buffer) controller.enqueue(buffer);
  95. }
  96. });
  97. };
  98. export const convertMessagesToHistory = (messages) => {
  99. const history = {
  100. messages: {},
  101. currentId: null
  102. };
  103. let parentMessageId = null;
  104. let messageId = null;
  105. for (const message of messages) {
  106. messageId = uuidv4();
  107. if (parentMessageId !== null) {
  108. history.messages[parentMessageId].childrenIds = [
  109. ...history.messages[parentMessageId].childrenIds,
  110. messageId
  111. ];
  112. }
  113. history.messages[messageId] = {
  114. ...message,
  115. id: messageId,
  116. parentId: parentMessageId,
  117. childrenIds: []
  118. };
  119. parentMessageId = messageId;
  120. }
  121. history.currentId = messageId;
  122. return history;
  123. };
  124. export const getGravatarURL = (email) => {
  125. // Trim leading and trailing whitespace from
  126. // an email address and force all characters
  127. // to lower case
  128. const address = String(email).trim().toLowerCase();
  129. // Create a SHA256 hash of the final string
  130. const hash = sha256(address);
  131. // Grab the actual image URL
  132. return `https://www.gravatar.com/avatar/${hash}`;
  133. };
  134. export const canvasPixelTest = () => {
  135. // Test a 1x1 pixel to potentially identify browser/plugin fingerprint blocking or spoofing
  136. // Inspiration: https://github.com/kkapsner/CanvasBlocker/blob/master/test/detectionTest.js
  137. const canvas = document.createElement('canvas');
  138. const ctx = canvas.getContext('2d');
  139. canvas.height = 1;
  140. canvas.width = 1;
  141. const imageData = new ImageData(canvas.width, canvas.height);
  142. const pixelValues = imageData.data;
  143. // Generate RGB test data
  144. for (let i = 0; i < imageData.data.length; i += 1) {
  145. if (i % 4 !== 3) {
  146. pixelValues[i] = Math.floor(256 * Math.random());
  147. } else {
  148. pixelValues[i] = 255;
  149. }
  150. }
  151. ctx.putImageData(imageData, 0, 0);
  152. const p = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
  153. // Read RGB data and fail if unmatched
  154. for (let i = 0; i < p.length; i += 1) {
  155. if (p[i] !== pixelValues[i]) {
  156. console.log(
  157. 'canvasPixelTest: Wrong canvas pixel RGB value detected:',
  158. p[i],
  159. 'at:',
  160. i,
  161. 'expected:',
  162. pixelValues[i]
  163. );
  164. console.log('canvasPixelTest: Canvas blocking or spoofing is likely');
  165. return false;
  166. }
  167. }
  168. return true;
  169. };
  170. export const compressImage = async (imageUrl, maxWidth, maxHeight) => {
  171. return new Promise((resolve, reject) => {
  172. const img = new Image();
  173. img.onload = () => {
  174. const canvas = document.createElement('canvas');
  175. let width = img.width;
  176. let height = img.height;
  177. // Maintain aspect ratio while resizing
  178. if (maxWidth && maxHeight) {
  179. // Resize with both dimensions defined (preserves aspect ratio)
  180. if (width <= maxWidth && height <= maxHeight) {
  181. resolve(imageUrl);
  182. return;
  183. }
  184. if (width / height > maxWidth / maxHeight) {
  185. height = Math.round((maxWidth * height) / width);
  186. width = maxWidth;
  187. } else {
  188. width = Math.round((maxHeight * width) / height);
  189. height = maxHeight;
  190. }
  191. } else if (maxWidth) {
  192. // Only maxWidth defined
  193. if (width <= maxWidth) {
  194. resolve(imageUrl);
  195. return;
  196. }
  197. height = Math.round((maxWidth * height) / width);
  198. width = maxWidth;
  199. } else if (maxHeight) {
  200. // Only maxHeight defined
  201. if (height <= maxHeight) {
  202. resolve(imageUrl);
  203. return;
  204. }
  205. width = Math.round((maxHeight * width) / height);
  206. height = maxHeight;
  207. }
  208. canvas.width = width;
  209. canvas.height = height;
  210. const context = canvas.getContext('2d');
  211. context.drawImage(img, 0, 0, width, height);
  212. // Get compressed image URL
  213. const compressedUrl = canvas.toDataURL();
  214. resolve(compressedUrl);
  215. };
  216. img.onerror = (error) => reject(error);
  217. img.src = imageUrl;
  218. });
  219. };
  220. export const generateInitialsImage = (name) => {
  221. const canvas = document.createElement('canvas');
  222. const ctx = canvas.getContext('2d');
  223. canvas.width = 100;
  224. canvas.height = 100;
  225. if (!canvasPixelTest()) {
  226. console.log(
  227. 'generateInitialsImage: failed pixel test, fingerprint evasion is likely. Using default image.'
  228. );
  229. return '/user.png';
  230. }
  231. ctx.fillStyle = '#F39C12';
  232. ctx.fillRect(0, 0, canvas.width, canvas.height);
  233. ctx.fillStyle = '#FFFFFF';
  234. ctx.font = '40px Helvetica';
  235. ctx.textAlign = 'center';
  236. ctx.textBaseline = 'middle';
  237. const sanitizedName = name.trim();
  238. const initials =
  239. sanitizedName.length > 0
  240. ? sanitizedName[0] +
  241. (sanitizedName.split(' ').length > 1
  242. ? sanitizedName[sanitizedName.lastIndexOf(' ') + 1]
  243. : '')
  244. : '';
  245. ctx.fillText(initials.toUpperCase(), canvas.width / 2, canvas.height / 2);
  246. return canvas.toDataURL();
  247. };
  248. export const formatDate = (inputDate) => {
  249. const date = dayjs(inputDate);
  250. const now = dayjs();
  251. if (date.isToday()) {
  252. return `Today at ${date.format('LT')}`;
  253. } else if (date.isYesterday()) {
  254. return `Yesterday at ${date.format('LT')}`;
  255. } else {
  256. return `${date.format('L')} at ${date.format('LT')}`;
  257. }
  258. };
  259. export const copyToClipboard = async (text) => {
  260. let result = false;
  261. if (!navigator.clipboard) {
  262. const textArea = document.createElement('textarea');
  263. textArea.value = text;
  264. // Avoid scrolling to bottom
  265. textArea.style.top = '0';
  266. textArea.style.left = '0';
  267. textArea.style.position = 'fixed';
  268. document.body.appendChild(textArea);
  269. textArea.focus();
  270. textArea.select();
  271. try {
  272. const successful = document.execCommand('copy');
  273. const msg = successful ? 'successful' : 'unsuccessful';
  274. console.log('Fallback: Copying text command was ' + msg);
  275. result = true;
  276. } catch (err) {
  277. console.error('Fallback: Oops, unable to copy', err);
  278. }
  279. document.body.removeChild(textArea);
  280. return result;
  281. }
  282. result = await navigator.clipboard
  283. .writeText(text)
  284. .then(() => {
  285. console.log('Async: Copying to clipboard was successful!');
  286. return true;
  287. })
  288. .catch((error) => {
  289. console.error('Async: Could not copy text: ', error);
  290. return false;
  291. });
  292. return result;
  293. };
  294. export const compareVersion = (latest, current) => {
  295. return current === '0.0.0'
  296. ? false
  297. : current.localeCompare(latest, undefined, {
  298. numeric: true,
  299. sensitivity: 'case',
  300. caseFirst: 'upper'
  301. }) < 0;
  302. };
  303. export const findWordIndices = (text) => {
  304. const regex = /\[([^\]]+)\]/g;
  305. const matches = [];
  306. let match;
  307. while ((match = regex.exec(text)) !== null) {
  308. matches.push({
  309. word: match[1],
  310. startIndex: match.index,
  311. endIndex: regex.lastIndex - 1
  312. });
  313. }
  314. return matches;
  315. };
  316. export const removeLastWordFromString = (inputString, wordString) => {
  317. console.log('inputString', inputString);
  318. // Split the string by newline characters to handle lines separately
  319. const lines = inputString.split('\n');
  320. // Take the last line to operate only on it
  321. const lastLine = lines.pop();
  322. // Split the last line into an array of words
  323. const words = lastLine.split(' ');
  324. // Conditional to check for the last word removal
  325. if (words.at(-1) === wordString || (wordString === '' && words.at(-1) === '\\#')) {
  326. words.pop(); // Remove last word if condition is satisfied
  327. }
  328. // Join the remaining words back into a string and handle space correctly
  329. let updatedLastLine = words.join(' ');
  330. // Add a trailing space to the updated last line if there are still words
  331. if (updatedLastLine !== '') {
  332. updatedLastLine += ' ';
  333. }
  334. // Combine the lines together again, placing the updated last line back in
  335. const resultString = [...lines, updatedLastLine].join('\n');
  336. // Return the final string
  337. console.log('resultString', resultString);
  338. return resultString;
  339. };
  340. export const removeFirstHashWord = (inputString) => {
  341. // Split the string into an array of words
  342. const words = inputString.split(' ');
  343. // Find the index of the first word that starts with #
  344. const index = words.findIndex((word) => word.startsWith('#'));
  345. // Remove the first word with #
  346. if (index !== -1) {
  347. words.splice(index, 1);
  348. }
  349. // Join the remaining words back into a string
  350. const resultString = words.join(' ');
  351. return resultString;
  352. };
  353. export const transformFileName = (fileName) => {
  354. // Convert to lowercase
  355. const lowerCaseFileName = fileName.toLowerCase();
  356. // Remove special characters using regular expression
  357. const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');
  358. // Replace spaces with dashes
  359. const finalFileName = sanitizedFileName.replace(/\s+/g, '-');
  360. return finalFileName;
  361. };
  362. export const calculateSHA256 = async (file) => {
  363. // Create a FileReader to read the file asynchronously
  364. const reader = new FileReader();
  365. // Define a promise to handle the file reading
  366. const readFile = new Promise((resolve, reject) => {
  367. reader.onload = () => resolve(reader.result);
  368. reader.onerror = reject;
  369. });
  370. // Read the file as an ArrayBuffer
  371. reader.readAsArrayBuffer(file);
  372. try {
  373. // Wait for the FileReader to finish reading the file
  374. const buffer = await readFile;
  375. // Convert the ArrayBuffer to a Uint8Array
  376. const uint8Array = new Uint8Array(buffer);
  377. // Calculate the SHA-256 hash using Web Crypto API
  378. const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
  379. // Convert the hash to a hexadecimal string
  380. const hashArray = Array.from(new Uint8Array(hashBuffer));
  381. const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
  382. return `${hashHex}`;
  383. } catch (error) {
  384. console.error('Error calculating SHA-256 hash:', error);
  385. throw error;
  386. }
  387. };
  388. export const getImportOrigin = (_chats) => {
  389. // Check what external service chat imports are from
  390. if ('mapping' in _chats[0]) {
  391. return 'openai';
  392. }
  393. return 'webui';
  394. };
  395. export const getUserPosition = async (raw = false) => {
  396. // Get the user's location using the Geolocation API
  397. const position = await new Promise((resolve, reject) => {
  398. navigator.geolocation.getCurrentPosition(resolve, reject);
  399. }).catch((error) => {
  400. console.error('Error getting user location:', error);
  401. throw error;
  402. });
  403. if (!position) {
  404. return 'Location not available';
  405. }
  406. // Extract the latitude and longitude from the position
  407. const { latitude, longitude } = position.coords;
  408. if (raw) {
  409. return { latitude, longitude };
  410. } else {
  411. return `${latitude.toFixed(3)}, ${longitude.toFixed(3)} (lat, long)`;
  412. }
  413. };
  414. const convertOpenAIMessages = (convo) => {
  415. // Parse OpenAI chat messages and create chat dictionary for creating new chats
  416. const mapping = convo['mapping'];
  417. const messages = [];
  418. let currentId = '';
  419. let lastId = null;
  420. for (const message_id in mapping) {
  421. const message = mapping[message_id];
  422. currentId = message_id;
  423. try {
  424. if (
  425. messages.length == 0 &&
  426. (message['message'] == null ||
  427. (message['message']['content']['parts']?.[0] == '' &&
  428. message['message']['content']['text'] == null))
  429. ) {
  430. // Skip chat messages with no content
  431. continue;
  432. } else {
  433. const new_chat = {
  434. id: message_id,
  435. parentId: lastId,
  436. childrenIds: message['children'] || [],
  437. role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
  438. content:
  439. message['message']?.['content']?.['parts']?.[0] ||
  440. message['message']?.['content']?.['text'] ||
  441. '',
  442. model: 'gpt-3.5-turbo',
  443. done: true,
  444. context: null
  445. };
  446. messages.push(new_chat);
  447. lastId = currentId;
  448. }
  449. } catch (error) {
  450. console.log('Error with', message, '\nError:', error);
  451. }
  452. }
  453. const history: Record<PropertyKey, (typeof messages)[number]> = {};
  454. messages.forEach((obj) => (history[obj.id] = obj));
  455. const chat = {
  456. history: {
  457. currentId: currentId,
  458. messages: history // Need to convert this to not a list and instead a json object
  459. },
  460. models: ['gpt-3.5-turbo'],
  461. messages: messages,
  462. options: {},
  463. timestamp: convo['create_time'],
  464. title: convo['title'] ?? 'New Chat'
  465. };
  466. return chat;
  467. };
  468. const validateChat = (chat) => {
  469. // Because ChatGPT sometimes has features we can't use like DALL-E or might have corrupted messages, need to validate
  470. const messages = chat.messages;
  471. // Check if messages array is empty
  472. if (messages.length === 0) {
  473. return false;
  474. }
  475. // Last message's children should be an empty array
  476. const lastMessage = messages[messages.length - 1];
  477. if (lastMessage.childrenIds.length !== 0) {
  478. return false;
  479. }
  480. // First message's parent should be null
  481. const firstMessage = messages[0];
  482. if (firstMessage.parentId !== null) {
  483. return false;
  484. }
  485. // Every message's content should be a string
  486. for (const message of messages) {
  487. if (typeof message.content !== 'string') {
  488. return false;
  489. }
  490. }
  491. return true;
  492. };
  493. export const convertOpenAIChats = (_chats) => {
  494. // Create a list of dictionaries with each conversation from import
  495. const chats = [];
  496. let failed = 0;
  497. for (const convo of _chats) {
  498. const chat = convertOpenAIMessages(convo);
  499. if (validateChat(chat)) {
  500. chats.push({
  501. id: convo['id'],
  502. user_id: '',
  503. title: convo['title'],
  504. chat: chat,
  505. timestamp: convo['create_time']
  506. });
  507. } else {
  508. failed++;
  509. }
  510. }
  511. console.log(failed, 'Conversations could not be imported');
  512. return chats;
  513. };
  514. export const isValidHttpUrl = (string: string) => {
  515. let url;
  516. try {
  517. url = new URL(string);
  518. } catch (_) {
  519. return false;
  520. }
  521. return url.protocol === 'http:' || url.protocol === 'https:';
  522. };
  523. export const removeEmojis = (str: string) => {
  524. // Regular expression to match emojis
  525. const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;
  526. // Replace emojis with an empty string
  527. return str.replace(emojiRegex, '');
  528. };
  529. export const removeFormattings = (str: string) => {
  530. return (
  531. str
  532. // Block elements (remove completely)
  533. .replace(/(```[\s\S]*?```)/g, '') // Code blocks
  534. .replace(/^\|.*\|$/gm, '') // Tables
  535. // Inline elements (preserve content)
  536. .replace(/(?:\*\*|__)(.*?)(?:\*\*|__)/g, '$1') // Bold
  537. .replace(/(?:[*_])(.*?)(?:[*_])/g, '$1') // Italic
  538. .replace(/~~(.*?)~~/g, '$1') // Strikethrough
  539. .replace(/`([^`]+)`/g, '$1') // Inline code
  540. // Links and images
  541. .replace(/!?\[([^\]]*)\](?:\([^)]+\)|\[[^\]]*\])/g, '$1') // Links & images
  542. .replace(/^\[[^\]]+\]:\s*.*$/gm, '') // Reference definitions
  543. // Block formatting
  544. .replace(/^#{1,6}\s+/gm, '') // Headers
  545. .replace(/^\s*[-*+]\s+/gm, '') // Lists
  546. .replace(/^\s*(?:\d+\.)\s+/gm, '') // Numbered lists
  547. .replace(/^\s*>[> ]*/gm, '') // Blockquotes
  548. .replace(/^\s*:\s+/gm, '') // Definition lists
  549. // Cleanup
  550. .replace(/\[\^[^\]]*\]/g, '') // Footnotes
  551. .replace(/[-*_~]/g, '') // Remaining markers
  552. .replace(/\n{2,}/g, '\n')
  553. ); // Multiple newlines
  554. };
  555. export const cleanText = (content: string) => {
  556. return removeFormattings(removeEmojis(content.trim()));
  557. };
  558. export const removeDetails = (content, types) => {
  559. for (const type of types) {
  560. content = content.replace(
  561. new RegExp(`<details\\s+type="${type}"[^>]*>.*?<\\/details>`, 'gis'),
  562. ''
  563. );
  564. }
  565. return content;
  566. };
  567. // This regular expression matches code blocks marked by triple backticks
  568. const codeBlockRegex = /```[\s\S]*?```/g;
  569. export const extractSentences = (text: string) => {
  570. const codeBlocks: string[] = [];
  571. let index = 0;
  572. // Temporarily replace code blocks with placeholders and store the blocks separately
  573. text = text.replace(codeBlockRegex, (match) => {
  574. const placeholder = `\u0000${index}\u0000`; // Use a unique placeholder
  575. codeBlocks[index++] = match;
  576. return placeholder;
  577. });
  578. // Split the modified text into sentences based on common punctuation marks, avoiding these blocks
  579. let sentences = text.split(/(?<=[.!?])\s+/);
  580. // Restore code blocks and process sentences
  581. sentences = sentences.map((sentence) => {
  582. // Check if the sentence includes a placeholder for a code block
  583. return sentence.replace(/\u0000(\d+)\u0000/g, (_, idx) => codeBlocks[idx]);
  584. });
  585. return sentences.map(cleanText).filter(Boolean);
  586. };
  587. export const extractParagraphsForAudio = (text: string) => {
  588. const codeBlocks: string[] = [];
  589. let index = 0;
  590. // Temporarily replace code blocks with placeholders and store the blocks separately
  591. text = text.replace(codeBlockRegex, (match) => {
  592. const placeholder = `\u0000${index}\u0000`; // Use a unique placeholder
  593. codeBlocks[index++] = match;
  594. return placeholder;
  595. });
  596. // Split the modified text into paragraphs based on newlines, avoiding these blocks
  597. let paragraphs = text.split(/\n+/);
  598. // Restore code blocks and process paragraphs
  599. paragraphs = paragraphs.map((paragraph) => {
  600. // Check if the paragraph includes a placeholder for a code block
  601. return paragraph.replace(/\u0000(\d+)\u0000/g, (_, idx) => codeBlocks[idx]);
  602. });
  603. return paragraphs.map(cleanText).filter(Boolean);
  604. };
  605. export const extractSentencesForAudio = (text: string) => {
  606. return extractSentences(text).reduce((mergedTexts, currentText) => {
  607. const lastIndex = mergedTexts.length - 1;
  608. if (lastIndex >= 0) {
  609. const previousText = mergedTexts[lastIndex];
  610. const wordCount = previousText.split(/\s+/).length;
  611. const charCount = previousText.length;
  612. if (wordCount < 4 || charCount < 50) {
  613. mergedTexts[lastIndex] = previousText + ' ' + currentText;
  614. } else {
  615. mergedTexts.push(currentText);
  616. }
  617. } else {
  618. mergedTexts.push(currentText);
  619. }
  620. return mergedTexts;
  621. }, [] as string[]);
  622. };
  623. export const getMessageContentParts = (content: string, split_on: string = 'punctuation') => {
  624. content = removeDetails(content, ['reasoning', 'code_interpreter', 'tool_calls']);
  625. const messageContentParts: string[] = [];
  626. switch (split_on) {
  627. default:
  628. case TTS_RESPONSE_SPLIT.PUNCTUATION:
  629. messageContentParts.push(...extractSentencesForAudio(content));
  630. break;
  631. case TTS_RESPONSE_SPLIT.PARAGRAPHS:
  632. messageContentParts.push(...extractParagraphsForAudio(content));
  633. break;
  634. case TTS_RESPONSE_SPLIT.NONE:
  635. messageContentParts.push(cleanText(content));
  636. break;
  637. }
  638. return messageContentParts;
  639. };
  640. export const blobToFile = (blob, fileName) => {
  641. // Create a new File object from the Blob
  642. const file = new File([blob], fileName, { type: blob.type });
  643. return file;
  644. };
  645. export const getPromptVariables = (user_name, user_location) => {
  646. return {
  647. '{{USER_NAME}}': user_name,
  648. '{{USER_LOCATION}}': user_location || 'Unknown',
  649. '{{CURRENT_DATETIME}}': getCurrentDateTime(),
  650. '{{CURRENT_DATE}}': getFormattedDate(),
  651. '{{CURRENT_TIME}}': getFormattedTime(),
  652. '{{CURRENT_WEEKDAY}}': getWeekday(),
  653. '{{CURRENT_TIMEZONE}}': getUserTimezone(),
  654. '{{USER_LANGUAGE}}': localStorage.getItem('locale') || 'en-US'
  655. };
  656. };
  657. /**
  658. * @param {string} template - The template string containing placeholders.
  659. * @returns {string} The template string with the placeholders replaced by the prompt.
  660. */
  661. export const promptTemplate = (
  662. template: string,
  663. user_name?: string,
  664. user_location?: string
  665. ): string => {
  666. // Get the current date
  667. const currentDate = new Date();
  668. // Format the date to YYYY-MM-DD
  669. const formattedDate =
  670. currentDate.getFullYear() +
  671. '-' +
  672. String(currentDate.getMonth() + 1).padStart(2, '0') +
  673. '-' +
  674. String(currentDate.getDate()).padStart(2, '0');
  675. // Format the time to HH:MM:SS AM/PM
  676. const currentTime = currentDate.toLocaleTimeString('en-US', {
  677. hour: 'numeric',
  678. minute: 'numeric',
  679. second: 'numeric',
  680. hour12: true
  681. });
  682. // Get the current weekday
  683. const currentWeekday = getWeekday();
  684. // Get the user's timezone
  685. const currentTimezone = getUserTimezone();
  686. // Get the user's language
  687. const userLanguage = localStorage.getItem('locale') || 'en-US';
  688. // Replace {{CURRENT_DATETIME}} in the template with the formatted datetime
  689. template = template.replace('{{CURRENT_DATETIME}}', `${formattedDate} ${currentTime}`);
  690. // Replace {{CURRENT_DATE}} in the template with the formatted date
  691. template = template.replace('{{CURRENT_DATE}}', formattedDate);
  692. // Replace {{CURRENT_TIME}} in the template with the formatted time
  693. template = template.replace('{{CURRENT_TIME}}', currentTime);
  694. // Replace {{CURRENT_WEEKDAY}} in the template with the current weekday
  695. template = template.replace('{{CURRENT_WEEKDAY}}', currentWeekday);
  696. // Replace {{CURRENT_TIMEZONE}} in the template with the user's timezone
  697. template = template.replace('{{CURRENT_TIMEZONE}}', currentTimezone);
  698. // Replace {{USER_LANGUAGE}} in the template with the user's language
  699. template = template.replace('{{USER_LANGUAGE}}', userLanguage);
  700. if (user_name) {
  701. // Replace {{USER_NAME}} in the template with the user's name
  702. template = template.replace('{{USER_NAME}}', user_name);
  703. }
  704. if (user_location) {
  705. // Replace {{USER_LOCATION}} in the template with the current location
  706. template = template.replace('{{USER_LOCATION}}', user_location);
  707. } else {
  708. // Replace {{USER_LOCATION}} in the template with 'Unknown' if no location is provided
  709. template = template.replace('{{USER_LOCATION}}', 'LOCATION_UNKNOWN');
  710. }
  711. return template;
  712. };
  713. /**
  714. * This function is used to replace placeholders in a template string with the provided prompt.
  715. * The placeholders can be in the following formats:
  716. * - `{{prompt}}`: This will be replaced with the entire prompt.
  717. * - `{{prompt:start:<length>}}`: This will be replaced with the first <length> characters of the prompt.
  718. * - `{{prompt:end:<length>}}`: This will be replaced with the last <length> characters of the prompt.
  719. * - `{{prompt:middletruncate:<length>}}`: This will be replaced with the prompt truncated to <length> characters, with '...' in the middle.
  720. *
  721. * @param {string} template - The template string containing placeholders.
  722. * @param {string} prompt - The string to replace the placeholders with.
  723. * @returns {string} The template string with the placeholders replaced by the prompt.
  724. */
  725. export const titleGenerationTemplate = (template: string, prompt: string): string => {
  726. template = template.replace(
  727. /{{prompt}}|{{prompt:start:(\d+)}}|{{prompt:end:(\d+)}}|{{prompt:middletruncate:(\d+)}}/g,
  728. (match, startLength, endLength, middleLength) => {
  729. if (match === '{{prompt}}') {
  730. return prompt;
  731. } else if (match.startsWith('{{prompt:start:')) {
  732. return prompt.substring(0, startLength);
  733. } else if (match.startsWith('{{prompt:end:')) {
  734. return prompt.slice(-endLength);
  735. } else if (match.startsWith('{{prompt:middletruncate:')) {
  736. if (prompt.length <= middleLength) {
  737. return prompt;
  738. }
  739. const start = prompt.slice(0, Math.ceil(middleLength / 2));
  740. const end = prompt.slice(-Math.floor(middleLength / 2));
  741. return `${start}...${end}`;
  742. }
  743. return '';
  744. }
  745. );
  746. template = promptTemplate(template);
  747. return template;
  748. };
  749. export const approximateToHumanReadable = (nanoseconds: number) => {
  750. const seconds = Math.floor((nanoseconds / 1e9) % 60);
  751. const minutes = Math.floor((nanoseconds / 6e10) % 60);
  752. const hours = Math.floor((nanoseconds / 3.6e12) % 24);
  753. const results: string[] = [];
  754. if (seconds >= 0) {
  755. results.push(`${seconds}s`);
  756. }
  757. if (minutes > 0) {
  758. results.push(`${minutes}m`);
  759. }
  760. if (hours > 0) {
  761. results.push(`${hours}h`);
  762. }
  763. return results.reverse().join(' ');
  764. };
  765. export const getTimeRange = (timestamp) => {
  766. const now = new Date();
  767. const date = new Date(timestamp * 1000); // Convert Unix timestamp to milliseconds
  768. // Calculate the difference in milliseconds
  769. const diffTime = now.getTime() - date.getTime();
  770. const diffDays = diffTime / (1000 * 3600 * 24);
  771. const nowDate = now.getDate();
  772. const nowMonth = now.getMonth();
  773. const nowYear = now.getFullYear();
  774. const dateDate = date.getDate();
  775. const dateMonth = date.getMonth();
  776. const dateYear = date.getFullYear();
  777. if (nowYear === dateYear && nowMonth === dateMonth && nowDate === dateDate) {
  778. return 'Today';
  779. } else if (nowYear === dateYear && nowMonth === dateMonth && nowDate - dateDate === 1) {
  780. return 'Yesterday';
  781. } else if (diffDays <= 7) {
  782. return 'Previous 7 days';
  783. } else if (diffDays <= 30) {
  784. return 'Previous 30 days';
  785. } else if (nowYear === dateYear) {
  786. return date.toLocaleString('default', { month: 'long' });
  787. } else {
  788. return date.getFullYear().toString();
  789. }
  790. };
  791. /**
  792. * Extract frontmatter as a dictionary from the specified content string.
  793. * @param content {string} - The content string with potential frontmatter.
  794. * @returns {Object} - The extracted frontmatter as a dictionary.
  795. */
  796. export const extractFrontmatter = (content) => {
  797. const frontmatter = {};
  798. let frontmatterStarted = false;
  799. let frontmatterEnded = false;
  800. const frontmatterPattern = /^\s*([a-z_]+):\s*(.*)\s*$/i;
  801. // Split content into lines
  802. const lines = content.split('\n');
  803. // Check if the content starts with triple quotes
  804. if (lines[0].trim() !== '"""') {
  805. return {};
  806. }
  807. frontmatterStarted = true;
  808. for (let i = 1; i < lines.length; i++) {
  809. const line = lines[i];
  810. if (line.includes('"""')) {
  811. if (frontmatterStarted) {
  812. frontmatterEnded = true;
  813. break;
  814. }
  815. }
  816. if (frontmatterStarted && !frontmatterEnded) {
  817. const match = frontmatterPattern.exec(line);
  818. if (match) {
  819. const [, key, value] = match;
  820. frontmatter[key.trim()] = value.trim();
  821. }
  822. }
  823. }
  824. return frontmatter;
  825. };
  826. // Function to determine the best matching language
  827. export const bestMatchingLanguage = (supportedLanguages, preferredLanguages, defaultLocale) => {
  828. const languages = supportedLanguages.map((lang) => lang.code);
  829. const match = preferredLanguages
  830. .map((prefLang) => languages.find((lang) => lang.startsWith(prefLang)))
  831. .find(Boolean);
  832. return match || defaultLocale;
  833. };
  834. // Get the date in the format YYYY-MM-DD
  835. export const getFormattedDate = () => {
  836. const date = new Date();
  837. const year = date.getFullYear();
  838. const month = String(date.getMonth() + 1).padStart(2, '0');
  839. const day = String(date.getDate()).padStart(2, '0');
  840. return `${year}-${month}-${day}`;
  841. };
  842. // Get the time in the format HH:MM:SS
  843. export const getFormattedTime = () => {
  844. const date = new Date();
  845. return date.toTimeString().split(' ')[0];
  846. };
  847. // Get the current date and time in the format YYYY-MM-DD HH:MM:SS
  848. export const getCurrentDateTime = () => {
  849. return `${getFormattedDate()} ${getFormattedTime()}`;
  850. };
  851. // Get the user's timezone
  852. export const getUserTimezone = () => {
  853. return Intl.DateTimeFormat().resolvedOptions().timeZone;
  854. };
  855. // Get the weekday
  856. export const getWeekday = () => {
  857. const date = new Date();
  858. const weekdays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
  859. return weekdays[date.getDay()];
  860. };
  861. export const createMessagesList = (history, messageId) => {
  862. if (messageId === null) {
  863. return [];
  864. }
  865. const message = history.messages[messageId];
  866. if (message?.parentId) {
  867. return [...createMessagesList(history, message.parentId), message];
  868. } else {
  869. return [message];
  870. }
  871. };
  872. export const formatFileSize = (size) => {
  873. if (size == null) return 'Unknown size';
  874. if (typeof size !== 'number' || size < 0) return 'Invalid size';
  875. if (size === 0) return '0 B';
  876. const units = ['B', 'KB', 'MB', 'GB', 'TB'];
  877. let unitIndex = 0;
  878. while (size >= 1024 && unitIndex < units.length - 1) {
  879. size /= 1024;
  880. unitIndex++;
  881. }
  882. return `${size.toFixed(1)} ${units[unitIndex]}`;
  883. };
  884. export const getLineCount = (text) => {
  885. console.log(typeof text);
  886. return text ? text.split('\n').length : 0;
  887. };