index.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059
  1. import { v4 as uuidv4 } from 'uuid';
  2. import sha256 from 'js-sha256';
  3. import dayjs from 'dayjs';
  4. import relativeTime from 'dayjs/plugin/relativeTime';
  5. import isToday from 'dayjs/plugin/isToday';
  6. import isYesterday from 'dayjs/plugin/isYesterday';
  7. import localizedFormat from 'dayjs/plugin/localizedFormat';
  8. dayjs.extend(relativeTime);
  9. dayjs.extend(isToday);
  10. dayjs.extend(isYesterday);
  11. dayjs.extend(localizedFormat);
  12. import { WEBUI_BASE_URL } from '$lib/constants';
  13. import { TTS_RESPONSE_SPLIT } from '$lib/types';
  14. //////////////////////////
  15. // Helper functions
  16. //////////////////////////
  17. export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
  18. function escapeRegExp(string: string): string {
  19. return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  20. }
  21. export const replaceTokens = (content, sourceIds, char, user) => {
  22. const charToken = /{{char}}/gi;
  23. const userToken = /{{user}}/gi;
  24. const videoIdToken = /{{VIDEO_FILE_ID_([a-f0-9-]+)}}/gi; // Regex to capture the video ID
  25. const htmlIdToken = /{{HTML_FILE_ID_([a-f0-9-]+)}}/gi; // Regex to capture the HTML ID
  26. // Replace {{char}} if char is provided
  27. if (char !== undefined && char !== null) {
  28. content = content.replace(charToken, char);
  29. }
  30. // Replace {{user}} if user is provided
  31. if (user !== undefined && user !== null) {
  32. content = content.replace(userToken, user);
  33. }
  34. // Replace video ID tags with corresponding <video> elements
  35. content = content.replace(videoIdToken, (match, fileId) => {
  36. const videoUrl = `${WEBUI_BASE_URL}/api/v1/files/${fileId}/content`;
  37. return `<video src="${videoUrl}" controls></video>`;
  38. });
  39. // Replace HTML ID tags with corresponding HTML content
  40. content = content.replace(htmlIdToken, (match, fileId) => {
  41. const htmlUrl = `${WEBUI_BASE_URL}/api/v1/files/${fileId}/content/html`;
  42. return `<iframe src="${htmlUrl}" width="100%" frameborder="0" onload="this.style.height=(this.contentWindow.document.body.scrollHeight+20)+'px';"></iframe>`;
  43. });
  44. // Remove sourceIds from the content and replace them with <source_id>...</source_id>
  45. if (Array.isArray(sourceIds)) {
  46. sourceIds.forEach((sourceId, idx) => {
  47. // Create a token based on the exact `[sourceId]` string
  48. const sourceToken = `\\[${idx}\\]`; // Escape special characters for RegExp
  49. const sourceRegex = new RegExp(sourceToken, 'g'); // Match all occurrences of [sourceId]
  50. content = content.replace(sourceRegex, `<source_id data="${idx}" title="${sourceId}" />`);
  51. });
  52. }
  53. return content;
  54. };
  55. export const sanitizeResponseContent = (content: string) => {
  56. return content
  57. .replace(/<\|[a-z]*$/, '')
  58. .replace(/<\|[a-z]+\|$/, '')
  59. .replace(/<$/, '')
  60. .replaceAll(/<\|[a-z]+\|>/g, ' ')
  61. .replaceAll('<', '&lt;')
  62. .replaceAll('>', '&gt;')
  63. .trim();
  64. };
  65. export const processResponseContent = (content: string) => {
  66. return content.trim();
  67. };
  68. export function unescapeHtml(html: string) {
  69. const doc = new DOMParser().parseFromString(html, 'text/html');
  70. return doc.documentElement.textContent;
  71. }
  72. export const capitalizeFirstLetter = (string) => {
  73. return string.charAt(0).toUpperCase() + string.slice(1);
  74. };
  75. export const splitStream = (splitOn) => {
  76. let buffer = '';
  77. return new TransformStream({
  78. transform(chunk, controller) {
  79. buffer += chunk;
  80. const parts = buffer.split(splitOn);
  81. parts.slice(0, -1).forEach((part) => controller.enqueue(part));
  82. buffer = parts[parts.length - 1];
  83. },
  84. flush(controller) {
  85. if (buffer) controller.enqueue(buffer);
  86. }
  87. });
  88. };
  89. export const convertMessagesToHistory = (messages) => {
  90. const history = {
  91. messages: {},
  92. currentId: null
  93. };
  94. let parentMessageId = null;
  95. let messageId = null;
  96. for (const message of messages) {
  97. messageId = uuidv4();
  98. if (parentMessageId !== null) {
  99. history.messages[parentMessageId].childrenIds = [
  100. ...history.messages[parentMessageId].childrenIds,
  101. messageId
  102. ];
  103. }
  104. history.messages[messageId] = {
  105. ...message,
  106. id: messageId,
  107. parentId: parentMessageId,
  108. childrenIds: []
  109. };
  110. parentMessageId = messageId;
  111. }
  112. history.currentId = messageId;
  113. return history;
  114. };
  115. export const getGravatarURL = (email) => {
  116. // Trim leading and trailing whitespace from
  117. // an email address and force all characters
  118. // to lower case
  119. const address = String(email).trim().toLowerCase();
  120. // Create a SHA256 hash of the final string
  121. const hash = sha256(address);
  122. // Grab the actual image URL
  123. return `https://www.gravatar.com/avatar/${hash}`;
  124. };
  125. export const canvasPixelTest = () => {
  126. // Test a 1x1 pixel to potentially identify browser/plugin fingerprint blocking or spoofing
  127. // Inspiration: https://github.com/kkapsner/CanvasBlocker/blob/master/test/detectionTest.js
  128. const canvas = document.createElement('canvas');
  129. const ctx = canvas.getContext('2d');
  130. canvas.height = 1;
  131. canvas.width = 1;
  132. const imageData = new ImageData(canvas.width, canvas.height);
  133. const pixelValues = imageData.data;
  134. // Generate RGB test data
  135. for (let i = 0; i < imageData.data.length; i += 1) {
  136. if (i % 4 !== 3) {
  137. pixelValues[i] = Math.floor(256 * Math.random());
  138. } else {
  139. pixelValues[i] = 255;
  140. }
  141. }
  142. ctx.putImageData(imageData, 0, 0);
  143. const p = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
  144. // Read RGB data and fail if unmatched
  145. for (let i = 0; i < p.length; i += 1) {
  146. if (p[i] !== pixelValues[i]) {
  147. console.log(
  148. 'canvasPixelTest: Wrong canvas pixel RGB value detected:',
  149. p[i],
  150. 'at:',
  151. i,
  152. 'expected:',
  153. pixelValues[i]
  154. );
  155. console.log('canvasPixelTest: Canvas blocking or spoofing is likely');
  156. return false;
  157. }
  158. }
  159. return true;
  160. };
  161. export const compressImage = async (imageUrl, maxWidth, maxHeight) => {
  162. return new Promise((resolve, reject) => {
  163. const img = new Image();
  164. img.onload = () => {
  165. const canvas = document.createElement('canvas');
  166. let width = img.width;
  167. let height = img.height;
  168. // Maintain aspect ratio while resizing
  169. if (maxWidth && maxHeight) {
  170. // Resize with both dimensions defined (preserves aspect ratio)
  171. if (width <= maxWidth && height <= maxHeight) {
  172. resolve(imageUrl);
  173. return;
  174. }
  175. if (width / height > maxWidth / maxHeight) {
  176. height = Math.round((maxWidth * height) / width);
  177. width = maxWidth;
  178. } else {
  179. width = Math.round((maxHeight * width) / height);
  180. height = maxHeight;
  181. }
  182. } else if (maxWidth) {
  183. // Only maxWidth defined
  184. if (width <= maxWidth) {
  185. resolve(imageUrl);
  186. return;
  187. }
  188. height = Math.round((maxWidth * height) / width);
  189. width = maxWidth;
  190. } else if (maxHeight) {
  191. // Only maxHeight defined
  192. if (height <= maxHeight) {
  193. resolve(imageUrl);
  194. return;
  195. }
  196. width = Math.round((maxHeight * width) / height);
  197. height = maxHeight;
  198. }
  199. canvas.width = width;
  200. canvas.height = height;
  201. const context = canvas.getContext('2d');
  202. context.drawImage(img, 0, 0, width, height);
  203. // Get compressed image URL
  204. const compressedUrl = canvas.toDataURL();
  205. resolve(compressedUrl);
  206. };
  207. img.onerror = (error) => reject(error);
  208. img.src = imageUrl;
  209. });
  210. };
  211. export const generateInitialsImage = (name) => {
  212. const canvas = document.createElement('canvas');
  213. const ctx = canvas.getContext('2d');
  214. canvas.width = 100;
  215. canvas.height = 100;
  216. if (!canvasPixelTest()) {
  217. console.log(
  218. 'generateInitialsImage: failed pixel test, fingerprint evasion is likely. Using default image.'
  219. );
  220. return '/user.png';
  221. }
  222. ctx.fillStyle = '#F39C12';
  223. ctx.fillRect(0, 0, canvas.width, canvas.height);
  224. ctx.fillStyle = '#FFFFFF';
  225. ctx.font = '40px Helvetica';
  226. ctx.textAlign = 'center';
  227. ctx.textBaseline = 'middle';
  228. const sanitizedName = name.trim();
  229. const initials =
  230. sanitizedName.length > 0
  231. ? sanitizedName[0] +
  232. (sanitizedName.split(' ').length > 1
  233. ? sanitizedName[sanitizedName.lastIndexOf(' ') + 1]
  234. : '')
  235. : '';
  236. ctx.fillText(initials.toUpperCase(), canvas.width / 2, canvas.height / 2);
  237. return canvas.toDataURL();
  238. };
  239. export const formatDate = (inputDate) => {
  240. const date = dayjs(inputDate);
  241. const now = dayjs();
  242. if (date.isToday()) {
  243. return `Today at ${date.format('LT')}`;
  244. } else if (date.isYesterday()) {
  245. return `Yesterday at ${date.format('LT')}`;
  246. } else {
  247. return `${date.format('L')} at ${date.format('LT')}`;
  248. }
  249. };
  250. export const copyToClipboard = async (text) => {
  251. let result = false;
  252. if (!navigator.clipboard) {
  253. const textArea = document.createElement('textarea');
  254. textArea.value = text;
  255. // Avoid scrolling to bottom
  256. textArea.style.top = '0';
  257. textArea.style.left = '0';
  258. textArea.style.position = 'fixed';
  259. document.body.appendChild(textArea);
  260. textArea.focus();
  261. textArea.select();
  262. try {
  263. const successful = document.execCommand('copy');
  264. const msg = successful ? 'successful' : 'unsuccessful';
  265. console.log('Fallback: Copying text command was ' + msg);
  266. result = true;
  267. } catch (err) {
  268. console.error('Fallback: Oops, unable to copy', err);
  269. }
  270. document.body.removeChild(textArea);
  271. return result;
  272. }
  273. result = await navigator.clipboard
  274. .writeText(text)
  275. .then(() => {
  276. console.log('Async: Copying to clipboard was successful!');
  277. return true;
  278. })
  279. .catch((error) => {
  280. console.error('Async: Could not copy text: ', error);
  281. return false;
  282. });
  283. return result;
  284. };
  285. export const compareVersion = (latest, current) => {
  286. return current === '0.0.0'
  287. ? false
  288. : current.localeCompare(latest, undefined, {
  289. numeric: true,
  290. sensitivity: 'case',
  291. caseFirst: 'upper'
  292. }) < 0;
  293. };
  294. export const findWordIndices = (text) => {
  295. const regex = /\[([^\]]+)\]/g;
  296. const matches = [];
  297. let match;
  298. while ((match = regex.exec(text)) !== null) {
  299. matches.push({
  300. word: match[1],
  301. startIndex: match.index,
  302. endIndex: regex.lastIndex - 1
  303. });
  304. }
  305. return matches;
  306. };
  307. export const removeLastWordFromString = (inputString, wordString) => {
  308. console.log('inputString', inputString);
  309. // Split the string by newline characters to handle lines separately
  310. const lines = inputString.split('\n');
  311. // Take the last line to operate only on it
  312. const lastLine = lines.pop();
  313. // Split the last line into an array of words
  314. const words = lastLine.split(' ');
  315. // Conditional to check for the last word removal
  316. if (words.at(-1) === wordString || (wordString === '' && words.at(-1) === '\\#')) {
  317. words.pop(); // Remove last word if condition is satisfied
  318. }
  319. // Join the remaining words back into a string and handle space correctly
  320. let updatedLastLine = words.join(' ');
  321. // Add a trailing space to the updated last line if there are still words
  322. if (updatedLastLine !== '') {
  323. updatedLastLine += ' ';
  324. }
  325. // Combine the lines together again, placing the updated last line back in
  326. const resultString = [...lines, updatedLastLine].join('\n');
  327. // Return the final string
  328. console.log('resultString', resultString);
  329. return resultString;
  330. };
  331. export const removeFirstHashWord = (inputString) => {
  332. // Split the string into an array of words
  333. const words = inputString.split(' ');
  334. // Find the index of the first word that starts with #
  335. const index = words.findIndex((word) => word.startsWith('#'));
  336. // Remove the first word with #
  337. if (index !== -1) {
  338. words.splice(index, 1);
  339. }
  340. // Join the remaining words back into a string
  341. const resultString = words.join(' ');
  342. return resultString;
  343. };
  344. export const transformFileName = (fileName) => {
  345. // Convert to lowercase
  346. const lowerCaseFileName = fileName.toLowerCase();
  347. // Remove special characters using regular expression
  348. const sanitizedFileName = lowerCaseFileName.replace(/[^\w\s]/g, '');
  349. // Replace spaces with dashes
  350. const finalFileName = sanitizedFileName.replace(/\s+/g, '-');
  351. return finalFileName;
  352. };
  353. export const calculateSHA256 = async (file) => {
  354. // Create a FileReader to read the file asynchronously
  355. const reader = new FileReader();
  356. // Define a promise to handle the file reading
  357. const readFile = new Promise((resolve, reject) => {
  358. reader.onload = () => resolve(reader.result);
  359. reader.onerror = reject;
  360. });
  361. // Read the file as an ArrayBuffer
  362. reader.readAsArrayBuffer(file);
  363. try {
  364. // Wait for the FileReader to finish reading the file
  365. const buffer = await readFile;
  366. // Convert the ArrayBuffer to a Uint8Array
  367. const uint8Array = new Uint8Array(buffer);
  368. // Calculate the SHA-256 hash using Web Crypto API
  369. const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
  370. // Convert the hash to a hexadecimal string
  371. const hashArray = Array.from(new Uint8Array(hashBuffer));
  372. const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
  373. return `${hashHex}`;
  374. } catch (error) {
  375. console.error('Error calculating SHA-256 hash:', error);
  376. throw error;
  377. }
  378. };
  379. export const getImportOrigin = (_chats) => {
  380. // Check what external service chat imports are from
  381. if ('mapping' in _chats[0]) {
  382. return 'openai';
  383. }
  384. return 'webui';
  385. };
  386. export const getUserPosition = async (raw = false) => {
  387. // Get the user's location using the Geolocation API
  388. const position = await new Promise((resolve, reject) => {
  389. navigator.geolocation.getCurrentPosition(resolve, reject);
  390. }).catch((error) => {
  391. console.error('Error getting user location:', error);
  392. throw error;
  393. });
  394. if (!position) {
  395. return 'Location not available';
  396. }
  397. // Extract the latitude and longitude from the position
  398. const { latitude, longitude } = position.coords;
  399. if (raw) {
  400. return { latitude, longitude };
  401. } else {
  402. return `${latitude.toFixed(3)}, ${longitude.toFixed(3)} (lat, long)`;
  403. }
  404. };
  405. const convertOpenAIMessages = (convo) => {
  406. // Parse OpenAI chat messages and create chat dictionary for creating new chats
  407. const mapping = convo['mapping'];
  408. const messages = [];
  409. let currentId = '';
  410. let lastId = null;
  411. for (const message_id in mapping) {
  412. const message = mapping[message_id];
  413. currentId = message_id;
  414. try {
  415. if (
  416. messages.length == 0 &&
  417. (message['message'] == null ||
  418. (message['message']['content']['parts']?.[0] == '' &&
  419. message['message']['content']['text'] == null))
  420. ) {
  421. // Skip chat messages with no content
  422. continue;
  423. } else {
  424. const new_chat = {
  425. id: message_id,
  426. parentId: lastId,
  427. childrenIds: message['children'] || [],
  428. role: message['message']?.['author']?.['role'] !== 'user' ? 'assistant' : 'user',
  429. content:
  430. message['message']?.['content']?.['parts']?.[0] ||
  431. message['message']?.['content']?.['text'] ||
  432. '',
  433. model: 'gpt-3.5-turbo',
  434. done: true,
  435. context: null
  436. };
  437. messages.push(new_chat);
  438. lastId = currentId;
  439. }
  440. } catch (error) {
  441. console.log('Error with', message, '\nError:', error);
  442. }
  443. }
  444. const history: Record<PropertyKey, (typeof messages)[number]> = {};
  445. messages.forEach((obj) => (history[obj.id] = obj));
  446. const chat = {
  447. history: {
  448. currentId: currentId,
  449. messages: history // Need to convert this to not a list and instead a json object
  450. },
  451. models: ['gpt-3.5-turbo'],
  452. messages: messages,
  453. options: {},
  454. timestamp: convo['create_time'],
  455. title: convo['title'] ?? 'New Chat'
  456. };
  457. return chat;
  458. };
  459. const validateChat = (chat) => {
  460. // Because ChatGPT sometimes has features we can't use like DALL-E or might have corrupted messages, need to validate
  461. const messages = chat.messages;
  462. // Check if messages array is empty
  463. if (messages.length === 0) {
  464. return false;
  465. }
  466. // Last message's children should be an empty array
  467. const lastMessage = messages[messages.length - 1];
  468. if (lastMessage.childrenIds.length !== 0) {
  469. return false;
  470. }
  471. // First message's parent should be null
  472. const firstMessage = messages[0];
  473. if (firstMessage.parentId !== null) {
  474. return false;
  475. }
  476. // Every message's content should be a string
  477. for (const message of messages) {
  478. if (typeof message.content !== 'string') {
  479. return false;
  480. }
  481. }
  482. return true;
  483. };
  484. export const convertOpenAIChats = (_chats) => {
  485. // Create a list of dictionaries with each conversation from import
  486. const chats = [];
  487. let failed = 0;
  488. for (const convo of _chats) {
  489. const chat = convertOpenAIMessages(convo);
  490. if (validateChat(chat)) {
  491. chats.push({
  492. id: convo['id'],
  493. user_id: '',
  494. title: convo['title'],
  495. chat: chat,
  496. timestamp: convo['timestamp']
  497. });
  498. } else {
  499. failed++;
  500. }
  501. }
  502. console.log(failed, 'Conversations could not be imported');
  503. return chats;
  504. };
  505. export const isValidHttpUrl = (string: string) => {
  506. let url;
  507. try {
  508. url = new URL(string);
  509. } catch (_) {
  510. return false;
  511. }
  512. return url.protocol === 'http:' || url.protocol === 'https:';
  513. };
  514. export const removeEmojis = (str: string) => {
  515. // Regular expression to match emojis
  516. const emojiRegex = /[\uD800-\uDBFF][\uDC00-\uDFFF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDE4F]/g;
  517. // Replace emojis with an empty string
  518. return str.replace(emojiRegex, '');
  519. };
  520. export const removeFormattings = (str: string) => {
  521. return (
  522. str
  523. // Block elements (remove completely)
  524. .replace(/(```[\s\S]*?```)/g, '') // Code blocks
  525. .replace(/^\|.*\|$/gm, '') // Tables
  526. // Inline elements (preserve content)
  527. .replace(/(?:\*\*|__)(.*?)(?:\*\*|__)/g, '$1') // Bold
  528. .replace(/(?:[*_])(.*?)(?:[*_])/g, '$1') // Italic
  529. .replace(/~~(.*?)~~/g, '$1') // Strikethrough
  530. .replace(/`([^`]+)`/g, '$1') // Inline code
  531. // Links and images
  532. .replace(/!?\[([^\]]*)\](?:\([^)]+\)|\[[^\]]*\])/g, '$1') // Links & images
  533. .replace(/^\[[^\]]+\]:\s*.*$/gm, '') // Reference definitions
  534. // Block formatting
  535. .replace(/^#{1,6}\s+/gm, '') // Headers
  536. .replace(/^\s*[-*+]\s+/gm, '') // Lists
  537. .replace(/^\s*(?:\d+\.)\s+/gm, '') // Numbered lists
  538. .replace(/^\s*>[> ]*/gm, '') // Blockquotes
  539. .replace(/^\s*:\s+/gm, '') // Definition lists
  540. // Cleanup
  541. .replace(/\[\^[^\]]*\]/g, '') // Footnotes
  542. .replace(/[-*_~]/g, '') // Remaining markers
  543. .replace(/\n{2,}/g, '\n')
  544. ); // Multiple newlines
  545. };
  546. export const cleanText = (content: string) => {
  547. return removeFormattings(removeEmojis(content.trim()));
  548. };
  549. export const removeDetails = (content, types) => {
  550. for (const type of types) {
  551. content = content.replace(
  552. new RegExp(`<details\\s+type="${type}"[^>]*>.*?<\\/details>`, 'gis'),
  553. ''
  554. );
  555. }
  556. return content;
  557. };
  558. // This regular expression matches code blocks marked by triple backticks
  559. const codeBlockRegex = /```[\s\S]*?```/g;
  560. export const extractSentences = (text: string) => {
  561. const codeBlocks: string[] = [];
  562. let index = 0;
  563. // Temporarily replace code blocks with placeholders and store the blocks separately
  564. text = text.replace(codeBlockRegex, (match) => {
  565. const placeholder = `\u0000${index}\u0000`; // Use a unique placeholder
  566. codeBlocks[index++] = match;
  567. return placeholder;
  568. });
  569. // Split the modified text into sentences based on common punctuation marks, avoiding these blocks
  570. let sentences = text.split(/(?<=[.!?])\s+/);
  571. // Restore code blocks and process sentences
  572. sentences = sentences.map((sentence) => {
  573. // Check if the sentence includes a placeholder for a code block
  574. return sentence.replace(/\u0000(\d+)\u0000/g, (_, idx) => codeBlocks[idx]);
  575. });
  576. return sentences.map(cleanText).filter(Boolean);
  577. };
  578. export const extractParagraphsForAudio = (text: string) => {
  579. const codeBlocks: string[] = [];
  580. let index = 0;
  581. // Temporarily replace code blocks with placeholders and store the blocks separately
  582. text = text.replace(codeBlockRegex, (match) => {
  583. const placeholder = `\u0000${index}\u0000`; // Use a unique placeholder
  584. codeBlocks[index++] = match;
  585. return placeholder;
  586. });
  587. // Split the modified text into paragraphs based on newlines, avoiding these blocks
  588. let paragraphs = text.split(/\n+/);
  589. // Restore code blocks and process paragraphs
  590. paragraphs = paragraphs.map((paragraph) => {
  591. // Check if the paragraph includes a placeholder for a code block
  592. return paragraph.replace(/\u0000(\d+)\u0000/g, (_, idx) => codeBlocks[idx]);
  593. });
  594. return paragraphs.map(cleanText).filter(Boolean);
  595. };
  596. export const extractSentencesForAudio = (text: string) => {
  597. return extractSentences(text).reduce((mergedTexts, currentText) => {
  598. const lastIndex = mergedTexts.length - 1;
  599. if (lastIndex >= 0) {
  600. const previousText = mergedTexts[lastIndex];
  601. const wordCount = previousText.split(/\s+/).length;
  602. const charCount = previousText.length;
  603. if (wordCount < 4 || charCount < 50) {
  604. mergedTexts[lastIndex] = previousText + ' ' + currentText;
  605. } else {
  606. mergedTexts.push(currentText);
  607. }
  608. } else {
  609. mergedTexts.push(currentText);
  610. }
  611. return mergedTexts;
  612. }, [] as string[]);
  613. };
  614. export const getMessageContentParts = (content: string, split_on: string = 'punctuation') => {
  615. content = removeDetails(content, ['reasoning', 'code_interpreter']);
  616. const messageContentParts: string[] = [];
  617. switch (split_on) {
  618. default:
  619. case TTS_RESPONSE_SPLIT.PUNCTUATION:
  620. messageContentParts.push(...extractSentencesForAudio(content));
  621. break;
  622. case TTS_RESPONSE_SPLIT.PARAGRAPHS:
  623. messageContentParts.push(...extractParagraphsForAudio(content));
  624. break;
  625. case TTS_RESPONSE_SPLIT.NONE:
  626. messageContentParts.push(cleanText(content));
  627. break;
  628. }
  629. return messageContentParts;
  630. };
  631. export const blobToFile = (blob, fileName) => {
  632. // Create a new File object from the Blob
  633. const file = new File([blob], fileName, { type: blob.type });
  634. return file;
  635. };
  636. export const getPromptVariables = (user_name, user_location) => {
  637. return {
  638. '{{USER_NAME}}': user_name,
  639. '{{USER_LOCATION}}': user_location || 'Unknown',
  640. '{{CURRENT_DATETIME}}': getCurrentDateTime(),
  641. '{{CURRENT_DATE}}': getFormattedDate(),
  642. '{{CURRENT_TIME}}': getFormattedTime(),
  643. '{{CURRENT_WEEKDAY}}': getWeekday(),
  644. '{{CURRENT_TIMEZONE}}': getUserTimezone(),
  645. '{{USER_LANGUAGE}}': localStorage.getItem('locale') || 'en-US'
  646. };
  647. };
  648. /**
  649. * @param {string} template - The template string containing placeholders.
  650. * @returns {string} The template string with the placeholders replaced by the prompt.
  651. */
  652. export const promptTemplate = (
  653. template: string,
  654. user_name?: string,
  655. user_location?: string
  656. ): string => {
  657. // Get the current date
  658. const currentDate = new Date();
  659. // Format the date to YYYY-MM-DD
  660. const formattedDate =
  661. currentDate.getFullYear() +
  662. '-' +
  663. String(currentDate.getMonth() + 1).padStart(2, '0') +
  664. '-' +
  665. String(currentDate.getDate()).padStart(2, '0');
  666. // Format the time to HH:MM:SS AM/PM
  667. const currentTime = currentDate.toLocaleTimeString('en-US', {
  668. hour: 'numeric',
  669. minute: 'numeric',
  670. second: 'numeric',
  671. hour12: true
  672. });
  673. // Get the current weekday
  674. const currentWeekday = getWeekday();
  675. // Get the user's timezone
  676. const currentTimezone = getUserTimezone();
  677. // Get the user's language
  678. const userLanguage = localStorage.getItem('locale') || 'en-US';
  679. // Replace {{CURRENT_DATETIME}} in the template with the formatted datetime
  680. template = template.replace('{{CURRENT_DATETIME}}', `${formattedDate} ${currentTime}`);
  681. // Replace {{CURRENT_DATE}} in the template with the formatted date
  682. template = template.replace('{{CURRENT_DATE}}', formattedDate);
  683. // Replace {{CURRENT_TIME}} in the template with the formatted time
  684. template = template.replace('{{CURRENT_TIME}}', currentTime);
  685. // Replace {{CURRENT_WEEKDAY}} in the template with the current weekday
  686. template = template.replace('{{CURRENT_WEEKDAY}}', currentWeekday);
  687. // Replace {{CURRENT_TIMEZONE}} in the template with the user's timezone
  688. template = template.replace('{{CURRENT_TIMEZONE}}', currentTimezone);
  689. // Replace {{USER_LANGUAGE}} in the template with the user's language
  690. template = template.replace('{{USER_LANGUAGE}}', userLanguage);
  691. if (user_name) {
  692. // Replace {{USER_NAME}} in the template with the user's name
  693. template = template.replace('{{USER_NAME}}', user_name);
  694. }
  695. if (user_location) {
  696. // Replace {{USER_LOCATION}} in the template with the current location
  697. template = template.replace('{{USER_LOCATION}}', user_location);
  698. }
  699. return template;
  700. };
  701. /**
  702. * This function is used to replace placeholders in a template string with the provided prompt.
  703. * The placeholders can be in the following formats:
  704. * - `{{prompt}}`: This will be replaced with the entire prompt.
  705. * - `{{prompt:start:<length>}}`: This will be replaced with the first <length> characters of the prompt.
  706. * - `{{prompt:end:<length>}}`: This will be replaced with the last <length> characters of the prompt.
  707. * - `{{prompt:middletruncate:<length>}}`: This will be replaced with the prompt truncated to <length> characters, with '...' in the middle.
  708. *
  709. * @param {string} template - The template string containing placeholders.
  710. * @param {string} prompt - The string to replace the placeholders with.
  711. * @returns {string} The template string with the placeholders replaced by the prompt.
  712. */
  713. export const titleGenerationTemplate = (template: string, prompt: string): string => {
  714. template = template.replace(
  715. /{{prompt}}|{{prompt:start:(\d+)}}|{{prompt:end:(\d+)}}|{{prompt:middletruncate:(\d+)}}/g,
  716. (match, startLength, endLength, middleLength) => {
  717. if (match === '{{prompt}}') {
  718. return prompt;
  719. } else if (match.startsWith('{{prompt:start:')) {
  720. return prompt.substring(0, startLength);
  721. } else if (match.startsWith('{{prompt:end:')) {
  722. return prompt.slice(-endLength);
  723. } else if (match.startsWith('{{prompt:middletruncate:')) {
  724. if (prompt.length <= middleLength) {
  725. return prompt;
  726. }
  727. const start = prompt.slice(0, Math.ceil(middleLength / 2));
  728. const end = prompt.slice(-Math.floor(middleLength / 2));
  729. return `${start}...${end}`;
  730. }
  731. return '';
  732. }
  733. );
  734. template = promptTemplate(template);
  735. return template;
  736. };
  737. export const approximateToHumanReadable = (nanoseconds: number) => {
  738. const seconds = Math.floor((nanoseconds / 1e9) % 60);
  739. const minutes = Math.floor((nanoseconds / 6e10) % 60);
  740. const hours = Math.floor((nanoseconds / 3.6e12) % 24);
  741. const results: string[] = [];
  742. if (seconds >= 0) {
  743. results.push(`${seconds}s`);
  744. }
  745. if (minutes > 0) {
  746. results.push(`${minutes}m`);
  747. }
  748. if (hours > 0) {
  749. results.push(`${hours}h`);
  750. }
  751. return results.reverse().join(' ');
  752. };
  753. export const getTimeRange = (timestamp) => {
  754. const now = new Date();
  755. const date = new Date(timestamp * 1000); // Convert Unix timestamp to milliseconds
  756. // Calculate the difference in milliseconds
  757. const diffTime = now.getTime() - date.getTime();
  758. const diffDays = diffTime / (1000 * 3600 * 24);
  759. const nowDate = now.getDate();
  760. const nowMonth = now.getMonth();
  761. const nowYear = now.getFullYear();
  762. const dateDate = date.getDate();
  763. const dateMonth = date.getMonth();
  764. const dateYear = date.getFullYear();
  765. if (nowYear === dateYear && nowMonth === dateMonth && nowDate === dateDate) {
  766. return 'Today';
  767. } else if (nowYear === dateYear && nowMonth === dateMonth && nowDate - dateDate === 1) {
  768. return 'Yesterday';
  769. } else if (diffDays <= 7) {
  770. return 'Previous 7 days';
  771. } else if (diffDays <= 30) {
  772. return 'Previous 30 days';
  773. } else if (nowYear === dateYear) {
  774. return date.toLocaleString('default', { month: 'long' });
  775. } else {
  776. return date.getFullYear().toString();
  777. }
  778. };
  779. /**
  780. * Extract frontmatter as a dictionary from the specified content string.
  781. * @param content {string} - The content string with potential frontmatter.
  782. * @returns {Object} - The extracted frontmatter as a dictionary.
  783. */
  784. export const extractFrontmatter = (content) => {
  785. const frontmatter = {};
  786. let frontmatterStarted = false;
  787. let frontmatterEnded = false;
  788. const frontmatterPattern = /^\s*([a-z_]+):\s*(.*)\s*$/i;
  789. // Split content into lines
  790. const lines = content.split('\n');
  791. // Check if the content starts with triple quotes
  792. if (lines[0].trim() !== '"""') {
  793. return {};
  794. }
  795. frontmatterStarted = true;
  796. for (let i = 1; i < lines.length; i++) {
  797. const line = lines[i];
  798. if (line.includes('"""')) {
  799. if (frontmatterStarted) {
  800. frontmatterEnded = true;
  801. break;
  802. }
  803. }
  804. if (frontmatterStarted && !frontmatterEnded) {
  805. const match = frontmatterPattern.exec(line);
  806. if (match) {
  807. const [, key, value] = match;
  808. frontmatter[key.trim()] = value.trim();
  809. }
  810. }
  811. }
  812. return frontmatter;
  813. };
  814. // Function to determine the best matching language
  815. export const bestMatchingLanguage = (supportedLanguages, preferredLanguages, defaultLocale) => {
  816. const languages = supportedLanguages.map((lang) => lang.code);
  817. const match = preferredLanguages
  818. .map((prefLang) => languages.find((lang) => lang.startsWith(prefLang)))
  819. .find(Boolean);
  820. return match || defaultLocale;
  821. };
  822. // Get the date in the format YYYY-MM-DD
  823. export const getFormattedDate = () => {
  824. const date = new Date();
  825. return date.toISOString().split('T')[0];
  826. };
  827. // Get the time in the format HH:MM:SS
  828. export const getFormattedTime = () => {
  829. const date = new Date();
  830. return date.toTimeString().split(' ')[0];
  831. };
  832. // Get the current date and time in the format YYYY-MM-DD HH:MM:SS
  833. export const getCurrentDateTime = () => {
  834. return `${getFormattedDate()} ${getFormattedTime()}`;
  835. };
  836. // Get the user's timezone
  837. export const getUserTimezone = () => {
  838. return Intl.DateTimeFormat().resolvedOptions().timeZone;
  839. };
  840. // Get the weekday
  841. export const getWeekday = () => {
  842. const date = new Date();
  843. const weekdays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
  844. return weekdays[date.getDay()];
  845. };
  846. export const createMessagesList = (history, messageId) => {
  847. if (messageId === null) {
  848. return [];
  849. }
  850. const message = history.messages[messageId];
  851. if (message?.parentId) {
  852. return [...createMessagesList(history, message.parentId), message];
  853. } else {
  854. return [message];
  855. }
  856. };
  857. export const formatFileSize = (size) => {
  858. if (size == null) return 'Unknown size';
  859. if (typeof size !== 'number' || size < 0) return 'Invalid size';
  860. if (size === 0) return '0 B';
  861. const units = ['B', 'KB', 'MB', 'GB', 'TB'];
  862. let unitIndex = 0;
  863. while (size >= 1024 && unitIndex < units.length - 1) {
  864. size /= 1024;
  865. unitIndex++;
  866. }
  867. return `${size.toFixed(1)} ${units[unitIndex]}`;
  868. };
  869. export const getLineCount = (text) => {
  870. console.log(typeof text);
  871. return text ? text.split('\n').length : 0;
  872. };