refac: markdown rendering

Co-Authored-By: Jun Siang Cheah <me@jscheah.me>

src/lib/components/chat/Messages/MarkdownInlineTokens.svelte

@@ -0,0 +1,44 @@
+<script lang="ts">
+	import type { Token } from 'marked';
+	import { unescapeHtml } from '$lib/utils';
+	import { onMount } from 'svelte';
+	import { revertSanitizedResponseContent } from '$lib/utils/index.js';
+	import Image from '$lib/components/common/Image.svelte';
+
+	export let id: string;
+	export let tokens: Token[];
+
+	onMount(() => {
+		console.log('MarkdownInlineTokens', id, tokens, top);
+	});
+</script>
+
+{#each tokens as token}
+	{#if token.type === 'escape'}
+		{unescapeHtml(token.text)}
+	{:else if token.type === 'html'}
+		{@html token.text}
+	{:else if token.type === 'link'}
+		<a href={token.href} target="_blank" rel="nofollow" title={token.title}>{token.text}</a>
+	{:else if token.type === 'image'}
+		<Image src={token.href} alt={token.text} />
+	{:else if token.type === 'strong'}
+		<strong>
+			<svelte:self id={`${id}-strong`} tokens={token.tokens} />
+		</strong>
+	{:else if token.type === 'em'}
+		<em>
+			<svelte:self id={`${id}-em`} tokens={token.tokens} />
+		</em>
+	{:else if token.type === 'codespan'}
+		<code>{unescapeHtml(token.text.replaceAll('&amp;', '&'))}</code>
+	{:else if token.type === 'br'}
+		<br />
+	{:else if token.type === 'del'}
+		<del>
+			<svelte:self id={`${id}-del`} tokens={token.tokens} />
+		</del>
+	{:else if token.type === 'text'}
+		{unescapeHtml(token.text)}
+	{/if}
+{/each}

src/lib/components/chat/Messages/MarkdownTokens.svelte

@@ -0,0 +1,118 @@
+<script lang="ts">
+	import type { Token } from 'marked';
+	import { revertSanitizedResponseContent, unescapeHtml } from '$lib/utils';
+	import CodeBlock from '$lib/components/chat/Messages/CodeBlock.svelte';
+	import { onMount } from 'svelte';
+	import MarkdownInlineTokens from '$lib/components/chat/Messages/MarkdownInlineTokens.svelte';
+
+	export let id: string;
+	export let tokens: Token[];
+	export let top = true;
+
+	const headerComponent = (depth: number) => {
+		return 'h' + depth;
+	};
+
+	onMount(() => {
+		console.log('MarkdownTokens', id, tokens, top);
+	});
+</script>
+
+{#each tokens as token, tokenIdx}
+	{#if token.type === 'hr'}
+		<hr />
+	{:else if token.type === 'heading'}
+		<svelte:element this={headerComponent(token.depth)}>
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-h`} tokens={token.tokens} />
+		</svelte:element>
+	{:else if token.type === 'code'}
+		<CodeBlock
+			{id}
+			lang={token?.lang ?? ''}
+			code={revertSanitizedResponseContent(token?.text ?? '')}
+		/>
+	{:else if token.type === 'table'}
+		<table>
+			<thead>
+				<tr>
+					{#each token.header as header, headerIdx}
+						<th style={token.align[headerIdx] ? '' : `text-align: ${token.align[headerIdx]}`}>
+							<MarkdownInlineTokens
+								id={`${id}-${tokenIdx}-header-${headerIdx}`}
+								tokens={header.tokens}
+							/>
+						</th>
+					{/each}
+				</tr>
+			</thead>
+			<tbody>
+				{#each token.rows as row, rowIdx}
+					<tr>
+						{#each row ?? [] as cell, cellIdx}
+							<td style={token.align[cellIdx] ? '' : `text-align: ${token.align[cellIdx]}`}>
+								<MarkdownInlineTokens
+									id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`}
+									tokens={cell.tokens}
+								/>
+							</td>
+						{/each}
+					</tr>
+				{/each}
+			</tbody>
+		</table>
+	{:else if token.type === 'blockquote'}
+		<blockquote>
+			<svelte:self id={`${id}-${tokenIdx}`} tokens={token.tokens} />
+		</blockquote>
+	{:else if token.type === 'list'}
+		{#if token.ordered}
+			<ol start={token.start || 1}>
+				{#each token.items as item, itemIdx}
+					<li>
+						<svelte:self
+							id={`${id}-${tokenIdx}-${itemIdx}`}
+							tokens={item.tokens}
+							top={token.loose}
+						/>
+					</li>
+				{/each}
+			</ol>
+		{:else}
+			<ul>
+				{#each token.items as item, itemIdx}
+					<li>
+						<svelte:self
+							id={`${id}-${tokenIdx}-${itemIdx}`}
+							tokens={item.tokens}
+							top={token.loose}
+						/>
+					</li>
+				{/each}
+			</ul>
+		{/if}
+	{:else if token.type === 'html'}
+		{@html token.text}
+	{:else if token.type === 'paragraph'}
+		<p>
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
+		</p>
+	{:else if token.type === 'text'}
+		{#if top}
+			<p>
+				{#if token.tokens}
+					<MarkdownInlineTokens id={`${id}-${tokenIdx}-t`} tokens={token.tokens} />
+				{:else}
+					{unescapeHtml(token.text)}
+				{/if}
+			</p>
+		{:else if token.tokens}
+			<MarkdownInlineTokens id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} />
+		{:else}
+			{unescapeHtml(token.text)}
+		{/if}
+	{:else if token.type === 'space'}
+		{''}
+	{:else}
+		{console.log('Unknown token', token)}
+	{/if}
+{/each}

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -38,7 +38,7 @@
 	import Spinner from '$lib/components/common/Spinner.svelte';
 	import WebSearchResults from './ResponseMessage/WebSearchResults.svelte';
 	import Sparkles from '$lib/components/icons/Sparkles.svelte';
-	import TokenRenderer from './TokenRenderer.svelte';
+	import MarkdownTokens from './MarkdownTokens.svelte';
 
 	export let message;
 	export let siblings;
@@ -77,9 +77,7 @@
 
 	let selectedCitation = null;
 
-	$: tokens = marked.lexer(
-		replaceTokens(sanitizeResponseContent(message?.content), model?.name, $user?.name)
-	);
+	$: tokens = marked.lexer(sanitizeResponseContent(message?.content));
 
 	$: if (message) {
 		renderStyling();
@@ -413,7 +411,7 @@
 				{/if}
 
 				<div
-					class="prose chat-{message.role} w-full max-w-full dark:prose-invert prose-headings:my-0 prose-headings:-mb-4 prose-p:m-0 prose-p:-mb-6 prose-pre:my-0 prose-table:my-0 prose-blockquote:my-0 prose-img:my-0 prose-ul:-my-4 prose-ol:-my-4 prose-li:-my-3 prose-ul:-mb-6 prose-ol:-mb-8 prose-ol:p-0 prose-li:-mb-4 whitespace-pre-line"
+					class="prose chat-{message.role} w-full max-w-full dark:prose-invert whitespace-pre-line"
 				>
 					<div>
 						{#if (message?.statusHistory ?? [...(message?.status ? [message?.status] : [])]).length > 0}
@@ -499,9 +497,7 @@
 								{:else if message.content && message.error !== true}
 									<!-- always show message contents even if there's an error -->
 									<!-- unless message.error === true which is legacy error handling, where the error message is stored in message.content -->
-									{#each tokens as token, tokenIdx}
-										<TokenRenderer {token} {tokenIdx} id={message.id} />
-									{/each}
+									<MarkdownTokens id={message.id} {tokens} />
 								{/if}
 
 								{#if message.error}

src/lib/components/chat/Messages/TokenRenderer.svelte

@@ -1,97 +0,0 @@
-<script lang="ts">
-	import { revertSanitizedResponseContent } from '$lib/utils';
-
-	import { marked } from 'marked';
-	import CodeBlock from './CodeBlock.svelte';
-	import Image from '$lib/components/common/Image.svelte';
-
-	export let token;
-	export let tokenIdx = 0;
-	export let id;
-
-	let element;
-	let content;
-
-	const renderer = new marked.Renderer();
-
-	// For code blocks with simple backticks
-	renderer.codespan = (code) => {
-		return `<code>${code.replaceAll('&amp;', '&')}</code>`;
-	};
-
-	let codes = [];
-	renderer.code = (code, lang) => {
-		codes.push({ code, lang, id: codes.length });
-		codes = codes;
-		return `{{@CODE ${codes.length - 1}}}`;
-	};
-
-	let images = [];
-	renderer.image = (href, title, text) => {
-		images.push({ href, title, text });
-		images = images;
-		return `{{@IMAGE ${images.length - 1}}}`;
-	};
-
-	// Open all links in a new tab/window (from https://github.com/markedjs/marked/issues/655#issuecomment-383226346)
-	const origLinkRenderer = renderer.link;
-	renderer.link = (href, title, text) => {
-		const html = origLinkRenderer.call(renderer, href, title, text);
-		return html.replace(/^<a /, '<a target="_blank" rel="nofollow" ');
-	};
-
-	const { extensions, ...defaults } = marked.getDefaults() as marked.MarkedOptions & {
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		extensions: any;
-	};
-
-	$: if (token) {
-		images = [];
-		codes = [];
-		content = marked
-			.parse(token.raw, {
-				...defaults,
-				gfm: true,
-				breaks: true,
-				renderer
-			})
-			.split(/({{@IMAGE [^}]+}}|{{@CODE [^}]+}})/g);
-	}
-</script>
-
-<div bind:this={element}>
-	{#if token.type === 'code'}
-		{#if token.lang === 'mermaid'}
-			<pre class="mermaid">{revertSanitizedResponseContent(token.text)}</pre>
-		{:else}
-			<CodeBlock
-				id={`${id}-${tokenIdx}`}
-				lang={token?.lang ?? ''}
-				code={revertSanitizedResponseContent(token?.text ?? '')}
-			/>
-		{/if}
-	{:else if token.type === 'image'}
-		<Image src={token.href} alt={token.text} />
-	{:else}
-		{#each content as part}
-			{@html part.startsWith('{{@IMAGE ') || part.startsWith('{{@CODE ') ? '' : part}
-
-			{#if images.length > 0 && part.startsWith('{{@IMAGE ')}
-				{@const img = images[parseInt(part.match(/{{@IMAGE (\d+)}}/)[1])]}
-
-				<div class="mt-6">
-					<Image src={img.href} text={img.text} />
-				</div>
-			{:else if codes.length > 0 && part.startsWith('{{@CODE ')}
-				{@const _code = codes[parseInt(part.match(/{{@CODE (\d+)}}/)[1])]}
-				<div class="my-10 -mb-6">
-					<CodeBlock
-						id={`${id}-${tokenIdx}-${_code.id}`}
-						lang={_code.lang}
-						code={revertSanitizedResponseContent(_code.code)}
-					/>
-				</div>
-			{/if}
-		{/each}
-	{/if}
-</div>

src/lib/components/common/Image.svelte

@@ -17,9 +17,9 @@
 		on:click={() => {
 			showImagePreview = true;
 		}}
-		class="w-full"
+		class=" w-fit"
 	>
-		<img src={_src} {alt} class=" max-h-96 rounded-lg" draggable="false" data-cy="image" />
+		<img src={_src} {alt} class=" max-h-96 rounded-lg w-full" draggable="false" data-cy="image" />
 	</button>
 </div>
 <ImagePreview bind:show={showImagePreview} src={_src} {alt} />

src/lib/utils/index.ts

@@ -90,6 +90,11 @@ export const revertSanitizedResponseContent = (content: string) => {
 	return content.replaceAll('&lt;', '<').replaceAll('&gt;', '>');
 };
 
+export function unescapeHtml(html: string) {
+	const doc = new DOMParser().parseFromString(html, 'text/html');
+	return doc.documentElement.textContent;
+}
+
 export const capitalizeFirstLetter = (string) => {
 	return string.charAt(0).toUpperCase() + string.slice(1);
 };