Replace custom markdown parsing with marked, KaTeX, and some custom plugins for matrix things

.changeset/add_math_and_markdown.md

@@ -0,0 +1,13 @@
+---
+default: minor
+---
+
+# Markdown parser and render updates
+
+Migrated markdown parsing and rendering to use marked, which should fix most (all?) markdown issues involving lists/nested structures, inconsistent/inaccurate code blocks, escape sequences, and all the other bugs with literally everything.
+
+Added math rendering support via marked and KaTeX, uses standard `$$` and `$` delimiters. Only renders a subset of latex tags that will likely need to be expanded so feel free to make issues if needed.
+
+Also adds support for sending markdown tables (although they're rendered rather plainly at the moment).
+
+Fixes link previews appearing in code blocks, fixes pmp new line behavior, and fixes links not opening in new tabs.

package.json

@@ -70,8 +70,10 @@
     "immer": "^9.0.21",
     "is-hotkey": "^0.2.0",
     "jotai": "^2.18.0",
+    "katex": "^0.16.45",
     "linkify-react": "^4.3.2",
     "linkifyjs": "^4.3.2",
+    "marked": "^18.0.2",
     "matrix-js-sdk": "^38.4.0",
     "matrix-widget-api": "^1.16.1",
     "pdfjs-dist": "^5.4.624",

src/app/components/editor/getLinks.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from 'vitest';
+import { getLinks, toPlainText } from './output';
+import type { ParagraphElement } from './slate';
+import { BlockType } from './types';
+
+describe('getLinks', () => {
+  it('extracts URLs from text', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: 'Check out https://example.com for more info' }],
+    };
+    const links = getLinks([node]);
+    expect(links).toContain('https://example.com');
+  });
+
+  it('excludes URLs in angle brackets (Matrix HTML spoiler)', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: 'Check out <https://example.com> for more info' }],
+    };
+    const links = getLinks([node]);
+    expect(links).not.toContain('https://example.com');
+    expect(links).toHaveLength(0);
+  });
+
+  it('extracts markdown link URLs', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: 'Check [my link](https://example.com) for more info' }],
+    };
+    const links = getLinks([node]);
+    expect(links).toContain('https://example.com');
+  });
+
+  it('excludes URLs inside code blocks', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: '```' }, { text: 'https://example.com' }, { text: '```' }],
+    };
+    const links = getLinks([node]);
+    expect(links).toHaveLength(0);
+  });
+});
+
+describe('toPlainText spoiler handling', () => {
+  it('replaces ||spoilered text|| with [Spoiler]', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: 'Hello ||spoilered|| world' }],
+    };
+    const plain = toPlainText(node, true);
+    expect(plain).toContain('[Spoiler]');
+    expect(plain).not.toContain('||spoilered||');
+  });
+
+  it('replaces ||spoilered links|| with [Spoiler]', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [{ text: 'Hello ||https://example.com|| world' }],
+    };
+    const plain = toPlainText(node, true);
+    expect(plain).toContain('[Spoiler]');
+    expect(plain).not.toContain('||https://example.com||');
+  });
+
+  it('extracts non-spoilered markdown link URLs alongside spoilered ones', () => {
+    const node: ParagraphElement = {
+      type: BlockType.Paragraph,
+      children: [
+        {
+          text: 'Check [visible](https://visible.com) and ||https://hidden.com||',
+        },
+      ],
+    };
+    const links = getLinks([node]);
+    expect(links).toContain('https://visible.com');
+    expect(links).not.toContain('https://hidden.com');
+  });
+});