diff --git a/packages/host/tests/unit/marked-sync-test.ts b/packages/host/tests/unit/marked-sync-test.ts
index d994cb1b83a..3a06e75bb67 100644
--- a/packages/host/tests/unit/marked-sync-test.ts
+++ b/packages/host/tests/unit/marked-sync-test.ts
@@ -1,10 +1,31 @@
import { module, test } from 'qunit';
+import {
+ SEARCH_MARKER,
+ SEPARATOR_MARKER,
+ REPLACE_MARKER,
+} from '@cardstack/runtime-common';
+import { escapeHtmlOutsideCodeBlocks } from '@cardstack/runtime-common/helpers/html';
import {
markedSync,
markdownToHtml,
} from '@cardstack/runtime-common/marked-sync';
+import { parseHtmlContent } from '@cardstack/host/lib/formatted-message/utils';
+
+// The render path a bot message's code patch travels before the host applies
+// it: markdown body โ bodyHTML (message.ts) โ parseHtmlContent โ
+// codeData.searchReplaceBlock. The patch must survive this byte-identical โ
+// the applier matches it against the target file.
+function roundTripSearchReplaceBlock(body: string) {
+ let html = markdownToHtml(escapeHtmlOutsideCodeBlocks(body), {
+ sanitize: false,
+ escapeHtmlInCodeBlocks: true,
+ });
+ let parts = parseHtmlContent(html, 'room-1', 'event-1');
+ return parts.find((p) => p.type === 'pre_tag')?.codeData ?? null;
+}
+
module('Unit | marked-sync', function () {
test('markedSync converts markdown to HTML', function (assert) {
const markdown = '# Hello\n**Bold text**';
@@ -344,6 +365,127 @@ module('Unit | marked-sync', function () {
);
});
+ test('markdownToHtml prefixes decorative bullets with a list marker', function (assert) {
+ const markdown = '๐ First point\n๐ Second point';
+ const result = markdownToHtml(markdown);
+
+ assert.true(
+ result.includes('
๐ First point'),
+ 'emoji-led lines become list items',
+ );
+ assert.true(
+ result.includes('๐ Second point'),
+ 'each emoji-led line is its own list item',
+ );
+ });
+
+ test('markdownToHtml leaves fenced code block content verbatim when lines start with decorative bullets', function (assert) {
+ const markdown = [
+ '๐ A real list item',
+ '```gts',
+ " ",
+ ' ๐ง SITE UNDER CONSTRUCTION ๐ง',
+ ' ',
+ '```',
+ ].join('\n');
+ const result = markdownToHtml(markdown, { sanitize: false });
+
+ assert.true(
+ result.includes('๐ A real list item'),
+ 'bullet normalization still applies outside the fence',
+ );
+ assert.true(
+ result.includes(' ๐ง SITE UNDER CONSTRUCTION ๐ง'),
+ 'emoji-led line inside the fence is unchanged',
+ );
+ assert.false(
+ result.includes('* ๐ง'),
+ 'no list marker is inserted inside the fence',
+ );
+ });
+
+ test('a code patch round-trips byte-identical through render and extraction', function (assert) {
+ const search = [
+ " ",
+ ' ๐ง SITE UNDER CONSTRUCTION ๐ง BEST VIEWED IN',
+ '
',
+ ].join('\n');
+ const block = `${SEARCH_MARKER}\n${search}\n${SEPARATOR_MARKER}\nreplaced\n${REPLACE_MARKER}`;
+ const body = `Fixing the file now.\n\n\`\`\`gts\nhttps://example.test/hello-world.gts\n${block}\n\`\`\``;
+
+ const codeData = roundTripSearchReplaceBlock(body);
+ assert.ok(codeData, 'a code block was extracted');
+ assert.strictEqual(
+ codeData!.searchReplaceBlock,
+ block,
+ 'the extracted patch is byte-identical to what the bot authored',
+ );
+ });
+
+ test('a code patch round-trips intact from CRLF input', function (assert) {
+ const search = ' ๐ง SITE UNDER CONSTRUCTION ๐ง';
+ const block = `${SEARCH_MARKER}\n${search}\n${SEPARATOR_MARKER}\nreplaced\n${REPLACE_MARKER}`;
+ const body =
+ `Fixing the file now.\n\n\`\`\`gts\nhttps://example.test/hello-world.gts\n${block}\n\`\`\``.replace(
+ /\n/g,
+ '\r\n',
+ );
+
+ const codeData = roundTripSearchReplaceBlock(body);
+ assert.ok(codeData, 'a code block was extracted from CRLF input');
+ assert.ok(
+ codeData!.searchReplaceBlock?.includes(search),
+ 'the emoji-led search line survives unmutated',
+ );
+ assert.false(
+ (codeData!.searchReplaceBlock ?? '').includes('* ๐ง'),
+ 'no list marker was inserted into the CRLF fenced content',
+ );
+ });
+
+ test('a fence opened on a list-item line is respected and does not invert fence tracking', function (assert) {
+ const markdown = [
+ '- ```gts',
+ ' ๐ง SITE UNDER CONSTRUCTION ๐ง',
+ ' ```',
+ '',
+ '๐ after the list',
+ ].join('\n');
+ const result = markdownToHtml(markdown, { sanitize: false });
+
+ assert.false(
+ result.includes('* ๐ง'),
+ 'content of the list-nested fence is unchanged',
+ );
+ assert.true(
+ result.includes('๐ after the list'),
+ 'normalization still applies after the fence closes (state did not invert)',
+ );
+ });
+
+ test('a decorative bullet alone on its line still becomes a list item', function (assert) {
+ const result = markdownToHtml('๐\n\ntext after', { sanitize: false });
+
+ assert.true(
+ result.includes('๐'),
+ 'a bare decorative bullet line is normalized',
+ );
+ });
+
+ test('a decorative bullet indented four spaces under a list item renders as a nested list', function (assert) {
+ const markdown = '- item one\n ๐ nested point';
+ const result = markdownToHtml(markdown, { sanitize: false });
+
+ assert.true(
+ result.includes('๐ nested point'),
+ 'the indented decorative bullet becomes its own list item',
+ );
+ assert.true(
+ /item one[\s\S]*[\s\S]*๐ nested point/.test(result),
+ 'the decorative bullet nests as a sub-list under the parent item',
+ );
+ });
+
test('markdownToHtml preserves heading IDs through sanitization', function (assert) {
const markdown = '## Test Heading';
const result = markdownToHtml(markdown);
diff --git a/packages/runtime-common/marked-sync.ts b/packages/runtime-common/marked-sync.ts
index 494840f561a..beeb62280e5 100644
--- a/packages/runtime-common/marked-sync.ts
+++ b/packages/runtime-common/marked-sync.ts
@@ -70,9 +70,71 @@ bfmMarked.use({
},
});
+// The trailing `\s+|\r?$` alternation keeps a bullet that ends its line โ
+// with or without trailing text โ normalizable; split('\n') has already
+// consumed the newline the old whole-string pattern used to match.
const DECORATIVE_BULLET_PATTERN =
// eslint-disable-next-line no-misleading-character-class -- match pictographic symbols plus a few geometric glyphs not covered by the Unicode class
- /(^|\n)(\s*)([\p{Extended_Pictographic}โ
โขโชโโโฆโงโโฆโพโฝโฌขโฌกโโโ๏ธโคโโโกโ])(\s+)/gu;
+ /^(\s*)([\p{Extended_Pictographic}โ
โขโชโโโฆโงโโฆโพโฝโฌขโฌกโโโ๏ธโคโโโกโ])(\s+|\r?$)/u;
+
+// `.` never matches `\r`, so on CRLF input the greedy group stops before a
+// trailing `\r`; the explicit `\r?` lets `$` still anchor. Without it no
+// fence line would ever match CRLF content and the tracker would rewrite
+// fenced code.
+const CODE_FENCE_PATTERN = /^(\s*)(`{3,}|~{3,})(.*)\r?$/;
+// Markdown can open a fence on the same line as a list marker (`- ```gts`);
+// marked treats that as fenced code, so the tracker must too. Only openers
+// take this form โ a closing fence cannot carry an info string, let alone a
+// list marker.
+const LIST_PREFIXED_CODE_FENCE_PATTERN =
+ /^(\s*)(?:[-*+]|\d{1,9}[.)])\s+(`{3,}|~{3,})(.*)\r?$/;
+// Prefix decorative bullets with a standard list marker so marked treats them
+// as list items โ but never inside fenced code blocks. Fenced content must
+// survive rendering verbatim: search/replace patches are extracted back out
+// of the rendered HTML, and an inserted marker makes the patch text no longer
+// match the file it targets.
+//
+// Only *fenced* blocks are protected. A 4-space indented line is an indented
+// code block to marked in some contexts, but inside a list item the same
+// indentation is ordinary list content (a nested bullet); telling the two
+// apart needs block context that only the lexer has. Since the patch format
+// is always fenced, indented code blocks are left to the rewrite.
+function normalizeDecorativeBullets(markdown: string): string {
+ let inFence = false;
+ let fenceChar = '';
+ let fenceLength = 0;
+ return markdown
+ .split('\n')
+ .map((line) => {
+ if (inFence) {
+ let closeMatch = line.match(CODE_FENCE_PATTERN);
+ if (
+ closeMatch &&
+ closeMatch[2][0] === fenceChar &&
+ closeMatch[2].length >= fenceLength &&
+ closeMatch[3].trim() === ''
+ ) {
+ inFence = false;
+ }
+ return line;
+ }
+ let openMatch =
+ line.match(CODE_FENCE_PATTERN) ??
+ line.match(LIST_PREFIXED_CODE_FENCE_PATTERN);
+ if (openMatch) {
+ inFence = true;
+ fenceChar = openMatch[2][0];
+ fenceLength = openMatch[2].length;
+ return line;
+ }
+ return line.replace(
+ DECORATIVE_BULLET_PATTERN,
+ (_match, indentation, bullet, whitespace) =>
+ `${indentation}* ${bullet}${whitespace}`,
+ );
+ })
+ .join('\n');
+}
const DEFAULT_MARKED_SYNC_OPTIONS = {
escapeHtmlInCodeBlocks: true,
@@ -187,11 +249,7 @@ export function markdownToHtml(
return '';
}
// Marked only treats ASCII list markers, so prefix decorative bullets with a standard marker.
- let normalizedMarkdown = markdown.replace(
- DECORATIVE_BULLET_PATTERN,
- (_match, boundary, indentation, bullet, whitespace) =>
- `${boundary}${indentation}* ${bullet}${whitespace}`,
- );
+ let normalizedMarkdown = normalizeDecorativeBullets(markdown);
let html = markedSync(normalizedMarkdown, {
escapeHtmlInCodeBlocks: opts.escapeHtmlInCodeBlocks,
enableMonacoSyntaxHighlighting: opts.enableMonacoSyntaxHighlighting,