Skip decorative-bullet normalization inside fenced code blocks

packages/host/tests/unit/marked-sync-test.ts

@@ -1,10 +1,31 @@
 import { module, test } from 'qunit';
 
+import {
+  SEARCH_MARKER,
+  SEPARATOR_MARKER,
+  REPLACE_MARKER,
+} from '@cardstack/runtime-common';
+import { escapeHtmlOutsideCodeBlocks } from '@cardstack/runtime-common/helpers/html';
 import {
   markedSync,
   markdownToHtml,
 } from '@cardstack/runtime-common/marked-sync';
 
+import { parseHtmlContent } from '@cardstack/host/lib/formatted-message/utils';
+
+// The render path a bot message's code patch travels before the host applies
+// it: markdown body → bodyHTML (message.ts) → parseHtmlContent →
+// codeData.searchReplaceBlock. The patch must survive this byte-identical —
+// the applier matches it against the target file.
+function roundTripSearchReplaceBlock(body: string) {
+  let html = markdownToHtml(escapeHtmlOutsideCodeBlocks(body), {
+    sanitize: false,
+    escapeHtmlInCodeBlocks: true,
+  });
+  let parts = parseHtmlContent(html, 'room-1', 'event-1');
+  return parts.find((p) => p.type === 'pre_tag')?.codeData ?? null;
+}
+
 module('Unit | marked-sync', function () {
   test('markedSync converts markdown to HTML', function (assert) {
     const markdown = '# Hello\n**Bold text**';
@@ -344,6 +365,127 @@ module('Unit | marked-sync', function () {
     );
   });
 
+  test('markdownToHtml prefixes decorative bullets with a list marker', function (assert) {
+    const markdown = '🌟 First point\n🌟 Second point';
+    const result = markdownToHtml(markdown);
+
+    assert.true(
+      result.includes('<li>🌟 First point</li>'),
+      'emoji-led lines become list items',
+    );
+    assert.true(
+      result.includes('<li>🌟 Second point</li>'),
+      'each emoji-led line is its own list item',
+    );
+  });
+
+  test('markdownToHtml leaves fenced code block content verbatim when lines start with decorative bullets', function (assert) {
+    const markdown = [
+      '🌟 A real list item',
+      '```gts',
+      "  <span class='marquee-text'>",
+      '    🚧 SITE UNDER CONSTRUCTION 🚧',
+      '  </span>',
+      '```',
+    ].join('\n');
+    const result = markdownToHtml(markdown, { sanitize: false });
+
+    assert.true(
+      result.includes('<li>🌟 A real list item</li>'),
+      'bullet normalization still applies outside the fence',
+    );
+    assert.true(
+      result.includes('    🚧 SITE UNDER CONSTRUCTION 🚧'),
+      'emoji-led line inside the fence is unchanged',
+    );
+    assert.false(
+      result.includes('* 🚧'),
+      'no list marker is inserted inside the fence',
+    );
+  });
+
+  test('a code patch round-trips byte-identical through render and extraction', function (assert) {
+    const search = [
+      "        <div class='marquee-wrap'>",
+      '            🚧 SITE UNDER CONSTRUCTION 🚧 &nbsp;&nbsp;&nbsp; BEST VIEWED IN',
+      '        </div>',
+    ].join('\n');
+    const block = `${SEARCH_MARKER}\n${search}\n${SEPARATOR_MARKER}\nreplaced\n${REPLACE_MARKER}`;
+    const body = `Fixing the file now.\n\n\`\`\`gts\nhttps://example.test/hello-world.gts\n${block}\n\`\`\``;
+
+    const codeData = roundTripSearchReplaceBlock(body);
+    assert.ok(codeData, 'a code block was extracted');
+    assert.strictEqual(
+      codeData!.searchReplaceBlock,
+      block,
+      'the extracted patch is byte-identical to what the bot authored',
+    );
+  });
+
+  test('a code patch round-trips intact from CRLF input', function (assert) {
+    const search = '            🚧 SITE UNDER CONSTRUCTION 🚧';
+    const block = `${SEARCH_MARKER}\n${search}\n${SEPARATOR_MARKER}\nreplaced\n${REPLACE_MARKER}`;
+    const body =
+      `Fixing the file now.\n\n\`\`\`gts\nhttps://example.test/hello-world.gts\n${block}\n\`\`\``.replace(
+        /\n/g,
+        '\r\n',
+      );
+
+    const codeData = roundTripSearchReplaceBlock(body);
+    assert.ok(codeData, 'a code block was extracted from CRLF input');
+    assert.ok(
+      codeData!.searchReplaceBlock?.includes(search),
+      'the emoji-led search line survives unmutated',
+    );
+    assert.false(
+      (codeData!.searchReplaceBlock ?? '').includes('* 🚧'),
+      'no list marker was inserted into the CRLF fenced content',
+    );
+  });
+
+  test('a fence opened on a list-item line is respected and does not invert fence tracking', function (assert) {
+    const markdown = [
+      '- ```gts',
+      '  🚧 SITE UNDER CONSTRUCTION 🚧',
+      '  ```',
+      '',
+      '🌟 after the list',
+    ].join('\n');
+    const result = markdownToHtml(markdown, { sanitize: false });
+
+    assert.false(
+      result.includes('* 🚧'),
+      'content of the list-nested fence is unchanged',
+    );
+    assert.true(
+      result.includes('<li>🌟 after the list</li>'),
+      'normalization still applies after the fence closes (state did not invert)',
+    );
+  });
+
+  test('a decorative bullet alone on its line still becomes a list item', function (assert) {
+    const result = markdownToHtml('🌟\n\ntext after', { sanitize: false });
+
+    assert.true(
+      result.includes('<li>🌟</li>'),
+      'a bare decorative bullet line is normalized',
+    );
+  });
+
+  test('a decorative bullet indented four spaces under a list item renders as a nested list', function (assert) {
+    const markdown = '- item one\n    🌟 nested point';
+    const result = markdownToHtml(markdown, { sanitize: false });
+
+    assert.true(
+      result.includes('<li>🌟 nested point</li>'),
+      'the indented decorative bullet becomes its own list item',
+    );
+    assert.true(
+      /<li>item one[\s\S]*<ul>[\s\S]*🌟 nested point/.test(result),
+      'the decorative bullet nests as a sub-list under the parent item',
+    );
+  });
+
   test('markdownToHtml preserves heading IDs through sanitization', function (assert) {
     const markdown = '## Test Heading';
     const result = markdownToHtml(markdown);

packages/runtime-common/marked-sync.ts

@@ -70,9 +70,71 @@ bfmMarked.use({
   },
 });
 
+// The trailing `\s+|\r?$` alternation keeps a bullet that ends its line —
+// with or without trailing text — normalizable; split('\n') has already
+// consumed the newline the old whole-string pattern used to match.
 const DECORATIVE_BULLET_PATTERN =
   // eslint-disable-next-line no-misleading-character-class -- match pictographic symbols plus a few geometric glyphs not covered by the Unicode class
-  /(^|\n)(\s*)([\p{Extended_Pictographic}★•▪●❖✦✧◉◦◾◽⬢⬡☑✔☑️➤➔➜➡→])(\s+)/gu;
+  /^(\s*)([\p{Extended_Pictographic}★•▪●❖✦✧◉◦◾◽⬢⬡☑✔☑️➤➔➜➡→])(\s+|\r?$)/u;
+
+// `.` never matches `\r`, so on CRLF input the greedy group stops before a
+// trailing `\r`; the explicit `\r?` lets `$` still anchor. Without it no
+// fence line would ever match CRLF content and the tracker would rewrite
+// fenced code.
+const CODE_FENCE_PATTERN = /^(\s*)(`{3,}|~{3,})(.*)\r?$/;
+// Markdown can open a fence on the same line as a list marker (`- ```gts`);
+// marked treats that as fenced code, so the tracker must too. Only openers
+// take this form — a closing fence cannot carry an info string, let alone a
+// list marker.
+const LIST_PREFIXED_CODE_FENCE_PATTERN =
+  /^(\s*)(?:[-*+]|\d{1,9}[.)])\s+(`{3,}|~{3,})(.*)\r?$/;
+// Prefix decorative bullets with a standard list marker so marked treats them
+// as list items — but never inside fenced code blocks. Fenced content must
+// survive rendering verbatim: search/replace patches are extracted back out
+// of the rendered HTML, and an inserted marker makes the patch text no longer
+// match the file it targets.
+//
+// Only *fenced* blocks are protected. A 4-space indented line is an indented
+// code block to marked in some contexts, but inside a list item the same
+// indentation is ordinary list content (a nested bullet); telling the two
+// apart needs block context that only the lexer has. Since the patch format
+// is always fenced, indented code blocks are left to the rewrite.
+function normalizeDecorativeBullets(markdown: string): string {
+  let inFence = false;
+  let fenceChar = '';
+  let fenceLength = 0;
+  return markdown
+    .split('\n')
+    .map((line) => {
+      if (inFence) {
+        let closeMatch = line.match(CODE_FENCE_PATTERN);
+        if (
+          closeMatch &&
+          closeMatch[2][0] === fenceChar &&
+          closeMatch[2].length >= fenceLength &&
+          closeMatch[3].trim() === ''
+        ) {
+          inFence = false;
+        }
+        return line;
+      }
+      let openMatch =
+        line.match(CODE_FENCE_PATTERN) ??
+        line.match(LIST_PREFIXED_CODE_FENCE_PATTERN);
+      if (openMatch) {
+        inFence = true;
+        fenceChar = openMatch[2][0];
+        fenceLength = openMatch[2].length;
+        return line;
+      }
+      return line.replace(
+        DECORATIVE_BULLET_PATTERN,
+        (_match, indentation, bullet, whitespace) =>
+          `${indentation}* ${bullet}${whitespace}`,
+      );
+    })
+    .join('\n');
+}
 
 const DEFAULT_MARKED_SYNC_OPTIONS = {
   escapeHtmlInCodeBlocks: true,
@@ -187,11 +249,7 @@ export function markdownToHtml(
     return '';
   }
   // Marked only treats ASCII list markers, so prefix decorative bullets with a standard marker.
-  let normalizedMarkdown = markdown.replace(
-    DECORATIVE_BULLET_PATTERN,
-    (_match, boundary, indentation, bullet, whitespace) =>
-      `${boundary}${indentation}* ${bullet}${whitespace}`,
-  );
+  let normalizedMarkdown = normalizeDecorativeBullets(markdown);
   let html = markedSync(normalizedMarkdown, {
     escapeHtmlInCodeBlocks: opts.escapeHtmlInCodeBlocks,
     enableMonacoSyntaxHighlighting: opts.enableMonacoSyntaxHighlighting,