fix(docs): preserve source text through syntax highlighting

package.json

@@ -33,7 +33,7 @@
     "test:guidance": "node --test tests/background-capability-guidance.test.mjs tests/subprocess-integration-guidance.test.mjs",
     "test:codeql-build-graph": "node --test tests/codeql-build-graph.test.mjs",
     "test:release-validation": "node --test tests/release-validation-workflow.test.mjs",
-    "test:docs-site": "node --test tests/docs-site-toc.test.mjs",
+    "test:docs-site": "node --test tests/docs-site-*.test.mjs",
     "test:release-preflight": "node --test tests/release-preflight-contract.test.mjs",
     "test:background-certification": "scripts/test-background-certification.sh",
     "test:automation": "pnpm run setup:swift && PEEKABOO_INCLUDE_AUTOMATION_TESTS=true swift test --package-path Apps/CLI --no-parallel",

scripts/build-docs-site.mjs

@@ -724,28 +724,35 @@ function highlightCode(code, lang) {
   return escapeHtml(code);
 }
 
-function stashToken(idx) {
-  return String.fromCharCode(0xe000 + idx);
+// Keep highlighted fragments separate from source text so code never doubles
+// as an internal placeholder, and the number of matches is unbounded.
+function highlightFragments(fragments, pattern, renderMatch) {
+  return fragments.flatMap((fragment) => {
+    if (typeof fragment !== "string") return [fragment];
+    const result = [];
+    let previous = 0;
+    fragment.replace(pattern, (...args) => {
+      const match = args[0];
+      const offset = args.at(-2);
+      result.push(fragment.slice(previous, offset), { html: renderMatch(...args) });
+      previous = offset + match.length;
+      return match;
+    });
+    result.push(fragment.slice(previous));
+    return result;
+  });
 }
 
-function restoreStashTokens(value, stash) {
-  return value.replace(/[-]/g, (token) => {
-    const idx = token.charCodeAt(0) - 0xe000;
-    return stash[idx] ?? "";
-  });
+function renderFragments(fragments) {
+  return fragments.map((fragment) => typeof fragment === "string" ? escapeHtml(fragment) : fragment.html).join("");
 }
 
 function withStash(code, patterns) {
-  const stash = [];
-  let working = code;
-  for (const [re, cls] of patterns) {
-    working = working.replace(re, (match) => {
-      const idx = stash.length;
-      stash.push(`<span class="${cls}">${escapeHtml(match)}</span>`);
-      return stashToken(idx);
-    });
+  let fragments = [code];
+  for (const [pattern, cls] of patterns) {
+    fragments = highlightFragments(fragments, pattern, (match) => `<span class="${cls}">${escapeHtml(match)}</span>`);
   }
-  return restoreStashTokens(escapeHtml(working), stash);
+  return renderFragments(fragments);
 }
 
 function highlightShell(code) {
@@ -764,22 +771,17 @@ function highlightShell(code) {
 }
 
 function highlightShellLine(line) {
-  const stash = [];
-  const stashAdd = (match, cls) => {
-    const idx = stash.length;
-    stash.push(`<span class="${cls}">${escapeHtml(match)}</span>`);
-    return stashToken(idx);
-  };
-  let working = line;
-  working = working.replace(/(?:'[^']*'|"[^"]*")/g, (m) => stashAdd(m, "hl-s"));
-  working = working.replace(/\s#.*$/g, (m) => stashAdd(m, "hl-c"));
-  working = working.replace(/(^|\s)(--?[A-Za-z][A-Za-z0-9-]*)/g, (_, lead, flag) => `${escapeHtml(lead)}${stashAdd(flag, "hl-f")}`);
-  working = working.replace(
+  const span = (match, cls) => `<span class="${cls}">${escapeHtml(match)}</span>`;
+  let fragments = [line];
+  fragments = highlightFragments(fragments, /(?:'[^']*'|"[^"]*")/g, (match) => span(match, "hl-s"));
+  fragments = highlightFragments(fragments, /\s#.*$/g, (match) => span(match, "hl-c"));
+  fragments = highlightFragments(fragments, /(^|\s)(--?[A-Za-z][A-Za-z0-9-]*)/g,
+    (_, lead, flag) => `${escapeHtml(lead)}${span(flag, "hl-f")}`);
+  fragments = highlightFragments(fragments,
     /\b(peekaboo|brew|npx|npm|pnpm|yarn|node|swift|git|gh|make|sudo|cd|export|cat|curl|jq|ls|mv|cp|rm|mkdir|docker|tail)\b/g,
-    (m) => stashAdd(m, "hl-cmd"),
-  );
-  working = working.replace(/\b(\d+(?:\.\d+)?)\b/g, (m) => stashAdd(m, "hl-n"));
-  return restoreStashTokens(escapeHtml(working), stash);
+    (match) => span(match, "hl-cmd"));
+  fragments = highlightFragments(fragments, /\b(\d+(?:\.\d+)?)\b/g, (match) => span(match, "hl-n"));
+  return renderFragments(fragments);
 }
 
 function highlightJson(code) {

tests/docs-site-code-preservation.test.mjs

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import test from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+const projectRoot = fileURLToPath(new URL('..', import.meta.url));
+
+function renderCode(t, language, source) {
+  const fixtureRoot = mkdtempSync(path.join(os.tmpdir(), 'peekaboo-docs-code-'));
+  t.after(() => rmSync(fixtureRoot, { recursive: true, force: true }));
+  mkdirSync(path.join(fixtureRoot, 'docs'));
+  writeFileSync(path.join(fixtureRoot, 'docs', 'index.md'), `# Fixture\n\n\`\`\`${language}\n${source}\n\`\`\`\n`);
+  const result = spawnSync(process.execPath, [path.join(projectRoot, 'scripts', 'build-docs-site.mjs')], {
+    cwd: fixtureRoot, encoding: 'utf8'
+  });
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+  const html = readFileSync(path.join(fixtureRoot, '_site', 'index.html'), 'utf8');
+  const code = html.match(/<pre><code[^>]*>([\s\S]*?)<\/code><\/pre>/)?.[1];
+  assert.ok(code !== undefined, 'missing rendered code block');
+  // Remove only exact span wrappers that this renderer owns. Literal source
+  // markup remains escaped until these wrappers have been removed.
+  let text = code.replaceAll('</span>', '');
+  for (const cls of ['hl-s', 'hl-c', 'hl-p', 'hl-f', 'hl-cmd', 'hl-n', 'hl-k', 'hl-m', 'hl-t']) {
+    text = text.replaceAll(`<span class="${cls}">`, '');
+  }
+  const copyable = text.replace(/&quot;/g, '"').replace(/&#39;/g, "'")
+    .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&');
+  assert.equal(copyable, source);
+  return code;
+}
+
+test('syntax highlighting preserves private-use characters in valid shell paths and YAML keys', (t) => {
+  renderCode(t, 'bash', 'cat \ue000report\uf8ff.txt --lines 12');
+  renderCode(t, 'yaml', '\ue000key\uf8ff: value');
+});
+
+test('syntax highlighting preserves blocks with more than 6400 highlighted tokens', (t) => {
+  const source = JSON.stringify(Array.from({ length: 7000 }, (_, index) => `value-${index}`));
+  const html = renderCode(t, 'json', source);
+  assert.equal((html.match(/class="hl-s"/g) || []).length, 7000);
+});
+
+test('source text and existing highlighting survive ordered pattern passes', (t) => {
+  const shell = renderCode(t, 'bash', 'peekaboo --app "Finder" 42 # npm <example>');
+  assert.match(shell, /class="hl-cmd">peekaboo/);
+  assert.match(shell, /class="hl-f">--app/);
+  assert.match(shell, /class="hl-s">&quot;Finder&quot;/);
+  assert.match(shell, /class="hl-n">42/);
+  assert.match(shell, /class="hl-c"> # npm &lt;example&gt;/);
+  renderCode(t, 'bash', 'echo "<span class=\'hl-s\'>literal & text</span>"');
+  renderCode(t, 'js', 'const value = "npm 12"; // git --flag');
+  renderCode(t, 'swift', 'let value = "quoted <text>" // 42');
+  renderCode(t, 'json', '{"name":"<example>","count":42,"enabled":true}');
+});