Prose-gate unpaired additions and deletions

CHANGELOG.md

@@ -23,3 +23,7 @@
   stamped with `engine_model`/`engine_tool_version` from the state-file history (the record written by or
   immediately after the commit), `run.json` carries per-version totals (`engine_strata`), and
   `overwrites.json` entries are stamped the same way; syncs with no record read `unrecorded`.
+- Unpaired additions and deletions are prose-gated ([#14](https://github.com/QuantEcon/textstrata/issues/14)):
+  an unpaired line with no target-script content routes to `code-or-markup` instead of
+  `addition`/`deletion`, mirroring the paired-line rule, so the omission taxonomy counts prose only.
+  The total pair count is unchanged — only categories shift.

docs/method.md

@@ -66,7 +66,7 @@ stay distinguishable.
 
 - **Squash merges** hide human work done inside a machine-drafted PR. `ai-initial` means *as landed*; human effort is a lower bound.
 - **Last-toucher blame** credits a whole line to whoever changed one character of it. Human shares are an upper bound at line granularity; churn is therefore also reported in changed characters (`chars_changed` per pair, `prose_chars_added`/`prose_chars_deleted` per commit, `prose_char_churn_by_tier` per document), where a one-character fix counts as one character. Counts come from `SequenceMatcher` opcodes over the paired lines' raw text; unpaired additions and deletions count the full line.
-- **Line pairing** inside rewritten paragraphs is heuristic (similarity-matched within a hunk). Category counts are indicative.
+- **Line pairing** inside rewritten paragraphs is heuristic (similarity-matched within a hunk). Category counts are indicative. Unpaired lines with no target-script content — code, maths, markup, metadata arriving or leaving — are categorised `code-or-markup`, not `addition`/`deletion`, so the omission taxonomy counts prose only.
 - **Identity** is resolved by e-mail and GitHub noreply handle only; display names are ignored. Unresolved authors fall to `ai-assisted` and should be reviewed in `commits.jsonl`.
 - **Pre-engine history** has no recorded engine version and lands in the `unrecorded` stratum; do not read its rates as the shipping engine's.
 

src/textstrata/scan.py

@@ -318,18 +318,22 @@ def tier_for(f: str, sha: str) -> str:
                                       "before": o, "after": n})
                     if tier in HUMAN_TIERS and cat in ("terminology", "punctuation-width", "fluency"):
                         mine_substitutions(o, n, prose, subs, sub_examples)
+                # an unpaired line with no target-script content is code, math or
+                # markup arriving or leaving, not an omission-taxonomy edit
                 for n in adds:
                     if n.strip():
+                        cat = "addition" if prose.is_prose(n) else "code-or-markup"
                         pairs_out.append({"document": f, "sha": c.sha[:8], "date": c.date[:10],
                                           "tier": tier, "pr": pr.group(1) if pr else None,
-                                          "category": "addition", "taxonomy": "omission",
+                                          "category": cat, "taxonomy": CATEGORY_MAP[cat],
                                           "similarity": 0.0, "chars_changed": len(n),
                                           "before": "", "after": n})
                 for _ln, o in dels:
                     if o.strip():
+                        cat = "deletion" if prose.is_prose(o) else "code-or-markup"
                         pairs_out.append({"document": f, "sha": c.sha[:8], "date": c.date[:10],
                                           "tier": tier, "pr": pr.group(1) if pr else None,
-                                          "category": "deletion", "taxonomy": "omission",
+                                          "category": cat, "taxonomy": CATEGORY_MAP[cat],
                                           "similarity": 0.0, "chars_changed": len(o),
                                           "before": o, "after": ""})
         d.prose_churn_by_tier = dict(churn)

tests/test_baseline.py

@@ -58,10 +58,20 @@ def engine_repo(tmp_path):
     (repo / "lectures" / "c.md").write_text("# 讲座丙\n\n没有状态文件的文稿。\n", encoding="utf-8")
     shas["c_doc"] = commit(repo, "Add c.md by hand", "2026-05-01T09:00:00Z")
     # a later sync touches a.md and its state file: must not move a.md's moment
-    (repo / "lectures" / "a.md").write_text("# 讲座甲\n\n这是机器重新同步的稿子。\n", encoding="utf-8")
+    (repo / "lectures" / "a.md").write_text(
+        "# 讲座甲\n\n这是机器重新同步的稿子。\n\ny = 2\n", encoding="utf-8")
     (repo / ".translate" / "state" / "a.md.yml").write_text(
         "mode: UPDATE\nmodel: alpha-2\ntool-version: 0.2.0\n", encoding="utf-8")
     shas["sync"] = commit(repo, "[translation-sync] resync a.md", "2026-06-01T09:00:00Z")
+    # two editor-shaped commits (non-bot, non-sync, off-roster -> ai-assisted),
+    # one purely additive and one purely deletive, so every line is unpaired:
+    # a prose line and a code line arrive, then a code line leaves
+    (repo / "lectures" / "a.md").write_text(
+        "# 讲座甲\n\n这是机器重新同步的稿子。\n\ny = 2\n\n新增的一段人写文字。\n\nx = 1\n", encoding="utf-8")
+    shas["edit"] = commit(repo, "Edit a.md by hand", "2026-07-01T09:00:00Z")
+    (repo / "lectures" / "a.md").write_text(
+        "# 讲座甲\n\n这是机器重新同步的稿子。\n\n新增的一段人写文字。\n\nx = 1\n", encoding="utf-8")
+    shas["edit2"] = commit(repo, "Drop the scratch code line", "2026-07-02T09:00:00Z")
     return repo, shas
 
 
@@ -148,3 +158,20 @@ def test_engine_strata_in_scan(engine_repo, tmp_path):
     # per-version totals, ordered by first appearance
     assert [(s["model"], s["tool_version"], s["sync_commits"]) for s in run["engine_strata"]] == [
         ("beta-1", "0.1.5", 1), ("alpha-2", "0.2.0", 1)]
+
+
+def test_unpaired_lines_prose_gated(engine_repo, tmp_path):
+    repo, shas = engine_repo
+    cfg = make_cfg(repo)
+    cfg.machine.sync = [r"\[translation-sync\]"]
+    with open(os.devnull, "w") as devnull:
+        scan(cfg, tmp_path / "out", log=devnull)
+    pairs = [json.loads(ln) for ln in (tmp_path / "out" / "pairs.jsonl").open(encoding="utf-8")]
+    mine = [p for p in pairs if p["sha"] in (shas["edit"][:8], shas["edit2"][:8])]
+    got = {(p["category"], p["after"] or p["before"]) for p in mine}
+    # a prose line arriving is an addition; code arriving or leaving is code-or-markup
+    assert ("addition", "新增的一段人写文字。") in got
+    assert ("code-or-markup", "x = 1") in got
+    assert ("code-or-markup", "y = 2") in got
+    assert all(p["taxonomy"] == "formatting" for p in mine if p["category"] == "code-or-markup")
+    assert not any(p["category"] == "deletion" for p in mine)