Implement the state-file baseline strategy (#1)

CHANGELOG.md

@@ -11,3 +11,7 @@
 - Character-level churn ([#4](https://github.com/QuantEcon/textstrata/issues/4)): `chars_changed` per pair,
   `prose_chars_added`/`prose_chars_deleted` per commit and `prose_char_churn_by_tier` per document, so F1
   can be read per 1,000 characters as well as per 1,000 lines.
+- `state-file` baseline strategy ([#1](https://github.com/QuantEcon/textstrata/issues/1)): the translation
+  moment is the first revision of the document's state file — the engine's own record — instead of the
+  script-ratio jump. Requires `machine.state_dir`; a document without a state file is untranslated under it.
+  The programming.zh-cn reference config adopts it.

configs/quantecon/programming-zh-cn.yml

@@ -4,7 +4,12 @@ files: "lectures/*.md"
 source:
   repo: ../../../project-translation/repos/lecture-python-programming
 prose: {strategy: script, script: Han, threshold: 0.05}
-baseline: {strategy: script-jump}
+baseline:
+  strategy: state-file
+  overrides:
+    # shared admonition snippet translated by hand in #23; the engine keeps no
+    # state file for it, so the state-file strategy alone reads it as untranslated
+    lectures/_admonition/gpu.md: e002d4f
 machine:
   bots: ['\[bot\]', 'dependabot', 'github-actions']
   sync: ['\[translation-sync\]', '\[action-translation\]', 'resync']

docs/configuration.md

@@ -15,8 +15,9 @@ prose:
   threshold: 0.05                  # script ratio that marks the translation moment
   # punctuation_map: {",": "，"}   # width normalisation; defaults to a Han map for script: Han
 baseline:
-  strategy: script-jump            # script-jump | state-file (planned)
-  overrides:                       # document -> sha prefix, when the first script-bearing revision is wrong
+  strategy: script-jump            # script-jump | state-file (state-file needs machine.state_dir:
+                                   # the moment is the first revision of the document's state file)
+  overrides:                       # document -> sha prefix, when the strategy picks the wrong revision
     lectures/long_run_growth.md: cd9808c
 machine:
   bots: ['\[bot\]', 'dependabot', 'github-actions']     # author/e-mail regexes (AI agents excluded, see disclosure)

docs/method.md

@@ -25,7 +25,11 @@ Rules are applied in order; the first match wins. Content rules come before auth
 
 ## The translation moment
 
-Strategy `script-jump`: the first revision at which the document's ratio of target-script characters (whitespace removed) reaches `prose.threshold` (default 5%). This is computed from content, not commit messages, and survives generic PR titles. Documents whose early history is messy (a discarded draft, a regenerated translation) take a per-document override in `baseline.overrides`.
+Strategy `script-jump`: the first revision at which the document's ratio of target-script characters (whitespace removed) reaches `prose.threshold` (default 5%). This is computed from content, not commit messages, and survives generic PR titles.
+
+Strategy `state-file`: the first revision of the document's per-document state file (`machine.state_dir/<name>.yml`, renames followed) marks the moment — the engine's own record that it created the translation, available in repositories that are engine-managed from the start. When the engine landed the document and its state file as adjacent single-file commits, the moment is the last document revision not after the state file's creation. A document with no state file is untranslated under this strategy, and the scan log says so.
+
+Under both strategies, documents whose history defeats the rule (a discarded draft, a regenerated translation, a missing state file) take a per-document override in `baseline.overrides`.
 
 ## Prose-only measurement
 

src/textstrata/config.py

@@ -57,7 +57,7 @@ def pattern(self) -> re.Pattern[str]:
 
 @dataclass
 class BaselineConfig:
-    strategy: str = "script-jump"       # script-jump | state-file (planned)
+    strategy: str = "script-jump"       # script-jump | state-file
     overrides: dict[str, str] = field(default_factory=dict)   # file -> sha prefix
 
 
@@ -148,16 +148,17 @@ def load_config(path: str | Path) -> Config:
     baseline = _sub(BaselineConfig, raw.get("baseline"), "baseline")
     if baseline.strategy not in ("script-jump", "state-file"):
         raise ConfigError(f"baseline.strategy must be script-jump or state-file, got {baseline.strategy!r}")
-    if baseline.strategy == "state-file":
-        raise ConfigError("baseline.strategy state-file is planned but not implemented")
+    machine = _sub(MachineConfig, raw.get("machine"), "machine")
+    if baseline.strategy == "state-file" and not machine.state_dir:
+        raise ConfigError("baseline.strategy state-file requires machine.state_dir")
     cfg = Config(
         name=str(raw["name"]),
         repo=Path(raw["repo"]),
         files=str(raw.get("files", "lectures/*.md")),
         source=_sub(SourceConfig, raw.get("source"), "source"),
         prose=prose,
         baseline=baseline,
-        machine=_sub(MachineConfig, raw.get("machine"), "machine"),
+        machine=machine,
         disclosure=_sub(DisclosureConfig, raw.get("disclosure"), "disclosure"),
         people=_sub(PeopleConfig, raw.get("people"), "people"),
         review_state=_sub(ReviewStateConfig, raw.get("review_state"), "review_state"),

src/textstrata/scan.py

@@ -85,6 +85,52 @@ def read_state(repo: Path, state_dir: str | None, doc: str) -> dict[str, str]:
     return out
 
 
+def translation_moment(cfg: Config, repo: Path, prose: Prose, f: str,
+                       hist: list[Commit], log=sys.stderr) -> tuple[str | None, str | None]:
+    """The document's translation moment under the configured baseline strategy.
+
+    A `baseline.overrides` entry wins regardless of strategy: it is the reviewed
+    correction for a document whose history defeats the rule.
+    """
+    forced = cfg.baseline.overrides.get(f)
+    if forced:
+        t_sha, t_date = None, None
+        for c in hist:
+            if c.sha.startswith(forced):
+                t_sha, t_date = c.sha, c.date
+        if t_sha is None:
+            print(f"  {f}: baseline override {forced} matches no commit in its history; "
+                  "treated as untranslated", file=log)
+        return t_sha, t_date
+    if cfg.baseline.strategy == "state-file":
+        return _state_file_moment(cfg, repo, f, hist, log)
+    for c in hist:  # script-jump
+        if prose.is_translated(show(repo, c.sha, c.path)):
+            return c.sha, c.date
+    return None, None
+
+
+def _state_file_moment(cfg: Config, repo: Path, f: str, hist: list[Commit],
+                       log) -> tuple[str | None, str | None]:
+    state_path = cfg.machine.state_dir.rstrip("/") + "/" + Path(f).name + ".yml"  # type: ignore[union-attr]
+    st_hist = file_history(repo, state_path)
+    if not st_hist:
+        print(f"  {f}: no state file at {state_path}; untranslated under state-file "
+              "(a baseline.overrides entry corrects this)", file=log)
+        return None, None
+    first = st_hist[0]
+    for c in hist:
+        if c.sha == first.sha:
+            return c.sha, c.date
+    # the engine sometimes lands a document and its state file as adjacent
+    # single-file commits; the document revision the state file describes is
+    # then the last one not after the state file's creation
+    created = datetime.fromisoformat(first.date)
+    prior = [c for c in hist if datetime.fromisoformat(c.date) <= created]
+    c = prior[-1] if prior else hist[0]
+    return c.sha, c.date
+
+
 def scan(cfg: Config, out_dir: Path, log=sys.stderr) -> dict:
     repo = cfg.repo
     prose = Prose(cfg.prose)
@@ -104,15 +150,7 @@ def scan(cfg: Config, out_dir: Path, log=sys.stderr) -> dict:
     for f in files:
         hist = file_history(repo, f)
         histories[f] = hist
-        forced = cfg.baseline.overrides.get(f)
-        t_sha, t_date = None, None
-        for c in hist:
-            content = show(repo, c.sha, c.path)
-            if forced:
-                if c.sha.startswith(forced):
-                    t_sha, t_date = c.sha, c.date
-            elif t_sha is None and prose.is_translated(content):
-                t_sha, t_date = c.sha, c.date
+        t_sha, t_date = translation_moment(cfg, repo, prose, f, hist, log)
         before = True
         for c in hist:
             if c.sha == t_sha:

tests/test_baseline.py

@@ -0,0 +1,107 @@
+"""The state-file baseline strategy against a synthetic engine-era repository.
+
+The fixture reproduces the shapes observed in lecture-python-programming.zh-cn:
+a document and its state file landing in one commit (the `translate init` case),
+the pair landing as adjacent single-file commits (the autodiff case), and a
+document with no state file at all.
+"""
+import os
+import subprocess
+import sys
+from datetime import datetime
+
+import pytest
+
+from textstrata.config import Config, ProseConfig
+from textstrata.git import file_history
+from textstrata.prose import Prose
+from textstrata.scan import translation_moment
+
+
+def git(repo, *args, date=None):
+    env = dict(os.environ,
+               GIT_AUTHOR_NAME="Engine", GIT_AUTHOR_EMAIL="engine@example.org",
+               GIT_COMMITTER_NAME="Engine", GIT_COMMITTER_EMAIL="engine@example.org",
+               GIT_CONFIG_GLOBAL="/dev/null", GIT_CONFIG_SYSTEM="/dev/null")
+    if date:
+        env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = date
+    r = subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, text=True, check=False)
+    assert r.returncode == 0, r.stderr
+    return r.stdout.strip()
+
+
+def commit(repo, msg, date):
+    git(repo, "add", "-A")
+    git(repo, "commit", "-m", msg, date=date)
+    return git(repo, "rev-parse", "HEAD")
+
+
+@pytest.fixture
+def engine_repo(tmp_path):
+    repo = tmp_path / "repo"
+    (repo / "lectures").mkdir(parents=True)
+    (repo / ".translate" / "state").mkdir(parents=True)
+    git(repo, "init", "-q")
+    # a.md: document and state file in one commit
+    (repo / "lectures" / "a.md").write_text("# 讲座甲\n\n这是机器翻译的第一稿。\n", encoding="utf-8")
+    (repo / ".translate" / "state" / "a.md.yml").write_text("mode: NEW\n", encoding="utf-8")
+    shas = {"init": commit(repo, "Initial translation via translate init", "2026-03-20T10:00:00Z")}
+    # b.md: document first, state file one second later (the autodiff shape)
+    (repo / "lectures" / "b.md").write_text("# 讲座乙\n\n另一篇机器初稿。\n", encoding="utf-8")
+    shas["b_doc"] = commit(repo, "Update translation: lectures/b.md", "2026-04-09T04:40:27+00:00")
+    (repo / ".translate" / "state" / "b.md.yml").write_text("mode: NEW\n", encoding="utf-8")
+    shas["b_state"] = commit(repo, "Update translation: .translate/state/b.md.yml", "2026-04-09T04:40:28+00:00")
+    # c.md: no state file
+    (repo / "lectures" / "c.md").write_text("# 讲座丙\n\n没有状态文件的文稿。\n", encoding="utf-8")
+    shas["c_doc"] = commit(repo, "Add c.md by hand", "2026-05-01T09:00:00Z")
+    # a later sync touches a.md and its state file: must not move a.md's moment
+    (repo / "lectures" / "a.md").write_text("# 讲座甲\n\n这是机器重新同步的稿子。\n", encoding="utf-8")
+    (repo / ".translate" / "state" / "a.md.yml").write_text("mode: UPDATE\n", encoding="utf-8")
+    shas["sync"] = commit(repo, "[translation-sync] resync a.md", "2026-06-01T09:00:00Z")
+    return repo, shas
+
+
+def make_cfg(repo, overrides=None):
+    cfg = Config(name="t", repo=repo, base_dir=repo)
+    cfg.baseline.strategy = "state-file"
+    cfg.baseline.overrides = overrides or {}
+    cfg.machine.state_dir = ".translate/state"
+    return cfg
+
+
+def moment(cfg, repo, doc):
+    return translation_moment(cfg, repo, Prose(ProseConfig()), doc, file_history(repo, doc), sys.stderr)
+
+
+def same_instant(date, expected):
+    # git renders %aI's UTC offset as either Z or +00:00 depending on version;
+    # compare instants, not strings
+    return datetime.fromisoformat(date) == datetime.fromisoformat(expected)
+
+
+def test_state_file_moments(engine_repo, capsys):
+    repo, shas = engine_repo
+    cfg = make_cfg(repo)
+    # same-commit case: the moment is the state file's creating commit
+    sha, date = moment(cfg, repo, "lectures/a.md")
+    assert sha == shas["init"] and same_instant(date, "2026-03-20T10:00:00+00:00")
+    # adjacent-commit case: the state-creating commit is not in the document's
+    # history; the moment falls back to the document revision just before it
+    assert moment(cfg, repo, "lectures/b.md")[0] == shas["b_doc"]
+    # no state file: untranslated, and the log says so
+    assert moment(cfg, repo, "lectures/c.md") == (None, None)
+    assert "no state file" in capsys.readouterr().err
+
+
+def test_override_beats_state_file(engine_repo):
+    repo, shas = engine_repo
+    cfg = make_cfg(repo, overrides={"lectures/c.md": shas["c_doc"][:7]})
+    sha, date = moment(cfg, repo, "lectures/c.md")
+    assert sha == shas["c_doc"] and same_instant(date, "2026-05-01T09:00:00+00:00")
+
+
+def test_override_without_match_warns(engine_repo, capsys):
+    repo, _shas = engine_repo
+    cfg = make_cfg(repo, overrides={"lectures/a.md": "deadbeef"})
+    assert moment(cfg, repo, "lectures/a.md") == (None, None)
+    assert "matches no commit" in capsys.readouterr().err

tests/test_units.py

@@ -106,8 +106,13 @@ def test_noreply_handle_resolution():
 
 def test_baseline_strategy_validated(tmp_path):
     (tmp_path / ".git").mkdir()
-    for strategy, msg in (("state-file", "not implemented"), ("bogus", "must be script-jump")):
+    # state-file without a state directory, and unknown strategies, fail loudly
+    for strategy, msg in (("state-file", "requires machine.state_dir"), ("bogus", "must be script-jump")):
         p = tmp_path / f"{strategy}.yml"
         p.write_text(f"name: t\nrepo: {tmp_path}\nbaseline: {{strategy: {strategy}}}\n", encoding="utf-8")
         with pytest.raises(ConfigError, match=msg):
             load_config(p)
+    p = tmp_path / "ok.yml"
+    p.write_text(f"name: t\nrepo: {tmp_path}\nbaseline: {{strategy: state-file}}\n"
+                 "machine: {state_dir: .translate/state}\n", encoding="utf-8")
+    assert load_config(p).baseline.strategy == "state-file"