diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66fa79d..da65e34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,9 @@ jobs: tests/test_download_cps.py tests/test_hierarchical_pipeline.py tests/test_supabase_client.py + tests/test_policyengine_ledger.py + scripts/check_thesis_facts_append.py + scripts/canonical_json.py - name: Test Arch surface run: > @@ -86,6 +89,7 @@ jobs: tests/test_download_cps.py tests/test_hierarchical_pipeline.py tests/test_supabase_client.py + tests/test_policyengine_ledger.py -q - name: Build target input database diff --git a/.github/workflows/thesis-facts-append.yml b/.github/workflows/thesis-facts-append.yml new file mode 100644 index 0000000..a04a602 --- /dev/null +++ b/.github/workflows/thesis-facts-append.yml @@ -0,0 +1,58 @@ +name: Thesis facts append gate + +# Deterministic review for every change to the observation ledger. Resolver +# appends arrive as pull requests targeting codex/thesis-ledger-facts; this +# gate enforces the immutable frozen prefix, an append-only diff against the +# PR base, per-row schema and binding requirements, and explicit supersede +# semantics for corrections. Direct pushes get the same full-file checks. + +on: + pull_request: + branches: + - codex/thesis-ledger-facts + paths: + - "ledger/**" + - "scripts/check_thesis_facts_append.py" + - "scripts/canonical_json.py" + - ".github/workflows/thesis-facts-append.yml" + push: + branches: + - codex/thesis-ledger-facts + paths: + - "ledger/**" + +jobs: + append-gate: + name: Append gate + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Enforce append-only ledger invariants + run: | + set -euo pipefail + if [ -n "${{ github.base_ref }}" ]; then + git fetch origin "${{ github.base_ref }}" --depth=1 + python3 scripts/check_thesis_facts_append.py \ + --base-ref "origin/${{ github.base_ref }}" + else + python3 scripts/check_thesis_facts_append.py + fi + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.14" + + - name: Install dependencies + run: uv sync --locked --all-extras + + - name: Ledger observation invariants + run: uv run pytest tests/test_policyengine_ledger.py -q diff --git a/arch/core.py b/arch/core.py index 581a935..2d5ae6a 100644 --- a/arch/core.py +++ b/arch/core.py @@ -17,7 +17,16 @@ Scalar = str | int | float | bool | None -ALLOWED_PERIOD_TYPES = {"calendar_year", "tax_year", "fiscal_year", "month"} +ALLOWED_PERIOD_TYPES = { + "calendar_year", + "tax_year", + "fiscal_year", + "month", + # Sub-monthly and quarterly publisher prints recorded by the Thesis + # resolver (e.g. DOL UI weekly claims, BEA quarterly GDP). + "quarter", + "week_ending", +} ALLOWED_GEOGRAPHY_LEVELS = { "country", "region", @@ -43,6 +52,9 @@ "government", "dwelling", "institutional_sector", + # Macro publisher prints (GDP growth, industrial production, price + # indexes) describe the whole economy, not a person-level population. + "economy", } ALLOWED_AGGREGATIONS = { "count", @@ -52,6 +64,9 @@ "rate", "ratio", "share", + # A published series value taken as printed (policy rates, index + # levels, claims counts) — the publisher's own aggregate, not ours. + "level", } ALLOWED_CONSTRAINT_OPERATORS = {"==", "!=", ">", ">=", "<", "<=", "in"} ALLOWED_CONCEPT_RELATIONS = { diff --git a/ledger/immutable_prefix.json b/ledger/immutable_prefix.json new file mode 100644 index 0000000..2600092 --- /dev/null +++ b/ledger/immutable_prefix.json @@ -0,0 +1,204 @@ +{ + "schemaVersion": "thesis_facts_immutable_prefix_v1", + "note": "Frozen at contract-enforcement start (ledger architecture review, 2026-07-10). The first prefixLineCount lines of ledger/official_observations.jsonl are immutable: CI fails if any byte changes. Corrections append a new row whose assertionVersion.supersedes names the replaced version; the prefix is never edited. Extending the manifest to cover newly accepted rows is an explicit migration, reviewed like any other change.", + "frozenAtHead": "324382b862b3cdfa3fef073c98fd70bf6f71b57e", + "prefixLineCount": 128, + "prefixSha256": "581f681aa3467d7f58b3ede4f13a17ea80c187d0328e88ae743756cc2b67a48b", + "lineSha256s": [ + "e2a8133842e64bf975354a3da02105cf542961a826440db5dfcdb8c1eba3df66", + "89332f17aae6d742b45ff342ab6b535d524423900c622c0b406a50786b6f7e62", + "d79ae9c0cb993809d667fe7f9973270304e7f81e762e4f665b798a7a3cde6b13", + "be4da94287ab57ab9822e9b590c5ce11b71d504aa4d1c62f8c7b0dfc7f2bb1f2", + "7137623b15da972e19cbce0c4ff341449c0c59bec91ee42c783ef061d22faa78", + "c1220deef59d28f6b8e5ffb988005d034a280aefb832c4754a856e9971e52982", + "7383e2bd476525231ab1ab13b720b19fca44ea47dce0ef0e24f478d2f57483a4", + "deeef6d4cf44900b07f793fa2e07d64c4cc57baa548c77acf6cc67128c1a6c61", + "e67af588eaac44c254cfc20f7e747c2fe046a4f1d3313b5aca49429bc26d099f", + "02f4cf4ab33bee956831f7cbc5dbcf8a2a6b0b09980a49b4d5873e792dccb93f", + "ae0bea19e3bc146853ca9d3443f9991edc9fce515c33535fb3c1fae781e7bbd9", + "1474345208a131ad065a138b3b8f77428bd7247ddeffb34d8e24ca3a61f5c764", + "e0af919c10f7d1c45b9b2daaaa8aa062dd8d986435179f910af2f622bf77ec27", + "4a48bedf5a2e5ffaed56dd68e98ae622661068072bf5de02179827fcd3196701", + "7197b7a8c911407902035c02aa9bb73826074042f85d4ed8dff5466680715519", + "d0ae460cdeee286b98af966fba3cc1834fa5ffec2312098c21f0168c2e37b57a", + "f3866a032b569f37b2d17738141adb9c474f27a4138f7a152341e01eaf23a9da", + "5d91209a457703679fed8c4d0aa493216f6781183b7a8283d17a451e4c91d4b2", + "8356aeeef7f71ef27d8aefc32dcfb13fdc33a01dcbeb9a28aae869a4ffbd4beb", + "c22cb1009fdaf950c0d25214713c6ffc9bd7742823626eac63af65848336c915", + "87f5a18c7a82c976caa2654b1acf83633522931cf063e3e9297c41dff22deb07", + "ed555e1728dd8adfd39069807f08aa816f18890a3abcbf5a4b4c380a5647bc3f", + "3195de3ff6e0345a8bb95e496ab009f48f74529bf59a6accb49d8c4b8529d4a8", + "208cf0510eadc79f51bb0fd43ade604b1147e0797be4f9d82200e4cc34b342f3", + "254d282792a59b47863d48783d6ccf5ccba4b53aba28541c5b5233eda00deaa7", + "d92446cf44b86c4645971e7111626e5afa43742db7e2392a502a71ee5f817161", + "8b4a8d2ce4f97e36218de9abab9e557e59bbd11fee2e8bad17fcbb6990775db9", + "4db67a541b456843893f3d26a5ba66899cb100b43139be6d551aee8af3afd34b", + "f8810d830f384a8b245feaa7227f2fa654eb12e42ccc27be3c526dfd466b9acd", + "dd658d16ffbe64ae0dc6d527e6863ebb5cb2f283d185f95f6f0cdbc946c47d3c", + "d238afd394cad210dfb11d3163a10defd0ad0013f8ba16af53d76e20fc79c47b", + "c641b65b0e477d32db38381b26636e127e4ee87b8838abe69bca99f0119f5ada", + "53f14df26aeb3770fe2dcf77b15a67ea9147cd4be36e1fd28f74bf72fe494538", + "4c0d51a9fe3de4665d0571ea27218b2676adcfdde0b928d08e9aa090ac0182da", + "a7a504c4b2104a2beeb1ffc5a1cd29aaec5354d3bfe5481497ece4a7c84d8545", + "2708189c1c7f08fac6182c15cd9c1deb6db2045f1c5e8bd2cef7181c5ea582b1", + "7ca974b56bee900993cabd4e5598bd6aeb6362db096de08af60916342e777393", + "1da9308a0aff1040dc07df4950d82ffa8aad24cabbf41021f36135f785d4075f", + "5b716a4cbbc00ee6ec2053593bf7da35fa3b75103eabb7be14a295045285cc60", + "2a2a3a6a646563714e9cd9fc5eb804a1077d823788a83e83dc06090b590b8e78", + "558bbeb5f09a045e0cf2450b17339b26137e37530102f564acdb944097dd93f3", + "511790ed1f50569353c083db9cab6ece7d7995110d4ad960f0a21a9e8635e127", + "49eb60c1aa838003c3a4c0d5325959daf5c86dc065afdf52a949e48f23b90d4d", + "caa86ea0c24054963dc8c4bc71e7bfb50f7187e74c8b10b6a60b288dbc095039", + "b0dfa45b0a25ea58ba114974c197a994134502180095f58f0af0b2c302135668", + "9d152b82fc524453e5d7449afad8568f4d2a10e7cab9433b3c10c2aa30e795bf", + "616eb5fb6dfc21bf307eb980b37cf3df26b2dcb5af6c8d6b25b183acb23d41e1", + "055b61657aa9288e57b6df1e7be7c12fbb9d609a4a48d9ab5c0e1b18ddcfde94", + "13ef76a4002d12a5fa9d39a4bac78dd34d7656e6a3ccbd33c19172b01149edbb", + "d1013d09a28d1e68c6854cfb78ce00f6e30b295f05faea455c634995c9b9a223", + "e6779a867db2a900b2f05c0732c23fcacb7ac049a8affd2719f561dbc044bd75", + "ecf5c9edce6c806f9fafbf777cbe0730517ecb7f29766f06e930cbba738cbf6b", + "ae608386c7f373ac45bd543134d38c047c1e91b78ef12808939a0d2479f13f4a", + "d596985494554885e96290fec82a8f76767a81b0fdc91794e7e0770bfb698d4e", + "99818e5aa6c24da871d39cc67f1c76d5983a4bc24a0bb5cc84318d56569d4b12", + "b3017d39e2e615d9285e1ade62f55a95cf34011050385dfa89238877294e3f95", + "0d6421a3f37768ee1651a47fea3e682ec486a4fcd37e991a8c90298f0bdb2a01", + "202931e20f273633d412500609a8ab0560f5f70a7d5de2f3fe1902102066e42e", + "20dec413965efdf45af6977f5d6d09292bc8b47860b02309b4f4cd39bbdb166f", + "3c9f167638c8d25767d86718d36fe102f46e94bfd61b23744ee292cdd82d83a5", + "dc8b4c2a8c8f2999d41958a1537a3419553cc2d0a27c4d711a73b39a2b985224", + "d809f477a6f33826e5b88329a30f94f0993181e7dd04510bc3bd95699b44abbe", + "770db4bf3d8d8095b5c9ef101af5fa4d855aa1639aca1cf6012786ddb5af34bd", + "e9aa47eba15104c4f5ca47cf25642b1f0f5e200717f13ae560239f65c86d3235", + "2ea42440d3aefa1bc2cec0a6bb17afec153b8691e996b65586ad08ac5dae127d", + "1ab4c13e9ffc5fe76385ae81de75ca5210d1c7039dc330423fc0c6aa081585be", + "39c40cc7d7a70dd5790ee03385445c58e4f8560b0386340bd942cfa3c499b891", + "21af52b137d9d4bfc41e42c98ac85363c6f15400425a032073c04f288d27ceab", + "7359125c65ff29197675dece98311f4776e79cca7fa43d37912461cf29632507", + "92f43264210f5ce485de8165f6d4d5d1406ab46dd9c29e473d13484e722403c3", + "5fd4dbb664bb4d6e53a0c354dfffec9d6a9ce6dca4d5a752d8512f5a36b3af0b", + "a59827a6c5e914e19aa91d2a9f21a6375eae1ff9965fdda2ef3121a0e6e9854a", + "8720af39d24df290fa29daff967eb99310f723973328647eee77c7d8d26b9c05", + "874a4394ca7349c4217d7db328d159953573b463b515c5f32bcd676ffb78100b", + "d26d2bcb680e94a9666875a10bae57870d067d2fef1a705001828ed266d5b7b5", + "4e6efe75ac68f22f3826f783f3fbd3522115a4d71e6e8dffed0b4edea8a3bfab", + "44d33ddd1ab529d057ce2a43e3744696012ef2a8b64e7e4c46303b480900cc7f", + "89fd9e9581d0711b22b708b5413a3a9e8783569638673a3ef1f71d199b4a7d01", + "491b8e256686fbb0781dff826d50c60a8c256482b5f636aed344d9417302cb57", + "601fe14970c210ba215aad000d85313d8530e25143ca42dac53215c9d93bd9c1", + "3b2398a8d35a8969f148b1309b41a148e929a1381272f295800d63976d11a7bd", + "8df2aabfb952b5d2bcc0d83ac465416d80a66ff74442c66a4018b83d1a065764", + "f1b6a97887d3c46b35b2cb87d246aa69ff7f4a3dda115e8f677cc610b26e9b66", + "f9e1fc6c1d69f3dce44ed08f5e1224688b0c2669730eb9eff470a9f8b69bb17f", + "2d22a5b82491d055b8b1ddb863daf6cd635f144df51caadb3a19e6566b00b1a0", + "065ed276106f8e695c1292ea390c676eab49fa87d9a6a8858793d3900419189e", + "23b6056070626a95ed1e12526def837f4502bb629ac8e9026eab5663260affea", + "693b01b98eb90159f4a24bb24269e0e9bcdd25200f80c1fdf6289f309d183d73", + "7a08de264eaaf2648e8d9a9a5988bb9b0955d6289db7deb255f66aa7881fe3c2", + "f9863b9c713d346eeffa0b75303823019e1dde7e75e04c525b57e052ffb8a35d", + "57cce88eee41ab037f7026ef35dc9568572c32c0a8e8ceb1b6a804a9be1bd602", + "4691460e1044aec091554541e2043980a7756fcd73debac9d22bf702bb66dd58", + "e0542d505897b1f4000eb29d793c7d3589ae4c620a2425c063fbc7bdc07673c2", + "34044c976aa121247c8f4507f90d41d35b06a510b4b83486b11c105cad1ef357", + "1e6e435497f7a9aa3c289534310023b6332b903e99f21d63263385659c2c52ec", + "55053ca30efbf2d225b0ab1b5dd1ee4f7312de3d0c5eadfe623e87f185e845a2", + "6852e1458f4b174f6a965cf92d5a133c65ec73600245601a70a382968e7518af", + "2e2abd23faa65dd73c8ba5ce3ff7141b230e24f1e74f78e8359dbe809729dd2b", + "a7e626b232f10a118015a32cf4b7702cdf1c1c7bee0855ab8dc62770e6849bfb", + "77234a7b5cdc1660dba0802f91beefaa4db8a6b4a5c314e48e15334fdd947224", + "8b9bd91ec044e36d363a90f1ea2fd2ecc38fa5990dae0c21027df006e77e3995", + "d303d01c36eb51b1db45b306fa092f9c8fc1f2d506d600abb9759106aec8f7cd", + "a591e88848138b4d37de7c8b8e25502b8126de7a2bb43f9b5df8d9a72d1423a9", + "0c225c75d688a103022e6255908c0462b8c08c17734af41fe354473a249b32b5", + "3c1c71a7ab428cb21ba9d5ddf41c6f66aecf77c8748ceacbf8e5eed6e20b9b16", + "656367d0d5e186e6d3be064bdfa3e11d4c9913f73b2ec67db639f3fef47cb2e5", + "8bc27e4b6473edfa003a73453936e9c97ff0ca86d6980259b3b029201dc88d88", + "bcf8c6150262b388f6ff04635bab309a7e59142875f34edb5b1505b349e998f5", + "db4cd63aada9c790a36b7171e220b8ed5b71e8e048b2f82540fec6029ff2c971", + "29cc2cf12b0a77e55e8077c532b84a340a672fa68c3d78eba994642bb3edfa2e", + "433b9d966940b90d1ba8395d055618f410141e142a71536eb90359bc9b93be28", + "1bae0e820264f23e1fde5ada356b221dee69fab02cd22345ddec21a52c69429b", + "07341d6bf64234c320a53c80f3d77e9884b6e8cd48c3a67849b281c2416ab056", + "07d4f32877754b9fa1ba33a2a327586b4c1a13b9ed70cf4d4be4d2ef98a71b05", + "4172d67c6ebcde89b0dbb243ad8de9e4286db297a04e0442cc89c84384ee9a50", + "ab713d202baaa40659786805d785b0e811119e061faf9d8c48269cc39e8c879c", + "fe493074a7a4f5891dc9d4860e8a093e04d9900e7db86d7a3bf13e2fbaef1162", + "aa485c364455af5f9883d1756e766284cf3bc5d9a38865ba0467e4e1651d578e", + "796b7756d683df21b41cb9179787417bbd3470295ec258156ce17717e7fa0310", + "5fe58d7d6cf789accc366c7cef8ce6567ab9c5f3892fb2517c1553c46c94633c", + "12f062118d475e68989ede947700c703ed61af565a9f76dd6804cdb9abf7ae4b", + "9f7e2cc8a5ce9f79ac7d8f4110fa1d03a8df8d16c572703125db2d41127952a9", + "fb4235d5327e3358c0cda0528308068650ca5d0a1cc828abf4b2af048c48da69", + "40576bb94b8bb1ac7c2ff4b8f7d03bb3063527846856184f1a7e92a69ff090a6", + "5b3da07b8bd356ed5dfc93669dd980205e89e94eaa21f8aa19feb67f95056ac4", + "8922880ebeee41ec9664e665470a6f8972ae25d83a2b2de17e5c7a42cd814f45", + "8d07bf462ebe371c812348a11a29dc1685d21df18875f1fe2d00f1e789cbad30", + "3c6a99159ee9e6283e966ec7347de5d673dc6dea78d03f2510339920a9e20b78" + ], + "knownInPlaceRewrites": [ + { + "line": 1, + "sourceRecordId": "bls.ces.total_nonfarm_payroll_change.may_2026.first_print", + "rewriteCommit": "816e9a3ec51d71a5d2677e673fff033336214dbf" + }, + { + "line": 2, + "sourceRecordId": "bls.cps.unemployment_rate.may_2026.first_print", + "rewriteCommit": "816e9a3ec51d71a5d2677e673fff033336214dbf" + }, + { + "line": 3, + "sourceRecordId": "bls.ces.average_hourly_earnings_private.may_2026.first_print", + "rewriteCommit": "816e9a3ec51d71a5d2677e673fff033336214dbf" + }, + { + "line": 4, + "sourceRecordId": "statcan.lfs.unemployment_rate.canada.may_2026.first_print", + "rewriteCommit": "816e9a3ec51d71a5d2677e673fff033336214dbf" + }, + { + "line": 5, + "sourceRecordId": "statcan.lfs.employment_change.canada.may_2026.first_print", + "rewriteCommit": "816e9a3ec51d71a5d2677e673fff033336214dbf" + }, + { + "line": 14, + "sourceRecordId": "bls.cpi.u.headline_mom.may_2026.first_print", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 15, + "sourceRecordId": "bls.cpi.u.core_mom.may_2026.first_print", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 16, + "sourceRecordId": "bank_of_canada.overnight_rate.after_june_2026", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 18, + "sourceRecordId": "dol.eta.initial_claims.sa.week_ending_2026_06_06", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 19, + "sourceRecordId": "ecb.deposit_facility_rate.after_june_2026", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 20, + "sourceRecordId": "bls.ppi.final_demand_monthly_change.may_2026.first_print", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 21, + "sourceRecordId": "statcan.building_permits.total_value_mom.canada.april_2026.first_print", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + }, + { + "line": 22, + "sourceRecordId": "ons.gdp.monthly_growth.april_2026.first_print", + "rewriteCommit": "6c9209434048317c15355f42e019339efbd0d6ea" + } + ] +} diff --git a/scripts/canonical_json.py b/scripts/canonical_json.py new file mode 100644 index 0000000..a20a78f --- /dev/null +++ b/scripts/canonical_json.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Canonical JSON compatible with site/src/data/canonical-json.ts. + +Object keys are sorted by UTF-16 code units, not Unicode code points. Python's +normal string ordering differs for astral-plane keys, so callers must use the +key below rather than ``sort_keys=True``. Number formatting follows +ECMAScript JSON.stringify's fixed/scientific thresholds. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from decimal import Decimal +from typing import Any + + +def utf16_sort_key(value: str) -> bytes: + """Return the big-endian UTF-16 code units used by JavaScript ordering.""" + + return value.encode("utf-16-be", errors="surrogatepass") + + +def _serialize_string(value: str) -> str: + pieces = ['"'] + short_escapes = { + "\b": "\\b", + "\t": "\\t", + "\n": "\\n", + "\f": "\\f", + "\r": "\\r", + '"': '\\"', + "\\": "\\\\", + } + for char in value: + if char in short_escapes: + pieces.append(short_escapes[char]) + elif ord(char) < 0x20 or 0xD800 <= ord(char) <= 0xDFFF: + pieces.append(f"\\u{ord(char):04x}") + else: + pieces.append(char) + pieces.append('"') + return "".join(pieces) + + +def _serialize_float(value: float) -> str: + if not math.isfinite(value): + raise ValueError(f"Canonical JSON cannot serialize non-finite number: {value}") + if value == 0: + return "0" + + absolute = abs(value) + rendered = repr(value).lower() + if 1e-6 <= absolute < 1e21: + fixed = format(Decimal(rendered), "f") + if "." in fixed: + fixed = fixed.rstrip("0").rstrip(".") + return fixed + + if "e" not in rendered: + rendered = format(value, ".15e") + mantissa, exponent_text = rendered.split("e", 1) + mantissa = mantissa.rstrip("0").rstrip(".") + exponent = int(exponent_text) + sign = "+" if exponent >= 0 else "" + return f"{mantissa}e{sign}{exponent}" + + +def canonical_stringify(value: Any) -> str: + """Serialize JSON-compatible data exactly like canonicalStringify in TS.""" + + ancestors: set[int] = set() + + def serialize(item: Any) -> str: + if item is None: + return "null" + if item is True: + return "true" + if item is False: + return "false" + if isinstance(item, str): + return _serialize_string(item) + if isinstance(item, int): + if abs(item) > (2**53 - 1): + try: + return _serialize_float(float(item)) + except OverflowError as exc: + raise ValueError( + f"Canonical JSON integer exceeds Number range: {item}" + ) from exc + return str(item) + if isinstance(item, float): + return _serialize_float(item) + + if isinstance(item, (list, dict)): + identity = id(item) + if identity in ancestors: + raise ValueError("Canonical JSON cannot serialize circular structures") + ancestors.add(identity) + try: + if isinstance(item, list): + return "[" + ",".join(serialize(entry) for entry in item) + "]" + entries = [] + for key in sorted(item, key=utf16_sort_key): + if not isinstance(key, str): + raise TypeError("Canonical JSON object keys must be strings") + entries.append(f"{_serialize_string(key)}:{serialize(item[key])}") + return "{" + ",".join(entries) + "}" + finally: + ancestors.remove(identity) + + raise TypeError( + f"Canonical JSON cannot serialize value of type {type(item).__name__}" + ) + + return serialize(value) + + +def canonical_bytes(value: Any) -> bytes: + return canonical_stringify(value).encode("utf-8") + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--sha256", action="store_true", help="print SHA-256 instead of JSON" + ) + args = parser.parse_args() + value = json.load(sys.stdin) + output = canonical_sha256(value) if args.sha256 else canonical_stringify(value) + sys.stdout.write(output + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_thesis_facts_append.py b/scripts/check_thesis_facts_append.py new file mode 100644 index 0000000..77e18bc --- /dev/null +++ b/scripts/check_thesis_facts_append.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Gate every change to the thesis-facts observation ledger. + +The observation file is append-only with an immutable frozen prefix +(``ledger/immutable_prefix.json``). Resolver appends arrive as pull +requests; this checker is the deterministic review each proposal must pass +before merge: + +- the frozen prefix is byte-identical (no rewrite, no truncation); +- against a base ref, the change only appends whole lines; +- every appended row parses, and carries the post-quarantine bindings: + ``assertionVersion`` (content-addressed, recomputed here), ``retrievedAt``, + ``sourceVintage``, ``ledgerRepoSha``, and a ``responseArchive`` digest; +- ``targetContentHash`` and ``sourceBindingProjection`` appear together or + not at all, the projection's response digest matches the archive, and its + unit matches the row's measure unit; +- a duplicate ``source_record_id`` is legal only as an explicit correction: + the later row's ``assertionVersion.supersedes`` must name the version ID + of the row it replaces. + +Usage: + python3 scripts/check_thesis_facts_append.py [--base-ref REF] + +With ``--base-ref`` (CI: the pull request's base commit) the append-only +diff is enforced; without it only the full-file invariants run. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import pathlib +import re +import subprocess +import sys +from typing import Any + +from canonical_json import canonical_sha256 + +ROOT = pathlib.Path(__file__).resolve().parents[1] +LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" +PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" + +ASSERTION_CONTENT_KEYS = ( + "source_record_id", + "value", + "observed_at", + "period", + "geography", + "entity", + "aggregation", + "filters", + "domain", +) + + +class AppendError(ValueError): + """The proposed ledger change violates an append invariant.""" + + +def _lines(text: str) -> list[str]: + return [line for line in text.split("\n") if line.strip()] + + +def expected_assertion_version_id(row: dict[str, Any]) -> str: + """Recompute the content address the resolver must have written. + + Mirrors ``assertion_version`` in the Thesis resolver: the ID commits to + everything that changes what the assertion means, so an in-place edit + is detectable and a correction must supersede explicitly. + """ + measure = row.get("measure") or {} + source = row.get("source") or {} + projection = {key: row.get(key) for key in ASSERTION_CONTENT_KEYS} + projection["measure"] = { + "concept": measure.get("concept"), + "unit": measure.get("unit"), + } + projection["source"] = { + "source_name": source.get("source_name"), + "source_table": source.get("source_table"), + "url": source.get("url"), + "vintage": source.get("vintage"), + } + return f"av1:{canonical_sha256(projection)}" + + +def check_prefix(lines: list[str]) -> dict[str, Any]: + prefix = json.loads(PREFIX_PATH.read_text()) + if prefix.get("schemaVersion") != "thesis_facts_immutable_prefix_v1": + raise AppendError( + f"unsupported prefix manifest schema {prefix.get('schemaVersion')!r}" + ) + count = int(prefix["prefixLineCount"]) + hashes = prefix["lineSha256s"] + if len(hashes) != count: + raise AppendError("prefix manifest line hashes disagree with its count") + if len(lines) < count: + raise AppendError( + f"ledger has {len(lines)} rows but the immutable prefix " + f"requires at least {count}" + ) + for index in range(count): + digest = hashlib.sha256(lines[index].encode("utf-8")).hexdigest() + if digest != hashes[index]: + row_id = json.loads(lines[index]).get("source_record_id", "?") + raise AppendError( + f"immutable prefix line {index + 1} ({row_id}) was rewritten" + ) + joined = hashlib.sha256( + ("\n".join(lines[:count]) + "\n").encode("utf-8") + ).hexdigest() + if joined != prefix["prefixSha256"]: + raise AppendError("immutable prefix cumulative hash mismatch") + return prefix + + +def check_rows(lines: list[str], prefix_count: int) -> None: + versions: dict[str, int] = {} + active_by_record_id: dict[str, tuple[int, str | None]] = {} + for number, line in enumerate(lines, start=1): + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise AppendError(f"line {number} is not valid JSON: {exc}") from exc + if not isinstance(row, dict): + raise AppendError(f"line {number} is not a JSON object") + record_id = row.get("source_record_id") + if not record_id: + raise AppendError(f"line {number} lacks source_record_id") + if not isinstance(row.get("value"), (int, float)): + raise AppendError(f"line {number} ({record_id}) has no numeric value") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(row.get("observed_at", ""))): + raise AppendError(f"line {number} ({record_id}) has no observed_at date") + unit = (row.get("measure") or {}).get("unit") + if not unit: + raise AppendError(f"line {number} ({record_id}) has no measure unit") + + version = row.get("assertionVersion") + version_id = None + supersedes = None + if version is not None: + if not isinstance(version, dict): + raise AppendError(f"line {number} assertionVersion is not an object") + version_id = str(version.get("id", "")) + supersedes = version.get("supersedes") + expected = expected_assertion_version_id(row) + if version_id != expected: + raise AppendError( + f"line {number} ({record_id}) assertionVersion.id does not " + f"match its content ({version_id} != {expected})" + ) + if version_id in versions: + raise AppendError( + f"line {number} restates assertion version {version_id} " + f"from line {versions[version_id]}" + ) + versions[version_id] = number + + if number > prefix_count: + for field in ( + "retrievedAt", + "sourceVintage", + "ledgerRepoSha", + "responseArchive", + "assertionVersion", + ): + if not row.get(field): + raise AppendError( + f"appended line {number} ({record_id}) lacks {field}" + ) + archive = row["responseArchive"] + if not isinstance(archive, dict) or not archive.get("sha256"): + raise AppendError( + f"appended line {number} responseArchive lacks a digest" + ) + has_hash = bool(row.get("targetContentHash")) + projection = row.get("sourceBindingProjection") + if has_hash != bool(projection): + raise AppendError( + f"appended line {number} ({record_id}) must carry " + "targetContentHash and sourceBindingProjection together" + ) + if projection: + if projection.get("responseSha256") != archive.get("sha256"): + raise AppendError( + f"appended line {number} projection digest does not " + "match its archived response" + ) + if projection.get("unit") != unit: + raise AppendError( + f"appended line {number} projection unit " + f"{projection.get('unit')!r} contradicts the row unit " + f"{unit!r}" + ) + + previous = active_by_record_id.get(str(record_id)) + if previous is not None: + previous_line, previous_version = previous + if supersedes is None: + raise AppendError( + f"line {number} duplicates {record_id} (line " + f"{previous_line}) without superseding an assertion " + "version — corrections must be explicit" + ) + if supersedes != previous_version: + raise AppendError( + f"line {number} supersedes {supersedes} but the active " + f"version of {record_id} is {previous_version}" + ) + elif supersedes is not None: + raise AppendError( + f"line {number} supersedes {supersedes} but {record_id} has " + "no earlier row" + ) + # Rows that predate explicit versioning are still addressable: their + # version is the content address a correction must recompute. + active_by_record_id[str(record_id)] = ( + number, + version_id or expected_assertion_version_id(row), + ) + + +def check_append_only(base_ref: str, lines: list[str]) -> int: + relative = LEDGER_PATH.relative_to(ROOT).as_posix() + try: + base_text = subprocess.check_output( + ["git", "show", f"{base_ref}:{relative}"], cwd=ROOT, text=True + ) + except subprocess.CalledProcessError as exc: + raise AppendError(f"cannot read {relative} at base {base_ref}") from exc + base_lines = _lines(base_text) + if len(lines) < len(base_lines): + raise AppendError( + f"change truncates the ledger: {len(base_lines)} -> {len(lines)} rows" + ) + for index, line in enumerate(base_lines): + if lines[index] != line: + row_id = json.loads(line).get("source_record_id", "?") + raise AppendError( + f"change rewrites existing line {index + 1} ({row_id}); " + "the ledger is append-only — supersede instead" + ) + return len(lines) - len(base_lines) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-ref", + help="enforce an append-only diff against this git ref", + ) + args = parser.parse_args() + lines = _lines(LEDGER_PATH.read_text(encoding="utf-8")) + try: + prefix = check_prefix(lines) + check_rows(lines, int(prefix["prefixLineCount"])) + appended = None + if args.base_ref: + appended = check_append_only(args.base_ref, lines) + except AppendError as exc: + print(f"thesis-facts append check failed: {exc}", file=sys.stderr) + return 1 + suffix = f", +{appended} appended vs base" if appended is not None else "" + print( + f"thesis-facts append check OK: {len(lines)} rows, immutable prefix " + f"{prefix['prefixLineCount']}{suffix}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_policyengine_ledger.py b/tests/test_policyengine_ledger.py index 089ec22..ec6b1a0 100644 --- a/tests/test_policyengine_ledger.py +++ b/tests/test_policyengine_ledger.py @@ -1,9 +1,17 @@ -"""Tests for checked-in PolicyEngine Ledger observation facts.""" +"""Invariants for the checked-in PolicyEngine Ledger observation facts. + +The old version of this file hard-coded a closed inventory of 49 +source_record_ids, went stale at the first append, and was dropped from CI. +These tests assert the properties that must hold at ANY row count instead: +the frozen prefix is immutable, every row is a valid publisher fact, and +duplicate identities only exist as explicit supersede corrections. +""" from __future__ import annotations import json -import re +import subprocess +import sys from pathlib import Path from arch.core import ( @@ -17,71 +25,48 @@ validate_facts, ) - ROOT = Path(__file__).resolve().parents[1] LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" +PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" -EXPECTED_SOURCE_RECORD_IDS = { - "bank_of_canada.overnight_rate.after_june_2026", - "bls.ces.total_nonfarm_payroll_change.may_2026.first_print", - "bls.ces.average_hourly_earnings_private.may_2026.first_print", - "bls.cpi.u.core_mom.may_2026.first_print", - "bls.cpi.u.headline_mom.may_2026.first_print", - "bls.cps.unemployment_rate.may_2026.first_print", - "bls.import_price_index.all_imports_mom.may_2026.first_print", - "bls.ppi.final_demand_monthly_change.may_2026.first_print", - "boe.bank_rate.2026-06-18", - "boe.bank_rate.after_mpc_june_2026.first_print", - "boj.policy_rate_guideline.after_june_2026", - "census.housing_starts.saar.may_2026.first_print", - "census.marts.adv44x72.may_2026.monthly_change.advance", - "census.mtis.total_business_inventories_level.april_2026.first_print", - "cms.medicaid_pi.beneficiaries_renewed_total.california.feb_2026.original_submission", - "cms.medicaid_pi.beneficiaries_renewed_ex_parte.california.feb_2026.original_submission", - "cms.medicaid_pi.beneficiaries_disenrolled_total.california.feb_2026.original_submission", - "cms.medicaid_pi.beneficiaries_disenrolled_procedural.california.feb_2026.original_submission", - "dol.eta.initial_claims.sa.week_ending_2026_06_06", - "ecb.deposit_facility_rate.after_june_2026", - "estat.jp.cpi.core_exfreshfood.yoy.2026-05", - "eurostat.hicp.all_items_annual_rate.euro_area.may_2026.final_first_print", - "eurostat.industrial_production.euro_area.april_2026.first_print", - "fed.g17.capacity_utilization.total_industry.may_2026.first_print", - "fed.g17.industrial_production.total_index_mom.may_2026.first_print", - "fns.snap.application_processing_timeliness.california.fy2024.official_release", - "fns.snap.overpayment_payment_error_rate.us.fy2024.official_release", - "fns.snap.total_payment_error_rate.us.fy2024.official_release", - "fns.snap.underpayment_payment_error_rate.us.fy2024.official_release", - "ons.cpi.annual_rate.may_2026.first_print", - "ons.cpih.annual_rate.2026-05", - "ons.gdp.monthly_growth.april_2026.first_print", - "ons.hmrc.paye_payrolled_employees.may_2026.first_print", - "ons.labour.unemployment_rate.february_to_april_2026.first_print", - "ons.pusf.j5ii.public_sector_net_borrowing_ex_banks.may_2026.first_print", - "ons.retail_sales.volume_mom.may_2026.first_print", - "rba.cash_rate_target.after_june_2026", - "statcan.building_permits.total_value_mom.canada.april_2026.first_print", - "statcan.employment_insurance.regular_beneficiaries.canada.april_2026.first_print", - "statcan.lfs.employment_change.canada.may_2026.first_print", - "statcan.lfs.unemployment_rate.canada.may_2026.first_print", - "statcan.retail_trade.sales_mom.canada.april_2026.first_print", - "statcan.wholesale_trade.sales_mom_exclusions.canada.april_2026.first_print", - "statjp.cpi.all_items_annual_rate.japan.may_2026.first_print", - "treasury.mts.monthly_deficit.may_2026.first_print", - "us.census.housing_starts.total_saar.2026-05", - "us.dol.initial_claims.sa.week_2026-06-13", - "us.fed.fomc.target_range_upper.2026-06", - "us.frb.industrial_production.total.mom_sa.2026-05", -} +sys.path.insert(0, str(ROOT / "scripts")) +from check_thesis_facts_append import ( # noqa: E402 + check_prefix, + check_rows, + expected_assertion_version_id, +) -def _read_ledger_facts() -> list[dict]: + +def _read_lines() -> list[str]: return [ - json.loads(line) - for line in LEDGER_PATH.read_text(encoding="utf-8").splitlines() + line + for line in LEDGER_PATH.read_text(encoding="utf-8").split("\n") if line.strip() ] +def _read_ledger_facts() -> list[dict]: + return [json.loads(line) for line in _read_lines()] + + +LEDGER_ROW_KEYS = { + "label", + "observed_at", + "source_record_id", + "source_cell_keys", + "source_row_keys", + # Resolution provenance attached by the Thesis resolver. + "ledgerRepoSha", + "sourceVintage", + "retrievedAt", + "responseArchive", + "targetContentHash", + "sourceBindingProjection", + "assertionVersion", +} + + def _to_aggregate_fact(row: dict) -> AggregateFact: return AggregateFact( value=row["value"], @@ -100,21 +85,29 @@ def _to_aggregate_fact(row: dict) -> AggregateFact: ) -def test_official_observation_ledger_has_expected_rows(): - rows = _read_ledger_facts() +def test_immutable_prefix_is_intact(): + lines = _read_lines() - assert {row["source_record_id"] for row in rows} == EXPECTED_SOURCE_RECORD_IDS - assert len(rows) == len(EXPECTED_SOURCE_RECORD_IDS) + prefix = check_prefix(lines) + assert prefix["prefixLineCount"] >= 128 + assert len(lines) >= prefix["prefixLineCount"] + + +def test_every_row_satisfies_the_append_invariants(): + lines = _read_lines() + prefix = check_prefix(lines) + + # Raises on any malformed row, unexplained duplicate identity, + # mis-addressed assertion version, or missing post-prefix binding. + check_rows(lines, int(prefix["prefixLineCount"])) -def test_official_observation_ledger_contains_facts_not_predictions(): - rows = _read_ledger_facts() - for row in rows: +def test_official_observation_ledger_contains_facts_not_predictions(): + for row in _read_ledger_facts(): assert "prediction" not in json.dumps(row).lower() assert "forecast" not in json.dumps(row).lower() assert row["source_record_id"] - assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", row["observed_at"]) assert row["source"]["url"].startswith("https://") assert row["source"]["vintage"] @@ -125,22 +118,102 @@ def test_official_observations_validate_as_aggregate_facts(): report = validate_facts(facts) assert report.valid, report.to_dict() - assert report.counts["by_source"] == { - "bank_of_canada": 1, - "bls": 7, - "boe": 2, - "boj": 1, - "census": 4, - "cms": 4, - "dol": 2, - "ecb": 1, - "eurostat": 2, - "fed": 4, - "fns": 4, - "ons": 7, - "rba": 1, - "statcan": 6, - "statjp": 2, - "treasury": 1, - } assert report.counts["missing_lineage"]["count"] == 0 + + +def test_rows_carry_no_unknown_top_level_fields(): + known = set(AggregateFact.__dataclass_fields__) | LEDGER_ROW_KEYS + for number, row in enumerate(_read_ledger_facts(), start=1): + unknown = set(row) - known + assert not unknown, f"line {number} has unknown fields: {sorted(unknown)}" + + +def test_a_rewritten_prefix_line_is_detected(tmp_path): + lines = _read_lines() + row = json.loads(lines[0]) + row["value"] = 999999 + tampered = [json.dumps(row, separators=(",", ":")), *lines[1:]] + ledger_dir = tmp_path / "ledger" + ledger_dir.mkdir() + (ledger_dir / "official_observations.jsonl").write_text( + "\n".join(tampered) + "\n" + ) + (ledger_dir / "immutable_prefix.json").write_text(PREFIX_PATH.read_text()) + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + for name in ("check_thesis_facts_append.py", "canonical_json.py"): + (scripts_dir / name).write_text((ROOT / "scripts" / name).read_text()) + + completed = subprocess.run( + [sys.executable, str(scripts_dir / "check_thesis_facts_append.py")], + capture_output=True, + text=True, + ) + + assert completed.returncode == 1 + assert "was rewritten" in completed.stderr + + +def _appended_row(original: dict, *, value_delta: float = 0) -> dict: + row = { + **original, + "value": original["value"] + value_delta, + "retrievedAt": "2026-07-11T00:00:00Z", + "sourceVintage": "2026-07-11", + "ledgerRepoSha": "a" * 40, + "responseArchive": {"sha256": "b" * 64, "contentEncoding": "gzip"}, + } + row.pop("targetContentHash", None) + row.pop("sourceBindingProjection", None) + return row + + +def test_a_duplicate_identity_without_supersedes_is_detected(): + lines = _read_lines() + duplicate = _appended_row(json.loads(lines[-1])) + duplicate["assertionVersion"] = { + "id": expected_assertion_version_id(duplicate), + "supersedes": None, + } + try: + check_rows( + [*lines, json.dumps(duplicate, separators=(",", ":"))], len(lines) + ) + except ValueError as error: + assert "without superseding" in str(error) + else: + raise AssertionError("duplicate identity was accepted silently") + + +def test_an_explicit_supersede_correction_is_accepted(): + # A correction of a pre-versioning row supersedes that row's + # recomputable content address. + lines = _read_lines() + original = json.loads(lines[-1]) + correction = _appended_row(original, value_delta=1) + correction["assertionVersion"] = { + "id": expected_assertion_version_id(correction), + "supersedes": expected_assertion_version_id(original), + } + + check_rows( + [*lines, json.dumps(correction, separators=(",", ":"))], len(lines) + ) + + +def test_a_correction_naming_a_stale_version_is_rejected(): + lines = _read_lines() + original = json.loads(lines[-1]) + correction = _appended_row(original, value_delta=1) + correction["assertionVersion"] = { + "id": expected_assertion_version_id(correction), + "supersedes": "av1:" + "0" * 64, + } + try: + check_rows( + [*lines, json.dumps(correction, separators=(",", ":"))], len(lines) + ) + except ValueError as error: + assert "the active version" in str(error) + else: + raise AssertionError("stale supersedes target was accepted")