Skip to content

feat: contract the replay vocabulary and fix the allowlist removal path #475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
guangyu-reflexio merged 3 commits into from
Sep 1, 2026
Merged

feat: contract the replay vocabulary and fix the allowlist removal path #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca35a4b
feat(playbook): retire the replay optimizer vocabulary
guangyu-reflexio Aug 31, 2026
ed2c091
fix(offline-tuner): close three Phase 7 review findings in the OSS pa…
guangyu-reflexio Sep 1, 2026
fe85c8b
fix(storage): require every admissible optimizer kind, not a sample
guangyu-reflexio Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 37 additions & 9 deletions reflexio/models/api_schema/domain/entities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,9 +409,13 @@ class AgentPlaybook(BaseModel):
superseded_by: int | None = None


# 'offline_tuner_legacy' and 'optimizer_legacy_unknown' were never written by
# running code. They were assigned to pre-existing rows by a one-time heuristic
# backfill (supabase/data/tenant/20260723000000:54-61) and are retained
# permanently as HISTORICAL labels: dropping them would make those rows violate
# the tenant CHECK and abort the contract migration.
OptimizerKind = Literal[
"gepa",
"offline_tuner_replay",
"offline_tuner_open_world",
"offline_tuner_legacy",
"optimizer_legacy_unknown",
Expand All @@ -425,30 +429,55 @@ class AgentPlaybook(BaseModel):
"evidence_frozen",
"discovery_analyzed",
"candidate_generated",
"replay_running",
"replay_evaluated",
"held_out_analyzed",
"publishing",
"applied",
"abstained",
"failed",
]

# Phase 7 removed the four members whose NAMES contain 'replay'. Seven more were
# reachable only through that same replay arm and are now equally dead: no
# writer, no reader, no tenant routine that can set them. They are RETAINED
# rather than removed, the way 'offline_tuner_legacy' is retained above -- and
# for the same two reasons. Removing them narrows the tenant CHECK a second
# time, and a CHECK narrowing is a one-way door this branch has already spent
# once; and historical rows written before the replay retirement still carry
# them, so a narrowed CHECK would abort the validating migration on the first
# organization holding one.
#
# Read a member of this set as "an outcome an earlier era could record", never
# as a state the current tuner can reach. Pinned by
# tests/models/test_terminal_outcome_reachability.py so it cannot silently grow:
# a new member added here is a new outcome somebody must show is WRITABLE.
#
# 'deployment_unsupported' is the trap. The same spelling is also
# OfflineTunerUnavailableReason -- a config-enablement rejection code in
# reflexio_ext capability_status.py, with ~40 live references. Those are a
# different vocabulary on a different type, so "grep says it is used" does not
# make this member reachable.
RETAINED_UNREACHABLE_TERMINAL_OUTCOMES: frozenset[str] = frozenset(
{
"insufficient_negative_evidence",
"insufficient_positive_evidence",
"insufficient_coverage",
"deployment_unsupported",
"candidate_regressed",
"candidate_did_not_improve",
"publication_failed",
}
)

OptimizationTerminalOutcome = Literal[
"applied",
"insufficient_negative_evidence",
"insufficient_positive_evidence",
"insufficient_coverage",
"replay_unsupported",
"deployment_unsupported",
"incomplete_replay_scope",
"insufficient_replay_cases",
"replay_inconclusive",
"candidate_regressed",
"candidate_did_not_improve",
"incumbent_changed",
"generation_failed",
"replay_failed",
"publication_failed",
"governance_erased",
"no_grounded_hypothesis",
Expand All @@ -462,7 +491,6 @@ class AgentPlaybook(BaseModel):
OptimizationArtifactKind = Literal[
"expected_population_manifest",
"generation_selection",
"replay_manifest",
"candidate",
"candidate_search_projection",
"open_world_evidence_bundle",
Expand Down
24 changes: 16 additions & 8 deletions reflexio/server/services/playbook/publication.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,20 @@
"confirmed_online_support",
}
)
PublishableOptimizerKind = Literal[
"gepa", "offline_tuner_replay", "offline_tuner_open_world"
]
PublishableOptimizerKind = Literal["gepa", "offline_tuner_open_world"]
# 'offline_optimizer' remains in the union because it is a value already
# PERSISTED on user_playbooks.source rows. Phase 7 removed the only optimizer
# kind that produced it, and Task 10 took the tenant RPC's matching CASE arm
# with it: commit_user_playbook_publication now writes the staged optimizer kind
# straight through to user_playbooks.source (20260830020000). No path can write
# 'offline_optimizer' any more -- it is read-only history.
PublicationSource = Literal["gepa", "offline_optimizer"]

_LEGACY_PUBLICATION_OPTIMIZERS = frozenset({"gepa", "offline_tuner_replay"})
_DECISION_PROOF_OPTIMIZERS = frozenset(
{"gepa", "offline_tuner_replay", "offline_tuner_open_world"}
)
# Phase 7 retired 'offline_tuner_replay'. The legacy publication path is now
# GEPA-only: the open-world path publishes through its own provisional
# publisher, not through publication_source_for_optimizer.
_LEGACY_PUBLICATION_OPTIMIZERS = frozenset({"gepa"})
_DECISION_PROOF_OPTIMIZERS = frozenset({"gepa", "offline_tuner_open_world"})
_PROJECTION_SCHEMA_VERSION = "offline-tuner-candidate-search-projection-v1"
_USER_PLAYBOOK_FULL_VERSION_SCHEMA = "user-playbook-full-version-v1"
_CANONICAL_DECIMAL = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]*[1-9])?\Z")
Expand Down Expand Up @@ -183,8 +188,11 @@ def _validate_decision_proof_optimizer(value: object) -> None:
def publication_source_for_optimizer(
optimizer_kind: OptimizerKind,
) -> PublicationSource:
# The 'offline_optimizer' arm was reachable only for 'offline_tuner_replay',
# which Phase 7 retired. _validate_legacy_publication_optimizer now admits
# 'gepa' alone, so the branch is gone rather than left dead.
_validate_legacy_publication_optimizer(optimizer_kind)
return "offline_optimizer" if optimizer_kind != "gepa" else "gepa"
return "gepa"


def incumbent_user_playbook_semantic_digest(
Expand Down
131 changes: 107 additions & 24 deletions reflexio/server/services/storage/sqlite_storage/_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,23 @@
_TRAJECTORY_FETCH_SIZE = 256
_SESSION_OUTCOME_MIGRATION_BATCH_SIZE = 256
_MINIMUM_SQLITE_VERSION = (3, 35, 0)
# Literals Phase 7 retired. The rebuild's trigger asks TWO questions, not one:
# "is every required literal present?" (the expand direction, which was all it
# ever asked) AND "is any retired literal still present?" (the contract
# direction). Without the second, an existing database whose table_sql still
# holds every SURVIVING literal satisfies the first and returns early -- the
# rebuild never runs and the permissive CHECK survives forever.
_RETIRED_OPTIMIZER_JOB_LITERALS: tuple[str, ...] = (
"'offline_tuner_replay'",
"'replay_running'",
"'replay_evaluated'",
"'replay_unsupported'",
"'incomplete_replay_scope'",
"'insufficient_replay_cases'",
"'replay_inconclusive'",
"'replay_failed'",
)
_RETIRED_ARTIFACT_KIND_LITERALS: tuple[str, ...] = ("'replay_manifest'",)
_SQLITE_INITIALIZATION_LOCK_STRIPES = 64
_sqlite_initialization_locks = tuple(
threading.Lock() for _ in range(_SQLITE_INITIALIZATION_LOCK_STRIPES)
Expand Down Expand Up @@ -2197,10 +2214,21 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
if table_sql_row is None:
return
table_sql = table_sql_row["sql"]
# Every literal the FINAL constraints admit must appear, not just a
# representative few. A predicate that names some of them cannot tell
# "current" from "missing a kind": a schema whose optimizer_kind CHECK
# omits 'gepa' carries no retired literal, satisfies a partial list,
# and is retained forever with a constraint that rejects legitimate
# rows. The artifact rebuild below already enumerates its full set;
# this is that same rule.
required_checks = (
"CHECK (optimizer_kind IN",
"CHECK (stage IS NULL OR stage IN",
"CHECK (terminal_outcome IS NULL OR terminal_outcome IN",
"'gepa'",
"'offline_tuner_open_world'",
"'offline_tuner_legacy'",
"'optimizer_legacy_unknown'",
"'governance_erased'",
"'discovery_analyzed'",
"'held_out_analyzed'",
Expand All @@ -2211,8 +2239,8 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
"'governance_invalidated'",
"'infrastructure_failure'",
)
if all(check in table_sql for check in required_checks) and (
"'offline_tuner_open_world'" in table_sql
if all(check in table_sql for check in required_checks) and not any(
retired in table_sql for retired in _RETIRED_OPTIMIZER_JOB_LITERALS
):
return
foreign_keys_enabled = bool(
Expand All @@ -2235,7 +2263,6 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
optimizer_kind TEXT NOT NULL DEFAULT 'optimizer_legacy_unknown'
CHECK (optimizer_kind IN (
'gepa',
'offline_tuner_replay',
'offline_tuner_open_world',
'offline_tuner_legacy',
'optimizer_legacy_unknown'
Expand All @@ -2256,8 +2283,6 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
'evidence_frozen',
'discovery_analyzed',
'candidate_generated',
'replay_running',
'replay_evaluated',
'held_out_analyzed',
'publishing',
'applied',
Expand All @@ -2269,16 +2294,11 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
'insufficient_negative_evidence',
'insufficient_positive_evidence',
'insufficient_coverage',
'replay_unsupported',
'deployment_unsupported',
'incomplete_replay_scope',
'insufficient_replay_cases',
'replay_inconclusive',
'candidate_regressed',
'candidate_did_not_improve',
'incumbent_changed',
'generation_failed',
'replay_failed',
'publication_failed',
'governance_erased',
'no_grounded_hypothesis',
Expand All @@ -2299,6 +2319,28 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
)
"""
)
# Phase 7 review finding P4-2. The pre-existing legacy sweep in
# _classify_legacy_playbook_optimization_jobs retires every ACTIVE
# 'offline_tuner_legacy' / 'optimizer_legacy_unknown' job, but it
# runs BEFORE this relabel. Without this statement the relabel below
# turns a running replay job into a running 'offline_tuner_legacy'
# job -- a kind with no producer or consumer -- which then holds a
# uq_poj_active_target / uq_poj_active_discovery / uq_poj_active_attempt
# slot until the next storage open lets the sweep see it. Retire it
# here with exactly the status, reason and lease clearing that sweep
# uses, so one open is enough.
self.conn.execute(
"""
UPDATE playbook_optimization_jobs
SET status = 'skipped',
decision_reason = 'retired_by_replay_redesign',
lease_owner = NULL,
lease_expires_at = NULL,
updated_at = CAST(strftime('%s', 'now') AS INTEGER)
WHERE optimizer_kind = 'offline_tuner_replay'
AND status IN ('pending', 'running')
"""
)
self.conn.execute(
"""
INSERT INTO playbook_optimization_jobs_new (
Expand All @@ -2311,10 +2353,27 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
candidate_content_digest, search_projection_digest,
publication_scope_digest, created_at, updated_at
) SELECT
job_id, optimizer_kind, target_kind, target_id, status,
job_id,
-- Phase 7 remediation. The copy would otherwise fail the new
-- CHECK on any row still carrying a retired literal.
-- 'offline_tuner_legacy' is the accurate surviving label: these
-- jobs WERE offline-tuner jobs, and OQ-1 option A keeps that
-- literal admissible. stage and terminal_outcome are nullable,
-- so their retired values become NULL rather than a wrong one.
CASE WHEN optimizer_kind = 'offline_tuner_replay'
THEN 'offline_tuner_legacy'
ELSE optimizer_kind END,
target_kind, target_id, status,
best_candidate_id, successor_target_id, decision_reason,
metadata_json, discovery_key, attempt_key, lease_owner,
lease_fence, lease_expires_at, stage, terminal_outcome,
lease_fence, lease_expires_at,
CASE WHEN stage IN ('replay_running', 'replay_evaluated')
THEN NULL ELSE stage END,
CASE WHEN terminal_outcome IN (
'replay_unsupported', 'incomplete_replay_scope',
'insufficient_replay_cases', 'replay_inconclusive',
'replay_failed'
) THEN NULL ELSE terminal_outcome END,
expected_population_manifest_digest,
generation_selection_manifest_digest, replay_manifest_digest,
candidate_content_digest, search_projection_digest,
Expand Down Expand Up @@ -2379,15 +2438,18 @@ def _enforce_playbook_optimization_artifact_constraints(self) -> None:
artifact_kinds = (
"'expected_population_manifest'",
"'generation_selection'",
"'replay_manifest'",
"'candidate'",
"'candidate_search_projection'",
"'open_world_evidence_bundle'",
"'open_world_discovery_memo'",
"'open_world_candidate'",
"'open_world_attempt_decision'",
)
if all(artifact_kind in table_sql for artifact_kind in artifact_kinds):
if all(
artifact_kind in table_sql for artifact_kind in artifact_kinds
) and not any(
retired in table_sql for retired in _RETIRED_ARTIFACT_KIND_LITERALS
):
return

foreign_keys_enabled = bool(
Expand All @@ -2413,7 +2475,6 @@ def _enforce_playbook_optimization_artifact_constraints(self) -> None:
artifact_kind TEXT NOT NULL CHECK (artifact_kind IN (
'expected_population_manifest',
'generation_selection',
'replay_manifest',
'candidate',
'candidate_search_projection',
'open_world_evidence_bundle',
Expand All @@ -2432,6 +2493,37 @@ def _enforce_playbook_optimization_artifact_constraints(self) -> None:
)
"""
)
# artifact_kind is NOT NULL and part of UNIQUE (job_id,
# artifact_kind), so there is no surviving value to map
# 'replay_manifest' onto. Local development SQLite only -- no tenant
# Postgres row is touched by this path.
#
# Phase 7 review finding P4-3: this is the OPPOSITE strategy from the
# sibling optimizer_kind vocabulary in
# _enforce_playbook_optimization_job_constraints, where OQ-1 option A
# RETAINS 'offline_tuner_legacy' / 'optimizer_legacy_unknown'
# permanently as historical labels precisely so historical rows
# survive. Retention was available here too and was rejected: the
# tenant contract (supabase/data/tenant/20260830020000, constraint
# playbook_optimization_artifacts_artifact_kind_check) drops
# 'replay_manifest' outright, so retaining it on SQLite alone would
# leave the two backends disagreeing about the artifact
# vocabulary -- the dual-expression divergence the rest of
# Phase 7 exists to remove. The rows are destroyed rather than
# migrated, so the deletion is logged with its count instead of being
# silent.
deleted_replay_manifests = self.conn.execute(
"DELETE FROM playbook_optimization_artifacts "
"WHERE artifact_kind = 'replay_manifest'"
).rowcount
if deleted_replay_manifests > 0:
logger.warning(
"Phase 7 artifact contraction destroyed %d "
"'replay_manifest' artifact row(s) with their content_json: "
"the retired kind has no surviving artifact_kind to map onto "
"and the tenant contract drops it as well",
deleted_replay_manifests,
)
self.conn.execute(
"""
INSERT INTO playbook_optimization_artifacts_new (
Expand Down Expand Up @@ -3416,7 +3508,6 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
optimizer_kind TEXT NOT NULL DEFAULT 'optimizer_legacy_unknown'
CHECK (optimizer_kind IN (
'gepa',
'offline_tuner_replay',
'offline_tuner_open_world',
'offline_tuner_legacy',
'optimizer_legacy_unknown'
Expand All @@ -3437,8 +3528,6 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
'evidence_frozen',
'discovery_analyzed',
'candidate_generated',
'replay_running',
'replay_evaluated',
'held_out_analyzed',
'publishing',
'applied',
Expand All @@ -3450,16 +3539,11 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
'insufficient_negative_evidence',
'insufficient_positive_evidence',
'insufficient_coverage',
'replay_unsupported',
'deployment_unsupported',
'incomplete_replay_scope',
'insufficient_replay_cases',
'replay_inconclusive',
'candidate_regressed',
'candidate_did_not_improve',
'incumbent_changed',
'generation_failed',
'replay_failed',
'publication_failed',
'governance_erased',
'no_grounded_hypothesis',
Expand Down Expand Up @@ -3524,7 +3608,6 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
artifact_kind TEXT NOT NULL CHECK (artifact_kind IN (
'expected_population_manifest',
'generation_selection',
'replay_manifest',
'candidate',
'candidate_search_projection',
'open_world_evidence_bundle',
Expand Down
Loading
Loading