feat: add open-world tuner analysis foundations

reflexio/models/api_schema/domain/entities.py

@@ -7,6 +7,7 @@
 
 from pydantic import (
     BaseModel,
+    ConfigDict,
     Field,
     PrivateAttr,
     field_validator,
@@ -122,6 +123,11 @@
     "PlaybookOptimizationCandidate",
     "PlaybookOptimizationEvaluation",
     "PlaybookOptimizationEvent",
+    "OpenWorldQualificationClass",
+    "OPEN_WORLD_QUALIFICATION_CLASSES",
+    "OPEN_WORLD_QUALIFICATION_RECORD_SCHEMA_VERSION",
+    "OpenWorldQualificationClassCount",
+    "OpenWorldQualificationRecord",
     "AgentPlaybookSourceWindow",
     "agent_playbook_to_snapshot",
     "RunPlaybookAggregationRequest",
@@ -412,9 +418,11 @@ class AgentPlaybook(BaseModel):
 
 OptimizationJobStage = Literal[
     "evidence_frozen",
+    "discovery_analyzed",
     "candidate_generated",
     "replay_running",
     "replay_evaluated",
+    "held_out_analyzed",
     "publishing",
     "applied",
     "abstained",
@@ -438,6 +446,12 @@ class AgentPlaybook(BaseModel):
     "replay_failed",
     "publication_failed",
     "governance_erased",
+    "no_grounded_hypothesis",
+    "analyst_unqualified",
+    "heldout_evidence_failed",
+    "stale_incumbent",
+    "governance_invalidated",
+    "infrastructure_failure",
 ]
 
 OptimizationArtifactKind = Literal[
@@ -447,6 +461,9 @@ class AgentPlaybook(BaseModel):
     "candidate",
     "candidate_search_projection",
     "open_world_evidence_bundle",
+    "open_world_discovery_memo",
+    "open_world_candidate",
+    "open_world_attempt_decision",
 ]
 
 Sha256Digest = str
@@ -603,6 +620,149 @@ class PlaybookOptimizationEvent(BaseModel):
     created_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp()))
 
 
+OpenWorldQualificationClass = Literal[
+    "citation_fidelity",
+    "abstention",
+    "support",
+    "refutation",
+    "insufficiency",
+    "unsupported_causal_claim_rejection",
+    "prompt_injection_resistance",
+]
+
+OPEN_WORLD_QUALIFICATION_CLASSES: Final[tuple[OpenWorldQualificationClass, ...]] = (
+    "citation_fidelity",
+    "abstention",
+    "support",
+    "refutation",
+    "insufficiency",
+    "unsupported_causal_claim_rejection",
+    "prompt_injection_resistance",
+)
+
+OPEN_WORLD_QUALIFICATION_RECORD_SCHEMA_VERSION: Final[str] = (
+    "offline-tuner-open-world-qualification-result-v1"
+)
+
+
+def _validate_lowercase_sha256(label: str, value: str) -> str:
+    """Return ``value`` when it is a lowercase SHA-256 hex digest.
+
+    Args:
+        label (str): Field name used in the raised error message.
+        value (str): Candidate digest.
+
+    Returns:
+        str: The validated digest.
+
+    Raises:
+        ValueError: If ``value`` is not 64 lowercase hex characters.
+    """
+    if len(value) != 64 or any(char not in "0123456789abcdef" for char in value):
+        raise ValueError(f"{label} must be lowercase SHA-256 hex")
+    return value
+
+
+class OpenWorldQualificationClassCount(BaseModel):
+    """Diagnostic required/passed counts for one safety-critical class.
+
+    Counts are never combined into a score: pass-all qualification is decided
+    by the reducer, and these values exist only to explain one result.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+    qualification_class: OpenWorldQualificationClass
+    required: int = Field(ge=0)
+    passed_required: int = Field(ge=0)
+
+    @model_validator(mode="after")
+    def validate_passed_within_required(self) -> Self:
+        if self.passed_required > self.required:
+            raise ValueError("qualification passed_required may not exceed required")
+        return self
+
+
+class OpenWorldQualificationRecord(BaseModel):
+    """One immutable pass-all qualification result for an analyst identity.
+
+    The record carries no customer data or model output: only the pinned
+    component identity, the suite it was measured against, the canonical
+    result digest, per-class diagnostic counts for every one of the seven
+    safety-critical classes in canonical order, and the sorted, unique
+    digests of the observations that produced it.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+    schema_version: Literal["offline-tuner-open-world-qualification-result-v1"] = (
+        "offline-tuner-open-world-qualification-result-v1"
+    )
+    component_identity_digest: Sha256Digest
+    suite_digest: Sha256Digest
+    result_digest: Sha256Digest
+    class_counts: tuple[OpenWorldQualificationClassCount, ...]
+    passed: bool
+    observation_digests: tuple[Sha256Digest, ...] = ()
+    created_at: int = Field(
+        default_factory=lambda: int(datetime.now(UTC).timestamp()), ge=0
+    )
+
+    @field_validator("component_identity_digest", "suite_digest", "result_digest")
+    @classmethod
+    def validate_identity_digests(cls, value: str) -> str:
+        return _validate_lowercase_sha256("qualification digest", value)
+
+    @field_validator("class_counts")
+    @classmethod
+    def validate_class_counts(
+        cls, value: tuple[OpenWorldQualificationClassCount, ...]
+    ) -> tuple[OpenWorldQualificationClassCount, ...]:
+        observed = tuple(count.qualification_class for count in value)
+        if observed != OPEN_WORLD_QUALIFICATION_CLASSES:
+            raise ValueError(
+                "qualification class_counts must list every safety-critical "
+                "class exactly once in canonical order"
+            )
+        return value
+
+    @field_validator("observation_digests")
+    @classmethod
+    def validate_observation_digests(cls, value: tuple[str, ...]) -> tuple[str, ...]:
+        for digest in value:
+            _validate_lowercase_sha256("qualification observation digest", digest)
+        if list(value) != sorted(set(value)):
+            raise ValueError(
+                "qualification observation digests must be sorted and unique"
+            )
+        return value
+
+    @model_validator(mode="after")
+    def validate_passed_requires_all_class_counts(self) -> Self:
+        if self.passed and any(
+            count.passed_required != count.required for count in self.class_counts
+        ):
+            raise ValueError(
+                "qualification passed=true requires every class to pass required"
+            )
+        return self
+
+    def semantic_key(self) -> tuple[Any, ...]:
+        """Return the conflict-detection identity, excluding ``created_at``."""
+        return (
+            self.schema_version,
+            self.component_identity_digest,
+            self.suite_digest,
+            self.result_digest,
+            tuple(
+                (count.qualification_class, count.required, count.passed_required)
+                for count in self.class_counts
+            ),
+            self.passed,
+            self.observation_digests,
+        )
+
+
 class AgentPlaybookSourceWindow(BaseModel):
     """Replayable source window snapshotted when an agent playbook is generated."""
 

reflexio/server/llm/__init__.py

@@ -9,6 +9,7 @@
     LiteLLMClient,
     LiteLLMClientError,
     LiteLLMConfig,
+    ProviderRequestGuardError,
     StructuredOutputRepairError,
     StructuredOutputValidator,
     ToolCallingChatResponse,
@@ -24,6 +25,7 @@
     "LiteLLMClient",
     "LiteLLMConfig",
     "LiteLLMClientError",
+    "ProviderRequestGuardError",
     "StructuredOutputRepairError",
     "StructuredOutputValidator",
     "ModelRole",

reflexio/server/llm/litellm_client.py

@@ -63,6 +63,7 @@
     _PromptTokenDetailsSnapshot as _PromptTokenDetailsSnapshot,
 )
 from reflexio.server.llm._litellm_text_generation import (
+    ProviderRequestGuardError,
     StructuredOutputValidator,
     TextGenerationMixin,
 )
@@ -101,6 +102,7 @@
     "LiteLLMConfig",
     "LiteLLMClientError",
     "StructuredOutputRepairError",
+    "ProviderRequestGuardError",
     "StructuredOutputValidator",
     "ToolCallingChatResponse",
     "create_litellm_client",

reflexio/server/services/storage/error.py

@@ -30,6 +30,10 @@ class OptimizationArtifactIntegrityError(StorageError):
     """Raised when a durable optimizer artifact is malformed or conflicts."""
 
 
+class OpenWorldQualificationConflictError(StorageError):
+    """Raised when a cached qualification key resolves to a conflicting result."""
+
+
 def require_non_empty_session_id(value: Any) -> str:
     """Return a stripped, non-empty request ``session_id`` or raise ``StorageError``.
 

reflexio/server/services/storage/sqlite_storage/_base.py

@@ -2202,9 +2202,17 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
             "CHECK (stage IS NULL OR stage IN",
             "CHECK (terminal_outcome IS NULL OR terminal_outcome IN",
             "'governance_erased'",
+            "'discovery_analyzed'",
+            "'held_out_analyzed'",
+            "'no_grounded_hypothesis'",
+            "'analyst_unqualified'",
+            "'heldout_evidence_failed'",
+            "'stale_incumbent'",
+            "'governance_invalidated'",
+            "'infrastructure_failure'",
         )
         if all(check in table_sql for check in required_checks) and (
-            "'offline_tuner_open_world'" not in table_sql
+            "'offline_tuner_open_world'" in table_sql
         ):
             return
         foreign_keys_enabled = bool(
@@ -2228,6 +2236,7 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
                     CHECK (optimizer_kind IN (
                         'gepa',
                         'offline_tuner_replay',
+                        'offline_tuner_open_world',
                         'offline_tuner_legacy',
                         'optimizer_legacy_unknown'
                     )),
@@ -2245,9 +2254,11 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
                 lease_expires_at INTEGER,
                 stage TEXT CHECK (stage IS NULL OR stage IN (
                     'evidence_frozen',
+                    'discovery_analyzed',
                     'candidate_generated',
                     'replay_running',
                     'replay_evaluated',
+                    'held_out_analyzed',
                     'publishing',
                     'applied',
                     'abstained',
@@ -2269,7 +2280,13 @@ def _enforce_playbook_optimization_job_constraints(self) -> None:
                     'generation_failed',
                     'replay_failed',
                     'publication_failed',
-                    'governance_erased'
+                    'governance_erased',
+                    'no_grounded_hypothesis',
+                    'analyst_unqualified',
+                    'heldout_evidence_failed',
+                    'stale_incumbent',
+                    'governance_invalidated',
+                    'infrastructure_failure'
                 )),
                 expected_population_manifest_digest TEXT,
                 generation_selection_manifest_digest TEXT,
@@ -2366,6 +2383,9 @@ def _enforce_playbook_optimization_artifact_constraints(self) -> None:
             "'candidate'",
             "'candidate_search_projection'",
             "'open_world_evidence_bundle'",
+            "'open_world_discovery_memo'",
+            "'open_world_candidate'",
+            "'open_world_attempt_decision'",
         )
         if all(artifact_kind in table_sql for artifact_kind in artifact_kinds):
             return
@@ -2396,7 +2416,10 @@ def _enforce_playbook_optimization_artifact_constraints(self) -> None:
                         'replay_manifest',
                         'candidate',
                         'candidate_search_projection',
-                        'open_world_evidence_bundle'
+                        'open_world_evidence_bundle',
+                        'open_world_discovery_memo',
+                        'open_world_candidate',
+                        'open_world_attempt_decision'
                     )),
                     content_json TEXT NOT NULL,
                     content_digest TEXT NOT NULL,
@@ -3394,6 +3417,7 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
         CHECK (optimizer_kind IN (
             'gepa',
             'offline_tuner_replay',
+            'offline_tuner_open_world',
             'offline_tuner_legacy',
             'optimizer_legacy_unknown'
         )),
@@ -3411,9 +3435,11 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
     lease_expires_at INTEGER,
     stage TEXT CHECK (stage IS NULL OR stage IN (
         'evidence_frozen',
+        'discovery_analyzed',
         'candidate_generated',
         'replay_running',
         'replay_evaluated',
+        'held_out_analyzed',
         'publishing',
         'applied',
         'abstained',
@@ -3435,7 +3461,13 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
         'generation_failed',
         'replay_failed',
         'publication_failed',
-        'governance_erased'
+        'governance_erased',
+        'no_grounded_hypothesis',
+        'analyst_unqualified',
+        'heldout_evidence_failed',
+        'stale_incumbent',
+        'governance_invalidated',
+        'infrastructure_failure'
     )),
     expected_population_manifest_digest TEXT,
     generation_selection_manifest_digest TEXT,
@@ -3495,7 +3527,10 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
         'replay_manifest',
         'candidate',
         'candidate_search_projection',
-        'open_world_evidence_bundle'
+        'open_world_evidence_bundle',
+        'open_world_discovery_memo',
+        'open_world_candidate',
+        'open_world_attempt_decision'
     )),
     content_json TEXT NOT NULL,
     content_digest TEXT NOT NULL,
@@ -3508,6 +3543,28 @@ def clear_user_data(self, user_id: str) -> dict[str, int]:
 CREATE INDEX IF NOT EXISTS idx_poa_job
     ON playbook_optimization_artifacts(job_id);
 
+CREATE TABLE IF NOT EXISTS offline_tuner_open_world_qualifications (
+    component_identity_digest TEXT NOT NULL,
+    suite_digest TEXT NOT NULL,
+    schema_version TEXT NOT NULL,
+    result_digest TEXT NOT NULL,
+    passed INTEGER NOT NULL CHECK (passed IN (0, 1)),
+    class_counts_json TEXT NOT NULL,
+    observation_digests_json TEXT NOT NULL,
+    created_at INTEGER NOT NULL,
+    PRIMARY KEY (component_identity_digest, suite_digest)
+);
+CREATE TRIGGER IF NOT EXISTS offline_tuner_open_world_qualifications_no_update
+BEFORE UPDATE ON offline_tuner_open_world_qualifications
+BEGIN
+    SELECT RAISE(ABORT, 'open-world qualification records are immutable');
+END;
+CREATE TRIGGER IF NOT EXISTS offline_tuner_open_world_qualifications_no_delete
+BEFORE DELETE ON offline_tuner_open_world_qualifications
+BEGIN
+    SELECT RAISE(ABORT, 'open-world qualification records are immutable');
+END;
+
 CREATE TABLE IF NOT EXISTS playbook_optimization_candidates (
     candidate_id INTEGER PRIMARY KEY AUTOINCREMENT,
     job_id INTEGER NOT NULL,