From 460a7062c75a6c3cbf6af4fd6aa7edc92f5423b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro?= Date: Tue, 1 Sep 2026 17:49:16 +0100 Subject: [PATCH] feat(runs)!: add governed Builder successor retries BREAKING CHANGE: Builder runs can no longer use generic run resume or conversation continuation. Clients must create an eligible immutable successor through POST /v1/runs/:runId/retry. --- .env.example | 3 + apps/docs/docs/self-host/production.md | 13 + .../0045_governed_builder_retry_lineage.sql | 161 +++ packages/db/src/schema.ts | 12 + packages/db/test/db.test.ts | 178 ++++ packages/sdk/openapi.json | 369 +++++++ packages/sdk/src/routes.ts | 1 + packages/sdk/src/schema.d.ts | 184 ++++ services/api/src/builder-plan-freshness.ts | 7 +- services/api/src/builder-plan-policy.ts | 55 +- services/api/src/config.ts | 2 + services/api/src/executors.ts | 20 + services/api/src/governed-builder-retry.ts | 940 +++++++++++++++++ services/api/src/routes/v1/conversations.ts | 21 +- services/api/src/routes/v1/runs.ts | 55 +- services/api/src/routes/v1/shared.ts | 1 + services/api/src/sandbox/orchestrator.ts | 134 ++- services/api/src/types.ts | 6 + services/api/test/api.test.ts | 225 +++- .../builder-plan-policy.integration.test.ts | 976 +++++++++++++++++- .../builder-plan-producer-inventory.test.ts | 24 +- services/api/test/config.test.ts | 10 + 22 files changed, 3328 insertions(+), 69 deletions(-) create mode 100644 packages/db/migrations/0045_governed_builder_retry_lineage.sql create mode 100644 services/api/src/governed-builder-retry.ts diff --git a/.env.example b/.env.example index 4386519b..e5adc948 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,9 @@ FACILITY_RUNNER_IMAGE=facility-runner:dev # Two-phase rollout only: enable after every API replica supports durable # repository-write leases and the old replicas have been drained. FACILITY_REPOSITORY_WRITE_TRACKING_PROMOTION=0 +# Two-phase rollout only: enable after every worker understands immutable +# governed Builder successors and every old worker has been drained. +FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION=0 # URLs as resolved from inside a sandbox, not necessarily browser-facing URLs. SANDBOX_API_URL=http://host.docker.internal:4400 SANDBOX_GATEWAY_URL=http://host.docker.internal:4410 diff --git a/apps/docs/docs/self-host/production.md b/apps/docs/docs/self-host/production.md index 56096aab..57866f95 100644 --- a/apps/docs/docs/self-host/production.md +++ b/apps/docs/docs/self-host/production.md @@ -85,6 +85,19 @@ with compose. binary until all version-1 runs are terminal and drained. The promotion flag controls only new `/hello` negotiation: a new API always enforces an existing version-1 marker even when the flag is off. + + Governed Builder successor retries use a second, independent two-phase gate: + + 1. Deploy the migration, API, and worker everywhere with + `FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION=0` (the default). Drain every + worker that predates immutable retry-lineage validation. + 2. Set `FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION=1` on every API replica. + Only then can `POST /v1/runs/:runId/retry` create successor rows. + + Once any successor row exists, do not reintroduce an old API or worker binary + until every successor is terminal and drained. Turning the promotion flag + off stops creation only; upgraded workers continue to enforce and dispatch + already-created successors. 6. Bootstrap the first owner and issue an API key. On an empty installation, run `facility instance bootstrap`, then open `https:///api/auth/login`; the configured GitHub user signs into the organization already created by bootstrap. With the optional web diff --git a/packages/db/migrations/0045_governed_builder_retry_lineage.sql b/packages/db/migrations/0045_governed_builder_retry_lineage.sql new file mode 100644 index 00000000..4b4ada11 --- /dev/null +++ b/packages/db/migrations/0045_governed_builder_retry_lineage.sql @@ -0,0 +1,161 @@ +-- Immutable, tenant-scoped lineage for governed Builder successor attempts. +ALTER TABLE runs + ADD COLUMN IF NOT EXISTS retry_of_run_id text; + +CREATE UNIQUE INDEX IF NOT EXISTS runs_org_project_id_uidx + ON runs (org_id, project_id, id); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'runs_retry_parent_fk' + AND conrelid = 'runs'::regclass + ) THEN + ALTER TABLE runs + ADD CONSTRAINT runs_retry_parent_fk + FOREIGN KEY (org_id, project_id, retry_of_run_id) + REFERENCES runs (org_id, project_id, id); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'runs_retry_not_self_check' + AND conrelid = 'runs'::regclass + ) THEN + ALTER TABLE runs + ADD CONSTRAINT runs_retry_not_self_check + CHECK (retry_of_run_id IS NULL OR retry_of_run_id <> id); + END IF; +END $$; + +-- A parent has at most one direct successor. Repeated attempts form a linear +-- root -> child -> child chain rather than a tree of competing executions. +CREATE UNIQUE INDEX IF NOT EXISTS runs_retry_of_run_uidx + ON runs (org_id, project_id, retry_of_run_id) + WHERE retry_of_run_id IS NOT NULL; + +-- Proposal and Architect-run uniqueness identify only the canonical root. +-- Governed successors deliberately carry the byte-identical accepted trigger. +DROP INDEX IF EXISTS runs_plan_acceptance_proposal_uidx; +CREATE UNIQUE INDEX runs_plan_acceptance_proposal_uidx + ON runs (org_id, ((trigger->>'proposalId'))) + WHERE trigger->>'source' = 'plan_acceptance' + AND retry_of_run_id IS NULL; + +DROP INDEX IF EXISTS runs_plan_acceptance_architect_run_uidx; +CREATE UNIQUE INDEX runs_plan_acceptance_architect_run_uidx + ON runs (org_id, ((trigger->>'architectRunId'))) + WHERE trigger->>'source' = 'plan_acceptance' + AND retry_of_run_id IS NULL; + +CREATE OR REPLACE FUNCTION enforce_governed_run_retry_lineage() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + parent runs%ROWTYPE; + has_child boolean; +BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.retry_of_run_id IS NULL THEN + RETURN NEW; + END IF; + + IF NEW.retry_of_run_id = NEW.id THEN + RAISE EXCEPTION 'run retry cannot reference itself' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_not_self_check'; + END IF; + + SELECT * INTO parent + FROM runs + WHERE id = NEW.retry_of_run_id + AND org_id = NEW.org_id + AND project_id = NEW.project_id + FOR SHARE; + IF NOT FOUND THEN + RAISE EXCEPTION 'run retry parent is not in the same organization and project' + USING ERRCODE = '23503', CONSTRAINT = 'runs_retry_parent_fk'; + END IF; + IF parent.status <> 'failed' + OR parent.mode NOT IN ('builder', 'codex-builder') + OR parent.trigger->>'source' IS DISTINCT FROM 'plan_acceptance' + THEN + RAISE EXCEPTION 'run retry parent is not a failed plan-linked Builder' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_parent_state_check'; + END IF; + IF NEW.status <> 'queued' + OR NEW.trigger IS DISTINCT FROM parent.trigger + OR NEW.agent_def_id IS DISTINCT FROM parent.agent_def_id + OR NEW.mode IS DISTINCT FROM parent.mode + OR NEW.engine IS DISTINCT FROM parent.engine + THEN + RAISE EXCEPTION 'run retry identity must exactly match its parent' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_identity_check'; + END IF; + RETURN NEW; + END IF; + + IF NEW.retry_of_run_id IS DISTINCT FROM OLD.retry_of_run_id THEN + RAISE EXCEPTION 'run retry lineage is immutable' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_lineage_immutable'; + END IF; + + -- A failed plan-linked Builder is a sealed execution record, whether or not + -- its successor has committed yet. Freezing the parent unconditionally + -- closes the concurrent INSERT-child / UPDATE-parent snapshot race: whichever + -- statement obtains the parent row lock first, the other statement either + -- observes a non-failed parent and rejects or this UPDATE rejects here. + IF OLD.status = 'failed' + AND OLD.mode IN ('builder', 'codex-builder') + AND OLD.trigger->>'source' = 'plan_acceptance' + THEN + IF NEW.status IS DISTINCT FROM OLD.status THEN + RAISE EXCEPTION 'failed plan-linked Builder status is immutable' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_parent_status_immutable'; + END IF; + IF NEW.org_id IS DISTINCT FROM OLD.org_id + OR NEW.project_id IS DISTINCT FROM OLD.project_id + OR NEW.trigger IS DISTINCT FROM OLD.trigger + OR NEW.agent_def_id IS DISTINCT FROM OLD.agent_def_id + OR NEW.mode IS DISTINCT FROM OLD.mode + OR NEW.engine IS DISTINCT FROM OLD.engine + THEN + RAISE EXCEPTION 'failed plan-linked Builder execution identity is immutable' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_parent_identity_immutable'; + END IF; + END IF; + + SELECT EXISTS ( + SELECT 1 FROM runs child + WHERE child.org_id = OLD.org_id + AND child.project_id = OLD.project_id + AND child.retry_of_run_id = OLD.id + ) INTO has_child; + IF (OLD.retry_of_run_id IS NOT NULL OR has_child) + AND ( + NEW.org_id IS DISTINCT FROM OLD.org_id + OR NEW.project_id IS DISTINCT FROM OLD.project_id + OR NEW.trigger IS DISTINCT FROM OLD.trigger + OR NEW.agent_def_id IS DISTINCT FROM OLD.agent_def_id + OR NEW.mode IS DISTINCT FROM OLD.mode + OR NEW.engine IS DISTINCT FROM OLD.engine + ) + THEN + RAISE EXCEPTION 'run retry execution identity is immutable' + USING ERRCODE = '23514', CONSTRAINT = 'runs_retry_identity_immutable'; + END IF; + RETURN NEW; +END $$; + +DROP TRIGGER IF EXISTS runs_governed_retry_lineage_guard ON runs; +CREATE TRIGGER runs_governed_retry_lineage_guard +BEFORE INSERT OR UPDATE OF retry_of_run_id, org_id, project_id, trigger, agent_def_id, mode, engine, status +ON runs +FOR EACH ROW +EXECUTE FUNCTION enforce_governed_run_retry_lineage(); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 40b11bd2..54fca5ad 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -485,6 +485,10 @@ export const runs = pgTable( mode: text("mode").notNull(), engine: text("engine").notNull(), status: text("status").notNull().default("queued"), + // Governed Builder retries are immutable successor rows. Composite scope + // and identity enforcement lives in migration 0045 because the parent is + // another row in this table. + retryOfRunId: text("retry_of_run_id"), // Zero write-lease rows are meaningful only after the upgraded runner has // explicitly negotiated this protocol during /hello. repositoryWriteTrackingVersion: integer("repository_write_tracking_version") @@ -518,6 +522,14 @@ export const runs = pgTable( uniqueIndex("runs_org_ci_repair_key_uidx") .on(table.orgId, table.ciRepairKey) .where(sql`${table.ciRepairKey} is not null`), + uniqueIndex("runs_org_project_id_uidx").on(table.orgId, table.projectId, table.id), + uniqueIndex("runs_retry_of_run_uidx") + .on(table.orgId, table.projectId, table.retryOfRunId) + .where(sql`${table.retryOfRunId} is not null`), + check( + "runs_retry_not_self_check", + sql`${table.retryOfRunId} is null or ${table.retryOfRunId} <> ${table.id}`, + ), check( "runs_repository_write_tracking_version_check", sql`${table.repositoryWriteTrackingVersion} in (0, 1)`, diff --git a/packages/db/test/db.test.ts b/packages/db/test/db.test.ts index bbb1e564..82e12d04 100644 --- a/packages/db/test/db.test.ts +++ b/packages/db/test/db.test.ts @@ -1242,6 +1242,177 @@ describe("db", async () => { ).toHaveLength(1); }); + it("enforces tenant-scoped immutable linear Builder retry lineage", async () => { + const orgId = newId("org"); + const projectId = newId("proj"); + const otherProjectId = newId("proj"); + await db.insert(schema.orgs).values({ + id: orgId, + name: "Retry Lineage", + slug: `retry-lineage-${orgId}`, + settings: {}, + }); + await db.insert(schema.projects).values([ + { id: projectId, orgId, name: "Retry Lineage", slug: `retry-${projectId}`, settings: {} }, + { + id: otherProjectId, + orgId, + name: "Other Retry Lineage", + slug: `retry-${otherProjectId}`, + settings: {}, + }, + ]); + for (const source of [undefined, null] as const) { + const invalidTrigger = source === undefined ? {} : { source }; + const invalidParent = ( + await db + .insert(schema.runs) + .values({ + id: newId("run"), + orgId, + projectId, + mode: "builder", + engine: "codex", + status: "failed", + trigger: invalidTrigger, + createdBy: { type: "system", id: "invalid-parent" }, + }) + .returning() + )[0]; + if (!invalidParent) throw new Error("invalid retry parent fixture missing"); + await expectCheckViolation( + db.insert(schema.runs).values({ + id: newId("run"), + orgId, + projectId, + retryOfRunId: invalidParent.id, + mode: invalidParent.mode, + engine: invalidParent.engine, + trigger: invalidTrigger, + createdBy: { type: "system", id: "invalid-child" }, + }), + { constraintName: "runs_retry_parent_state_check" }, + ); + } + const trigger = { + source: "plan_acceptance", + proposalId: newId("prop"), + architectRunId: newId("run"), + approvedPlan: "Immutable plan", + }; + const root = ( + await db + .insert(schema.runs) + .values({ + id: newId("run"), + orgId, + projectId, + mode: "builder", + engine: "codex", + status: "failed", + trigger, + createdBy: { type: "system", id: "test" }, + }) + .returning() + )[0]; + if (!root) throw new Error("retry root fixture missing"); + for (const status of ["queued", "running"] as const) { + await expectCheckViolation( + db.update(schema.runs).set({ status }).where(eq(schema.runs.id, root.id)), + { constraintName: "runs_retry_parent_status_immutable" }, + ); + } + const child = ( + await db + .insert(schema.runs) + .values({ + id: newId("run"), + orgId, + projectId, + retryOfRunId: root.id, + mode: root.mode, + engine: root.engine, + trigger, + createdBy: { type: "system", id: "test" }, + }) + .returning() + )[0]; + expect(child?.retryOfRunId).toBe(root.id); + + await expect( + db.insert(schema.runs).values({ + id: newId("run"), + orgId, + projectId, + retryOfRunId: root.id, + mode: root.mode, + engine: root.engine, + trigger, + createdBy: { type: "system", id: "duplicate" }, + }), + ).rejects.toMatchObject({ + cause: { code: "23505", constraint_name: "runs_retry_of_run_uidx" }, + }); + await expectCheckViolation( + db.insert(schema.runs).values({ + id: newId("run"), + orgId, + projectId, + retryOfRunId: root.id, + mode: root.mode, + engine: root.engine, + trigger: { ...trigger, approvedPlan: "forged" }, + createdBy: { type: "system", id: "forged" }, + }), + { constraintName: "runs_retry_identity_check" }, + ); + await expect( + db.insert(schema.runs).values({ + id: newId("run"), + orgId, + projectId: otherProjectId, + retryOfRunId: root.id, + mode: root.mode, + engine: root.engine, + trigger, + createdBy: { type: "system", id: "cross-project" }, + }), + ).rejects.toMatchObject({ cause: { code: "23503", constraint_name: "runs_retry_parent_fk" } }); + if (!child) throw new Error("retry child fixture missing"); + await expectCheckViolation( + db.update(schema.runs).set({ mode: "codex-builder" }).where(eq(schema.runs.id, child.id)), + { constraintName: "runs_retry_identity_immutable" }, + ); + await expectCheckViolation( + db + .update(schema.runs) + .set({ trigger: { ...trigger, approvedPlan: "mutated root" } }) + .where(eq(schema.runs.id, root.id)), + { constraintName: "runs_retry_parent_identity_immutable" }, + ); + + await db + .update(schema.runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(schema.runs.id, child.id)); + const grandchild = ( + await db + .insert(schema.runs) + .values({ + id: newId("run"), + orgId, + projectId, + retryOfRunId: child.id, + mode: child.mode, + engine: child.engine, + trigger, + createdBy: { type: "system", id: "test" }, + }) + .returning() + )[0]; + expect(grandchild?.retryOfRunId).toBe(child.id); + }); + it("applies metering precision and index migrations in order", async () => { const columns = (await db.execute( sql` @@ -1253,6 +1424,7 @@ describe("db", async () => { OR (table_name = 'provider_credentials' AND column_name = 'auth_mode') OR (table_name = 'projects' AND column_name = 'builder_plan_policy') OR (table_name = 'runs' AND column_name = 'workspace_base_sha') + OR (table_name = 'runs' AND column_name = 'retry_of_run_id') OR (table_name = 'run_deliveries' AND column_name = 'base_sha') `, )) as Iterable<{ table_name: string; column_name: string; data_type: string }>; @@ -1270,6 +1442,7 @@ describe("db", async () => { expect(columnTypes.get("provider_credentials.auth_mode")).toBe("text"); expect(columnTypes.get("projects.builder_plan_policy")).toBe("text"); expect(columnTypes.get("runs.workspace_base_sha")).toBe("text"); + expect(columnTypes.get("runs.retry_of_run_id")).toBe("text"); expect(columnTypes.get("run_deliveries.base_sha")).toBe("text"); const indexes = (await db.execute( sql` @@ -1294,6 +1467,7 @@ describe("db", async () => { 'registry_versions_one_active_uidx', 'runs_plan_acceptance_proposal_uidx', 'runs_plan_acceptance_architect_run_uidx', + 'runs_retry_of_run_uidx', 'webhook_deliveries_pending_idx', 'webhook_deliveries_org_created_idx', 'idempotency_records_expiry_idx', @@ -1333,6 +1507,7 @@ describe("db", async () => { "runs_plan_acceptance_proposal_uidx", // Duplicate proposals for one architect plan cannot double-dispatch (migration 0020). "runs_plan_acceptance_architect_run_uidx", + "runs_retry_of_run_uidx", // Durable integration outbox and API replay records (migrations 0021-0022). "webhook_deliveries_pending_idx", "webhook_deliveries_org_created_idx", @@ -1402,6 +1577,9 @@ describe("db", async () => { expect(Array.from(applied).map((row) => row.name)).toContain( "0044_run_repository_write_leases.sql", ); + expect(Array.from(applied).map((row) => row.name)).toContain( + "0045_governed_builder_retry_lineage.sql", + ); expect(Array.from(applied).map((row) => row.name)).toContain("0043_builder_plan_policy.sql"); expect(Array.from(applied).map((row) => row.name)).toContain( "0042_run_base_sha_provenance.sql", diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 17366a7c..2564b80c 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -6594,6 +6594,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -6728,6 +6732,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -6928,6 +6933,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -7062,6 +7071,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -7428,6 +7438,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -7562,6 +7576,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -7893,6 +7908,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -8047,6 +8066,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -8209,6 +8229,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -8343,6 +8367,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -8461,6 +8486,335 @@ "x-facility-permission": "runs:read" } }, + "/v1/runs/{runId}/retry": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "default": {}, + "nullable": true, + "type": "object", + "properties": { + "reason": { + "type": "string", + "maxLength": 500 + } + }, + "additionalProperties": false + } + } + } + }, + "parameters": [ + { + "schema": { + "type": "string" + }, + "in": "path", + "name": "runId", + "required": true + }, + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "description": "Replays the original response for the same principal, path, key, and request body for 24 hours.", + "schema": { + "type": "string", + "minLength": 8, + "maxLength": 200 + } + } + ], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "orgId": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "agentDefId": { + "nullable": true, + "type": "string" + }, + "mode": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "status": { + "type": "string" + }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, + "trigger": { + "type": "object", + "additionalProperties": {} + }, + "sandbox": { + "type": "object", + "additionalProperties": {} + }, + "receipt": { + "nullable": true, + "type": "object", + "properties": { + "usage": { + "type": "object", + "properties": { + "input_tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "output_tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "cache_read": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "cache_write": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "cost_cents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "cost_source": { + "type": "string" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "cost_cents", + "cost_source" + ], + "additionalProperties": {} + }, + "events": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "checks": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "count", + "checks" + ], + "additionalProperties": {} + } + }, + "additionalProperties": {} + }, + "gh": { + "type": "object", + "additionalProperties": {} + }, + "engineSessionId": { + "nullable": true, + "type": "string" + }, + "transcriptUri": { + "nullable": true, + "type": "string" + }, + "sessionStateUri": { + "nullable": true, + "type": "string" + }, + "workspaceBaseSha": { + "nullable": true, + "type": "string" + }, + "error": { + "nullable": true, + "type": "string" + }, + "queuedAt": { + "type": "string", + "format": "date-time" + }, + "startedAt": { + "nullable": true, + "type": "string", + "format": "date-time" + }, + "endedAt": { + "nullable": true, + "type": "string", + "format": "date-time" + }, + "createdBy": { + "type": "object", + "additionalProperties": {} + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "updatedAt": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "orgId", + "projectId", + "agentDefId", + "mode", + "engine", + "status", + "retryOfRunId", + "trigger", + "sandbox", + "receipt", + "gh", + "engineSessionId", + "transcriptUri", + "sessionStateUri", + "workspaceBaseSha", + "error", + "queuedAt", + "startedAt", + "endedAt", + "createdBy", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "The request is invalid.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "Authentication is required or invalid.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "403": { + "description": "The authenticated principal lacks the required permission.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "The requested resource was not found or is outside the principal scope.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "409": { + "description": "The request conflicts with current resource state.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "429": { + "description": "The request rate limit was exceeded.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "An unexpected server error occurred.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "A required service is unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + }, + "operationId": "postRunsByRunIdRetry", + "summary": "Retry run", + "tags": [ + "Runs" + ], + "security": [ + { + "bearerAuth": [] + }, + { + "sessionCookie": [] + } + ], + "x-facility-permission": "runs:trigger" + } + }, "/v1/runs/{runId}/cancel": { "post": { "parameters": [ @@ -8514,6 +8868,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -8648,6 +9006,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -9794,6 +10153,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -9928,6 +10291,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", @@ -13470,6 +13834,10 @@ "status": { "type": "string" }, + "retryOfRunId": { + "nullable": true, + "type": "string" + }, "trigger": { "type": "object", "additionalProperties": {} @@ -13604,6 +13972,7 @@ "mode", "engine", "status", + "retryOfRunId", "trigger", "sandbox", "receipt", diff --git a/packages/sdk/src/routes.ts b/packages/sdk/src/routes.ts index 9621c2b8..b2cf65c5 100644 --- a/packages/sdk/src/routes.ts +++ b/packages/sdk/src/routes.ts @@ -140,6 +140,7 @@ export const FACILITY_V1_ROUTES = [ "POST /v1/runs/:runId/interrupt", "POST /v1/runs/:runId/kb-checkpoint", "POST /v1/runs/:runId/resume", + "POST /v1/runs/:runId/retry", "POST /v1/runs/:runId/steer", "POST /v1/sandbox-profiles", "POST /v1/tasks/:taskId/propose", diff --git a/packages/sdk/src/schema.d.ts b/packages/sdk/src/schema.d.ts index 19d7d086..6cba4bb4 100644 --- a/packages/sdk/src/schema.d.ts +++ b/packages/sdk/src/schema.d.ts @@ -543,6 +543,23 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/runs/{runId}/retry": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Retry run */ + post: operations["postRunsByRunIdRetry"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/runs/{runId}/cancel": { parameters: { query?: never; @@ -6110,6 +6127,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -6276,6 +6294,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -6541,6 +6560,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -6792,6 +6812,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -6946,6 +6967,166 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; + trigger: { + [key: string]: unknown; + }; + sandbox: { + [key: string]: unknown; + }; + receipt: ({ + usage?: { + input_tokens: number; + output_tokens: number; + cache_read?: number; + cache_write?: number; + cost_cents: number; + cost_source: string; + } & { + [key: string]: unknown; + }; + events?: { + count: number; + checks: number; + } & { + [key: string]: unknown; + }; + } & { + [key: string]: unknown; + }) | null; + gh: { + [key: string]: unknown; + }; + engineSessionId: string | null; + transcriptUri: string | null; + sessionStateUri: string | null; + workspaceBaseSha: string | null; + error: string | null; + /** Format: date-time */ + queuedAt: string; + /** Format: date-time */ + startedAt: string | null; + /** Format: date-time */ + endedAt: string | null; + createdBy: { + [key: string]: unknown; + }; + /** Format: date-time */ + createdAt: string; + /** Format: date-time */ + updatedAt: string; + }; + }; + }; + /** @description The request is invalid. */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description Authentication is required or invalid. */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description The authenticated principal lacks the required permission. */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description The requested resource was not found or is outside the principal scope. */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description The request conflicts with current resource state. */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description The request rate limit was exceeded. */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description An unexpected server error occurred. */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description A required service is unavailable. */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + }; + }; + postRunsByRunIdRetry: { + parameters: { + query?: never; + header?: { + /** @description Replays the original response for the same principal, path, key, and request body for 24 hours. */ + "Idempotency-Key"?: string; + }; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + reason?: string; + } | null; + }; + }; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + id: string; + orgId: string; + projectId: string; + agentDefId: string | null; + mode: string; + engine: string; + status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -7098,6 +7279,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -7877,6 +8059,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; @@ -9664,6 +9847,7 @@ export interface operations { mode: string; engine: string; status: string; + retryOfRunId: string | null; trigger: { [key: string]: unknown; }; diff --git a/services/api/src/builder-plan-freshness.ts b/services/api/src/builder-plan-freshness.ts index bd3dba14..572bc949 100644 --- a/services/api/src/builder-plan-freshness.ts +++ b/services/api/src/builder-plan-freshness.ts @@ -122,9 +122,10 @@ export async function resolveBuilderPlanFreshnessForProposal( if (error instanceof ApiError && error.code === "builder_plan_freshness_unavailable") { throw error; } - throw freshnessUnavailable( - error instanceof Error ? `github_freshness_error:${error.message}` : "github_freshness_error", - ); + // Provider errors can include URLs, installation identifiers, or request + // fragments. Keep the public/audited reason stable and sanitized; the + // original exception remains available to process-local observability. + throw freshnessUnavailable("github_freshness_error"); } } diff --git a/services/api/src/builder-plan-policy.ts b/services/api/src/builder-plan-policy.ts index e49325a2..22291bad 100644 --- a/services/api/src/builder-plan-policy.ts +++ b/services/api/src/builder-plan-policy.ts @@ -13,7 +13,7 @@ import { runs, } from "@facility/db"; import { agentDefTriggersBuilder, isBuilderMode } from "@facility/run-objective"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, isNull, sql } from "drizzle-orm"; import { ApiError } from "./errors.js"; export type BuilderPlanPolicy = "optional" | "required"; @@ -45,6 +45,8 @@ export type BuilderPlanDispatchInput = { trigger: unknown; gh?: unknown; runId?: string | null; + /** Canonical root that consumed Gate 1 when dispatching an immutable retry child. */ + acceptanceRunId?: string | null; actor?: AuditInsert["actor"]; source?: string; /** Trusted, live evidence resolved by the canonical executor/worker, never request JSON. */ @@ -110,6 +112,44 @@ export function builderIdentity(mode: string, agentName?: string | null) { return isBuilderMode(mode) || (agentName ? isBuilderMode(agentName) : false); } +/** + * Generic session resume is intentionally unavailable to every Builder, + * regardless of the project's current Gate 1 policy. Builder recovery must + * create a governed immutable successor so the original attempt stays sealed. + */ +export async function assertGenericRunResumeAllowed( + db: FacilityDb, + run: Pick, +): Promise { + const trigger = objectValue(run.trigger); + let isBuilder = builderIdentity(run.mode) || trigger.source === "plan_acceptance"; + if (!isBuilder && run.agentDefId) { + const agent = ( + await db + .select({ name: agentDefs.name, triggers: agentDefs.triggers }) + .from(agentDefs) + .where( + and( + eq(agentDefs.orgId, run.orgId), + eq(agentDefs.projectId, run.projectId), + eq(agentDefs.id, run.agentDefId), + ), + ) + .limit(1) + )[0]; + isBuilder = Boolean( + agent && (builderIdentity(run.mode, agent.name) || agentDefTriggersBuilder(agent.triggers)), + ); + } + if (isBuilder) { + throw new ApiError( + 409, + "builder_resume_forbidden", + "Builder runs must be recovered through a governed immutable retry", + ); + } +} + export function isBuilderPlanDenialError(error: unknown): error is ApiError { return error instanceof ApiError && builderPlanDenialCode(error.code) !== null; } @@ -394,6 +434,7 @@ async function validatePlanAcceptance( ) { return invalid("builder_plan_expired", "proposal_expired"); } + const acceptanceRunId = input.acceptanceRunId ?? input.runId; const linkedRuns = await db .select({ id: runs.id }) .from(runs) @@ -403,10 +444,11 @@ async function validatePlanAcceptance( eq(runs.projectId, input.projectId), sql`${runs.trigger}->>'source' = 'plan_acceptance'`, sql`${runs.trigger}->>'proposalId' = ${proposal.id}`, + isNull(runs.retryOfRunId), ), ) .limit(2); - if (linkedRuns.some((run) => run.id !== input.runId)) { + if (linkedRuns.some((run) => run.id !== acceptanceRunId)) { return invalid("builder_plan_already_consumed", "proposal_linked_to_another_run"); } const architectLinkedRuns = await db @@ -418,16 +460,17 @@ async function validatePlanAcceptance( eq(runs.projectId, input.projectId), sql`${runs.trigger}->>'source' = 'plan_acceptance'`, sql`${runs.trigger}->>'architectRunId' = ${architectRunId}`, + isNull(runs.retryOfRunId), ), ) .limit(2); - if (architectLinkedRuns.some((run) => run.id !== input.runId)) { + if (architectLinkedRuns.some((run) => run.id !== acceptanceRunId)) { return invalid("builder_plan_already_consumed", "architect_plan_linked_to_another_run"); } const dispatchingLinkedRun = Boolean( - input.runId && - linkedRuns.some((run) => run.id === input.runId) && - architectLinkedRuns.some((run) => run.id === input.runId), + acceptanceRunId && + linkedRuns.some((run) => run.id === acceptanceRunId) && + architectLinkedRuns.some((run) => run.id === acceptanceRunId), ); if (proposal.state === "executed" && !dispatchingLinkedRun) { return invalid("builder_plan_context_invalid", "executed_proposal_missing_linked_run"); diff --git a/services/api/src/config.ts b/services/api/src/config.ts index 260863c6..afe8dbc2 100644 --- a/services/api/src/config.ts +++ b/services/api/src/config.ts @@ -50,6 +50,7 @@ const EnvSchema = z // seeded default sandbox profile and `facility doctor` both key off this. FACILITY_RUNNER_IMAGE: z.string().default("facility-runner:dev"), FACILITY_REPOSITORY_WRITE_TRACKING_PROMOTION: z.enum(["0", "1"]).default("0"), + FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION: z.enum(["0", "1"]).default("0"), // Driver the seeded default sandbox profile uses. Must match the deployment: // "docker" for local/self-host, "vercel" for managed Sandboxes, or "aws" // for the CodeBuild development provider. @@ -273,6 +274,7 @@ export function readConfig(env = process.env): AppConfig { sandboxRunnerImage: parsed.FACILITY_RUNNER_IMAGE, repositoryWriteTrackingPromotionEnabled: parsed.FACILITY_REPOSITORY_WRITE_TRACKING_PROMOTION === "1", + governedBuilderRetryPromotionEnabled: parsed.FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION === "1", sandboxDriver: parsed.FACILITY_SANDBOX_DRIVER, authIdentityProvider: parsed.AUTH_IDENTITY_PROVIDER, authCallbackUrl: parsed.AUTH_CALLBACK_URL ?? `${webUrl.replace(/\/$/, "")}/api/auth/callback`, diff --git a/services/api/src/executors.ts b/services/api/src/executors.ts index 745bb908..e312b8a4 100644 --- a/services/api/src/executors.ts +++ b/services/api/src/executors.ts @@ -41,6 +41,7 @@ import { } from "./builder-plan-freshness.js"; import { architectRunIdentityValid, + assertGenericRunResumeAllowed, builderPlanDenialCode, builderPlanRequired, lockBuilderPlanPolicy, @@ -627,6 +628,7 @@ export async function loadPlanBuilderRun(db: Db, proposal: typeof proposals.$inf eq(runs.orgId, proposal.orgId), eq(runs.projectId, proposal.projectId ?? ""), inArray(runs.mode, ["builder", "codex-builder"]), + isNull(runs.retryOfRunId), sql`${runs.trigger} @> ${JSON.stringify({ source: "plan_acceptance", proposalId: proposal.id, @@ -648,6 +650,7 @@ async function loadArchitectBuilderRun(db: Db, proposal: typeof proposals.$infer eq(runs.orgId, proposal.orgId), eq(runs.projectId, proposal.projectId ?? ""), inArray(runs.mode, ["builder", "codex-builder"]), + isNull(runs.retryOfRunId), sql`${runs.trigger} @> ${JSON.stringify({ source: "plan_acceptance", architectRunId: proposal.runId, @@ -1032,6 +1035,7 @@ async function executeKnownMcpTool( .limit(1) )[0]; if (!parent) throw new Error("run_not_found"); + await assertGenericRunResumeAllowed(db, parent); if (!TERMINAL_RUN_STATUSES.includes(parent.status as (typeof TERMINAL_RUN_STATUSES)[number])) { throw new Error("run_not_terminal"); } @@ -1176,6 +1180,22 @@ async function executeKnownMcpTool( .returning() )[0]; if (!conversation) throw new Error("conversation_turn_in_flight"); + if (conversation.engineSessionId && conversation.lastRunId) { + const parent = ( + await tx + .select() + .from(runs) + .where( + and( + eq(runs.orgId, conversation.orgId), + eq(runs.projectId, conversation.projectId), + eq(runs.id, conversation.lastRunId), + ), + ) + .limit(1) + )[0]; + if (parent) await assertGenericRunResumeAllowed(tx, parent); + } const rows = await tx .select({ max: sql`coalesce(max(${conversationMessages.seq}), 0)` }) .from(conversationMessages) diff --git a/services/api/src/governed-builder-retry.ts b/services/api/src/governed-builder-retry.ts new file mode 100644 index 00000000..586c2665 --- /dev/null +++ b/services/api/src/governed-builder-retry.ts @@ -0,0 +1,940 @@ +import { createHash } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { newId } from "@facility/core"; +import { + agentDefs, + apiKeys, + type FacilityDb, + insertAuditEvent, + outcomes, + proposals, + repos, + runDeliveries, + runEvents, + runRepositoryWriteLeases, + runs, + virtualKeys, +} from "@facility/db"; +import { agentDefTriggersBuilder, isBuilderMode } from "@facility/run-objective"; +import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { + type BuilderPlanFreshnessEvidence, + type BuilderPlanFreshnessOptions, + resolveBuilderPlanFreshnessForRun, +} from "./builder-plan-freshness.js"; +import { + assertBuilderPlanDispatch, + builderPlanDenialCode, + builderPlanRequired, + lockBuilderPlanPolicy, + recordBuilderPlanDenial, +} from "./builder-plan-policy.js"; +import { ApiError } from "./errors.js"; +import { laneFor } from "./github/agent-routing.js"; +import { + createGithubClientFactory, + type FacilityGithubClient, + type GithubClientFactory, +} from "./github/client.js"; +import { createGithubClientForRepo } from "./github/kickstart.js"; +import { + inspectRemoteRepositoryWriteOutput, + repositoryWriteLeaseEligibility, +} from "./repository-write-lease.js"; +import { acquireExclusiveRunTransitionTransactionLease } from "./run-api-key-lease.js"; +import { readSandbox } from "./sandbox/state.js"; +import type { AppConfig } from "./types.js"; + +type RunRow = typeof runs.$inferSelect; +type RetryActor = { type: "user" | "key" | "system"; id: string }; +type EligibleRepositoryWriteEvidence = Extract< + ReturnType, + { eligible: true } +>; + +export type GovernedRetryExternalOptions = BuilderPlanFreshnessOptions & { + config?: AppConfig; + githubFactory?: GithubClientFactory; + repositoryWriteClient?: { + repoId: string; + client: Pick< + FacilityGithubClient, + "assertRepositoryAccessible" | "getRef" | "listPullRequestsForHead" + >; + }; +}; + +export type GovernedRetryLineage = { + attempt: RunRow; + parent: RunRow | null; + root: RunRow; + /** Current attempt followed by every immutable ancestor through the root. */ + attempts: RunRow[]; + /** Number of immutable retry edges between this attempt and the root. */ + depth: number; +}; + +export type GovernedRetryEvidence = { + freshness: BuilderPlanFreshnessEvidence; + /** Digest of every completed attempt's repository-write evidence in lineage order. */ + repositoryLeaseChainDigest: string; +}; + +export type GovernedRetryCreation = { + run: RunRow; + created: boolean; +}; + +const FINGERPRINT_MAX_AGE_MS = 5 * 60_000; +const MAX_LINEAGE_DEPTH = 100; +const GOVERNED_RETRY_DENIAL_CODES = new Set([ + "governed_retry_agent_invalid", + "governed_retry_cleanup_incomplete", + "governed_retry_durable_output", + "governed_retry_lane_invalid", + "governed_retry_lineage_invalid", + "governed_retry_output_indeterminate", + "governed_retry_parent_not_found", + "governed_retry_parent_not_retryable", + "governed_retry_policy_required", + "governed_retry_repository_write_blocked", + "governed_retry_requires_fresh_gate1", +]); +const GOVERNED_RETRY_DENIAL_REASONS = new Set([ + "ancestor_not_failed", + "base_or_issue_revision_changed", + "builder_agent_disabled_or_changed", + "delivery_or_outcome_exists", + "execution_identity_changed", + "github_client_repo_mismatch", + "github_client_unavailable", + "github_issue_identity_missing", + "legacy_resume_descendant_exists", + "legacy_repository_write_tracking_unavailable", + "lineage_changed", + "lineage_cycle_or_depth", + "lineage_depth_exceeded", + "parent_not_failed", + "parent_not_failed_plan_builder", + "parent_not_found", + "project_policy_not_required", + "proposal_not_found", + "proposal_repository_invalid", + "recorded_repository_output", + "remote_branch_or_pull_request_exists", + "remote_state_unavailable", + "repository_fingerprint_unverified", + "repository_issue_changed", + "repository_lane_changed", + "repository_not_found", + "repository_write_credential_persistent", + "repository_write_credential_unexpired", + "repository_write_lease_ambiguous", + "repository_write_lease_malformed", + "repository_write_lease_reserved", + "repository_write_tracking_unavailable", + "retry_parent_missing", + "root_not_plan_builder", + "run_credentials_live", + "sandbox_not_destroyed", + "successor_not_clean", + "write_base_changed", + "write_evidence_changed", +]); + +/** Resolve and revalidate the immutable root -> ... -> attempt chain. */ +export async function resolveGovernedRetryLineage( + db: FacilityDb, + attempt: RunRow, +): Promise { + const seen = new Set([attempt.id]); + const attempts = [attempt]; + let child = attempt; + let directParent: RunRow | null = null; + let depth = 0; + while (child.retryOfRunId) { + if (depth >= MAX_LINEAGE_DEPTH || seen.has(child.retryOfRunId)) { + throw retryError("governed_retry_lineage_invalid", "lineage_cycle_or_depth"); + } + const parent = ( + await db + .select() + .from(runs) + .where( + and( + eq(runs.orgId, attempt.orgId), + eq(runs.projectId, attempt.projectId), + eq(runs.id, child.retryOfRunId), + ), + ) + .limit(1) + )[0]; + if (!parent) throw retryError("governed_retry_lineage_invalid", "parent_not_found"); + if (parent.status !== "failed") { + throw retryError("governed_retry_lineage_invalid", "ancestor_not_failed"); + } + assertSameExecutionIdentity(child, parent); + if (!directParent) directParent = parent; + seen.add(parent.id); + attempts.push(parent); + child = parent; + depth += 1; + } + if (objectValue(child.trigger).source !== "plan_acceptance" || !isBuilderMode(child.mode)) { + throw retryError("governed_retry_lineage_invalid", "root_not_plan_builder"); + } + return { attempt, parent: directParent, root: child, attempts, depth }; +} + +/** + * Resolve GitHub freshness and every repository-write lease for all completed + * attempts in a retry chain. Tracking version zero is rejected before any + * network lookup. + */ +export async function resolveGovernedRetryEvidence( + db: FacilityDb, + completedAttempts: readonly RunRow[], + root: RunRow, + options: GovernedRetryExternalOptions = {}, +): Promise { + const parent = completedAttempts[0]; + if (!parent) throw retryError("governed_retry_lineage_invalid", "parent_not_found"); + assertRetryableParent(parent); + await assertNoLegacyResumeDescendants(db, completedAttempts); + + const leaseEvidence: Array<{ + attempt: RunRow; + rows: (typeof runRepositoryWriteLeases.$inferSelect)[]; + eligibility: EligibleRepositoryWriteEvidence; + }> = []; + const now = new Date(); + await assertNoDurableRetryOutput(db, completedAttempts); + const leasesByAttempt = await loadRepositoryWriteLeases(db, completedAttempts); + for (const attempt of completedAttempts) { + const rows = leasesByAttempt.get(attempt.id) ?? []; + const eligibility = repositoryWriteLeaseEligibility(rows, now, { + trackingVersion: attempt.repositoryWriteTrackingVersion, + }); + if (!eligibility.eligible) { + if (eligibility.reason === "repository_write_tracking_unavailable") { + throw new ApiError( + 409, + "governed_retry_requires_fresh_gate1", + "This historical Builder run cannot prove complete repository-write tracking", + { + reason: "legacy_repository_write_tracking_unavailable", + requiredAction: "run_architect_and_approve_new_plan", + }, + ); + } + throw retryError("governed_retry_repository_write_blocked", eligibility.reason); + } + leaseEvidence.push({ attempt, rows, eligibility }); + } + + const freshness = await resolveBuilderPlanFreshnessForRun(db, root, options); + for (const { attempt, eligibility } of leaseEvidence) { + for (const inspection of eligibility.remoteInspections) { + if (inspection.baseSha.toLowerCase() !== freshness.baseSha.toLowerCase()) { + throw retryError("governed_retry_repository_write_blocked", "write_base_changed"); + } + const repo = ( + await db + .select() + .from(repos) + .where( + and( + eq(repos.orgId, attempt.orgId), + eq(repos.projectId, attempt.projectId), + eq(repos.id, inspection.repoId), + ), + ) + .limit(1) + )[0]; + if (!repo) throw retryError("governed_retry_output_indeterminate", "repository_not_found"); + const client = await repositoryWriteClient(db, repo, options); + const remote = await inspectRemoteRepositoryWriteOutput( + client, + inspection.authorizedBranch, + inspection.baseSha, + ); + if (remote.state === "indeterminate") { + throw retryError("governed_retry_output_indeterminate", "remote_state_unavailable"); + } + if (remote.state === "durable_output") { + throw retryError("governed_retry_durable_output", "remote_branch_or_pull_request_exists"); + } + } + } + return { + freshness, + repositoryLeaseChainDigest: repositoryLeaseChainDigest(leaseEvidence), + }; +} + +/** Revalidate all durable state after the project and parent locks are held. */ +export async function assertGovernedRetryLockedAdmission( + db: FacilityDb, + completedAttempts: readonly RunRow[], + root: RunRow, + evidence: GovernedRetryEvidence, + actor: RetryActor, + source: string, +): Promise { + const parent = completedAttempts[0]; + if (!parent) throw retryError("governed_retry_lineage_invalid", "parent_not_found"); + assertRetryableParent(parent); + await assertNoLegacyResumeDescendants(db, completedAttempts); + const now = new Date(); + const leaseEvidence: Array<{ + attempt: RunRow; + rows: (typeof runRepositoryWriteLeases.$inferSelect)[]; + }> = []; + await assertNoDurableRetryOutput(db, completedAttempts); + const leasesByAttempt = await loadRepositoryWriteLeases(db, completedAttempts); + for (const attempt of completedAttempts) { + const rows = leasesByAttempt.get(attempt.id) ?? []; + const eligibility = repositoryWriteLeaseEligibility(rows, now, { + trackingVersion: attempt.repositoryWriteTrackingVersion, + }); + if (!eligibility.eligible) { + throw retryError("governed_retry_repository_write_blocked", "write_evidence_changed"); + } + leaseEvidence.push({ attempt, rows }); + } + if (repositoryLeaseChainDigest(leaseEvidence) !== evidence.repositoryLeaseChainDigest) { + throw retryError("governed_retry_repository_write_blocked", "write_evidence_changed"); + } + if (!(await builderPlanRequired(db, parent.orgId, parent.projectId))) { + throw retryError("governed_retry_policy_required", "project_policy_not_required"); + } + const trigger = objectValue(root.trigger); + const proposalId = stringValue(trigger.proposalId); + const proposal = proposalId + ? ( + await db + .select() + .from(proposals) + .where( + and( + eq(proposals.orgId, parent.orgId), + eq(proposals.projectId, parent.projectId), + eq(proposals.id, proposalId), + ), + ) + .limit(1) + )[0] + : undefined; + if (!proposal) throw retryError("governed_retry_lineage_invalid", "proposal_not_found"); + const payload = objectValue(proposal.payload); + const repoId = stringValue(payload.repoId); + const repo = repoId + ? ( + await db + .select() + .from(repos) + .where( + and( + eq(repos.orgId, parent.orgId), + eq(repos.projectId, parent.projectId), + eq(repos.id, repoId), + ), + ) + .limit(1) + )[0] + : undefined; + if (!repo) throw retryError("governed_retry_lineage_invalid", "proposal_repository_invalid"); + const gh = objectValue(root.gh); + if ( + gh.owner !== repo.owner || + gh.repo !== repo.name || + positiveInteger(gh.issueNumber) !== positiveInteger(payload.issueNumber) + ) { + throw retryError("governed_retry_lineage_invalid", "repository_issue_changed"); + } + const agent = root.agentDefId + ? ( + await db + .select() + .from(agentDefs) + .where( + and( + eq(agentDefs.orgId, root.orgId), + eq(agentDefs.projectId, root.projectId), + eq(agentDefs.id, root.agentDefId), + ), + ) + .limit(1) + )[0] + : undefined; + if ( + !agent?.enabled || + agent.engine !== root.engine || + !( + isBuilderMode(root.mode) || + isBuilderMode(agent.name) || + agentDefTriggersBuilder(agent.triggers) + ) + ) { + throw retryError("governed_retry_agent_invalid", "builder_agent_disabled_or_changed"); + } + if (laneFor(repo, root.mode) !== "platform") { + throw retryError("governed_retry_lane_invalid", "repository_lane_changed"); + } + const verifiedAt = repo.fingerprintVerifiedAt?.getTime() ?? Number.NaN; + if ( + !repo.fingerprint || + repo.fingerprintStatus !== "ok" || + !Number.isFinite(verifiedAt) || + verifiedAt < Date.now() - FINGERPRINT_MAX_AGE_MS || + verifiedAt > Date.now() + 30_000 + ) { + throw retryError("governed_retry_lane_invalid", "repository_fingerprint_unverified"); + } + await assertBuilderPlanDispatch(db, { + orgId: root.orgId, + projectId: root.projectId, + mode: root.mode, + agentDefId: root.agentDefId, + trigger: root.trigger, + gh: root.gh, + runId: root.id, + actor, + source, + freshnessEvidence: evidence.freshness, + }); +} + +export async function createGovernedBuilderRetry( + db: FacilityDb, + transitionDb: FacilityDb, + input: { + orgId: string; + projectId?: string | null; + parentRunId: string; + actor: RetryActor; + reason?: string; + }, + options: GovernedRetryExternalOptions = {}, +): Promise { + const parent = await loadScopedRun(db, input.orgId, input.projectId, input.parentRunId); + const existing = await loadRetryChild(db, parent); + if (existing) { + assertSameExecutionIdentity(existing, parent); + return { run: existing, created: false }; + } + let lineage: GovernedRetryLineage | undefined; + let evidence: GovernedRetryEvidence | undefined; + try { + lineage = await resolveGovernedRetryLineage(db, parent); + if (lineage.depth >= MAX_LINEAGE_DEPTH) { + throw retryError("governed_retry_lineage_invalid", "lineage_depth_exceeded"); + } + evidence = await resolveGovernedRetryEvidence(db, lineage.attempts, lineage.root, options); + const admissionEvidence = evidence; + + return await transitionDb.transaction(async (tx) => { + const lockedDb = tx as unknown as FacilityDb; + await lockBuilderPlanPolicy(lockedDb, parent.orgId, parent.projectId); + await acquireExclusiveRunTransitionTransactionLease(lockedDb, parent.id); + const lockedParent = ( + await lockedDb + .select() + .from(runs) + .where( + and( + eq(runs.orgId, parent.orgId), + eq(runs.projectId, parent.projectId), + eq(runs.id, parent.id), + ), + ) + .for("update") + .limit(1) + )[0]; + if (!lockedParent) throw retryError("governed_retry_parent_not_found", "parent_not_found"); + const winningChild = await loadRetryChild(lockedDb, lockedParent); + if (winningChild) { + assertSameExecutionIdentity(winningChild, lockedParent); + return { run: winningChild, created: false }; + } + const lockedLineage = await resolveGovernedRetryLineage(lockedDb, lockedParent); + if (lockedLineage.depth >= MAX_LINEAGE_DEPTH) { + throw retryError("governed_retry_lineage_invalid", "lineage_depth_exceeded"); + } + await assertGovernedRetryLockedAdmission( + lockedDb, + lockedLineage.attempts, + lockedLineage.root, + admissionEvidence, + input.actor, + "governed_retry_admission", + ); + + // builder-plan-preflight: governed_builder_retry + const child = ( + await lockedDb + .insert(runs) + .values({ + id: newId("run"), + orgId: lockedParent.orgId, + projectId: lockedParent.projectId, + agentDefId: lockedParent.agentDefId, + mode: lockedParent.mode, + engine: lockedParent.engine, + retryOfRunId: lockedParent.id, + trigger: lockedParent.trigger, + gh: minimalGithubIdentity(lockedLineage.root.gh), + createdBy: input.actor, + }) + .returning() + )[0]; + if (!child) throw new ApiError(500, "run_create_failed", "Retry run could not be created"); + const trigger = objectValue(lockedLineage.root.trigger); + await lockedDb.insert(runEvents).values({ + orgId: child.orgId, + runId: child.id, + seq: 1, + type: "queued", + data: { + queue: "runs.dispatch", + source: "governed_retry", + rootRunId: lockedLineage.root.id, + parentRunId: lockedParent.id, + proposalId: stringValue(trigger.proposalId), + architectRunId: stringValue(trigger.architectRunId), + }, + }); + await insertAuditEvent(lockedDb, { + orgId: child.orgId, + projectId: child.projectId, + actor: input.actor, + action: "run.retried", + target: { type: "run", id: child.id }, + payload: { + rootRunId: lockedLineage.root.id, + parentRunId: lockedParent.id, + childRunId: child.id, + proposalId: stringValue(trigger.proposalId), + architectRunId: stringValue(trigger.architectRunId), + planSha256: sha256Value(trigger.planSha256), + baseSha: admissionEvidence.freshness.baseSha, + issueRevisionSha256: admissionEvidence.freshness.issueRevisionSha256, + ...(sanitizedReason(input.reason) ? { reason: sanitizedReason(input.reason) } : {}), + }, + }); + return { run: child, created: true }; + }); + } catch (error) { + const planCode = error instanceof ApiError ? builderPlanDenialCode(error.code) : null; + if (planCode && lineage) { + await recordBuilderPlanDenial( + db, + { + orgId: lineage.root.orgId, + projectId: lineage.root.projectId, + mode: lineage.root.mode, + agentDefId: lineage.root.agentDefId, + trigger: lineage.root.trigger, + gh: lineage.root.gh, + runId: lineage.root.id, + actor: input.actor, + source: "governed_retry_admission", + freshnessEvidence: evidence?.freshness, + }, + planCode, + governedReason(error, "transactional_retry_preflight_denied"), + ).catch(() => undefined); + } + await recordGovernedRetryDenial( + db, + parent, + input.actor, + "governed_retry_admission", + error, + ).catch(() => undefined); + throw error; + } +} + +export function governedRetryDenial(error: unknown): { code: string; reason: string } | null { + if (!(error instanceof ApiError) || !GOVERNED_RETRY_DENIAL_CODES.has(error.code)) return null; + return { code: error.code, reason: governedReason(error, "denied") }; +} + +export async function recordGovernedRetryDenial( + db: FacilityDb, + run: Pick, + actor: RetryActor, + source: string, + error: unknown, +): Promise { + const denial = governedRetryDenial(error); + if (!denial) return; + await insertAuditEvent(db, { + orgId: run.orgId, + projectId: run.projectId, + actor, + action: "run.governed_retry_denied", + target: { type: "run", id: run.id }, + payload: { + code: denial.code, + reason: denial.reason, + source, + parentRunId: run.retryOfRunId, + }, + }); +} + +export async function validateGovernedRetryForDispatch( + db: FacilityDb, + run: RunRow, + options: GovernedRetryExternalOptions = {}, +): Promise<{ lineage: GovernedRetryLineage; evidence: GovernedRetryEvidence }> { + if (!run.retryOfRunId) { + throw retryError("governed_retry_lineage_invalid", "retry_parent_missing"); + } + const lineage = await resolveGovernedRetryLineage(db, run); + if (lineage.parent?.status !== "failed") { + throw retryError("governed_retry_lineage_invalid", "parent_not_failed"); + } + assertSameExecutionIdentity(run, lineage.parent); + if (["queued", "provisioning"].includes(run.status)) + assertUnprovisionedRetryChild(run, lineage.root); + const evidence = await resolveGovernedRetryEvidence( + db, + lineage.attempts.slice(1), + lineage.root, + options, + ); + return { lineage, evidence }; +} + +export async function assertGovernedRetryDispatchState( + db: FacilityDb, + run: RunRow, + expected: { lineage: GovernedRetryLineage; evidence: GovernedRetryEvidence }, +): Promise { + const currentLineage = await resolveGovernedRetryLineage(db, run); + if ( + !isDeepStrictEqual( + currentLineage.attempts.map((attempt) => attempt.id), + expected.lineage.attempts.map((attempt) => attempt.id), + ) + ) { + throw retryError("governed_retry_lineage_invalid", "lineage_changed"); + } + assertUnprovisionedRetryChild(run, currentLineage.root); + if (!currentLineage.parent) { + throw retryError("governed_retry_lineage_invalid", "parent_not_found"); + } + await assertGovernedRetryLockedAdmission( + db, + currentLineage.attempts.slice(1), + currentLineage.root, + expected.evidence, + { type: "system", id: "runs.dispatch" }, + "governed_retry_worker", + ); +} + +async function loadScopedRun( + db: FacilityDb, + orgId: string, + projectId: string | null | undefined, + runId: string, +) { + const row = ( + await db + .select() + .from(runs) + .where( + and( + eq(runs.orgId, orgId), + eq(runs.id, runId), + ...(projectId ? [eq(runs.projectId, projectId)] : []), + ), + ) + .limit(1) + )[0]; + if (!row) throw new ApiError(404, "run_not_found", "Run not found"); + return row; +} + +async function loadRetryChild(db: FacilityDb, parent: RunRow) { + return ( + await db + .select() + .from(runs) + .where( + and( + eq(runs.orgId, parent.orgId), + eq(runs.projectId, parent.projectId), + eq(runs.retryOfRunId, parent.id), + ), + ) + .limit(1) + )[0]; +} + +async function loadRepositoryWriteLeases(db: FacilityDb, attempts: readonly RunRow[]) { + const first = assertSameLineageScope(attempts); + const rows = await db + .select() + .from(runRepositoryWriteLeases) + .where( + and( + eq(runRepositoryWriteLeases.orgId, first.orgId), + eq(runRepositoryWriteLeases.projectId, first.projectId), + inArray( + runRepositoryWriteLeases.runId, + attempts.map((attempt) => attempt.id), + ), + ), + ); + const byAttempt = new Map(); + for (const row of rows) { + const attemptRows = byAttempt.get(row.runId) ?? []; + attemptRows.push(row); + byAttempt.set(row.runId, attemptRows); + } + return byAttempt; +} + +async function assertNoLegacyResumeDescendants( + db: FacilityDb, + completedAttempts: readonly RunRow[], +) { + const first = assertSameLineageScope(completedAttempts); + const attemptIds = completedAttempts.map((attempt) => attempt.id); + const legacy = await db + .select({ id: runs.id }) + .from(runs) + .where( + and( + eq(runs.orgId, first.orgId), + eq(runs.projectId, first.projectId), + isNull(runs.retryOfRunId), + inArray(sql`${runs.trigger}->>'resumeOf'`, attemptIds), + ), + ) + .limit(1); + if (legacy.length) { + throw retryError("governed_retry_durable_output", "legacy_resume_descendant_exists"); + } +} + +async function assertNoDurableRetryOutput(db: FacilityDb, attempts: readonly RunRow[]) { + const first = assertSameLineageScope(attempts); + const attemptIds = attempts.map((attempt) => attempt.id); + for (const attempt of attempts) { + const gh = objectValue(attempt.gh); + const receiptGithub = objectValue(objectValue(attempt.receipt).github); + if ( + stringValue(gh.branch) || + stringValue(gh.headSha) || + Object.keys(objectValue(gh.pr)).length > 0 || + positiveInteger(receiptGithub.pr) + ) { + throw retryError("governed_retry_durable_output", "recorded_repository_output"); + } + const sandbox = readSandbox(attempt.sandbox); + if (sandbox.ref && (!sandbox.destroyedAt || sandbox.lastStatus !== "destroyed")) { + throw retryError("governed_retry_cleanup_incomplete", "sandbox_not_destroyed"); + } + } + const [delivery, outcome, livePlatformKey, liveVirtualKey] = await Promise.all([ + db + .select({ runId: runDeliveries.runId }) + .from(runDeliveries) + .where(and(eq(runDeliveries.orgId, first.orgId), inArray(runDeliveries.runId, attemptIds))) + .limit(1), + db + .select({ id: outcomes.id }) + .from(outcomes) + .where(and(eq(outcomes.orgId, first.orgId), inArray(outcomes.runId, attemptIds))) + .limit(1), + db + .select({ id: apiKeys.id }) + .from(apiKeys) + .where( + and( + eq(apiKeys.orgId, first.orgId), + inArray(apiKeys.runId, attemptIds), + isNull(apiKeys.revokedAt), + ), + ) + .limit(1), + db + .select({ id: virtualKeys.id }) + .from(virtualKeys) + .where( + and( + eq(virtualKeys.orgId, first.orgId), + inArray(virtualKeys.runId, attemptIds), + isNull(virtualKeys.revokedAt), + ), + ) + .limit(1), + ]); + if (delivery.length || outcome.length) { + throw retryError("governed_retry_durable_output", "delivery_or_outcome_exists"); + } + if (livePlatformKey.length || liveVirtualKey.length) { + throw retryError("governed_retry_cleanup_incomplete", "run_credentials_live"); + } +} + +function assertSameLineageScope(attempts: readonly RunRow[]) { + const first = attempts[0]; + if ( + !first || + attempts.some( + (attempt) => attempt.orgId !== first.orgId || attempt.projectId !== first.projectId, + ) + ) { + throw retryError("governed_retry_lineage_invalid", "lineage_changed"); + } + return first; +} + +function assertRetryableParent(parent: RunRow) { + if ( + parent.status !== "failed" || + !isBuilderMode(parent.mode) || + objectValue(parent.trigger).source !== "plan_acceptance" + ) { + throw retryError("governed_retry_parent_not_retryable", "parent_not_failed_plan_builder"); + } +} + +function assertSameExecutionIdentity(child: RunRow, parent: RunRow) { + if ( + child.orgId !== parent.orgId || + child.projectId !== parent.projectId || + child.retryOfRunId !== parent.id || + child.agentDefId !== parent.agentDefId || + child.mode !== parent.mode || + child.engine !== parent.engine || + !isDeepStrictEqual(child.trigger, parent.trigger) + ) { + throw retryError("governed_retry_lineage_invalid", "execution_identity_changed"); + } +} + +function assertUnprovisionedRetryChild(child: RunRow, root: RunRow) { + if ( + !["queued", "provisioning"].includes(child.status) || + !isDeepStrictEqual(objectValue(child.gh), minimalGithubIdentity(root.gh)) || + Object.keys(readSandbox(child.sandbox)).length > 0 || + child.receipt !== null || + child.engineSessionId !== null || + child.transcriptUri !== null || + child.sessionStateUri !== null || + child.workspaceBaseSha !== null || + child.error !== null + ) { + throw retryError("governed_retry_lineage_invalid", "successor_not_clean"); + } +} + +function minimalGithubIdentity(value: unknown) { + const gh = objectValue(value); + const owner = stringValue(gh.owner); + const repo = stringValue(gh.repo); + const issueNumber = positiveInteger(gh.issueNumber); + if (!owner || !repo || !issueNumber) { + throw retryError("governed_retry_lineage_invalid", "github_issue_identity_missing"); + } + return { owner, repo, issueNumber }; +} + +async function repositoryWriteClient( + db: FacilityDb, + repo: typeof repos.$inferSelect, + options: GovernedRetryExternalOptions, +) { + if (options.repositoryWriteClient) { + if (options.repositoryWriteClient.repoId !== repo.id) { + throw retryError("governed_retry_output_indeterminate", "github_client_repo_mismatch"); + } + return options.repositoryWriteClient.client; + } + const factory = + options.githubFactory ?? + (options.config?.githubAppId && options.config.githubAppPrivateKey + ? createGithubClientFactory(options.config) + : null); + if (!factory) { + throw retryError("governed_retry_output_indeterminate", "github_client_unavailable"); + } + try { + return await createGithubClientForRepo(db, factory, repo); + } catch { + throw retryError("governed_retry_output_indeterminate", "github_client_unavailable"); + } +} + +function repositoryLeaseChainDigest( + evidence: readonly { + attempt: RunRow; + rows: readonly (typeof runRepositoryWriteLeases.$inferSelect)[]; + }[], +) { + return createHash("sha256") + .update( + JSON.stringify( + evidence.map(({ attempt, rows }) => ({ + runId: attempt.id, + trackingVersion: attempt.repositoryWriteTrackingVersion, + leases: [...rows] + .sort((left, right) => left.id.localeCompare(right.id)) + .map((row) => ({ + id: row.id, + repoId: row.repoId, + provider: row.provider, + status: row.status, + requestedBranch: row.requestedBranch, + authorizedBranch: row.authorizedBranch, + baseSha: row.baseSha, + permissions: row.permissions, + issuedAt: row.issuedAt?.toISOString() ?? null, + expiresAt: row.expiresAt?.toISOString() ?? null, + failureReason: row.failureReason, + })), + })), + ), + ) + .digest("hex"); +} + +function sanitizedReason(value: string | undefined) { + const normalized = value?.trim().replaceAll(/\p{Cc}/gu, " "); + return normalized ? normalized.slice(0, 500) : null; +} + +function retryError(code: string, reason: string) { + return new ApiError(409, code, "Governed Builder retry was denied", { reason }); +} + +function governedReason(error: unknown, fallback: string) { + const reason = stringValue(objectValue(error instanceof ApiError ? error.details : null).reason); + return reason && GOVERNED_RETRY_DENIAL_REASONS.has(reason) ? reason : fallback; +} + +function objectValue(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function stringValue(value: unknown) { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function positiveInteger(value: unknown) { + return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; +} + +function sha256Value(value: unknown) { + return typeof value === "string" && /^[a-f0-9]{64}$/i.test(value) ? value.toLowerCase() : null; +} diff --git a/services/api/src/routes/v1/conversations.ts b/services/api/src/routes/v1/conversations.ts index 840ef685..a360a7c2 100644 --- a/services/api/src/routes/v1/conversations.ts +++ b/services/api/src/routes/v1/conversations.ts @@ -10,7 +10,10 @@ import { import { and, desc, eq, sql } from "drizzle-orm"; import type { FastifyInstance } from "fastify"; import { z } from "zod"; -import { withBuilderPlanPreflight } from "../../builder-plan-policy.js"; +import { + assertGenericRunResumeAllowed, + withBuilderPlanPreflight, +} from "../../builder-plan-policy.js"; import { ApiError, notFound } from "../../errors.js"; import { assertBareRowProjectScope, @@ -221,6 +224,22 @@ export async function registerConversationsRoutes(app: FastifyInstance, context: .returning() )[0]; if (!claimed) return null; + if (claimed.engineSessionId && claimed.lastRunId) { + const parent = ( + await tx + .select() + .from(runs) + .where( + and( + eq(runs.orgId, claimed.orgId), + eq(runs.projectId, claimed.projectId), + eq(runs.id, claimed.lastRunId), + ), + ) + .limit(1) + )[0]; + if (parent) await assertGenericRunResumeAllowed(tx, parent); + } const rows = await tx .select({ max: sql`coalesce(max(seq), 0)` }) .from(conversationMessages) diff --git a/services/api/src/routes/v1/runs.ts b/services/api/src/routes/v1/runs.ts index 2f315312..30efa590 100644 --- a/services/api/src/routes/v1/runs.ts +++ b/services/api/src/routes/v1/runs.ts @@ -14,9 +14,13 @@ import { and, desc, eq, notInArray, sql } from "drizzle-orm"; import type { FastifyInstance, FastifyReply } from "fastify"; import postgres from "postgres"; import { z } from "zod"; -import { withBuilderPlanPreflight } from "../../builder-plan-policy.js"; +import { + assertGenericRunResumeAllowed, + withBuilderPlanPreflight, +} from "../../builder-plan-policy.js"; import { readTranscriptObject } from "../../envelopes.js"; import { ApiError, notFound } from "../../errors.js"; +import { createGovernedBuilderRetry } from "../../governed-builder-retry.js"; import { acquireExclusiveRunTransitionTransactionLease } from "../../run-api-key-lease.js"; import { cancelRun, revokeRunKeys } from "../../sandbox/orchestrator.js"; import { @@ -499,6 +503,54 @@ export async function registerRunsRoutes(app: FastifyInstance, context: V1RouteC }, ); + app.post( + "/v1/runs/:runId/retry", + { + config: { permission: "runs:trigger", idempotent: true, runLifecycle: true }, + schema: { + params: IdParams, + body: z + .object({ reason: z.string().max(500).optional() }) + .strict() + .nullable() + .default({}), + response: { 200: RunSchema }, + }, + }, + async (request) => { + if (!config.governedBuilderRetryPromotionEnabled) { + throw new ApiError( + 409, + "governed_retry_promotion_disabled", + "Governed Builder retry is not enabled on this Facility deployment", + { requiredAction: "complete_retry_worker_rollout_and_enable_promotion" }, + ); + } + const p = principal(request); + const { runId } = request.params as { runId: string }; + const body = (request.body ?? {}) as { reason?: string }; + const result = await createGovernedBuilderRetry( + db, + app.runTransitionDb, + { + orgId: p.orgId, + projectId: p.projectId, + parentRunId: runId, + actor: auditActor(p), + reason: body.reason, + }, + { + config, + githubFactory: app.githubClientFactory, + }, + ); + if (result.run.status === "queued") { + await app.enqueue("runs.dispatch", { runId: result.run.id, orgId: result.run.orgId }); + } + return redactRunSecrets(result.run); + }, + ); + app.post( "/v1/runs/:runId/cancel", { @@ -805,6 +857,7 @@ export async function registerRunsRoutes(app: FastifyInstance, context: V1RouteC const p = principal(request); const { runId } = request.params as { runId: string }; const parent = await loadRun(p, runId); + await assertGenericRunResumeAllowed(db, parent); if (!terminalStatus(parent.status)) { throw new ApiError(409, "run_not_terminal", "Only terminal runs can be resumed"); } diff --git a/services/api/src/routes/v1/shared.ts b/services/api/src/routes/v1/shared.ts index 656cc7a3..7fe8d3db 100644 --- a/services/api/src/routes/v1/shared.ts +++ b/services/api/src/routes/v1/shared.ts @@ -170,6 +170,7 @@ export const RunSchema = z.object({ mode: z.string(), engine: z.string(), status: z.string(), + retryOfRunId: z.string().nullable(), trigger: AnyObject, sandbox: AnyObject, receipt: z diff --git a/services/api/src/sandbox/orchestrator.ts b/services/api/src/sandbox/orchestrator.ts index 5f630636..167a2c35 100644 --- a/services/api/src/sandbox/orchestrator.ts +++ b/services/api/src/sandbox/orchestrator.ts @@ -47,6 +47,7 @@ import { } from "../builder-plan-freshness.js"; import { assertBuilderPlanDispatch, + assertGenericRunResumeAllowed, builderPlanDenialCode, builderPlanRequired, recordBuilderPlanDenial, @@ -80,6 +81,13 @@ import { sanitizeSecurityReport, syncSecurityFindings, } from "../github/security-findings.js"; +import { + assertGovernedRetryDispatchState, + type GovernedRetryExternalOptions, + governedRetryDenial, + recordGovernedRetryDenial, + validateGovernedRetryForDispatch, +} from "../governed-builder-retry.js"; import { harnessFragmentForBundle, validateProjectKb } from "../harness.js"; import { assertPreviewProvisioningAvailable, @@ -119,6 +127,7 @@ type DispatchRunDeps = { sandboxDriver?: (name: SandboxDriverName) => Promise; githubFactory?: GithubClientFactory; githubClient?: BuilderPlanFreshnessOptions["githubClient"]; + repositoryWriteClient?: GovernedRetryExternalOptions["repositoryWriteClient"]; }; type ArchitectPlanPublicationJob = { @@ -178,21 +187,41 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis let launchedSandbox: { driver: SandboxDriver; ref: string } | undefined; let run: RunRow | undefined; let freshnessFailureSource: "worker_initial_freshness" | "worker_claimed_freshness" | null = null; + let governedRetryFailureSource: + | "worker_initial_governed_retry" + | "worker_claimed_governed_retry" + | null = null; try { run = await loadRun(db, job.orgId, job.runId); if (run?.status !== "queued") return; - const requiredBuilderPlan = - isBuilderMode(run.mode) && (await builderPlanRequired(db, run.orgId, run.projectId)); - const requiredPlanFreshness = - requiredBuilderPlan && objectOrEmpty(run.trigger).source === "plan_acceptance"; - freshnessFailureSource = requiredPlanFreshness ? "worker_initial_freshness" : null; - const initialFreshness = requiredPlanFreshness - ? await resolveBuilderPlanFreshnessForRun(db, run, { + governedRetryFailureSource = run.retryOfRunId ? "worker_initial_governed_retry" : null; + freshnessFailureSource = run.retryOfRunId ? "worker_initial_freshness" : null; + const initialGovernedRetry = run.retryOfRunId + ? await validateGovernedRetryForDispatch(db, run, { config, githubFactory: deps.githubFactory, githubClient: deps.githubClient, + repositoryWriteClient: deps.repositoryWriteClient, }) : undefined; + const planRoot = initialGovernedRetry?.lineage.root ?? run; + const requiredBuilderPlan = + isBuilderMode(run.mode) && (await builderPlanRequired(db, run.orgId, run.projectId)); + const requiredPlanFreshness = + requiredBuilderPlan && objectOrEmpty(planRoot.trigger).source === "plan_acceptance"; + freshnessFailureSource = requiredPlanFreshness ? "worker_initial_freshness" : null; + const initialFreshness = initialGovernedRetry + ? initialGovernedRetry.evidence.freshness + : requiredPlanFreshness + ? await resolveBuilderPlanFreshnessForRun(db, planRoot, { + config, + githubFactory: deps.githubFactory, + githubClient: deps.githubClient, + }) + : undefined; + // assertBuilderPlanDispatch persists its own denial. Keep this source only + // around the external freshness lookup so the outer catch cannot duplicate + // that durable audit. freshnessFailureSource = null; await assertBuilderPlanDispatch(db, { orgId: run.orgId, @@ -202,6 +231,7 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis trigger: run.trigger, gh: run.gh, runId: run.id, + acceptanceRunId: initialGovernedRetry?.lineage.root.id, actor: { type: "system", id: "runs.dispatch" }, source: "worker_dispatch", freshnessEvidence: initialFreshness, @@ -220,14 +250,33 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis // claiming first prevents another worker racing ahead while this final // check fails. The outer failure boundary marks the row failed before any // credential or sandbox side effect is created. - freshnessFailureSource = requiredPlanFreshness ? "worker_claimed_freshness" : null; - const claimedFreshness = requiredPlanFreshness - ? await resolveBuilderPlanFreshnessForRun(db, run, { + const claimedCurrentRun = await loadRun(db, run.orgId, run.id); + if (!claimedCurrentRun) return; + governedRetryFailureSource = claimedCurrentRun.retryOfRunId + ? "worker_claimed_governed_retry" + : governedRetryFailureSource; + freshnessFailureSource = claimedCurrentRun.retryOfRunId + ? "worker_claimed_freshness" + : requiredPlanFreshness + ? "worker_claimed_freshness" + : null; + const claimedGovernedRetry = claimedCurrentRun.retryOfRunId + ? await validateGovernedRetryForDispatch(db, claimedCurrentRun, { config, githubFactory: deps.githubFactory, githubClient: deps.githubClient, + repositoryWriteClient: deps.repositoryWriteClient, }) : undefined; + const claimedFreshness = claimedGovernedRetry + ? claimedGovernedRetry.evidence.freshness + : requiredPlanFreshness + ? await resolveBuilderPlanFreshnessForRun(db, planRoot, { + config, + githubFactory: deps.githubFactory, + githubClient: deps.githubClient, + }) + : undefined; freshnessFailureSource = null; const claimedRunScope = { orgId: run.orgId, runId: run.id }; const dispatchSnapshot = await withBuilderPlanPreflight( @@ -240,6 +289,7 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis trigger: run.trigger, gh: run.gh, runId: run.id, + acceptanceRunId: claimedGovernedRetry?.lineage.root.id, actor: { type: "system", id: "runs.dispatch" }, source: "worker_claimed_dispatch", freshnessEvidence: claimedFreshness, @@ -247,6 +297,9 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis async (tx, admission) => { let claimedRun = await loadRun(tx, claimedRunScope.orgId, claimedRunScope.runId); if (claimedRun?.status !== "provisioning") return null; + if (claimedGovernedRetry) { + await assertGovernedRetryDispatchState(tx, claimedRun, claimedGovernedRetry); + } if (claimedRun.mode !== admission.mode) { const sealed = ( await tx @@ -429,6 +482,7 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis await updateGithubRunProgress(db, run.id, "provisioning", { config }).catch(() => undefined); } catch (error) { const builderPlanCode = error instanceof ApiError ? builderPlanDenialCode(error.code) : null; + const governedRetry = governedRetryDenial(error); if (builderPlanCode && freshnessFailureSource && run) { await recordBuilderPlanDenial( db, @@ -448,12 +502,28 @@ export async function dispatchRun(config: AppConfig, job: DispatchJob, deps: Dis "freshness_resolution_failed", ).catch(() => undefined); } + if (governedRetry && run) { + await recordGovernedRetryDenial( + db, + run, + { type: "system", id: "runs.dispatch" }, + governedRetryFailureSource ?? "worker_governed_retry", + error, + ).catch(() => undefined); + } + const stableFailure = governedRetry + ? `${governedRetry.code}:${governedRetry.reason}` + : (builderPlanCode ?? errorMessage(error)); await failRun( db, job.orgId, job.runId, - builderPlanCode ?? errorMessage(error), - builderPlanCode ? "builder_plan_denied" : "provision_failed", + stableFailure, + governedRetry + ? "governed_retry_denied" + : builderPlanCode + ? "builder_plan_denied" + : "provision_failed", ).catch(() => undefined); await updateGithubRunProgress(db, job.runId, "failed", { config }).catch(() => undefined); // failRun revokes by the persisted sandbox, which on a pre-persist failure @@ -2761,6 +2831,7 @@ async function resumeForRun(db: ReturnType["db"], run: RunRow) if (!parentId) return null; const parent = await loadResumeParent(db, run, parentId); if (!parent?.engineSessionId) return null; + await assertGenericRunResumeAllowed(db, parent); return { sessionId: parent.engineSessionId, sessionStateFrom: parent.id, @@ -2792,6 +2863,7 @@ async function resumeForRun(db: ReturnType["db"], run: RunRow) if (!conversation?.engineSessionId || !parentId) return null; const parent = await loadResumeParent(db, run, parentId); if (!parent) return null; + await assertGenericRunResumeAllowed(db, parent); return { sessionId: conversation.engineSessionId, sessionStateFrom: parent.id, @@ -3240,6 +3312,44 @@ async function canonicalRunReceipt( } async function previousReceiptDigest(db: ReturnType["db"], run: RunRow) { + if (run.retryOfRunId) { + const seen = new Set([run.id]); + let ancestorId: string | null = run.retryOfRunId; + for (let depth = 0; ancestorId && depth < 100; depth += 1) { + if (seen.has(ancestorId)) return null; + seen.add(ancestorId); + const ancestor: + | { + id: string; + retryOfRunId: string | null; + receipt: unknown; + } + | undefined = ( + await db + .select({ + id: runs.id, + retryOfRunId: runs.retryOfRunId, + receipt: runs.receipt, + }) + .from(runs) + .where( + and( + eq(runs.orgId, run.orgId), + eq(runs.projectId, run.projectId), + eq(runs.id, ancestorId), + ), + ) + .limit(1) + )[0]; + if (!ancestor) return null; + const parsed = FacilityReceiptSchema.safeParse(ancestor.receipt); + if (parsed.success && verifyFacilityReceipt(parsed.data)) { + return parsed.data.integrity?.payload_sha256 ?? null; + } + ancestorId = ancestor.retryOfRunId; + } + return null; + } const candidates = await db .select({ receipt: runs.receipt }) .from(runs) diff --git a/services/api/src/types.ts b/services/api/src/types.ts index 9d52f96f..1b57724f 100644 --- a/services/api/src/types.ts +++ b/services/api/src/types.ts @@ -50,6 +50,12 @@ export type AppConfig = { * old /push-token handler and create unrecorded write authority. */ repositoryWriteTrackingPromotionEnabled?: boolean; + /** + * Phase-two rollout gate for creating governed Builder successor rows. Keep + * false until every worker understands and revalidates immutable retry + * lineage. Existing successor rows remain enforceable when this is false. + */ + governedBuilderRetryPromotionEnabled?: boolean; // Driver the seeded default sandbox profile uses. sandboxDriver: "docker" | "aws" | "vercel"; authIdentityProvider?: "github" | "oidc"; diff --git a/services/api/test/api.test.ts b/services/api/test/api.test.ts index b6f9a68f..46020a6e 100644 --- a/services/api/test/api.test.ts +++ b/services/api/test/api.test.ts @@ -296,7 +296,7 @@ describe("api", async () => { : 0), 0, ), - ).toBe(140); + ).toBe(141); expect(document.paths["/v1/projects"]?.get?.security).toEqual([ { bearerAuth: [] }, { sessionCookie: [] }, @@ -312,6 +312,13 @@ describe("api", async () => { expect(document.paths["/v1/projects"]?.post?.parameters).toContainEqual( expect.objectContaining({ name: "Idempotency-Key", in: "header" }), ); + expect(document.paths["/v1/runs/{runId}/retry"]?.post).toMatchObject({ + "x-facility-permission": "runs:trigger", + tags: ["Runs"], + }); + expect(document.paths["/v1/runs/{runId}/retry"]?.post?.parameters).toContainEqual( + expect.objectContaining({ name: "Idempotency-Key", in: "header" }), + ); expect(document.paths["/health"]?.get?.security).toEqual([]); expect( document.paths["/v1/projects/{projectId}/previews/{previewId}/open"]?.get?.[ @@ -2132,26 +2139,77 @@ describe("api", async () => { data: { source: "plan_acceptance", architectRunId: architectRun.id }, }); - // Optional projects preserve the legacy resume shape: provenance is used - // only for the policy preflight and is not copied into the new row, where - // the plan-acceptance uniqueness indexes would reject it. + // Builder attempts are immutable even while the project policy is optional. + // Generic session resume must never bypass the governed successor route. await db .update(runs) .set({ status: "failed", engine: "claude_code", engineSessionId: "plan-resume-session" }) .where(eq(runs.id, builderRuns[0]?.id ?? "")); + + const originalRetryEnqueue = app.enqueue; + const originalRetryFactory = app.githubClientFactory; + const retryEnqueues: Array<{ queue: string; data: Record }> = []; + let retryGithubCalls = 0; + app.enqueue = async (queue, data) => { + retryEnqueues.push({ queue, data }); + return null; + }; + app.githubClientFactory = async () => { + retryGithubCalls += 1; + throw new Error("legacy retry reached GitHub"); + }; + try { + const promotionDisabled = await app.inject({ + method: "POST", + url: `/v1/runs/${builderRuns[0]?.id}/retry`, + headers: { cookie, "idempotency-key": `retry-disabled-${Date.now()}` }, + }); + expect(promotionDisabled.statusCode, promotionDisabled.body).toBe(409); + expect(promotionDisabled.json().error.code).toBe("governed_retry_promotion_disabled"); + expect( + await db + .select({ id: runs.id }) + .from(runs) + .where(eq(runs.retryOfRunId, builderRuns[0]?.id ?? "")), + ).toHaveLength(0); + expect(retryEnqueues).toHaveLength(0); + + config.governedBuilderRetryPromotionEnabled = true; + const legacyDenied = await app.inject({ + method: "POST", + url: `/v1/runs/${builderRuns[0]?.id}/retry`, + headers: { cookie, "idempotency-key": `retry-legacy-${Date.now()}` }, + }); + expect(legacyDenied.statusCode, legacyDenied.body).toBe(409); + expect(legacyDenied.json().error).toMatchObject({ + code: "governed_retry_requires_fresh_gate1", + details: { + reason: "legacy_repository_write_tracking_unavailable", + requiredAction: "run_architect_and_approve_new_plan", + }, + }); + expect(retryGithubCalls).toBe(0); + expect(retryEnqueues).toHaveLength(0); + expect( + await db + .select({ id: runs.id }) + .from(runs) + .where(eq(runs.retryOfRunId, builderRuns[0]?.id ?? "")), + ).toHaveLength(0); + } finally { + config.governedBuilderRetryPromotionEnabled = false; + app.enqueue = originalRetryEnqueue; + app.githubClientFactory = originalRetryFactory; + } + const resumedPlanRun = await app.inject({ method: "POST", url: `/v1/runs/${builderRuns[0]?.id}/resume`, headers: { cookie }, payload: { message: "Continue the optional legacy run" }, }); - expect(resumedPlanRun.statusCode).toBe(200); - expect(resumedPlanRun.json().trigger).toMatchObject({ - type: "resume", - resumeOf: builderRuns[0]?.id, - }); - expect(resumedPlanRun.json().trigger).not.toHaveProperty("source"); - expect(resumedPlanRun.json().trigger).not.toHaveProperty("proposalId"); + expect(resumedPlanRun.statusCode).toBe(409); + expect(resumedPlanRun.json().error.code).toBe("builder_resume_forbidden"); const duplicateProposal = await createInternalPlanProposal( architectRun.id, @@ -3067,7 +3125,11 @@ describe("api", async () => { throw new Error("MCP Builder denial reached GitHub"); }) as unknown as GithubClientFactory; - const executeDenied = async (toolName: string, args: Record) => { + const executeDenied = async ( + toolName: string, + args: Record, + expectedError = "builder_plan_required", + ) => { const before = await db .select({ id: runs.id }) .from(runs) @@ -3094,7 +3156,7 @@ describe("api", async () => { expect(approved.statusCode, approved.body).toBe(200); expect(approved.json()).toMatchObject({ state: "execution_failed", - executionError: "builder_plan_required", + executionError: expectedError, }); expect( await db.select({ id: runs.id }).from(runs).where(eq(runs.projectId, target.projectId)), @@ -3113,14 +3175,79 @@ describe("api", async () => { number: 204, agentName: "builder", }); - await executeDenied("facility_resume_run", { - runId: terminalBuilder.id, - message: "Attempt governed MCP resume", + await executeDenied( + "facility_resume_run", + { + runId: terminalBuilder.id, + message: "Attempt governed MCP resume", + }, + "builder_resume_forbidden", + ); + await db + .update(conversations) + .set({ + lastRunId: terminalBuilder.id, + engineSessionId: terminalBuilder.engineSessionId, + status: "idle", + }) + .where(eq(conversations.id, governedConversation.id)); + // Conversation continuation is an implicit session resume. Verify both + // producers reject it synchronously even while Gate 1 is optional, and + // leave no message/run/status churn for a worker to clean up later. + await db + .update(projects) + .set({ builderPlanPolicy: "optional" }) + .where(and(eq(projects.orgId, orgId), eq(projects.id, target.projectId))); + const conversationRunsBefore = await db + .select({ id: runs.id }) + .from(runs) + .where(eq(runs.projectId, target.projectId)); + const conversationMessagesBefore = await db + .select({ id: conversationMessages.id }) + .from(conversationMessages) + .where(eq(conversationMessages.conversationId, governedConversation.id)); + await executeDenied( + "facility_send_conversation_message", + { + conversationId: governedConversation.id, + body: "Attempt governed Builder conversation turn", + }, + "builder_resume_forbidden", + ); + const restConversationDenied = await app.inject({ + method: "POST", + url: `/v1/conversations/${governedConversation.id}/messages`, + headers: { cookie }, + payload: { body: "Attempt governed Builder REST conversation turn" }, }); - await executeDenied("facility_send_conversation_message", { - conversationId: governedConversation.id, - body: "Attempt governed Builder conversation turn", + expect(restConversationDenied.statusCode, restConversationDenied.body).toBe(409); + expect(restConversationDenied.json().error.code).toBe("builder_resume_forbidden"); + expect( + await db.select({ id: runs.id }).from(runs).where(eq(runs.projectId, target.projectId)), + ).toHaveLength(conversationRunsBefore.length); + expect( + await db + .select({ id: conversationMessages.id }) + .from(conversationMessages) + .where(eq(conversationMessages.conversationId, governedConversation.id)), + ).toHaveLength(conversationMessagesBefore.length); + expect( + ( + await db + .select() + .from(conversations) + .where(eq(conversations.id, governedConversation.id)) + .limit(1) + )[0], + ).toMatchObject({ + status: "idle", + lastRunId: terminalBuilder.id, + engineSessionId: terminalBuilder.engineSessionId, }); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(and(eq(projects.orgId, orgId), eq(projects.id, target.projectId))); const beforeRepos = await db .select({ id: repos.id }) .from(repos) @@ -3167,13 +3294,9 @@ describe("api", async () => { .filter((event) => event.action === "run.builder_plan_denied") .map((event) => (event.payload as { source?: unknown }).source); expect(sources).toEqual( - expect.arrayContaining([ - "mcp_trigger_run", - "mcp_trigger_github_issue", - "mcp_resume_run", - "mcp_conversation_message", - ]), + expect.arrayContaining(["mcp_trigger_run", "mcp_trigger_github_issue"]), ); + expect(sources).not.toContain("mcp_conversation_message"); } finally { app.enqueue = originalEnqueue; app.githubClientFactory = originalFactory; @@ -3259,8 +3382,7 @@ describe("api", async () => { id: newId("run"), orgId, projectId: target.projectId, - agentDefId: target.agent.id, - mode: "builder", + mode: "analyst", engine: "claude_code", status: "running", createdBy: { type: "test", id: "mcp-interactive" }, @@ -3282,16 +3404,11 @@ describe("api", async () => { id: newId("run"), orgId, projectId: target.projectId, - agentDefId: target.agent.id, - mode: "builder", + mode: "analyst", engine: "claude_code", engineSessionId: "session_mcp_resume", status: "succeeded", - trigger: { - source: "plan_acceptance", - proposalId: newId("prop"), - architectRunId: newId("run"), - }, + trigger: { type: "manual" }, createdBy: { type: "test", id: "mcp-interactive" }, }) .returning() @@ -4669,7 +4786,7 @@ describe("api", async () => { orgId, projectId: target.projectId, agentDefId: target.agent.id, - mode: "builder", + mode: "analyst", engine: "claude_code", status: "running", sandbox: { runnerTokenHash: await hashKey(parentToken) }, @@ -6957,7 +7074,7 @@ describe("api", async () => { orgId, projectId: target.projectId, agentDefId: target.agent.id, - mode: "builder", + mode: "analyst", engine: "claude_code", status: "succeeded", engineSessionId: "sess_resume_1", @@ -6983,7 +7100,7 @@ describe("api", async () => { { queue: "runs.dispatch", data: { runId: resumed.json().id, orgId } }, ]); - const runningParent = ( + const builderParent = ( await db .insert(runs) .values({ @@ -6993,6 +7110,29 @@ describe("api", async () => { agentDefId: target.agent.id, mode: "builder", engine: "claude_code", + status: "failed", + engineSessionId: "sess_builder_resume_forbidden", + createdBy: { type: "user", id: "test" }, + }) + .returning() + )[0]; + const optionalBuilderResume = await app.inject({ + method: "POST", + url: `/v1/runs/${builderParent?.id}/resume`, + headers: { cookie }, + }); + expect(optionalBuilderResume.statusCode).toBe(409); + expect(optionalBuilderResume.json().error.code).toBe("builder_resume_forbidden"); + + const runningParent = ( + await db + .insert(runs) + .values({ + id: newId("run"), + orgId, + projectId: target.projectId, + mode: "analyst", + engine: "claude_code", status: "running", engineSessionId: "sess_resume_2", createdBy: { type: "user", id: "test" }, @@ -7013,8 +7153,7 @@ describe("api", async () => { id: newId("run"), orgId, projectId: target.projectId, - agentDefId: target.agent.id, - mode: "builder", + mode: "analyst", engine: "codex", status: "succeeded", engineSessionId: "sess_codex", @@ -7030,9 +7169,7 @@ describe("api", async () => { expect(codex.statusCode).toBe(409); expect(codex.json().error.code).toBe("not_resumable"); - // A resume is a new Builder row, not a second consumption of the - // parent's plan acceptance. Required projects therefore deny it before - // insertion instead of copying provenance into a non-canonical trigger. + // The same Builder guard is independent of project policy. await db .update(projects) .set({ builderPlanPolicy: "required" }) @@ -7044,12 +7181,12 @@ describe("api", async () => { const dispatchedBeforeRequiredResume = dispatched.length; const governedResume = await app.inject({ method: "POST", - url: `/v1/runs/${parent?.id}/resume`, + url: `/v1/runs/${builderParent?.id}/resume`, headers: { cookie }, payload: { message: "attempt a governed resume" }, }); expect(governedResume.statusCode, governedResume.body).toBe(409); - expect(governedResume.json().error.code).toBe("builder_plan_required"); + expect(governedResume.json().error.code).toBe("builder_resume_forbidden"); expect( await db.select({ id: runs.id }).from(runs).where(eq(runs.projectId, target.projectId)), ).toHaveLength(beforeRequiredResume.length); diff --git a/services/api/test/builder-plan-policy.integration.test.ts b/services/api/test/builder-plan-policy.integration.test.ts index 223f3ea6..8ce6d091 100644 --- a/services/api/test/builder-plan-policy.integration.test.ts +++ b/services/api/test/builder-plan-policy.integration.test.ts @@ -1,8 +1,9 @@ import { createHash } from "node:crypto"; -import { newId, sealFacilityReceipt } from "@facility/core"; +import { generateApiKey, newId, sealFacilityReceipt } from "@facility/core"; import { actionTypes, agentDefs, + apiKeys, auditEvents, createDb, type FacilityDb, @@ -13,12 +14,19 @@ import { proposalEvents, proposals, registryItems, + registryVersions, repos, + roles, + runEvents, + runRepositoryWriteLeases, runs, + sandboxProfiles, + virtualKeys, } from "@facility/db"; import { and, desc, eq } from "drizzle-orm"; import postgres from "postgres"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { buildApp } from "../src/app.js"; import { assertBuilderPlanDispatch, lockBuilderPlanPolicy, @@ -28,6 +36,11 @@ import { ApiError } from "../src/errors.js"; import { githubIssueRevisionSha256 } from "../src/github/issue-revision.js"; import { syncRepoFacilityConfig } from "../src/github/kickstart.js"; import { routeTrigger, type TriggerPayload } from "../src/github/router.js"; +import { + createGovernedBuilderRetry, + validateGovernedRetryForDispatch, +} from "../src/governed-builder-retry.js"; +import type { SandboxDriver } from "../src/sandbox/driver.js"; import { dispatchRun } from "../src/sandbox/orchestrator.js"; import type { AppConfig } from "../src/types.js"; @@ -653,6 +666,967 @@ describe("builder plan policy integration", async () => { await expect(lastDenialCode(fixture.orgId)).resolves.toBe(expected); }); + it("creates one clean governed successor under concurrent retry requests and revalidates it for dispatch", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("governed retry root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + + const repository = fixture.payload.repository; + const owner = repository?.owner?.login; + const repo = repository?.name; + if (!owner || !repo) throw new Error("governed retry repository fixture missing"); + const options = { githubClient: { owner, repo, client: fixture.client } }; + const attempts = await Promise.all( + Array.from({ length: 10 }, () => + createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: routed.runId as string, + actor: { type: "user", id: "approver" }, + reason: "Retry the failed tracked Builder", + }, + options, + ), + ), + ); + expect(new Set(attempts.map((attempt) => attempt.run.id)).size).toBe(1); + expect(attempts.filter((attempt) => attempt.created)).toHaveLength(1); + const child = attempts[0]?.run; + if (!child) throw new Error("governed retry child missing"); + expect(child).toMatchObject({ + retryOfRunId: routed.runId, + status: "queued", + sandbox: {}, + receipt: null, + engineSessionId: null, + transcriptUri: null, + sessionStateUri: null, + gh: { owner, repo, issueNumber: 204 }, + }); + expect(await db.select().from(runEvents).where(eq(runEvents.runId, child.id))).toEqual([ + expect.objectContaining({ seq: 1, type: "queued" }), + ]); + const retryAudits = await db + .select() + .from(auditEvents) + .where(and(eq(auditEvents.orgId, fixture.orgId), eq(auditEvents.action, "run.retried"))); + expect( + retryAudits.filter((event) => (event.target as { id?: unknown }).id === child.id), + ).toHaveLength(1); + + const dispatchValidation = await validateGovernedRetryForDispatch(db, child, options); + expect(dispatchValidation.lineage).toMatchObject({ + parent: { id: routed.runId, status: "failed" }, + root: { id: routed.runId }, + depth: 1, + }); + await db + .update(runs) + .set({ gh: { owner, repo, issueNumber: 204, branch: "forged/output" } }) + .where(eq(runs.id, child.id)); + const forged = (await db.select().from(runs).where(eq(runs.id, child.id)).limit(1))[0]; + if (!forged) throw new Error("forged governed retry child missing"); + await expect(validateGovernedRetryForDispatch(db, forged, options)).rejects.toMatchObject({ + code: "governed_retry_lineage_invalid", + details: { reason: "successor_not_clean" }, + }); + await db + .update(runs) + .set({ gh: { owner, repo, issueNumber: 204 } }) + .where(eq(runs.id, child.id)); + + const retryConfig: AppConfig = { + databaseUrl, + secretMasterKey: Buffer.alloc(32, 7).toString("base64"), + port: 0, + publicUrl: "http://127.0.0.1:4400", + webUrl: "http://127.0.0.1:4400", + sandboxApiUrl: "http://127.0.0.1:4400", + sandboxGatewayUrl: "http://127.0.0.1:4410", + gatewayUrl: "http://127.0.0.1:4410", + sandboxRunnerImage: "facility-runner:test", + sandboxDriver: "docker", + facilityInsecureDev: true, + packageRegistryToken: "package-token", + governedBuilderRetryPromotionEnabled: true, + logLevel: "silent", + }; + const roleId = newId("role"); + await db.insert(roles).values({ + id: roleId, + orgId: fixture.orgId, + name: `governed-retry-${crypto.randomUUID()}`, + permissions: ["runs:trigger"], + }); + const retryKey = await generateApiKey("fak"); + await db.insert(apiKeys).values({ + id: retryKey.id, + orgId: fixture.orgId, + name: "governed retry integration", + prefix: retryKey.lookup, + last4: retryKey.last4, + hash: retryKey.hash, + scopeType: "project", + projectId: fixture.projectId, + roleId, + createdBy: "integration-test", + }); + const foreignOrgId = newId("org"); + const foreignProjectId = newId("proj"); + const foreignRoleId = newId("role"); + await db.insert(orgs).values({ + id: foreignOrgId, + name: "Foreign retry tenant", + slug: `foreign-retry-${crypto.randomUUID()}`, + }); + await db.insert(projects).values({ + id: foreignProjectId, + orgId: foreignOrgId, + name: "Foreign retry project", + slug: `foreign-retry-${crypto.randomUUID()}`, + }); + await db.insert(roles).values({ + id: foreignRoleId, + orgId: foreignOrgId, + name: `foreign-retry-${crypto.randomUUID()}`, + permissions: ["runs:trigger"], + }); + const foreignKey = await generateApiKey("fak"); + await db.insert(apiKeys).values({ + id: foreignKey.id, + orgId: foreignOrgId, + name: "foreign governed retry integration", + prefix: foreignKey.lookup, + last4: foreignKey.last4, + hash: foreignKey.hash, + scopeType: "project", + projectId: foreignProjectId, + roleId: foreignRoleId, + createdBy: "integration-test", + }); + const app = await buildApp(retryConfig); + let enqueueAttempts = 0; + app.enqueue = async () => { + enqueueAttempts += 1; + if (enqueueAttempts === 1) throw new Error("simulated_broker_outage"); + return `job_${enqueueAttempts}`; + }; + await app.ready(); + try { + const crossTenant = await app.inject({ + method: "POST", + url: `/v1/runs/${routed.runId}/retry`, + headers: { + authorization: `Bearer ${foreignKey.secret}`, + "idempotency-key": `governed-retry-foreign-${crypto.randomUUID()}`, + }, + }); + expect(crossTenant.statusCode).toBe(404); + expect(crossTenant.json().error.code).toBe("run_not_found"); + expect(enqueueAttempts).toBe(0); + const idempotencyKey = `governed-retry-adopt-${crypto.randomUUID()}`; + const failedEnqueue = await app.inject({ + method: "POST", + url: `/v1/runs/${routed.runId}/retry`, + headers: { + authorization: `Bearer ${retryKey.secret}`, + "idempotency-key": idempotencyKey, + }, + }); + expect(failedEnqueue.statusCode).toBe(500); + const adopted = await app.inject({ + method: "POST", + url: `/v1/runs/${routed.runId}/retry`, + headers: { + authorization: `Bearer ${retryKey.secret}`, + "idempotency-key": idempotencyKey, + }, + }); + expect(adopted.statusCode, adopted.body).toBe(200); + expect(adopted.json()).toMatchObject({ id: child.id, retryOfRunId: routed.runId }); + expect(enqueueAttempts).toBe(2); + expect( + await db + .select({ id: runs.id }) + .from(runs) + .where(eq(runs.retryOfRunId, routed.runId as string)), + ).toHaveLength(1); + } finally { + await app.close(); + } + + const builderAgent = ( + await db + .select() + .from(agentDefs) + .where(eq(agentDefs.id, child.agentDefId ?? "")) + .limit(1) + )[0]; + if (!builderAgent) throw new Error("governed retry Builder agent missing"); + const profileId = newId("sbx"); + await db.insert(sandboxProfiles).values({ + id: profileId, + orgId: fixture.orgId, + projectId: fixture.projectId, + name: "Governed retry integration", + driver: "docker", + image: "facility-runner:test", + setup: {}, + resources: {}, + network: {}, + }); + await db + .update(agentDefs) + .set({ sandboxProfileId: profileId }) + .where(eq(agentDefs.id, builderAgent.id)); + await db.insert(registryVersions).values({ + id: newId("ver"), + orgId: fixture.orgId, + itemId: builderAgent.contractItemId, + version: 1, + content: "Implement only the approved plan.", + contentHash: createHash("sha256").update("Implement only the approved plan.").digest("hex"), + status: "active", + createdBy: "integration-test", + }); + let launches = 0; + const driver: SandboxDriver = { + name: "docker", + launch: async () => { + launches += 1; + return { ref: `governed-retry-${launches}` }; + }, + status: async () => "running", + async *logs() {}, + stop: async () => undefined, + destroy: async () => undefined, + }; + await Promise.all([ + dispatchRun( + retryConfig, + { runId: child.id, orgId: fixture.orgId }, + { + githubClient: options.githubClient, + sandboxDriver: async () => driver, + }, + ), + dispatchRun( + retryConfig, + { runId: child.id, orgId: fixture.orgId }, + { + githubClient: options.githubClient, + sandboxDriver: async () => driver, + }, + ), + ]); + expect(launches).toBe(1); + expect( + await db + .select({ id: virtualKeys.id }) + .from(virtualKeys) + .where(eq(virtualKeys.runId, child.id)), + ).toHaveLength(1); + expect( + await db.select({ id: apiKeys.id }).from(apiKeys).where(eq(apiKeys.runId, child.id)), + ).toHaveLength(1); + expect( + ( + await db + .select({ status: runs.status, sandbox: runs.sandbox }) + .from(runs) + .where(eq(runs.id, child.id)) + .limit(1) + )[0], + ).toMatchObject({ status: "provisioning", sandbox: { ref: "governed-retry-1" } }); + }); + + it("fails a governed retry worker on post-creation drift before credentials or sandbox launch", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("governed retry drift root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + const owner = fixture.payload.repository?.owner?.login; + const repo = fixture.payload.repository?.name; + if (!owner || !repo) throw new Error("governed retry drift repository missing"); + const child = ( + await createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: routed.runId, + actor: { type: "user", id: "approver" }, + }, + { githubClient: { owner, repo, client: fixture.client } }, + ) + ).run; + await db + .update(repos) + .set({ fingerprintStatus: "drifted", fingerprintVerifiedAt: new Date() }) + .where(eq(repos.projectId, fixture.projectId)); + + let launches = 0; + const driver: SandboxDriver = { + name: "docker", + launch: async () => { + launches += 1; + return { ref: "must-not-launch" }; + }, + status: async () => "running", + async *logs() {}, + stop: async () => undefined, + destroy: async () => undefined, + }; + await expect( + dispatchRun( + { databaseUrl } as AppConfig, + { runId: child.id, orgId: fixture.orgId }, + { + githubClient: { owner, repo, client: fixture.client }, + sandboxDriver: async () => driver, + }, + ), + ).rejects.toMatchObject({ + code: "governed_retry_lane_invalid", + details: { reason: "repository_fingerprint_unverified" }, + }); + expect(launches).toBe(0); + expect( + await db + .select({ id: virtualKeys.id }) + .from(virtualKeys) + .where(eq(virtualKeys.runId, child.id)), + ).toHaveLength(0); + expect( + await db.select({ id: apiKeys.id }).from(apiKeys).where(eq(apiKeys.runId, child.id)), + ).toHaveLength(0); + expect( + ( + await db + .select({ status: runs.status, error: runs.error, sandbox: runs.sandbox }) + .from(runs) + .where(eq(runs.id, child.id)) + .limit(1) + )[0], + ).toEqual({ + status: "failed", + error: "governed_retry_lane_invalid:repository_fingerprint_unverified", + sandbox: {}, + }); + const denials = await db + .select({ target: auditEvents.target, payload: auditEvents.payload }) + .from(auditEvents) + .where( + and( + eq(auditEvents.orgId, fixture.orgId), + eq(auditEvents.action, "run.governed_retry_denied"), + ), + ); + expect(denials.filter((event) => (event.target as { id?: unknown }).id === child.id)).toEqual([ + expect.objectContaining({ + payload: expect.objectContaining({ + code: "governed_retry_lane_invalid", + reason: "repository_fingerprint_unverified", + source: "worker_claimed_governed_retry", + }), + }), + ]); + }); + + it("rejects a forged successor of a legacy tracking-zero run before GitHub or launch", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("legacy governed retry root was not created"); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + const parent = (await db.select().from(runs).where(eq(runs.id, routed.runId)).limit(1))[0]; + const repository = ( + await db.select().from(repos).where(eq(repos.projectId, fixture.projectId)).limit(1) + )[0]; + if (!parent || !repository) throw new Error("legacy governed retry fixture missing"); + await db.insert(runRepositoryWriteLeases).values({ + id: newId("rwl"), + orgId: fixture.orgId, + projectId: fixture.projectId, + runId: parent.id, + repoId: repository.id, + provider: "github_installation", + status: "issued", + requestedBranch: "facility/legacy-fabricated", + authorizedBranch: "facility/legacy-fabricated", + baseSha: String( + (parent.trigger as { planProvenance?: { workspaceBaseSha?: unknown } }).planProvenance + ?.workspaceBaseSha, + ), + permissions: ["contents"], + issuedAt: new Date(Date.now() - 60 * 60_000), + expiresAt: new Date(Date.now() - 30 * 60_000), + }); + const child = ( + await db + .insert(runs) + .values({ + id: newId("run"), + orgId: parent.orgId, + projectId: parent.projectId, + retryOfRunId: parent.id, + agentDefId: parent.agentDefId, + mode: parent.mode, + engine: parent.engine, + trigger: parent.trigger, + gh: { owner: repository.owner, repo: repository.name, issueNumber: 204 }, + createdBy: { type: "user", id: "forged-legacy-test" }, + }) + .returning() + )[0]; + if (!child) throw new Error("forged legacy child missing"); + let githubCalls = 0; + let launches = 0; + const githubClient = { + getDefaultBranchSha: async () => { + githubCalls += 1; + throw new Error("legacy retry reached GitHub"); + }, + getIssue: async () => { + githubCalls += 1; + throw new Error("legacy retry reached GitHub"); + }, + listIssueComments: async () => { + githubCalls += 1; + throw new Error("legacy retry reached GitHub"); + }, + } as never; + const driver: SandboxDriver = { + name: "docker", + launch: async () => { + launches += 1; + return { ref: "must-not-launch" }; + }, + status: async () => "running", + async *logs() {}, + stop: async () => undefined, + destroy: async () => undefined, + }; + await expect( + dispatchRun( + { databaseUrl } as AppConfig, + { runId: child.id, orgId: child.orgId }, + { + githubClient: { owner: repository.owner, repo: repository.name, client: githubClient }, + sandboxDriver: async () => driver, + }, + ), + ).rejects.toMatchObject({ + code: "governed_retry_requires_fresh_gate1", + details: { + reason: "legacy_repository_write_tracking_unavailable", + requiredAction: "run_architect_and_approve_new_plan", + }, + }); + expect(githubCalls).toBe(0); + expect(launches).toBe(0); + expect( + await db + .select({ id: virtualKeys.id }) + .from(virtualKeys) + .where(eq(virtualKeys.runId, child.id)), + ).toHaveLength(0); + expect( + await db.select({ id: apiKeys.id }).from(apiKeys).where(eq(apiKeys.runId, child.id)), + ).toHaveLength(0); + }); + + it("revalidates repository output across every retry ancestor before worker credentials", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("ancestor evidence root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + const repository = ( + await db.select().from(repos).where(eq(repos.projectId, fixture.projectId)).limit(1) + )[0]; + const owner = fixture.payload.repository?.owner?.login; + const repo = fixture.payload.repository?.name; + if (!repository || !owner || !repo) throw new Error("ancestor evidence repository missing"); + const baseSha = fixture.live.baseSha; + const authorizedBranch = `facility/ancestor-${crypto.randomUUID()}`; + await db.insert(runRepositoryWriteLeases).values({ + id: newId("rwl"), + orgId: fixture.orgId, + projectId: fixture.projectId, + runId: routed.runId, + repoId: repository.id, + provider: "github_installation", + status: "issued", + requestedBranch: authorizedBranch, + authorizedBranch, + baseSha, + permissions: ["contents"], + issuedAt: new Date(Date.now() - 2 * 60 * 60_000), + expiresAt: new Date(Date.now() - 60 * 60_000), + }); + let remoteHead = baseSha; + const repositoryWriteClient = { + repoId: repository.id, + client: { + assertRepositoryAccessible: async () => undefined, + getRef: async () => remoteHead, + listPullRequestsForHead: async () => ({ pullRequests: [], hasNextPage: false }), + }, + }; + const options = { + githubClient: { owner, repo, client: fixture.client }, + repositoryWriteClient, + }; + const firstChild = ( + await createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: routed.runId, + actor: { type: "user", id: "approver" }, + }, + options, + ) + ).run; + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, firstChild.id)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, firstChild.id)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, firstChild.id)); + const grandchild = ( + await createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: firstChild.id, + actor: { type: "user", id: "approver" }, + }, + options, + ) + ).run; + + remoteHead = "b".repeat(40); + let launches = 0; + const driver: SandboxDriver = { + name: "docker", + launch: async () => { + launches += 1; + return { ref: "must-not-launch" }; + }, + status: async () => "running", + async *logs() {}, + stop: async () => undefined, + destroy: async () => undefined, + }; + await expect( + dispatchRun( + { databaseUrl } as AppConfig, + { runId: grandchild.id, orgId: fixture.orgId }, + { + githubClient: options.githubClient, + repositoryWriteClient, + sandboxDriver: async () => driver, + }, + ), + ).rejects.toMatchObject({ + code: "governed_retry_durable_output", + details: { reason: "remote_branch_or_pull_request_exists" }, + }); + expect(launches).toBe(0); + expect( + await db + .select({ id: virtualKeys.id }) + .from(virtualKeys) + .where(eq(virtualKeys.runId, grandchild.id)), + ).toHaveLength(0); + expect( + await db.select({ id: apiKeys.id }).from(apiKeys).where(eq(apiKeys.runId, grandchild.id)), + ).toHaveLength(0); + expect( + ( + await db + .select({ status: runs.status, error: runs.error, sandbox: runs.sandbox }) + .from(runs) + .where(eq(runs.id, grandchild.id)) + .limit(1) + )[0], + ).toEqual({ + status: "failed", + error: "governed_retry_durable_output:remote_branch_or_pull_request_exists", + sandbox: {}, + }); + const denial = ( + await db + .select({ payload: auditEvents.payload }) + .from(auditEvents) + .where( + and( + eq(auditEvents.orgId, fixture.orgId), + eq(auditEvents.action, "run.governed_retry_denied"), + ), + ) + .orderBy(desc(auditEvents.seq)) + .limit(1) + )[0]; + expect(denial?.payload).toMatchObject({ + code: "governed_retry_durable_output", + reason: "remote_branch_or_pull_request_exists", + source: "worker_initial_governed_retry", + }); + }); + + it("denies a parallel governed retry when a legacy resume descendant already exists", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("legacy descendant root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + const parent = (await db.select().from(runs).where(eq(runs.id, routed.runId)).limit(1))[0]; + if (!parent) throw new Error("legacy descendant parent missing"); + await db.insert(runs).values({ + id: newId("run"), + orgId: parent.orgId, + projectId: parent.projectId, + agentDefId: parent.agentDefId, + mode: parent.mode, + engine: parent.engine, + status: "succeeded", + trigger: { type: "resume", resumeOf: parent.id }, + gh: { + owner: fixture.payload.repository?.owner?.login, + repo: fixture.payload.repository?.name, + issueNumber: 204, + branch: "facility/legacy-resume", + pr: { number: 991 }, + }, + createdBy: { type: "user", id: "legacy-resume-user" }, + endedAt: new Date(), + }); + const owner = fixture.payload.repository?.owner?.login; + const repo = fixture.payload.repository?.name; + if (!owner || !repo) throw new Error("legacy descendant repository missing"); + let githubCalls = 0; + const noNetworkClient = { + getDefaultBranchSha: async () => { + githubCalls += 1; + return fixture.live.baseSha; + }, + getIssue: async () => { + githubCalls += 1; + return null; + }, + listIssueComments: async () => { + githubCalls += 1; + return []; + }, + } as never; + + await expect( + createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: parent.id, + actor: { type: "user", id: "approver" }, + }, + { githubClient: { owner, repo, client: noNetworkClient } }, + ), + ).rejects.toMatchObject({ + code: "governed_retry_durable_output", + details: { reason: "legacy_resume_descendant_exists" }, + }); + expect(githubCalls).toBe(0); + expect( + await db.select({ id: runs.id }).from(runs).where(eq(runs.retryOfRunId, parent.id)), + ).toHaveLength(0); + }); + + it("persists one sanitized Gate 1 denial after the retry transaction rolls back", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("stale retry root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + fixture.live.issueBody = "The issue changed after the approved plan"; + const owner = fixture.payload.repository?.owner?.login; + const repo = fixture.payload.repository?.name; + if (!owner || !repo) throw new Error("stale retry repository missing"); + + await expect( + createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId: routed.runId, + actor: { type: "user", id: "approver" }, + }, + { githubClient: { owner, repo, client: fixture.client } }, + ), + ).rejects.toMatchObject({ code: "builder_plan_stale" }); + expect( + await db.select({ id: runs.id }).from(runs).where(eq(runs.retryOfRunId, routed.runId)), + ).toHaveLength(0); + const denials = await db + .select({ target: auditEvents.target, payload: auditEvents.payload }) + .from(auditEvents) + .where( + and( + eq(auditEvents.orgId, fixture.orgId), + eq(auditEvents.action, "run.builder_plan_denied"), + ), + ); + expect( + denials.filter((event) => (event.target as { id?: unknown }).id === routed.runId), + ).toEqual([ + expect.objectContaining({ + payload: expect.objectContaining({ + code: "builder_plan_stale", + reason: "base_or_issue_revision_changed", + source: "governed_retry_admission", + }), + }), + ]); + }); + + it("allows the bounded lineage depth and rejects the next successor before GitHub", async () => { + const fixture = await githubRouteFixture("open"); + await db + .update(projects) + .set({ builderPlanPolicy: "required" }) + .where(eq(projects.id, fixture.projectId)); + await db + .update(repos) + .set({ + fingerprint: { files: [] }, + fingerprintStatus: "ok", + fingerprintVerifiedAt: new Date(), + }) + .where(eq(repos.projectId, fixture.projectId)); + const routed = await routeTrigger(db, fixture.orgId, fixture.client, fixture.payload); + if (!routed.runId) throw new Error("depth retry root was not created"); + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, routed.runId)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, routed.runId)); + const owner = fixture.payload.repository?.owner?.login; + const repo = fixture.payload.repository?.name; + if (!owner || !repo) throw new Error("depth retry repository missing"); + let githubCalls = 0; + const countingClient = { + getDefaultBranchSha: async () => { + githubCalls += 1; + return fixture.live.baseSha; + }, + getIssue: async (number: number) => { + githubCalls += 1; + return { + number, + title: "Require a plan", + body: "Implement it", + state: "open", + user: { login: "requester" }, + labels: [], + html_url: `https://github.test/${owner}/${repo}/issues/${number}`, + }; + }, + listIssueComments: async () => { + githubCalls += 1; + return [ + { + id: 204, + author: "maintainer", + authorType: "User", + body: "/builder", + createdAt: "2026-08-26T10:00:00Z", + url: "https://github.test/comments/204", + }, + ]; + }, + } as never; + let parentRunId = routed.runId; + for (let depth = 1; depth <= 100; depth += 1) { + const next = ( + await createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId, + actor: { type: "user", id: "approver" }, + reason: `bounded depth ${depth}`, + }, + { githubClient: { owner, repo, client: countingClient } }, + ) + ).run; + await db.update(runs).set({ status: "provisioning" }).where(eq(runs.id, next.id)); + await db + .update(runs) + .set({ status: "running", repositoryWriteTrackingVersion: 1 }) + .where(eq(runs.id, next.id)); + await db + .update(runs) + .set({ status: "failed", endedAt: new Date() }) + .where(eq(runs.id, next.id)); + parentRunId = next.id; + } + const githubCallsAtLimit = githubCalls; + await expect( + createGovernedBuilderRetry( + db, + db, + { + orgId: fixture.orgId, + projectId: fixture.projectId, + parentRunId, + actor: { type: "user", id: "approver" }, + }, + { githubClient: { owner, repo, client: countingClient } }, + ), + ).rejects.toMatchObject({ + code: "governed_retry_lineage_invalid", + details: { reason: "lineage_depth_exceeded" }, + }); + expect(githubCalls).toBe(githubCallsAtLimit); + expect( + await db.select({ id: runs.id }).from(runs).where(eq(runs.retryOfRunId, parentRunId)), + ).toHaveLength(0); + }, 60_000); + it("recovers an executing GitHub plan and a crash after the exact row was created", async () => { const executing = await githubRouteFixture("executing"); const recovered = await routeTrigger( diff --git a/services/api/test/builder-plan-producer-inventory.test.ts b/services/api/test/builder-plan-producer-inventory.test.ts index 55af3246..848c4733 100644 --- a/services/api/test/builder-plan-producer-inventory.test.ts +++ b/services/api/test/builder-plan-producer-inventory.test.ts @@ -8,6 +8,7 @@ const expectedProducerCounts = new Map([ ["executors.ts", 5], ["github/processor.ts", 1], ["github/router.ts", 1], + ["governed-builder-retry.ts", 1], ["integrations/inbound.ts", 1], ["learning.ts", 1], ["routes/v1/assistant.ts", 1], @@ -54,7 +55,7 @@ describe("Builder plan producer inventory", () => { }); } - expect(inserts).toHaveLength(19); + expect(inserts).toHaveLength(20); expect( new Map( [...new Set(inserts.map((insert) => insert.relativeFile))].map((file) => [ @@ -117,6 +118,16 @@ function persistsAdmissionMode( /admittedMode\s*=\s*admission\.mode\b/.test(body) && /mode\s*:\s*admittedMode\b/.test(body) ); } + if ( + relativeFile === "governed-builder-retry.ts" && + ts.isPropertyAccessExpression(call.expression) && + call.expression.name.text === "transaction" + ) { + return ( + body.includes("assertGovernedRetryLockedAdmission(") && + /mode\s*:\s*lockedParent\.mode\b/.test(body) + ); + } } return false; } @@ -141,6 +152,17 @@ function hasTransactionalAdmissionAncestor( const body = current.getText(sourceFile); return body.includes("lockBuilderPlanPolicy(") && body.includes("assertBuilderPlanDispatch("); } + if ( + relativeFile === "governed-builder-retry.ts" && + ts.isPropertyAccessExpression(call.expression) && + call.expression.name.text === "transaction" + ) { + const body = current.getText(sourceFile); + return ( + body.includes("lockBuilderPlanPolicy(") && + body.includes("assertGovernedRetryLockedAdmission(") + ); + } } return false; } diff --git a/services/api/test/config.test.ts b/services/api/test/config.test.ts index 50339d34..d22ce97c 100644 --- a/services/api/test/config.test.ts +++ b/services/api/test/config.test.ts @@ -27,6 +27,16 @@ const validProductionOauthEnv = { }; describe("API configuration", () => { + it("keeps governed Builder retry promotion fail-closed until explicitly enabled", () => { + expect(readConfig(validEnv).governedBuilderRetryPromotionEnabled).toBe(false); + expect( + readConfig({ + ...validEnv, + FACILITY_GOVERNED_BUILDER_RETRY_PROMOTION: "1", + }).governedBuilderRetryPromotionEnabled, + ).toBe(true); + }); + it("accepts a master key that decodes to exactly 32 bytes", () => { expect(readConfig(validEnv).secretMasterKey).toBe(validEnv.SECRET_MASTER_KEY); });