From 0670ecce884955b7e90b2d1743db82f76996807f Mon Sep 17 00:00:00 2001 From: kjgbot Date: Tue, 21 Jul 2026 21:34:13 +0200 Subject: [PATCH 1/2] feat: make feature guardian conversational --- .../factory-feature-guardian/agent.test.ts | 119 +- .../agents/factory-feature-guardian/agent.ts | 653 ++++- .../manifest-contract.test.ts | 1 + .../factory-feature-guardian/persona.json | 49 +- .agentworkforce/features/manifest.yaml | 38 +- .agentworkforce/features/verify/procedures.md | 26 +- package.json | 4 + src/feature-guardian/conversation.test.ts | 715 ++++++ src/feature-guardian/conversation.ts | 2228 +++++++++++++++++ src/feature-guardian/index.ts | 1 + src/index.ts | 1 + 11 files changed, 3790 insertions(+), 45 deletions(-) create mode 100644 src/feature-guardian/conversation.test.ts create mode 100644 src/feature-guardian/conversation.ts create mode 100644 src/feature-guardian/index.ts diff --git a/.agentworkforce/agents/factory-feature-guardian/agent.test.ts b/.agentworkforce/agents/factory-feature-guardian/agent.test.ts index 05c5ca2..c6f2ebe 100644 --- a/.agentworkforce/agents/factory-feature-guardian/agent.test.ts +++ b/.agentworkforce/agents/factory-feature-guardian/agent.test.ts @@ -16,9 +16,14 @@ import guardian, { SLACK_WRITEBACK_TIMEOUT_MS, createSdkProgressStore, deliveredSlackTs, + factoryFeatureGuardianAdapters, featurePostIdempotencyKey, + gateFactoryGuardianTier, + parseFactoryGuardianTestCounts, resolveManifestPath, + resolveFactoryGuardianProcedure, runGuardian, + runFactoryGuardianProcedure, type ProgressState, } from './agent.js'; @@ -424,8 +429,8 @@ describe('factory-feature-guardian runtime paths', () => { it('declares bounded manifest and memory reads plus configured Slack output', () => { expect(persona.integrations.github?.relayfileMount).toEqual({ - requiredReadPaths: ['/github/repos/AgentWorkforce/factory/.agentworkforce/features/**'], - writeOnlyPaths: [], + requiredReadPaths: ['/github/repos/AgentWorkforce/factory/**'], + writeOnlyPaths: ['/github/repos/AgentWorkforce/factory/issues/**'], }); expect(persona.memory).toEqual({ enabled: true, @@ -436,8 +441,8 @@ describe('factory-feature-guardian runtime paths', () => { optional: true, enabledByInput: 'SLACK_CHANNEL', relayfileMount: { - requiredReadPaths: [], - writeOnlyPaths: ['/slack/channels/${SLACK_CHANNEL}/**'], + requiredReadPaths: ['/slack/channels/${SLACK_CHANNEL}/messages/**'], + writeOnlyPaths: ['/slack/channels/${SLACK_CHANNEL}/messages/**'], }, }); }); @@ -583,7 +588,7 @@ describe('factory-feature-guardian runtime paths', () => { featureId: 'broker-status', ts: '1710000001.000100', }); - expect(ctx.files.write).toHaveBeenCalledTimes(2); + expect(ctx.files.write).toHaveBeenCalledTimes(3); } finally { restore(); } @@ -749,13 +754,23 @@ describe('factory-feature-guardian runtime paths', () => { } }); - it('scopes provider idempotency to a feature within one cycle', () => { - expect(featurePostIdempotencyKey('cycle-a', 'start-broker')).toBe( - featurePostIdempotencyKey('cycle-a', 'start-broker') - ); - expect(featurePostIdempotencyKey('cycle-a', 'start-broker')).not.toBe( - featurePostIdempotencyKey('cycle-b', 'start-broker') - ); + it('scopes provider idempotency to an exact feature revision and generation', () => { + const revision = { manifestRevision: 'manifest-a', procedureRevision: 'procedures-a', generation: 1 }; + const key = featurePostIdempotencyKey('cycle-a', 'start-broker', revision); + expect(key).toBe(featurePostIdempotencyKey('cycle-a', 'start-broker', revision)); + expect(key).not.toBe(featurePostIdempotencyKey('cycle-b', 'start-broker', revision)); + expect(key).not.toBe(featurePostIdempotencyKey('cycle-a', 'start-broker', { + ...revision, + manifestRevision: 'manifest-b', + })); + expect(key).not.toBe(featurePostIdempotencyKey('cycle-a', 'start-broker', { + ...revision, + procedureRevision: 'procedures-b', + })); + expect(key).not.toBe(featurePostIdempotencyKey('cycle-a', 'start-broker', { + ...revision, + generation: 2, + })); }); it('requires a delivered Slack ts instead of a draft receipt id', () => { @@ -1113,3 +1128,83 @@ describe('factory-feature-guardian delayed Slack receipts', () => { } }); }); + +describe('Factory feature guardian conversation adapters', () => { + const snapshot = { + id: 'verification-procedure-routing', + name: 'Manifest-to-Procedure Routing', + category: 'release-verification', + api: 'manifest.yaml#verification.categories', + description: 'Routes features to exact procedures.', + locations: ['.agentworkforce/features/manifest.yaml'], + procedure: 'cli-and-package', + tier: 1, + criticality: 'critical', + }; + + it.each([ + ['all passing', ' Test Files 2 passed (2)\n Tests 42 passed (42)', { passed: 42, failed: 0 }], + ['mixed', ' Test Files 1 failed | 2 passed (3)\n Tests 3 failed | 39 passed (42)', { passed: 39, failed: 3 }], + ['build only', 'compiled successfully', { passed: 0, failed: 0 }], + ['ANSI summary', '\u001b[32m Tests 7 passed (7)\u001b[39m', { passed: 7, failed: 0 }], + ])('parses %s test evidence without inventing failures', (_label, output, expected) => { + expect(parseFactoryGuardianTestCounts(output)).toEqual(expected); + }); + + it('fails confirmation authority closed until a configured actor matches', () => { + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + expect(factoryFeatureGuardianAdapters.isAuthorizedConfirmer(ctx, 'U-ANY')).toBe(false); + (ctx.persona.inputs as Record).SLACK_USER_KHALIQ = 'U-KHALIQ'; + expect(factoryFeatureGuardianAdapters.isAuthorizedConfirmer(ctx, 'U-KHALIQ')).toBe(true); + expect(factoryFeatureGuardianAdapters.isAuthorizedConfirmer(ctx, 'U-ANY')).toBe(false); + }); + + it('reports missing installed dependencies as SKIP instead of a defect', async () => { + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + ctx.sandbox.exec = vi.fn(async () => ({ output: '', exitCode: 1 })); + const gate = await gateFactoryGuardianTier(ctx, snapshot, { + name: 'cli-and-package', + path: '.agentworkforce/features/verify/procedures.md', + prerequisites: 'source checkout and npm ci', + body: 'npm run build', + }); + expect(gate).toEqual({ + outcome: 'skip', + reason: expect.stringContaining('installed Node dependencies'), + }); + }); + + it('selects the complete documented command and runs it in a disposable checkout', async () => { + const procedures = readFileSync( + new URL('../../features/verify/procedures.md', import.meta.url), + 'utf8', + ); + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + ctx.sandbox.readFile = vi.fn(async () => procedures); + const exec = vi.fn(async () => ({ + output: ' Test Files 2 passed (2)\n Tests 42 passed (42)\n__FACTORY_GUARDIAN_CLEANUP_OK__\n', + exitCode: 0, + })); + ctx.sandbox.exec = exec; + const procedure = await resolveFactoryGuardianProcedure(ctx, snapshot); + const result = await runFactoryGuardianProcedure(ctx, snapshot, procedure, 'adapter-test'); + + expect(result).toMatchObject({ + outcome: 'passed', + result: 'positive', + tests: { passed: 42, failed: 0 }, + cleanup: ['Observed removal of the unique temporary checkout.'], + }); + const script = String(exec.mock.calls[0]?.[0]); + expect(script).toContain('tar --exclude=.git --exclude=node_modules') + expect(script).toContain('cd "$TMP/repo"') + expect(script).toContain('node bin/factory.mjs --help') + expect(script).toContain('node bin/factory.mjs featuremap check') + }); + + it('routes every remediation only to the Factory repository', () => { + expect(factoryFeatureGuardianAdapters.repositoryForFeature(snapshot)).toBe( + 'AgentWorkforce/factory', + ); + }); +}); diff --git a/.agentworkforce/agents/factory-feature-guardian/agent.ts b/.agentworkforce/agents/factory-feature-guardian/agent.ts index 6d3edbf..dd486dd 100644 --- a/.agentworkforce/agents/factory-feature-guardian/agent.ts +++ b/.agentworkforce/agents/factory-feature-guardian/agent.ts @@ -12,7 +12,6 @@ * After the full manifest is covered, the cycle resets. */ import { - defineAgent, type RelayfileCredentials, type WorkforceCtx, type WorkforceEvent, @@ -28,9 +27,26 @@ import { slackClient, type WritebackResult } from '@relayfile/relay-helpers'; import { randomUUID } from 'node:crypto'; import { parseManifestFeatures, + featureLocations, + validateFeatureManifest, type FeatureCriticality as Criticality, type ManifestFeature, } from '../../../src/featuremap/validate.js'; +import { + defineFeatureGuardianAgent, + guardianContentRevision, + registerGuardianQuestion, + remediationMarker, + runGuardianConversationTurn, + type FeatureGuardianAdapters, + type GuardianConversationDependencies, + type GuardianDefectKind, + type GuardianFeatureSnapshot, + type GuardianIssuePolicy, + type GuardianManifestCatalog, + type GuardianProcedure, + type GuardianProcedureResult, +} from '../../../src/feature-guardian/conversation.js'; export { parseManifestFeatures }; @@ -38,7 +54,41 @@ export { parseManifestFeatures }; type Feature = ManifestFeature; +function factoryFeatureSnapshot(feature: Feature): GuardianFeatureSnapshot { + if (!feature.procedure) { + throw new Error(`Factory manifest feature ${feature.id} is missing its named procedure`); + } + return { + id: feature.id, + name: feature.name, + category: feature.category, + ...(feature.cli ? { cli: feature.cli } : {}), + ...(feature.api ? { api: feature.api } : {}), + description: feature.desc, + locations: featureLocations(feature), + procedure: feature.procedure, + tier: feature.tier, + criticality: feature.criticality, + }; +} + +function snapshotManifestFeature(feature: GuardianFeatureSnapshot): Feature { + return { + id: feature.id, + name: feature.name, + category: feature.category, + ...(feature.cli ? { cli: feature.cli } : {}), + ...(feature.api ? { api: feature.api } : {}), + desc: feature.description, + location: feature.locations.join(', '), + tier: feature.tier, + criticality: feature.criticality as Criticality, + procedure: feature.procedure, + }; +} + const MANIFEST_RELPATH = '.agentworkforce/features/manifest.yaml'; +const PROCEDURES_RELPATH = '.agentworkforce/features/verify/procedures.md'; const FACTORY_REPO_RELPATH = 'github/repos/AgentWorkforce/factory'; /** @@ -51,11 +101,32 @@ export function resolveManifestPath(workspaceDir: string): string { return `${workspaceDir}/${FACTORY_REPO_RELPATH}/${MANIFEST_RELPATH}`; } -/** Load and parse features from the Factory checkout mounted in the proactive workspace. */ -async function loadFeatures(ctx: WorkforceCtx): Promise { +export function resolveProceduresPath(workspaceDir: string): string { + return `${workspaceDir}/${FACTORY_REPO_RELPATH}/${PROCEDURES_RELPATH}`; +} + +/** Load the exact manifest/procedure revisions from the scoped Factory checkout. */ +export async function loadFactoryGuardianCatalog( + ctx: WorkforceCtx, +): Promise { const absPath = resolveManifestPath(ctx.sandbox.cwd); - const raw = await ctx.sandbox.readFile(absPath); - return parseManifestFeatures(raw); + const [raw, procedures] = await Promise.all([ + ctx.sandbox.readFile(absPath), + ctx.sandbox.readFile(resolveProceduresPath(ctx.sandbox.cwd)), + ]); + const validated = validateFeatureManifest(raw); + return { + manifestRevision: guardianContentRevision(raw), + manifestVersion: validated.version, + procedureRevision: guardianContentRevision(procedures), + features: validated.features.map(factoryFeatureSnapshot), + }; +} + +/** Compatibility loader retained for the scheduled feature ordering path. */ +async function loadFeatures(ctx: WorkforceCtx): Promise { + const catalog = await loadFactoryGuardianCatalog(ctx); + return catalog.features.map(snapshotManifestFeature); } // ── progress tracking ───────────────────────────────────────────────────────── @@ -523,9 +594,20 @@ function createProgressStore(ctx: WorkforceCtx): ProgressStore { throw new Error('exact Relayfile credentials are required for guardian cycle state'); } -/** Build the stable Slack idempotency key for one feature in one guardian cycle. */ -export function featurePostIdempotencyKey(cycleStartedAt: string, featureId: string): string { - return `factory-feature-guardian:${cycleStartedAt}:${featureId}`; +/** Build the stable Slack key for one exact feature revision and cycle generation. */ +export function featurePostIdempotencyKey( + cycleStartedAt: string, + featureId: string, + revision?: { + manifestRevision: string; + procedureRevision: string; + generation: number; + }, +): string { + const revisionKey = revision + ? guardianContentRevision(JSON.stringify(revision)).replace(/^sha256:/u, '') + : 'legacy'; + return `factory-feature-guardian:${cycleStartedAt}:${featureId}:${revisionKey}`; } /** Extract and validate the delivered Slack timestamp from a writeback receipt. */ @@ -537,6 +619,15 @@ export function deliveredSlackTs(result: WritebackResult | null | undefined): st return isSlackTs(ts) ? ts : ''; } +/** Convert Slack's provider timestamp into a deterministic canonical time. */ +function slackTsTimestamp(ts: string): string { + const seconds = Number(ts); + if (!Number.isFinite(seconds) || seconds <= 0) { + throw new Error('Slack provider timestamp cannot identify the question time'); + } + return new Date(seconds * 1_000).toISOString(); +} + // ── feature selection ───────────────────────────────────────────────────────── /** Select the highest-priority unchecked feature using criticality and tier ordering. */ @@ -608,6 +699,8 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise export interface GuardianDependencies { createSlackClient?: typeof slackClient; + registerQuestion?: typeof registerGuardianQuestion; + conversationDependencies?: Pick; } /** Execute one fail-closed guardian tick from manifest load through durable Slack checkpoint. */ @@ -624,9 +717,11 @@ export async function runGuardian( } // Load the live feature list from the manifest + let catalog: GuardianManifestCatalog; let features: Feature[]; try { - features = await loadFeatures(ctx); + catalog = await loadFactoryGuardianCatalog(ctx); + features = catalog.features.map(snapshotManifestFeature); } catch (err) { const absPath = resolveManifestPath(ctx.sandbox.cwd); ctx.log('error', 'factory-feature-guardian.manifest-load-failed', { path: absPath, err: String(err) }); @@ -795,7 +890,11 @@ export async function runGuardian( { channelId: channel }, { text: message, - idempotencyKey: featurePostIdempotencyKey(progress.state.cycleStartedAt, feature.id), + idempotencyKey: featurePostIdempotencyKey(progress.state.cycleStartedAt, feature.id, { + manifestRevision: catalog.manifestRevision, + procedureRevision: catalog.procedureRevision, + generation: progress.state.generation, + }), } ); } catch (err) { @@ -812,6 +911,34 @@ export async function runGuardian( throw new Error(`Slack post failed: no timestamp returned for feature ${feature.id}`); } + // Bind the exact provider thread to the exact manifest/procedure revisions + // before advancing the scheduled cycle. If this checkpoint fails, the next + // tick replays the same Slack idempotency key and retries registration. + try { + await (dependencies.registerQuestion ?? registerGuardianQuestion)( + ctx, + { + feature: factoryFeatureSnapshot(feature), + manifestRevision: catalog.manifestRevision, + manifestVersion: catalog.manifestVersion, + procedureRevision: catalog.procedureRevision, + generation: progress.state.generation, + channelId: channel, + threadTs: ts, + askedAt: slackTsTimestamp(ts), + }, + dependencies.conversationDependencies, + ); + } catch (err) { + ctx.log('error', 'factory-feature-guardian.conversation-checkpoint-failed', { + channel, + feature: feature.id, + ts, + err: String(err), + }); + return; + } + // Checkpoint immediately after the confirmed provider receipt. The stable // idempotency key makes a retry safe if this save times out or the run caps. checkedIds.add(feature.id); @@ -844,7 +971,509 @@ export async function runGuardian( }); } -export default defineAgent({ +const FACTORY_PROCEDURE_COMMANDS: Record = { + 'cli-and-package': [ + 'npm run build', + 'npm test -- --run src/cli/fleet.test.ts src/featuremap/validate.test.ts', + 'node bin/factory.mjs --help | tee "$TMP/help.txt"', + 'node bin/factory.mjs --version | tee "$TMP/version.txt"', + 'node bin/factory.mjs featuremap check | tee "$TMP/featuremap.json"', + 'node -e \'const p=require("./package.json"); if (!p.version) process.exit(1)\'', + "grep -Fq 'featuremap check' \"$TMP/help.txt\"", + "grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+' \"$TMP/version.txt\"", + 'node -e \'const r=require(process.argv[1]); if (!r.ok || r.featureCount < 1) process.exit(1)\' "$TMP/featuremap.json"', + ].join('\n'), + 'fleet-execution': [ + 'npm run build', + 'npx vitest run \\', + ' src/cli/fleet.test.ts \\', + ' src/fleet/create-fleet.test.ts \\', + ' src/fleet/ensure-relay-broker.test.ts \\', + ' src/fleet/internal-fleet-client.test.ts \\', + ' src/fleet/relay-fleet-client.test.ts \\', + ' src/node/factory-node.test.ts', + ].join('\n'), + 'provider-discovery': [ + 'factory triage "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/triage.json"', + 'factory canary "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/canary.json"', + 'factory run-once --config "$CONFIG" --dry-run | tee "$TMP/discovery.json"', + "node -e 'const r=require(process.argv[1]); if (!r.ok) process.exit(1)' \"$TMP/canary.json\"", + ].join('\n'), + 'triage-and-configuration': [ + 'npx vitest run \\', + ' src/config/schema.test.ts \\', + ' src/config/local-clone-paths.test.ts \\', + ' src/triage/triage.test.ts \\', + ' src/safety/factory-scope.test.ts', + ].join('\n'), + 'issue-dispatch-lifecycle': [ + 'npx vitest run \\', + ' src/orchestrator/batch-tracker.test.ts \\', + ' src/orchestrator/factory.test.ts \\', + ' src/dispatch/templates.test.ts \\', + ' src/git/agent-worktree.test.ts \\', + ' src/state/file-state-store.test.ts \\', + ' src/state/github-lifecycle-identity.test.ts', + ].join('\n'), + 'human-clarification': [ + 'npx vitest run \\', + ' src/config/schema.test.ts \\', + ' src/orchestrator/coalesced-task-queue.test.ts \\', + ' src/orchestrator/factory.test.ts \\', + ' src/state/file-state-store.test.ts', + ].join('\n'), + 'pull-request-lifecycle': [ + 'npx vitest run \\', + ' src/github/merge-gate.test.ts \\', + ' src/github/probe-closer.test.ts \\', + ' src/github/standalone-babysitter.test.ts \\', + ' src/orchestrator/factory.test.ts \\', + ' src/state/file-state-store.test.ts', + ].join('\n'), + 'safety-boundaries': [ + 'npx vitest run \\', + ' src/safety/factory-scope.test.ts \\', + ' src/github/merge-gate.test.ts \\', + ' src/github/probe-closer.test.ts \\', + ' src/node/factory-node.test.ts \\', + ' src/__tests__/mount-delete-callsite-invariant.test.ts \\', + ' src/__tests__/writefile-callsite-invariant.test.ts', + ].join('\n'), + 'integrations-and-writeback': [ + 'npx vitest run \\', + ' src/mount/local-mount-preflight.test.ts \\', + ' src/mount/relayfile-binary.test.ts \\', + ' src/mount/relayfile-cloud-mount-client.test.ts \\', + ' src/mount/relayfile-github-connection-write.test.ts \\', + ' src/mount/relayfile-integration-preflight.test.ts \\', + ' src/writeback/writeback.test.ts', + ].join('\n'), + 'event-intake': [ + 'npx vitest run \\', + ' src/webhook/handler.test.ts \\', + ' src/webhook/registrar.test.ts \\', + ' src/subscriptions/__tests__/globs.test.ts \\', + ' src/subscriptions/__tests__/linear-filter.test.ts \\', + ' src/subscriptions/__tests__/slack-filter.test.ts \\', + ' src/subscriptions/__tests__/specs.test.ts \\', + ' src/subscriptions/__tests__/event-client.test.ts', + ].join('\n'), + 'public-api': [ + 'npm run build', + 'npx vitest run src/__tests__/dist-entrypoints.test.ts', + 'npm pack --pack-destination "$TMP"', + 'mkdir "$TMP/consumer" && cd "$TMP/consumer"', + 'npm init -y >/dev/null', + 'npm install --ignore-scripts "$TMP"/*.tgz >/dev/null', + "node --input-type=module <<'NODE'", + "for (const subpath of ['', '/observability', '/testing', '/writeback', '/featuremap', '/feature-guardian', '/hosted', '/environments']) {", + ' const mod = await import(`@agent-relay/factory${subpath}`)', + " if (Object.keys(mod).length === 0) throw new Error(`empty export ${subpath || '/'}`)", + '}', + "const node = await import('@agent-relay/factory/node')", + "if (!node.default) throw new Error('node default export missing')", + 'NODE', + ].join('\n'), + 'hosted-control-plane': [ + 'npx vitest run \\', + ' src/hosted/orchestrator.test.ts \\', + ' src/hosted/state-store.test.ts \\', + ' src/hosted/worker-safety.test.ts', + ].join('\n'), + 'cloud-observability': [ + 'npx vitest run \\', + ' src/observability/events.test.ts \\', + ' src/observability/instance-identity.test.ts \\', + ' src/observability/outbox.test.ts \\', + ' src/observability/cloud-reporter.test.ts \\', + ' src/cli/fleet.test.ts', + ].join('\n'), + 'release-verification': [ + 'npm run build', + 'npm run featuremap:check', + 'npm test', + 'node bin/factory.mjs --help >/dev/null', + ].join('\n'), + 'proactive-health': [ + 'npx vitest run \\', + ' .agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts \\', + ' .agentworkforce/agents/factory-feature-guardian/agent.test.ts \\', + ' src/feature-guardian/conversation.test.ts', + ].join('\n'), + 'loop-and-recovery': [ + 'npx vitest run \\', + ' src/orchestrator/factory.test.ts \\', + ' src/orchestrator/process-identity.test.ts \\', + ' src/orchestrator/reaper.test.ts \\', + ' src/state/file-state-store.test.ts', + ].join('\n'), + 'verification-environments': [ + 'npm run build', + 'npx vitest run \\', + ' src/environments/verification-stack-descriptor.test.ts \\', + ' src/environments/verification-stack-deployer.test.ts \\', + ' src/environments/kubernetes-provider.test.ts \\', + ' src/environments/load-harness.test.ts \\', + ' src/environments/verification-pipeline.test.ts \\', + ' src/orchestrator/environment-reaper.test.ts', + ].join('\n'), +}; + +/** Resolve one named manifest procedure without allowing a heading escape. */ +export async function resolveFactoryGuardianProcedure( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, +): Promise { + if (!/^[a-z0-9-]+$/u.test(feature.procedure)) { + throw new Error(`Factory feature ${feature.id} has an invalid procedure name`); + } + const path = resolveProceduresPath(ctx.sandbox.cwd); + const raw = await ctx.sandbox.readFile(path); + const escaped = feature.procedure.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&'); + const match = raw.match( + new RegExp(`^## ${escaped}\\s*$([\\s\\S]*?)(?=^## |(?![\\s\\S]))`, 'mu'), + ); + if (!match?.[1]) { + throw new Error(`Missing Factory verification procedure: ${feature.procedure}`); + } + const body = match[1].trim(); + const prerequisites = body.match(/\*\*Prerequisites:\*\*\s*([\s\S]*?)(?=\n```|\n\n)/u)?.[1] + ?.replace(/\s+/gu, ' ') + .trim(); + if (!prerequisites) { + throw new Error(`Factory procedure ${feature.procedure} has no explicit prerequisites`); + } + return { name: feature.procedure, path: PROCEDURES_RELPATH, prerequisites, body }; +} + +/** Keep live/provider tiers opt-in and never turn missing prerequisites into confirmation. */ +export async function gateFactoryGuardianTier( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, + procedure: GuardianProcedure, +) { + const command = FACTORY_PROCEDURE_COMMANDS[procedure.name]; + if (!command) { + return { outcome: 'manual' as const, reason: `MANUAL: ${procedure.name} has no bounded automation command.` }; + } + const repository = `${ctx.sandbox.cwd}/${FACTORY_REPO_RELPATH}`; + const packagePrerequisites = await ctx.sandbox.exec( + 'test -f package.json && test -d node_modules && command -v node >/dev/null && command -v npm >/dev/null', + { cwd: repository, timeoutMs: 10_000 }, + ); + if (packagePrerequisites.exitCode !== 0) { + return { + outcome: 'skip' as const, + reason: 'SKIP: the scoped Factory checkout or its installed Node dependencies are unavailable.', + }; + } + const maximum = Number(factoryGuardianInput(ctx, 'GUARDIAN_VERIFY_MAX_TIER') ?? '2'); + if (!Number.isInteger(maximum) || feature.tier > maximum) { + return { + outcome: 'manual' as const, + reason: `MANUAL: tier ${feature.tier} exceeds the configured automated tier ${Number.isInteger(maximum) ? maximum : 2}.`, + }; + } + if (feature.tier >= 6) { + return { outcome: 'manual' as const, reason: 'MANUAL: tier 6 requires a selected disposable live issue or pull request.' }; + } + if (feature.tier >= 3) { + const optIn = new Set( + (factoryGuardianInput(ctx, 'GUARDIAN_LIVE_VERIFY_OPT_IN') ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean), + ); + if (!optIn.has(feature.id) && !optIn.has(feature.procedure)) { + return { + outcome: 'manual' as const, + reason: `MANUAL: tier ${feature.tier} needs explicit opt-in for ${feature.id} or ${feature.procedure}.`, + }; + } + } + if ( + feature.tier === 3 && + (!process.env.FACTORY_VERIFY_CANARY_ISSUE || + !process.env.FACTORY_VERIFY_CONFIG || + (await ctx.sandbox.exec('command -v factory >/dev/null', { + cwd: repository, + timeoutMs: 5_000, + })).exitCode !== 0) + ) { + return { outcome: 'skip' as const, reason: 'SKIP: the disposable provider issue/config or installed Factory CLI prerequisite is unavailable.' }; + } + if (feature.tier === 4 && process.env.FACTORY_GUARDIAN_FLEET_OPT_IN !== 'true') { + return { outcome: 'skip' as const, reason: 'SKIP: no explicitly authorized disposable fleet is available.' }; + } + if ( + feature.tier === 5 && + (!ctx.credentials.tryRequire() || process.env.FACTORY_GUARDIAN_CLOUD_OPT_IN !== 'true') + ) { + return { outcome: 'skip' as const, reason: 'SKIP: cloud credentials and writable disposable mount opt-in are unavailable.' }; + } + return { outcome: 'available' as const, reason: `Tier ${feature.tier} prerequisites are available.` }; +} + +/** Execute only the exact bounded command documented by the named procedure. */ +export async function runFactoryGuardianProcedure( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, + procedure: GuardianProcedure, + runKey: string, +): Promise { + const command = FACTORY_PROCEDURE_COMMANDS[procedure.name]; + if (!command || !normalizedWhitespace(procedure.body).includes(normalizedWhitespace(command))) { + return { + source: 'procedure', + result: 'negative', + outcome: 'failed', + verifier: 'factory-feature-guardian:procedure-runner', + summary: `Procedure drift: ${procedure.name} no longer contains the allowlisted exact tier-safe command.`, + commands: [], + positiveAssertions: [], + negativeAssertions: ['Documented command did not match the Factory adapter allowlist.'], + tests: { passed: 0, failed: 0 }, + cleanup: ['No command executed.'], + }; + } + const run = runKey.replace(/[^a-zA-Z0-9_.-]/gu, '-').slice(-80); + const repository = `${ctx.sandbox.cwd}/${FACTORY_REPO_RELPATH}`; + const wrapped = [ + 'set -Eeuo pipefail', + 'TMP="$(mktemp -d)"', + 'cleanup() {', + ' status=$?', + ' cd /', + ' rm -rf "$TMP"', + ' if [[ -e "$TMP" ]]; then', + ' printf "__FACTORY_GUARDIAN_CLEANUP_FAILED__\\n" >&2', + ' status=1', + ' else', + ' printf "__FACTORY_GUARDIAN_CLEANUP_OK__\\n"', + ' fi', + ' exit "$status"', + '}', + 'trap cleanup EXIT', + `RUN="${run}"`, + 'CONFIG="${FACTORY_VERIFY_CONFIG:-$TMP/factory.config.json}"', + 'export TMP RUN CONFIG', + `SOURCE=${shellQuote(repository)}`, + 'mkdir "$TMP/repo"', + '(cd "$SOURCE" && tar --exclude=.git --exclude=node_modules --exclude=dist --exclude=artifacts -cf - .) | tar -xf - -C "$TMP/repo"', + 'ln -s "$SOURCE/node_modules" "$TMP/repo/node_modules"', + 'cd "$TMP/repo"', + command, + ].join('\n'); + const execution = await ctx.sandbox.exec(wrapped, { + cwd: repository, + timeoutMs: 240_000, + }); + const counts = parseFactoryGuardianTestCounts(execution.output); + const output = boundedText(execution.output.trim(), 2_000); + const cleaned = execution.output.includes('__FACTORY_GUARDIAN_CLEANUP_OK__'); + if (execution.exitCode !== 0 || !cleaned) { + return { + source: 'procedure', + result: 'negative', + outcome: 'failed', + verifier: 'factory-feature-guardian:procedure-runner', + summary: `${procedure.name} failed with exit ${execution.exitCode}: ${output || '(no output)'}`, + commands: [command], + positiveAssertions: ['Exact tier-safe documented command and isolated cleanup trap were selected.'], + negativeAssertions: [ + `Command exited ${execution.exitCode}.`, + ...(!cleaned ? ['Isolated workspace cleanup was not positively observed.'] : []), + ], + tests: counts, + cleanup: [cleaned + ? 'Observed removal of the unique temporary checkout.' + : 'Temporary-checkout removal was not confirmed.'], + }; + } + return { + source: 'procedure', + result: 'positive', + outcome: 'passed', + verifier: 'factory-feature-guardian:procedure-runner', + summary: `${procedure.name} passed${counts.passed ? ` with ${counts.passed} tests` : ''}.`, + commands: [command], + positiveAssertions: [ + 'The exact tier-safe checked-in procedure command exited 0.', + 'The named procedure and command allowlist matched.', + ], + negativeAssertions: [], + tests: counts, + cleanup: ['Observed removal of the unique temporary checkout.'], + }; +} + +export const factoryFeatureGuardianAdapters: FeatureGuardianAdapters = { + loadCatalog: loadFactoryGuardianCatalog, + resolveProcedure: resolveFactoryGuardianProcedure, + gateTier: gateFactoryGuardianTier, + runProcedure: runFactoryGuardianProcedure, + isAuthorizedConfirmer: (ctx, actorId) => { + const authorities = [ + factoryGuardianInput(ctx, 'SLACK_USER_WILL'), + factoryGuardianInput(ctx, 'SLACK_USER_KHALIQ'), + ].filter((value): value is string => Boolean(value)); + return authorities.includes(actorId); + }, + clarification: (feature, procedure, turn, turnNumber) => { + const surface = [feature.cli && `CLI \`${feature.cli}\``, feature.api && `API/config \`${feature.api}\``] + .filter(Boolean) + .join(' / '); + const focus = turnNumber === 1 + ? 'What exact command or user-visible path did you try, and what did you observe?' + : 'Please paste the positive and negative assertion results plus any cleanup evidence; if it was not run, say which prerequisite is unavailable.'; + return [ + `I can't confirm *${feature.name}* from that response yet (clarification ${turnNumber}).`, + `${surface}; source ${feature.locations.map((path) => `\`${path}\``).join(', ')}.`, + `Use \`${procedure.name}\` at tier ${feature.tier}: ${procedure.prerequisites}`, + focus, + ].join(' '); + }, + classifyDefect: (_feature, turn, evidence) => classifyFactoryDefect(turn.text, evidence), + repositoryForFeature: () => 'AgentWorkforce/factory', + issuePolicy: ({ feature, conversation, defectKind, slackBacklink }) => + factoryGuardianIssuePolicy(feature, conversation, defectKind, slackBacklink), +}; + +function factoryGuardianIssuePolicy( + feature: GuardianFeatureSnapshot, + conversation: Parameters[0]['conversation'], + defectKind: GuardianDefectKind, + slackBacklink: string, +): GuardianIssuePolicy { + const dedupeKey = `factory:${feature.id}:${feature.procedure}`; + const commands = conversation.evidence.flatMap((entry) => entry.commands ?? []); + const observed = conversation.evidence + .filter((entry) => entry.result === 'negative') + .map((entry) => `- ${entry.summary}`) + .join('\n') || '- The scoped guardian response reported a defect.'; + const evidence = conversation.evidence.map((entry) => + `- ${entry.source}/${entry.result}: ${entry.summary}`, + ).join('\n'); + return { + repository: 'AgentWorkforce/factory', + defectKind, + dedupeKey, + labels: ['factory-ready'], + title: `[Feature guardian] ${feature.name}: ${defectKind} defect`, + body: [ + remediationMarker(dedupeKey), + '## Guardian finding', + '', + `Feature \`${feature.id}\` has an established **${defectKind}** defect.`, + `Manifest revision: \`${conversation.manifestRevision}\`; generation: \`${conversation.generation}\`.`, + '', + '## Expected behavior', + '', + feature.description, + '', + '## Observed behavior', + '', + observed, + '', + '## Reproduction and evidence', + '', + evidence, + ...(commands.length > 0 ? ['', '```bash', ...commands, '```'] : []), + '', + '## Affected surfaces', + '', + `- Feature: \`${feature.id}\` (${feature.category})`, + `- Procedure: \`${PROCEDURES_RELPATH}#${feature.procedure}\``, + ...feature.locations.map((path) => `- Source: \`${path}\``), + `- Verification tier: ${feature.tier}`, + `- Slack thread: ${slackBacklink}`, + '', + '## Definition of done', + '', + `- [ ] Correct the ${defectKind} defect at the affected Factory surface.`, + `- [ ] Add or update a regression test that fails before the fix and passes after it.`, + `- [ ] Run the exact \`${feature.procedure}\` procedure and record positive/negative assertions and cleanup.`, + '- [ ] Reconcile the manifest, procedure, and runbook so they describe the implemented behavior.', + '- [ ] Re-run the feature guardian check and produce one exact-revision confirmation record.', + ].join('\n'), + }; +} + +function classifyFactoryDefect( + text: string, + evidence: readonly { source?: string; result?: string; summary: string }[], +): GuardianDefectKind { + if (evidence.some((entry) => + entry.source === 'procedure' && + entry.result === 'negative' && + /\bprocedure drift\b/iu.test(entry.summary), + )) return 'procedure'; + const actorEvidence = evidence + .filter((entry) => entry.source === 'actor') + .map((entry) => entry.summary) + .join('\n'); + const haystack = `${text}\n${actorEvidence}`.toLowerCase(); + if (/\b(manifest|catalog|feature map|featuremap)\b/u.test(haystack)) return 'manifest'; + if (/\b(procedure|runbook|verification steps?)\b/u.test(haystack)) return 'procedure'; + if (/\b(documentation|docs?|readme)\b/u.test(haystack)) return 'documentation'; + if (/\b(test|spec|fixture|assertion)\b/u.test(haystack)) return 'test'; + return 'implementation'; +} + +function factoryGuardianInput(ctx: WorkforceCtx, name: string): string | undefined { + const spec = ctx.persona?.inputSpecs?.[name]; + const value = process.env[spec?.env ?? name] ?? ctx.persona?.inputs?.[name] ?? spec?.default; + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +function normalizedWhitespace(value: string): string { + return value.replace(/\s+/gu, ' ').trim(); +} + +export function parseFactoryGuardianTestCounts(output: string): { passed: number; failed: number } { + const summary = output + .replace(/\u001b\[[0-9;]*m/gu, '') + .split(/\r?\n/u) + .filter((line) => /^\s*Tests(?:\s|:)/u.test(line)) + .at(-1) ?? ''; + const passed = Number(summary.match(/\b(\d+)\s+passed\b/iu)?.[1] ?? 0); + const failed = Number(summary.match(/\b(\d+)\s+failed\b/iu)?.[1] ?? 0); + return { passed, failed }; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function boundedText(value: string, max: number): string { + return value.length <= max ? value : `${value.slice(0, max - 1)}…`; +} + +export interface FactoryGuardianHandlerDependencies { + schedule?: GuardianDependencies; + conversation?: GuardianConversationDependencies; +} + +export async function runFactoryFeatureGuardian( + ctx: WorkforceCtx, + event: WorkforceEvent, + dependencies: FactoryGuardianHandlerDependencies = {}, +): Promise { + if (event.type === 'cron.tick') { + await runGuardian(ctx, event, dependencies.schedule); + return; + } + await runGuardianConversationTurn( + ctx, + event, + factoryFeatureGuardianAdapters, + dependencies.conversation, + ); +} + +export default defineFeatureGuardianAgent({ + adapters: factoryFeatureGuardianAdapters, + scheduled: runGuardian, + channelInput: 'SLACK_CHANNEL', schedules: [{ name: 'hourly-check', cron: '0 * * * *', tz: 'America/New_York' }], - handler: runGuardian, }); diff --git a/.agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts index 7cf11dd..981a484 100644 --- a/.agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts +++ b/.agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts @@ -189,6 +189,7 @@ describe('Factory feature manifest contract', () => { './writeback': 'LinearWriteback / GithubWriteback / SlackWriteback', './node': 'createFactoryNodeDefinition()', './featuremap': '@agent-relay/factory/featuremap', + './feature-guardian': '@agent-relay/factory/feature-guardian', './hosted': '@agent-relay/factory/hosted', './verification-stack.schema.json': '@agent-relay/factory/verification-stack.schema.json', './kubernetes-environment-stack.schema.json': '@agent-relay/factory/kubernetes-environment-stack.schema.json', diff --git a/.agentworkforce/agents/factory-feature-guardian/persona.json b/.agentworkforce/agents/factory-feature-guardian/persona.json index d70456d..8d1bbec 100644 --- a/.agentworkforce/agents/factory-feature-guardian/persona.json +++ b/.agentworkforce/agents/factory-feature-guardian/persona.json @@ -2,7 +2,7 @@ "id": "factory-feature-guardian", "intent": "relay-orchestrator", "tags": ["factory", "verification", "proactive", "health", "release-safety", "durability"], - "description": "Reads the validated Factory feature catalog from a scoped repository clone, then cycles through every @agent-relay/factory feature hourly. Each provider-confirmed, idempotent Slack check names the exact source surface, named end-to-end procedure, verification tier, and supported safety/topology boundary; exact revisioned progress persists across the exhaustive manifest and resets only after a complete cycle.", + "description": "Cycles through every Factory feature hourly, then owns exact-message Slack reactions and replies through clarification, safely gated procedure evidence, immutable exact-revision confirmation, or one deduplicated factory-ready remediation issue.", "cloud": true, "harness": "opencode", "model": "deepseek-v4-flash-free", @@ -13,25 +13,30 @@ "integrations": { "github": { "scope": { - "repo": "AgentWorkforce/factory" + "repo": "AgentWorkforce/factory", + "resources": ["issues"] }, "relayfileMount": { "requiredReadPaths": [ - "/github/repos/AgentWorkforce/factory/.agentworkforce/features/**" + "/github/repos/AgentWorkforce/factory/**" ], - "writeOnlyPaths": [] + "writeOnlyPaths": [ + "/github/repos/AgentWorkforce/factory/issues/**" + ] } }, "slack": { "optional": true, "enabledByInput": "SLACK_CHANNEL", "scope": { - "paths": "/slack/channels/**" + "paths": "/slack/channels/${SLACK_CHANNEL}/messages/**" }, "relayfileMount": { - "requiredReadPaths": [], + "requiredReadPaths": [ + "/slack/channels/${SLACK_CHANNEL}/messages/**" + ], "writeOnlyPaths": [ - "/slack/channels/${SLACK_CHANNEL}/**" + "/slack/channels/${SLACK_CHANNEL}/messages/**" ] } } @@ -63,6 +68,36 @@ "provider": "slack", "resource": "users" } + }, + "GUARDIAN_VERIFY_MAX_TIER": { + "description": "Highest verification tier the guardian may automate; tiers above it remain MANUAL.", + "env": "GUARDIAN_VERIFY_MAX_TIER", + "default": "2" + }, + "GUARDIAN_LIVE_VERIFY_OPT_IN": { + "description": "Comma-separated feature IDs or named procedures explicitly authorized for tier 3-5 disposable checks.", + "env": "GUARDIAN_LIVE_VERIFY_OPT_IN", + "optional": true + }, + "FACTORY_VERIFY_CANARY_ISSUE": { + "description": "Disposable provider issue selected for explicitly opted-in tier-3 checks.", + "env": "FACTORY_VERIFY_CANARY_ISSUE", + "optional": true + }, + "FACTORY_VERIFY_CONFIG": { + "description": "Disposable Factory config selected for explicitly opted-in provider checks.", + "env": "FACTORY_VERIFY_CONFIG", + "optional": true + }, + "FACTORY_GUARDIAN_FLEET_OPT_IN": { + "description": "Explicit authorization to use a disposable fleet for a tier-4 procedure.", + "env": "FACTORY_GUARDIAN_FLEET_OPT_IN", + "optional": true + }, + "FACTORY_GUARDIAN_CLOUD_OPT_IN": { + "description": "Explicit authorization to use disposable writable Cloud resources for a tier-5 procedure.", + "env": "FACTORY_GUARDIAN_CLOUD_OPT_IN", + "optional": true } }, "memory": { diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 283d8a6..4425eed 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -2,13 +2,13 @@ version: '1.1' updated: '2026-07-21' catalog: category_count: 24 - feature_count: 300 + feature_count: 304 tier_counts: - 1: 43 + 1: 45 2: 123 3: 11 4: 51 - 5: 57 + 5: 59 6: 15 # Every user-facing feature in @agent-relay/factory, categorized and scored. @@ -1348,6 +1348,13 @@ categories: location: src/featuremap/index.ts, src/featuremap/validate.ts, src/featuremap/check.ts, package.json verify_tier: 1 + - id: api-feature-guardian + name: Reusable Feature Guardian API + api: '@agent-relay/factory/feature-guardian' + description: Define repository-adapted scheduled guardians with shared Slack correlation, exact-revision state transitions, immutable confirmations, and remediation writeback contracts + location: src/feature-guardian/index.ts, src/feature-guardian/conversation.ts, src/index.ts, package.json + verify_tier: 1 + - id: api-linear-state-resolution name: Linear State Resolution API api: resolveFactoryStates() / stateResolutionFromIds() @@ -1678,7 +1685,7 @@ categories: proactive-health: name: Proactive Feature Guardian - description: Scheduled, exhaustive, durable Slack questions over the live Factory feature manifest + description: Scheduled, exhaustive Slack questions owned through exact-revision confirmation, safe deferral, or deduplicated remediation criticality: hot features: - id: guardian-hourly-feature-check @@ -1698,10 +1705,31 @@ categories: - id: guardian-idempotent-slack-delivery name: Idempotent Guardian Slack Delivery api: slack.messages.write({ idempotencyKey }) - description: Retry an ambiguous scheduled run with a stable cycle-and-feature key, require a real provider Slack timestamp, and advance progress only after the confirmed receipt + description: Retry an ambiguous scheduled run with a stable cycle, feature, generation, manifest, and procedure key; require a real provider Slack timestamp; and advance progress only after the confirmed receipt location: .agentworkforce/agents/factory-feature-guardian/agent.ts verify_tier: 5 + - id: guardian-conversation-state + name: Correlated Guardian Conversation State + api: defineFeatureGuardianAgent() / runGuardianConversationTurn() + description: Correlate exact-message reactions and threaded replies, fence duplicate and delayed events, resume pending work before later responses, and preserve bounded CAS-checked multi-turn state + location: src/feature-guardian/conversation.ts, src/feature-guardian/conversation.test.ts, .agentworkforce/agents/factory-feature-guardian/agent.ts + verify_tier: 1 + + - id: guardian-confirmation-evidence + name: Immutable Guardian Confirmation Evidence + api: /memory/workspace/factory-feature-guardian/confirmations/*.json + description: Append one exact-revision confirmation containing feature, generation, actor or verifier, provider event time, command and test evidence, and Slack thread identity before posting the final summary + location: src/feature-guardian/conversation.ts, .agentworkforce/agents/factory-feature-guardian/agent.ts + verify_tier: 5 + + - id: guardian-remediation-writeback + name: Deduplicated Guardian Remediation + api: GuardianIssueWriter / factoryFeatureGuardianAdapters.issuePolicy + description: Classify proven implementation, test, manifest, procedure, or documentation defects and write one provider-confirmed factory-ready issue with reproduction, affected paths, definition of done, and Slack backlink + location: src/feature-guardian/conversation.ts, .agentworkforce/agents/factory-feature-guardian/agent.ts, .agentworkforce/agents/factory-feature-guardian/persona.json + verify_tier: 5 + config-intake: name: Workspace Intake Configuration description: Factory source, capacity, subscription, and retry controls in factory.config.json diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 5f2f236..d5a95e7 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -688,7 +688,7 @@ mkdir "$TMP/consumer" && cd "$TMP/consumer" npm init -y >/dev/null npm install --ignore-scripts "$TMP"/*.tgz >/dev/null node --input-type=module <<'NODE' -for (const subpath of ['', '/observability', '/testing', '/writeback', '/featuremap', '/hosted', '/environments']) { +for (const subpath of ['', '/observability', '/testing', '/writeback', '/featuremap', '/feature-guardian', '/hosted', '/environments']) { const mod = await import(`@agent-relay/factory${subpath}`) if (Object.keys(mod).length === 0) throw new Error(`empty export ${subpath || '/'}`) } @@ -698,9 +698,10 @@ NODE ``` Back in the checkout, run the focused tests named by each API entry. Assert root -types and all eight package export keys resolve, hosted remains worker-safe, -feature-map validation includes procedure routing, dependency/worktree ports are -exported, and fake clients support a hermetic consumer. Clean only `$TMP`. +types and all nine package export keys resolve, hosted remains worker-safe, +feature-map validation includes procedure routing, the reusable feature guardian +is exported, dependency/worktree ports are exported, and fake clients support a +hermetic consumer. Clean only `$TMP`. ## hosted-control-plane @@ -766,6 +767,7 @@ SHAs from the reviewed event. npm run build npm run featuremap:check npm test +node bin/factory.mjs --help >/dev/null FACTORY_E2E_HEAD_SHA="$(git rev-parse HEAD)" \ FACTORY_E2E_BASE_SHA="$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD^)" \ npm run verify:e2e @@ -797,7 +799,8 @@ Slack channel. ```bash npx vitest run \ .agentworkforce/agents/factory-feature-guardian/manifest-contract.test.ts \ - .agentworkforce/agents/factory-feature-guardian/agent.test.ts + .agentworkforce/agents/factory-feature-guardian/agent.test.ts \ + src/feature-guardian/conversation.test.ts ``` In preview, begin with no state, inject a checkpoint failure after a confirmed @@ -809,10 +812,15 @@ shrink/multiple retirements fail closed, and complete cycles increment exactly once. Reject malformed, duplicate, oversized, stale-revision, authorization, timeout, corrupt-readback, and receiptless states without advancing. Force LLM failure and require the fallback to retain CLI/API, source, tier, and procedure. -In deployment, assert the manifest is read only from -`/github/repos/AgentWorkforce/factory/.agentworkforce/features/**`, Slack can -write only to `${SLACK_CHANNEL}`, and one scheduled tick posts one question with -a real provider `ts` before the exact state checkpoint advances. +Exercise affirmative, wrench, untested, ambiguous, duplicate, delayed, +unauthorized, two-turn restart, confirmation, deferral, and remediation paths, +including provider-write-before-CAS retries and manifest/procedure revision +changes. +In deployment, assert the catalog comes from the scoped Factory checkout, the +procedure runner copies that checkout into a disposable workspace before +execution, Slack reads and writes only `${SLACK_CHANNEL}/messages/**`, and one +scheduled tick posts one question with a real provider `ts` before the exact +state checkpoint advances. ## loop-and-recovery diff --git a/package.json b/package.json index 12e0d5c..578238a 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,10 @@ "types": "./dist/featuremap/index.d.ts", "import": "./dist/featuremap/index.js" }, + "./feature-guardian": { + "types": "./dist/feature-guardian/index.d.ts", + "import": "./dist/feature-guardian/index.js" + }, "./hosted": { "types": "./dist/hosted/index.d.ts", "import": "./dist/hosted/index.js" diff --git a/src/feature-guardian/conversation.test.ts b/src/feature-guardian/conversation.test.ts new file mode 100644 index 0000000..c252e0e --- /dev/null +++ b/src/feature-guardian/conversation.test.ts @@ -0,0 +1,715 @@ +import type { WorkforceCtx, WorkforceEvent } from '@agentworkforce/runtime' +import { slackClient, type GithubClient } from '@relayfile/relay-helpers' +import { describe, expect, it, vi } from 'vitest' + +import { + GuardianStateConflictError, + classifyGuardianResponse, + createGithubIssueWriter, + createSdkConversationStore, + guardianConfirmationPath, + guardianContentRevision, + guardianConversationId, + registerGuardianQuestion, + remediationMarker, + runGuardianConversationTurn, + type ConversationSnapshot, + type ConversationStore, + type FeatureGuardianAdapters, + type GuardianConfirmationRecord, + type GuardianConversationRecord, + type GuardianConversationState, + type GuardianIssuePolicy, + type GuardianIssueReceipt, + type GuardianManifestCatalog, + type GuardianProcedureResult, + type ImmutableConfirmationStore, +} from './conversation.js' + +const CHANNEL = 'C-GUARDIAN' +const ROOT_TS = '1710000000.000100' +const AUTHORIZED = 'U-AUTHORIZED' + +const feature = { + id: 'verification-procedure-routing', + name: 'Manifest-to-Procedure Routing', + category: 'release-verification', + api: 'manifest.yaml#verification.categories', + description: 'Routes every feature to an exact named procedure.', + locations: [ + '.agentworkforce/features/manifest.yaml', + '.agentworkforce/features/verify/procedures.md', + ], + procedure: 'release-verification', + tier: 1, + criticality: 'critical', +} + +const catalog: GuardianManifestCatalog = { + manifestRevision: guardianContentRevision('manifest-a'), + manifestVersion: '1.1', + procedureRevision: guardianContentRevision('procedures-a'), + features: [feature], +} + +const question = { + feature, + manifestRevision: catalog.manifestRevision, + manifestVersion: catalog.manifestVersion, + procedureRevision: catalog.procedureRevision, + generation: 7, + channelId: CHANNEL, + threadTs: ROOT_TS, + askedAt: slackTime(ROOT_TS), +} + +class MemoryConversationStore implements ConversationStore { + snapshot: ConversationSnapshot | null = null + saveCalls = 0 + failSaveCalls = new Set() + + async load(): Promise { + return this.snapshot ? structuredClone(this.snapshot) : null + } + + async save( + state: GuardianConversationState, + expected: ConversationSnapshot | null, + ): Promise { + this.saveCalls += 1 + if (this.failSaveCalls.delete(this.saveCalls)) throw new GuardianStateConflictError() + if (expected?.revision !== this.snapshot?.revision) throw new GuardianStateConflictError() + this.snapshot = { + state: structuredClone(state), + revision: `memory-${this.saveCalls}`, + } + return structuredClone(this.snapshot) + } + + record(): GuardianConversationRecord { + const record = this.snapshot?.state.records[0] + if (!record) throw new Error('missing test conversation') + return record + } +} + +class MemoryConfirmations implements ImmutableConfirmationStore { + readonly records = new Map() + calls = 0 + + async append(record: GuardianConfirmationRecord): Promise { + this.calls += 1 + const path = guardianConfirmationPath(record) + const existing = this.records.get(path) + if (existing && JSON.stringify(existing) !== JSON.stringify(record)) { + throw new Error('immutable confirmation differs') + } + this.records.set(path, structuredClone(record)) + return path + } +} + +class MemorySlack { + readonly posts: Array<{ text: string; key: string }> = [] + private readonly receipts = new Map() + + readonly create = (() => ({ + replies: { + write: vi.fn(async (_target: unknown, body: { text: string; idempotencyKey: string }) => { + let ts = this.receipts.get(body.idempotencyKey) + if (!ts) { + ts = `1710000099.${String(this.receipts.size + 1).padStart(6, '0')}` + this.receipts.set(body.idempotencyKey, ts) + this.posts.push({ text: body.text, key: body.idempotencyKey }) + } + return { + path: '/slack/reply.json', + absolutePath: '/slack/reply.json', + receipt: { externalId: 'not-a-provider-ts', ts }, + } + }), + }, + })) as unknown as typeof slackClient +} + +function procedureResult(outcome: GuardianProcedureResult['outcome']): GuardianProcedureResult { + const result = { + passed: 'positive', + failed: 'negative', + skip: 'skip', + manual: 'manual', + } as const + return { + source: 'procedure', + result: result[outcome], + outcome, + verifier: 'test-procedure-runner', + summary: `${feature.procedure} ${outcome}`, + commands: ['npm run build', 'npm test'], + positiveAssertions: outcome === 'passed' ? ['command exited 0'] : [], + negativeAssertions: outcome === 'failed' ? ['command exited 1'] : [], + tests: { passed: outcome === 'passed' ? 42 : 0, failed: outcome === 'failed' ? 1 : 0 }, + cleanup: ['isolated workspace removed'], + } +} + +function adapters(options: { + currentCatalog?: GuardianManifestCatalog + result?: GuardianProcedureResult + issueWriter?: (policy: GuardianIssuePolicy) => Promise +} = {}): FeatureGuardianAdapters { + return { + loadCatalog: vi.fn(async () => options.currentCatalog ?? catalog), + resolveProcedure: vi.fn(async () => ({ + name: feature.procedure, + path: '.agentworkforce/features/verify/procedures.md', + prerequisites: 'source checkout with installed dependencies', + body: 'npm run build\nnpm test', + })), + gateTier: vi.fn(async () => ({ outcome: 'available', reason: 'available' })), + runProcedure: vi.fn(async () => options.result ?? procedureResult('passed')), + isAuthorizedConfirmer: (_ctx, actorId) => actorId === AUTHORIZED, + clarification: (snapshot, procedure, _turn, number) => + `Clarification ${number}: test ${snapshot.api} at ${snapshot.locations.join(', ')} with ${procedure.name} tier ${snapshot.tier}.`, + classifyDefect: (_snapshot, turn) => { + if (/manifest/iu.test(turn.text)) return 'manifest' + if (/procedure|runbook/iu.test(turn.text)) return 'procedure' + if (/docs?|documentation/iu.test(turn.text)) return 'documentation' + if (/test/iu.test(turn.text)) return 'test' + return 'implementation' + }, + repositoryForFeature: () => 'AgentWorkforce/factory', + issuePolicy: ({ conversation, defectKind, slackBacklink }) => { + const dedupeKey = `factory:${feature.id}:${feature.procedure}` + return { + repository: 'AgentWorkforce/factory', + title: `[Feature guardian] ${feature.name}: ${defectKind}`, + body: `${remediationMarker(dedupeKey)}\n${conversation.manifestRevision}\n${slackBacklink}`, + labels: ['factory-ready'], + defectKind, + dedupeKey, + } + }, + } +} + +function context(): WorkforceCtx { + const files = new Map() + return { + agent: { id: 'sim-agent' }, + deployment: { id: 'sim-deployment' }, + persona: { inputs: { SLACK_CHANNEL: CHANNEL }, inputSpecs: {} }, + credentials: { tryRequire: vi.fn(() => null) }, + files: { + read: vi.fn(async (path: string) => { + const value = files.get(path) + if (value === undefined) throw new Error(`ENOENT: ${path}`) + return value + }), + write: vi.fn(async (path: string, value: string) => { + files.set(path, value) + }), + }, + sandbox: { + cwd: '/workspace', + readFile: vi.fn(async (path: string) => { + throw new Error(`ENOENT: ${path}`) + }), + }, + log: vi.fn(), + } as unknown as WorkforceCtx +} + +function slackTime(ts: string): string { + return new Date(Number(ts) * 1_000).toISOString() +} + +function reply( + envelopeId: string, + messageTs: string, + text: string, + actorId = AUTHORIZED, + channel = CHANNEL, +): WorkforceEvent { + return { + id: envelopeId, + type: 'slack.message.created', + occurredAt: slackTime(messageTs), + resource: { path: `/slack/channels/${channel}/messages/${ROOT_TS.replace('.', '_')}` }, + expand: vi.fn(async () => ({ + data: { + event: { + channel, + ts: messageTs, + thread_ts: ROOT_TS, + user: actorId, + text, + event_ts: messageTs, + }, + }, + })), + } as unknown as WorkforceEvent +} + +function reaction( + envelopeId: string, + eventTs: string, + emoji: string, + actorId = AUTHORIZED, +): WorkforceEvent { + return { + id: envelopeId, + type: 'slack.reaction.added', + occurredAt: slackTime(eventTs), + resource: { path: `/slack/channels/${CHANNEL}/messages/${ROOT_TS.replace('.', '_')}` }, + expand: vi.fn(async () => ({ + data: { + event: { + item: { channel: CHANNEL, ts: ROOT_TS }, + reaction: emoji, + user: actorId, + event_ts: eventTs, + }, + }, + })), + } as unknown as WorkforceEvent +} + +async function fixture(options: Parameters[0] = {}) { + const store = new MemoryConversationStore() + const confirmations = new MemoryConfirmations() + const slack = new MemorySlack() + const ctx = context() + await registerGuardianQuestion(ctx, question, { conversationStore: store }) + const issueWrites: GuardianIssuePolicy[] = [] + const issueWriter = { + upsert: vi.fn(async (policy: GuardianIssuePolicy) => { + issueWrites.push(policy) + if (options.issueWriter) return options.issueWriter(policy) + return { + repository: policy.repository, + number: 1700, + url: 'https://github.com/AgentWorkforce/factory/issues/1700', + dedupeKey: policy.dedupeKey, + } + }), + } + const deps = { + conversationStore: store, + confirmationStore: confirmations, + issueWriter, + createSlackClient: slack.create, + } + return { store, confirmations, slack, ctx, deps, issueWriter, issueWrites } +} + +describe('feature guardian response classification', () => { + it.each([ + [{ reaction: 'white_check_mark' }, 'affirmative'], + [{ reaction: 'wrench' }, 'failure'], + [{ text: 'untested' }, 'untested'], + [{ text: 'I looked at it briefly' }, 'ambiguous'], + [{ text: "tested but it doesn't work" }, 'failure'], + [{ text: 'not broken' }, 'affirmative'], + [{ text: 'manual, credentials are unavailable' }, 'deferred'], + ] as const)('classifies %o as %s', (input, expected) => { + expect(classifyGuardianResponse(input)).toBe(expected) + }) +}) + +describe('feature guardian conversation ownership', () => { + it('holds two clarification turns across restart and stores one exact confirmation', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn(f.ctx, reply('env-1', '1710000001.000100', 'maybe'), a, f.deps) + expect(f.store.record()).toMatchObject({ + status: 'discussing', + turnCount: 1, + clarificationCount: 1, + }) + expect(f.slack.posts[0]?.text).toContain('Clarification 1') + + await runGuardianConversationTurn( + f.ctx, + reply('duplicate-envelope', '1710000001.000100', 'maybe'), + a, + f.deps, + ) + expect(f.slack.posts).toHaveLength(1) + + await runGuardianConversationTurn(f.ctx, reply('env-2', '1710000002.000100', 'maybe still'), a, f.deps) + expect(f.store.record()).toMatchObject({ turnCount: 2, clarificationCount: 2 }) + expect(f.slack.posts[1]?.text).toContain('Clarification 2') + + await runGuardianConversationTurn( + f.ctx, + reaction('env-3', '1710000003.000100', 'white_check_mark'), + a, + f.deps, + ) + expect(f.store.record().status).toBe('confirmed') + expect(f.confirmations.records).toHaveLength(1) + const confirmation = [...f.confirmations.records.values()][0]! + expect(confirmation).toMatchObject({ + featureId: feature.id, + manifestRevision: catalog.manifestRevision, + procedureRevision: catalog.procedureRevision, + generation: 7, + actor: { id: AUTHORIZED }, + verifier: AUTHORIZED, + timestamp: slackTime('1710000003.000100'), + slack: { channelId: CHANNEL, threadTs: ROOT_TS, questionTs: ROOT_TS }, + }) + }) + + it('clarifies a wrench, verifies the next failure, and opens one remediation issue', async () => { + const f = await fixture({ result: procedureResult('failed') }) + const a = adapters({ result: procedureResult('failed') }) + await runGuardianConversationTurn(f.ctx, reaction('wrench-1', '1710000001.000100', 'wrench'), a, f.deps) + expect(f.store.record().status).toBe('discussing') + expect(a.runProcedure).not.toHaveBeenCalled() + + const failure = reply('failure-2', '1710000002.000100', 'the implementation is still failing') + await runGuardianConversationTurn(f.ctx, failure, a, f.deps) + expect(f.store.record()).toMatchObject({ + status: 'remediation-open', + issue: { repository: 'AgentWorkforce/factory', number: 1700 }, + }) + expect(f.issueWriter.upsert).toHaveBeenCalledTimes(1) + expect(f.issueWrites[0]).toMatchObject({ + repository: 'AgentWorkforce/factory', + labels: ['factory-ready'], + defectKind: 'implementation', + }) + expect(f.slack.posts.at(-1)?.text).toContain('/issues/1700') + + await runGuardianConversationTurn(f.ctx, failure, a, f.deps) + expect(f.issueWriter.upsert).toHaveBeenCalledTimes(1) + }) + + it('runs an untested check only after clarification and records automated evidence', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn(f.ctx, reply('untested-1', '1710000001.000100', 'untested'), a, f.deps) + expect(a.runProcedure).not.toHaveBeenCalled() + await runGuardianConversationTurn(f.ctx, reply('untested-2', '1710000002.000100', 'still untested'), a, f.deps) + expect(a.runProcedure).toHaveBeenCalledTimes(1) + expect(f.store.record().status).toBe('confirmed') + expect([...f.confirmations.records.values()][0]).toMatchObject({ + verifier: 'factory-feature-guardian:procedure-runner', + commands: ['npm run build', 'npm test'], + tests: { passed: 42, failed: 0 }, + }) + }) + + it('requires automation for an unauthorized affirmative actor', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn( + f.ctx, + reaction('unauthorized-check', '1710000001.000100', 'white_check_mark', 'U-OTHER'), + a, + f.deps, + ) + expect(a.runProcedure).toHaveBeenCalledTimes(1) + expect([...f.confirmations.records.values()][0]?.verifier).toBe( + 'factory-feature-guardian:procedure-runner', + ) + }) + + it('preserves exact SKIP and MANUAL outcomes without confirmation', async () => { + const skip = await fixture() + const skipAdapters = adapters() + skipAdapters.gateTier = vi.fn(async () => ({ outcome: 'skip', reason: 'SKIP: no fixture' })) + await runGuardianConversationTurn(skip.ctx, reply('skip-1', '1710000001.000100', 'untested'), skipAdapters, skip.deps) + await runGuardianConversationTurn(skip.ctx, reply('skip-2', '1710000002.000100', 'still untested'), skipAdapters, skip.deps) + expect(skip.store.record().status).toBe('deferred') + expect(skip.slack.posts.at(-1)?.text).toContain('Result: SKIP') + expect(skip.confirmations.records).toHaveLength(0) + + const manual = await fixture() + await runGuardianConversationTurn( + manual.ctx, + reply('manual-1', '1710000001.000100', 'manual, credentials are unavailable'), + adapters(), + manual.deps, + ) + expect(manual.slack.posts.at(-1)?.text).toContain('Result: MANUAL') + }) + + it('defers when the manifest or procedure revision changes after the question', async () => { + const f = await fixture() + const changed = { ...catalog, manifestRevision: guardianContentRevision('manifest-b') } + const a = adapters({ currentCatalog: changed }) + await runGuardianConversationTurn(f.ctx, reply('changed-1', '1710000001.000100', 'yes'), a, f.deps) + expect(f.store.record().status).toBe('deferred') + expect(f.confirmations.records).toHaveLength(0) + expect(f.slack.posts.at(-1)?.text).toContain('different revision') + }) + + it('ignores unrelated channels, threads, bots, and delayed older events', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn( + f.ctx, + reply('wrong-channel', '1710000001.000100', 'yes', AUTHORIZED, 'C-OTHER'), + a, + f.deps, + ) + const wrongThread = reply('wrong-thread', '1710000001.000200', 'yes') as WorkforceEvent & { + expand: WorkforceEvent['expand'] + } + wrongThread.expand = vi.fn(async () => ({ + data: { event: { channel: CHANNEL, ts: '1710000001.000200', thread_ts: '1710000000.999999', user: AUTHORIZED, text: 'yes' } }, + })) + await runGuardianConversationTurn(f.ctx, wrongThread, a, f.deps) + expect(f.store.record().status).toBe('asked') + + await runGuardianConversationTurn(f.ctx, reply('newer', '1710000002.000100', 'maybe'), a, f.deps) + await runGuardianConversationTurn(f.ctx, reply('older', '1710000001.000100', 'maybe'), a, f.deps) + expect(f.store.record()).toMatchObject({ turnCount: 1, clarificationCount: 1 }) + expect(f.slack.posts).toHaveLength(1) + }) + + it('replays a clarification after a post-before-CAS conflict without duplicating Slack', async () => { + const f = await fixture() + const a = adapters() + f.store.failSaveCalls.add(f.store.saveCalls + 1) + const event = reply('cas-1', '1710000001.000100', 'maybe') + await expect(runGuardianConversationTurn(f.ctx, event, a, f.deps)).rejects.toBeInstanceOf( + GuardianStateConflictError, + ) + expect(f.slack.posts).toHaveLength(1) + await runGuardianConversationTurn(f.ctx, event, a, f.deps) + expect(f.slack.posts).toHaveLength(1) + expect(f.store.record()).toMatchObject({ status: 'discussing', turnCount: 1 }) + }) + + it('resumes an issue-write failure without rerunning the procedure', async () => { + let attempts = 0 + const f = await fixture({ + result: procedureResult('failed'), + issueWriter: async (policy) => { + attempts += 1 + if (attempts === 1) throw new Error('simulated issue admission failure') + return { + repository: policy.repository, + number: 1701, + url: 'https://github.com/AgentWorkforce/factory/issues/1701', + dedupeKey: policy.dedupeKey, + } + }, + }) + const a = adapters({ result: procedureResult('failed') }) + await runGuardianConversationTurn(f.ctx, reaction('issue-1', '1710000001.000100', 'wrench'), a, f.deps) + const event = reply('issue-2', '1710000002.000100', 'implementation is failing') + await expect(runGuardianConversationTurn(f.ctx, event, a, f.deps)).rejects.toThrow( + 'issue admission failure', + ) + expect(f.store.record().status).toBe('verifying') + await runGuardianConversationTurn(f.ctx, event, a, f.deps) + expect(a.runProcedure).toHaveBeenCalledTimes(1) + expect(f.store.record().status).toBe('remediation-open') + }) + + it('stores confirmation once when terminal checkpoint retries after Slack delivery', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn(f.ctx, reply('confirm-1', '1710000001.000100', 'untested'), a, f.deps) + // verifying, result, confirmation-recorded, confirmation path, then terminal + f.store.failSaveCalls.add(f.store.saveCalls + 5) + const event = reply('confirm-2', '1710000002.000100', 'still untested') + await expect(runGuardianConversationTurn(f.ctx, event, a, f.deps)).rejects.toBeInstanceOf( + GuardianStateConflictError, + ) + expect(f.store.record().status).toBe('confirmation-recorded') + await runGuardianConversationTurn(f.ctx, event, a, f.deps) + expect(f.confirmations.records).toHaveLength(1) + expect(f.store.record().status).toBe('confirmed') + expect(f.slack.posts.filter((post) => post.text.startsWith('✅'))).toHaveLength(1) + }) + + it('opens remediation for repeated documentation drift even when implementation checks pass', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn( + f.ctx, + reply('docs-1', '1710000001.000100', 'the documentation is wrong'), + a, + f.deps, + ) + await runGuardianConversationTurn( + f.ctx, + reply('docs-2', '1710000002.000100', 'the documentation is still wrong'), + a, + f.deps, + ) + expect(f.store.record().status).toBe('remediation-open') + expect(f.issueWrites[0]?.defectKind).toBe('documentation') + }) +}) + +describe('feature guardian malformed and bounded state', () => { + it('rejects an impossible pending state from the exact SDK store', async () => { + const id = guardianConversationId(question) + const malformed = { + kind: 'feature-guardian:conversations', + version: 1, + records: [{ + ...question, + id, + status: 'asked', + turnCount: 0, + clarificationCount: 0, + seenEventIds: [], + evidence: [], + pending: { + eventId: `slack:${'a'.repeat(64)}`, + eventOrder: `${slackTime('1710000001.000100')}:1710000001.000100:slack:${'a'.repeat(64)}`, + eventOccurredAt: slackTime('1710000001.000100'), + actorId: AUTHORIZED, + response: 'ambiguous', + text: 'maybe', + }, + updatedAt: slackTime('1710000001.000100'), + }], + } + const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ + path: '/memory/workspace/factory-feature-guardian/conversations.json', + revision: 'rev-1', + content: `${JSON.stringify(malformed)}\n`, + encoding: 'utf-8', + }), { status: 200, headers: { 'content-type': 'application/json' } })) as typeof fetch + const store = createSdkConversationStore( + { url: 'https://relayfile.test', token: 'token', workspaceId: 'workspace' }, + { fetchImpl }, + ) + await expect(store.load()).rejects.toThrow('asked conversation cannot retain a pending turn') + }) + + it('prunes oldest terminal records to satisfy count and byte bounds', async () => { + const store = new MemoryConversationStore() + const records = Array.from({ length: 512 }, (_, index): GuardianConversationRecord => { + const ts = `1720${String(index).padStart(6, '0')}.000100` + const historical = { ...question, threadTs: ts, askedAt: slackTime(ts) } + return { + ...historical, + id: guardianConversationId(historical), + status: 'confirmed', + turnCount: 1, + clarificationCount: 0, + seenEventIds: [`event-${index}`], + evidence: [], + confirmationPath: `/confirmations/${index}.json`, + finalReplyTs: ts, + updatedAt: slackTime(ts), + } + }) + store.snapshot = { + state: { kind: 'feature-guardian:conversations', version: 1, records }, + revision: 'memory-seed', + } + await registerGuardianQuestion(context(), question, { conversationStore: store }) + expect(store.snapshot?.state.records.length).toBeLessThanOrEqual(512) + expect(new TextEncoder().encode(JSON.stringify(store.snapshot?.state)).byteLength).toBeLessThanOrEqual( + 256 * 1024, + ) + expect(store.snapshot?.state.records.some((record) => record.id === records[0]?.id)).toBe(false) + expect(store.snapshot?.state.records.some((record) => record.id === guardianConversationId(question))).toBe(true) + }) +}) + +describe('feature guardian GitHub remediation durability', () => { + const policy: GuardianIssuePolicy = { + repository: 'AgentWorkforce/factory', + title: '[Feature guardian] routing defect', + body: `${remediationMarker(`factory:${feature.id}:${feature.procedure}`)}\ncomplete evidence`, + labels: ['factory-ready'], + defectKind: 'implementation', + dedupeKey: `factory:${feature.id}:${feature.procedure}`, + } + + function github(overrides: Partial = {}): GithubClient { + return { + issues: { list: vi.fn(async () => []) }, + 'issue-comments': { list: vi.fn(async () => []) }, + createIssue: vi.fn(async () => ({ + status: 'confirmed', + id: '1777', + url: 'https://github.com/AgentWorkforce/factory/issues/1777', + path: '/github/issues/draft.json', + receipt: { externalId: '1777' }, + })), + comment: vi.fn(), + ...overrides, + } as unknown as GithubClient + } + + it('persists a provider receipt and never creates the issue twice', async () => { + const ctx = context() + const client = github() + const writer = createGithubIssueWriter(ctx, client) + const first = await writer.upsert(policy) + const replay = await writer.upsert(policy) + expect(replay).toEqual(first) + expect(client.createIssue).toHaveBeenCalledTimes(1) + }) + + it('appends changed evidence to the deduplicated issue exactly once', async () => { + const ctx = context() + const client = github({ + comment: vi.fn(async () => ({ + status: 'confirmed', + id: 'comment-1', + url: 'https://github.com/AgentWorkforce/factory/issues/1777#issuecomment-1', + path: '/github/issues/1777/comments/draft.json', + })), + }) + const writer = createGithubIssueWriter(ctx, client) + const changedPolicy = { ...policy, body: `${policy.body}\nnew verification evidence` } + await writer.upsert(policy) + await writer.upsert(changedPolicy) + await writer.upsert(changedPolicy) + expect(client.createIssue).toHaveBeenCalledTimes(1) + expect(client.comment).toHaveBeenCalledTimes(1) + }) + + it('refuses a second create while an admitted submission is still uncorrelated', async () => { + const ctx = context() + const client = github({ + createIssue: vi.fn(async () => ({ + status: 'pending', + id: '/github/issues/draft.json', + url: '', + path: '/github/issues/draft.json', + })), + }) + const writer = createGithubIssueWriter(ctx, client) + await expect(writer.upsert(policy)).rejects.toThrow('pending, not provider-confirmed') + await expect(writer.upsert(policy)).rejects.toThrow('refusing a duplicate issue') + expect(client.createIssue).toHaveBeenCalledTimes(1) + }) + + it('finds an existing marker in the canonical nested issue mount', async () => { + const ctx = context() + ctx.sandbox.readFile = vi.fn(async (path: string) => { + if (path.endsWith('/issues/_index.json')) { + return JSON.stringify([{ id: '1778', number: 1778, title: 'Guardian routing defect' }]) + } + if (path.endsWith('/1778__guardian-routing-defect/meta.json')) { + return JSON.stringify({ + payload: { + number: 1778, + url: 'https://github.com/AgentWorkforce/factory/issues/1778', + body: policy.body, + }, + }) + } + throw new Error(`ENOENT: ${path}`) + }) + const client = github() + const receipt = await createGithubIssueWriter(ctx, client).upsert(policy) + expect(receipt.number).toBe(1778) + expect(client.createIssue).not.toHaveBeenCalled() + }) +}) diff --git a/src/feature-guardian/conversation.ts b/src/feature-guardian/conversation.ts new file mode 100644 index 0000000..a0e0db2 --- /dev/null +++ b/src/feature-guardian/conversation.ts @@ -0,0 +1,2228 @@ +import { createHash, randomUUID } from 'node:crypto' + +import type { + RelayfileCredentials, + WorkforceCtx, + WorkforceEvent, +} from '@agentworkforce/runtime' +import { defineAgent } from '@agentworkforce/runtime' +import { + RelayFileApiError, + RelayFileClient, + RevisionConflictError, + type OperationStatusResponse, +} from '@relayfile/sdk' +import { + githubClient, + slackClient, + type GithubClient, + type SlackClient, + type WritebackResult, +} from '@relayfile/relay-helpers' + +export const GUARDIAN_CONVERSATION_STATE_PATH = + '/memory/workspace/factory-feature-guardian/conversations.json' +export const GUARDIAN_CONFIRMATIONS_PATH = + '/memory/workspace/factory-feature-guardian/confirmations' +export const GUARDIAN_REMEDIATIONS_PATH = + '/memory/workspace/factory-feature-guardian/remediations' + +export const GUARDIAN_STATE_TIMEOUT_MS = 5_000 +export const GUARDIAN_WRITEBACK_TIMEOUT_MS = 15_000 +export const GUARDIAN_WRITEBACK_POLL_MS = 250 + +const MAX_STATE_BYTES = 256 * 1024 +const MAX_CONVERSATIONS = 512 +const MAX_SEEN_EVENTS = 128 +const MAX_EVIDENCE_ITEMS = 32 +const MAX_TURNS = 12 +const UTF8_ENCODER = new TextEncoder() + +export type GuardianConversationStatus = + | 'asked' + | 'discussing' + | 'verifying' + | 'confirmation-recorded' + | 'remediation-recorded' + | 'deferred-recorded' + | 'confirmed' + | 'remediation-open' + | 'deferred' + +export type GuardianResponseKind = + | 'affirmative' + | 'failure' + | 'untested' + | 'ambiguous' + | 'deferred' + +export type GuardianDefectKind = + | 'implementation' + | 'test' + | 'manifest' + | 'procedure' + | 'documentation' + +export type GuardianProcedureResultKind = 'passed' | 'failed' | 'skip' | 'manual' + +export interface GuardianFeatureSnapshot { + id: string + name: string + category: string + cli?: string + api?: string + description: string + locations: string[] + procedure: string + tier: number + criticality: string +} + +export interface GuardianEvidence { + source: 'actor' | 'procedure' | 'system' + result: 'positive' | 'negative' | 'skip' | 'manual' | 'unknown' + summary: string + commands?: string[] + positiveAssertions?: string[] + negativeAssertions?: string[] + tests?: { passed: number; failed: number } + cleanup?: string[] +} + +export interface GuardianProcedureResult extends GuardianEvidence { + source: 'procedure' + outcome: GuardianProcedureResultKind + verifier: string +} + +export interface GuardianQuestion { + feature: GuardianFeatureSnapshot + manifestRevision: string + manifestVersion: string + procedureRevision: string + generation: number + channelId: string + threadTs: string + askedAt: string +} + +export interface GuardianPendingTurn { + eventId: string + eventOrder: string + eventOccurredAt: string + actorId: string + response: GuardianResponseKind + text: string + reaction?: string + evidence?: GuardianEvidence + verification?: GuardianProcedureResult + defectKind?: GuardianDefectKind +} + +export interface GuardianIssueReceipt { + repository: string + number: number + url: string + dedupeKey: string +} + +export interface GuardianConversationRecord extends GuardianQuestion { + id: string + status: GuardianConversationStatus + turnCount: number + clarificationCount: number + seenEventIds: string[] + evidence: GuardianEvidence[] + pending?: GuardianPendingTurn + confirmationPath?: string + issue?: GuardianIssueReceipt + finalReplyTs?: string + lastProcessedEventOrder?: string + updatedAt: string +} + +export interface GuardianConversationState { + kind: 'feature-guardian:conversations' + version: 1 + records: GuardianConversationRecord[] +} + +export interface GuardianConfirmationRecord { + kind: 'feature-guardian:confirmation' + version: 1 + featureId: string + manifestRevision: string + manifestVersion: string + procedureRevision: string + generation: number + result: 'confirmed' + actor?: { id: string } + verifier: string + timestamp: string + evidence: GuardianEvidence[] + commands: string[] + tests: { passed: number; failed: number } + slack: { + channelId: string + threadTs: string + questionTs: string + } +} + +export interface GuardianManifestCatalog { + manifestRevision: string + manifestVersion: string + procedureRevision: string + features: GuardianFeatureSnapshot[] +} + +export interface GuardianProcedure { + name: string + path: string + prerequisites: string + body: string +} + +export interface GuardianTierGate { + outcome: 'available' | 'skip' | 'manual' + reason: string +} + +export interface GuardianIssuePolicy { + repository: string + title: string + body: string + labels: string[] + defectKind: GuardianDefectKind + dedupeKey: string +} + +export interface FeatureGuardianAdapters { + loadCatalog(ctx: WorkforceCtx): Promise + resolveProcedure( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, + ): Promise + gateTier( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, + procedure: GuardianProcedure, + ): Promise + runProcedure( + ctx: WorkforceCtx, + feature: GuardianFeatureSnapshot, + procedure: GuardianProcedure, + runKey: string, + ): Promise + isAuthorizedConfirmer(ctx: WorkforceCtx, actorId: string): boolean + clarification( + feature: GuardianFeatureSnapshot, + procedure: GuardianProcedure, + turn: GuardianPendingTurn, + turnNumber: number, + ): string + classifyDefect( + feature: GuardianFeatureSnapshot, + turn: GuardianPendingTurn, + evidence: readonly GuardianEvidence[], + ): GuardianDefectKind + repositoryForFeature(feature: GuardianFeatureSnapshot): string + issuePolicy(input: { + feature: GuardianFeatureSnapshot + conversation: GuardianConversationRecord + defectKind: GuardianDefectKind + slackBacklink: string + }): GuardianIssuePolicy +} + +export interface ConversationSnapshot { + state: GuardianConversationState + revision: string +} + +export interface ConversationStore { + load(): Promise + save( + state: GuardianConversationState, + expected: ConversationSnapshot | null, + ): Promise +} + +export interface ImmutableConfirmationStore { + append(record: GuardianConfirmationRecord): Promise +} + +export interface GuardianIssueWriter { + upsert(policy: GuardianIssuePolicy): Promise +} + +export interface GuardianConversationDependencies { + conversationStore?: ConversationStore + confirmationStore?: ImmutableConfirmationStore + issueWriter?: GuardianIssueWriter + createSlackClient?: typeof slackClient + now?: () => Date +} + +export interface DefineFeatureGuardianAgentOptions { + adapters: FeatureGuardianAdapters + scheduled: (ctx: WorkforceCtx, event: WorkforceEvent) => Promise | void + channelInput?: string + schedules?: ReadonlyArray<{ name: string; cron: string; tz?: string }> +} + +export class GuardianStateConflictError extends Error { + constructor(message = 'feature guardian conversation state revision conflict') { + super(message) + this.name = 'GuardianStateConflictError' + } +} + +/** + * Reusable feature-guardian factory. Repository adopters provide their catalog, + * procedures, gates, prompt, authority, and issue policy; this factory owns the + * scoped Slack listeners and dispatches every response through the shared state + * machine. + */ +export function defineFeatureGuardianAgent(options: DefineFeatureGuardianAgentOptions) { + const channelInput = options.channelInput ?? 'SLACK_CHANNEL' + const paths = [`/slack/channels/\${${channelInput}}/messages/**`] + return defineAgent({ + triggers: { + slack: [ + { on: 'message.created', paths, maxConcurrency: 1 }, + { on: 'reaction.added', paths, maxConcurrency: 1 }, + ], + }, + schedules: options.schedules ?? [ + { name: 'hourly-check', cron: '0 * * * *', tz: 'America/New_York' }, + ], + handler: async (ctx, event) => { + if (event.type === 'cron.tick') { + await options.scheduled(ctx, event) + return + } + await runGuardianConversationTurn(ctx, event, options.adapters) + }, + }) +} + +type FetchLike = typeof fetch + +/** A stable content revision suitable for binding a question to exact source text. */ +export function guardianContentRevision(content: string): string { + return `sha256:${createHash('sha256').update(content).digest('hex')}` +} + +/** Stable identity for one question in one manifest generation. */ +export function guardianConversationId(question: GuardianQuestion): string { + return [ + question.feature.id, + question.manifestRevision, + `g${question.generation}`, + question.channelId, + question.threadTs, + ].join(':') +} + +/** Stable key shared by retries of one threaded response. */ +export function guardianReplyIdempotencyKey( + record: Pick, + eventId: string, + purpose: string, +): string { + return `feature-guardian:${record.id}:${eventId}:${purpose}` +} + +/** Immutable confirmation path. The filename is a hash so provider IDs never become paths. */ +export function guardianConfirmationPath(record: GuardianConfirmationRecord): string { + const identity = [ + record.featureId, + record.manifestRevision, + `g${record.generation}`, + record.slack.channelId, + record.slack.threadTs, + ].join(':') + return `${GUARDIAN_CONFIRMATIONS_PATH}/${createHash('sha256').update(identity).digest('hex')}.json` +} + +/** Parse checkmark/wrench/question reactions and conversational text without an LLM guess. */ +export function classifyGuardianResponse(input: { + text?: string + reaction?: string +}): GuardianResponseKind { + const reaction = (input.reaction ?? '').trim().toLowerCase().replaceAll('-', '_') + if (['white_check_mark', 'heavy_check_mark', 'ballot_box_with_check', 'check'].includes(reaction)) { + return 'affirmative' + } + if (['wrench', 'hammer_and_wrench', 'hammer', 'x'].includes(reaction)) return 'failure' + if (['question', 'grey_question', 'interrobang'].includes(reaction)) return 'untested' + + const text = (input.text ?? '').trim().toLowerCase() + if (!text) return 'ambiguous' + if (/\b(defer(?:red)?|manual|skip(?:ped)?|cannot run|can't run|blocked by credentials)\b/u.test(text)) { + return 'deferred' + } + if (/\b(untested|not tested|haven't tested|have not tested|unknown|unsure|don't know|do not know)\b/u.test(text)) { + return 'untested' + } + if ( + /\b(?:does(?:n't| not)|is(?:n't| not)|not)\s+(?:work(?:ing)?|pass(?:ing)?)\b/u.test(text) + ) { + return 'failure' + } + if (/^(?:not broken|not failing|no problems?|no issues?|works? for me)[.!\s]*$/u.test(text)) { + return 'affirmative' + } + if (/\b(broken|breaks?|failing|fails?|failed|regression|drift(?:ed)?|wrong|bug|problem|off)\b/u.test(text)) { + return 'failure' + } + if (/^(?:yes|yep|yeah|confirmed|works?|working|passes?|passed|all good|looks good|✅)[.!\s]*$/u.test(text)) { + return 'affirmative' + } + if (/\b(tested|verified|confirmed)\b/u.test(text) && /\b(works?|working|passes?|passed|expected|good)\b/u.test(text)) { + return 'affirmative' + } + return 'ambiguous' +} + +interface ParsedSlackResponse { + eventId: string + eventOrder: string + occurredAt: string + channelId: string + threadTs: string + messageTs: string + actorId: string + text: string + reaction?: string + isReaction: boolean + isBot: boolean +} + +/** Expand and fail-closed parse one Slack reply/reaction event. */ +export async function parseGuardianSlackEvent( + event: WorkforceEvent, +): Promise { + if (!event.type.startsWith('slack.')) return null + const envelopeId = nonEmptyString(event.id) + if (!envelopeId || typeof event.expand !== 'function') return null + const expanded = await event.expand('full') + const root = asRecord(expanded?.data) + const payload = findSlackPayload(root) + if (!payload) return null + + const item = asRecord(payload.item) + const isReaction = event.type.includes('reaction') || nonEmptyString(payload.reaction) !== undefined + const channelId = + nonEmptyString(payload.channel) ?? + nonEmptyString(payload.channel_id) ?? + nonEmptyString(item?.channel) ?? + channelFromPath(event.resource?.path) + const messageTs = + nonEmptyString(payload.ts) ?? + nonEmptyString(payload.message_ts) ?? + nonEmptyString(item?.ts) ?? + messageTsFromPath(event.resource?.path) + const threadTs = isReaction + ? messageTs + : nonEmptyString(payload.thread_ts) ?? nonEmptyString(payload.threadTs) ?? '' + const actorId = + nonEmptyString(payload.user) ?? + nonEmptyString(payload.user_id) ?? + nonEmptyString(event.summary?.actor?.id) + if (!channelId || !messageTs || !threadTs || !actorId) return null + if (!isReaction && messageTs === threadTs) return null + const occurredAt = + canonicalTimestamp(event.occurredAt) ?? + timestampFromSlackTs(nonEmptyString(payload.event_ts) ?? messageTs) + if (!occurredAt) return null + const providerIdentity = isReaction + ? nonEmptyString(payload.event_ts) ?? envelopeId + : messageTs + const eventId = `slack:${createHash('sha256') + .update( + [ + channelId, + threadTs, + messageTs, + actorId, + nonEmptyString(payload.reaction) ?? '', + providerIdentity, + ].join(':'), + ) + .digest('hex')}` + const eventOrder = [occurredAt, messageTs, eventId].join(':') + + return { + eventId, + eventOrder, + occurredAt, + channelId, + threadTs, + messageTs, + actorId, + text: nonEmptyString(payload.text) ?? '', + ...(nonEmptyString(payload.reaction) ? { reaction: nonEmptyString(payload.reaction) } : {}), + isReaction, + isBot: + payload.user_is_bot === true || + Boolean(nonEmptyString(payload.bot_id)) || + nonEmptyString(payload.subtype) === 'bot_message', + } +} + +/** Register a provider-confirmed question before the scheduled cycle advances. */ +export async function registerGuardianQuestion( + ctx: WorkforceCtx, + question: GuardianQuestion, + dependencies: Pick = {}, +): Promise { + const store = dependencies.conversationStore ?? createConversationStore(ctx) + const now = (dependencies.now ?? (() => new Date()))().toISOString() + for (let attempt = 0; attempt < 3; attempt += 1) { + const snapshot = await store.load() + const state = snapshot?.state ?? emptyConversationState() + const id = guardianConversationId(question) + const existing = state.records.find((record) => record.id === id) + if (existing) { + if (!sameQuestion(existing, question)) { + throw new Error('guardian question identity collided with different question data') + } + return existing + } + const record: GuardianConversationRecord = { + ...question, + id, + status: 'asked', + turnCount: 0, + clarificationCount: 0, + seenEventIds: [], + evidence: [], + updatedAt: now, + } + const retained = retainConversationCapacity(state.records, record) + try { + await store.save({ ...state, records: [...retained, record] }, snapshot) + return record + } catch (error) { + if (!(error instanceof GuardianStateConflictError) || attempt === 2) throw error + } + } + throw new Error('unreachable guardian question registration retry') +} + +/** Own one correlated Slack response through a durable or explicitly deferred outcome. */ +export async function runGuardianConversationTurn( + ctx: WorkforceCtx, + event: WorkforceEvent, + adapters: FeatureGuardianAdapters, + dependencies: GuardianConversationDependencies = {}, +): Promise { + const parsed = await parseGuardianSlackEvent(event) + if (!parsed || parsed.isBot) return + const configuredChannel = resolvedInput(ctx, 'SLACK_CHANNEL') + if (!configuredChannel || parsed.channelId !== configuredChannel) { + ctx.log('warn', 'feature-guardian.event-rejected', { + reason: 'unauthorized-channel', + channel: parsed.channelId, + }) + return + } + if (parsed.actorId === resolvedInput(ctx, 'SLACK_BOT_USER')) { + ctx.log('info', 'feature-guardian.event-ignored', { reason: 'configured-bot-actor' }) + return + } + + const store = dependencies.conversationStore ?? createConversationStore(ctx) + const confirmationStore = dependencies.confirmationStore ?? createConfirmationStore(ctx) + const issueWriter = dependencies.issueWriter ?? createGithubIssueWriter(ctx) + const createSlackClient = dependencies.createSlackClient ?? slackClient + const now = dependencies.now ?? (() => new Date()) + + let snapshot = await store.load() + if (!snapshot) return + const correlated = findCorrelatedConversation(snapshot.state, parsed) + if (!correlated) { + ctx.log('info', 'feature-guardian.event-ignored', { + reason: 'unrelated-thread', + channel: parsed.channelId, + threadTs: parsed.threadTs, + messageTs: parsed.messageTs, + }) + return + } + let record: GuardianConversationRecord = correlated + if (isTerminal(record.status)) return + if (Date.parse(parsed.occurredAt) < Date.parse(record.askedAt)) { + ctx.log('info', 'feature-guardian.event-ignored', { + reason: 'event-predates-question', + eventId: parsed.eventId, + conversationId: record.id, + }) + return + } + + // A durable pending turn always owns the conversation. Resume it before + // considering a later delivery so a restart or out-of-order event cannot + // overwrite procedure evidence, a confirmation write, or an issue receipt. + const pendingBeforeResume = record.pending + if (pendingBeforeResume) { + await resumePendingTurn() + if (isTerminal(record.status) || record.pending) return + if (pendingBeforeResume.eventId === parsed.eventId || record.seenEventIds.includes(parsed.eventId)) { + return + } + if (Date.parse(parsed.occurredAt) <= Date.parse(pendingBeforeResume.eventOccurredAt)) { + ctx.log('info', 'feature-guardian.event-ignored', { + reason: 'superseded-out-of-order-event', + eventId: parsed.eventId, + conversationId: record.id, + }) + return + } + } + if (record.seenEventIds.includes(parsed.eventId)) return + if ( + record.lastProcessedEventOrder && + parsed.eventOrder <= record.lastProcessedEventOrder + ) { + ctx.log('info', 'feature-guardian.event-ignored', { + reason: 'delayed-event-precedes-conversation-watermark', + eventId: parsed.eventId, + conversationId: record.id, + }) + return + } + + const catalog = await adapters.loadCatalog(ctx) + if ( + catalog.manifestRevision !== record.manifestRevision || + catalog.procedureRevision !== record.procedureRevision + ) { + const turn = pendingTurn(parsed, 'deferred', { + source: 'system', + result: 'manual', + summary: 'The manifest or procedure changed after this question was posted; refusing to confirm a different revision.', + }) + snapshot = await checkpointPending(snapshot, record, turn, 'deferred-recorded') + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, turn.evidence?.summary ?? 'Revision changed before verification.') + return + } + + const response = classifyGuardianResponse({ text: parsed.text, reaction: parsed.reaction }) + const actorEvidence: GuardianEvidence = { + source: 'actor', + result: + response === 'affirmative' + ? 'positive' + : response === 'failure' + ? 'negative' + : response === 'deferred' + ? 'manual' + : 'unknown', + summary: parsed.reaction + ? `Slack reaction :${parsed.reaction}: from ${parsed.actorId}` + : `Slack reply from ${parsed.actorId}: ${bounded(parsed.text, 1_000)}`, + } + const turn = pendingTurn(parsed, response, actorEvidence) + + if (response === 'ambiguous') { + await clarify(turn) + return + } + + if ((response === 'failure' || response === 'untested') && record.turnCount === 0) { + await clarify(turn) + return + } + + if (response === 'affirmative' && adapters.isAuthorizedConfirmer(ctx, parsed.actorId)) { + snapshot = await checkpointPending(snapshot, record, turn, 'confirmation-recorded', true) + record = requireRecordById(snapshot.state, record.id) + await finishConfirmation(record, turn) + return + } + + if (response === 'deferred') { + snapshot = await checkpointPending(snapshot, record, turn, 'deferred-recorded', true) + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, actorEvidence.summary) + return + } + + const procedure = await adapters.resolveProcedure(ctx, record.feature) + const gate = await adapters.gateTier(ctx, record.feature, procedure) + if (gate.outcome !== 'available') { + const gateEvidence: GuardianEvidence = { + source: 'system', + result: gate.outcome, + summary: gate.reason, + } + const deferredTurn = { ...turn, response: 'deferred' as const, evidence: gateEvidence } + snapshot = await checkpointPending(snapshot, record, deferredTurn, 'deferred-recorded', true) + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, gate.reason) + return + } + + snapshot = await checkpointPending(snapshot, record, turn, 'verifying', true) + record = requireRecordById(snapshot.state, record.id) + await resumePendingTurn(procedure) + + async function resumePendingTurn(preResolvedProcedure?: GuardianProcedure): Promise { + if (!snapshot || !record.pending) return + const pending = record.pending + if (record.status === 'confirmation-recorded') { + await finishConfirmation(record, pending) + return + } + if (record.status === 'remediation-recorded') { + if (!record.issue) throw new Error('remediation-recorded state is missing its issue receipt') + await finishRemediation(record, record.issue) + return + } + if (record.status === 'deferred-recorded') { + await finishDeferred(record, pending.evidence?.summary ?? 'Verification deferred.') + return + } + if (record.status !== 'verifying') return + + const procedure = preResolvedProcedure ?? await adapters.resolveProcedure(ctx, record.feature) + const result = pending.verification ?? await adapters.runProcedure( + ctx, + record.feature, + procedure, + guardianReplyIdempotencyKey(record, pending.eventId, 'procedure'), + ) + if (!pending.verification) { + const next = replaceRecord(snapshot.state, record.id, { + ...record, + pending: { ...pending, verification: result }, + evidence: appendEvidence(record.evidence, result), + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + } + const evidence = record.evidence + if (result.outcome === 'passed') { + if (pending.response === 'failure') { + const reportedDefect = adapters.classifyDefect(record.feature, pending, evidence) + if ( + record.clarificationCount >= 1 && + ['test', 'manifest', 'procedure', 'documentation'].includes(reportedDefect) + ) { + await recordRemediation(result, reportedDefect) + return + } + const clarificationNumber = record.clarificationCount + 1 + const text = adapters.clarification( + record.feature, + procedure, + { ...pending, evidence: result, verification: result }, + clarificationNumber, + ) + const receipt = await postThreadReply( + createSlackClient, + record, + text, + guardianReplyIdempotencyKey(record, pending.eventId, `post-check-clarification-${clarificationNumber}`), + ) + const next = replaceRecord(snapshot.state, record.id, { + ...record, + status: 'discussing', + clarificationCount: clarificationNumber, + seenEventIds: appendSeen(record.seenEventIds, pending.eventId), + lastProcessedEventOrder: pending.eventOrder, + pending: undefined, + finalReplyTs: receipt, + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + return + } + const next = replaceRecord(snapshot.state, record.id, { + ...record, + status: 'confirmation-recorded', + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + await finishConfirmation(record, pending) + return + } + if (result.outcome === 'skip' || result.outcome === 'manual') { + const next = replaceRecord(snapshot.state, record.id, { + ...record, + status: 'deferred-recorded', + pending: { ...pending, response: 'deferred', evidence: result }, + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, result.summary) + return + } + + const defectKind = adapters.classifyDefect(record.feature, pending, evidence) + await recordRemediation(result, defectKind) + } + + async function recordRemediation( + result: GuardianProcedureResult, + defectKind: GuardianDefectKind, + ): Promise { + if (!snapshot || !record.pending) return + const pending = record.pending + const evidence = record.evidence + const policy = adapters.issuePolicy({ + feature: record.feature, + conversation: { ...record, evidence }, + defectKind, + slackBacklink: slackThreadBacklink(record.channelId, record.threadTs), + }) + const routedRepository = adapters.repositoryForFeature(record.feature) + if (policy.repository !== routedRepository) { + throw new Error( + `guardian issue policy routed ${policy.repository}, expected ${routedRepository}`, + ) + } + const receipt = await issueWriter.upsert(policy) + if (receipt.repository !== routedRepository || receipt.dedupeKey !== policy.dedupeKey) { + throw new Error('guardian issue receipt did not match the authorized repository/dedupe key') + } + const next = replaceRecord(snapshot.state, record.id, { + ...record, + status: 'remediation-recorded', + issue: receipt, + pending: { ...pending, defectKind, evidence: result }, + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + await finishRemediation(record, receipt) + } + + async function clarify(turn: GuardianPendingTurn): Promise { + if (!snapshot) return + if (record.turnCount >= MAX_TURNS) { + snapshot = await checkpointPending(snapshot, record, { + ...turn, + response: 'deferred', + evidence: { + source: 'system', + result: 'manual', + summary: `Clarification stopped after the bounded ${MAX_TURNS}-turn limit.`, + }, + }, 'deferred-recorded', true) + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, `Clarification stopped after ${MAX_TURNS} turns.`) + return + } + const procedure = await adapters.resolveProcedure(ctx, record.feature) + const clarificationNumber = record.clarificationCount + 1 + const text = adapters.clarification(record.feature, procedure, turn, clarificationNumber) + const receipt = await postThreadReply( + createSlackClient, + record, + text, + guardianReplyIdempotencyKey(record, turn.eventId, `clarification-${clarificationNumber}`), + ) + const nextRecord: GuardianConversationRecord = { + ...record, + status: 'discussing', + turnCount: record.turnCount + 1, + clarificationCount: clarificationNumber, + seenEventIds: appendSeen(record.seenEventIds, turn.eventId), + lastProcessedEventOrder: turn.eventOrder, + evidence: appendEvidence(record.evidence, turn.evidence), + finalReplyTs: receipt, + updatedAt: now().toISOString(), + } + snapshot = await store.save(replaceRecord(snapshot.state, record.id, nextRecord), snapshot) + record = requireRecordById(snapshot.state, record.id) + } + + async function finishConfirmation( + current: GuardianConversationRecord, + turn: GuardianPendingTurn, + ): Promise { + if (!snapshot) return + // Bind the immutable timestamp to the provider event, not the retry wall + // clock, so a post-before-checkpoint retry produces byte-identical evidence. + const confirmation = confirmationRecord(current, turn, turn.eventOccurredAt) + const confirmationPath = await confirmationStore.append(confirmation) + if (current.confirmationPath !== confirmationPath) { + const next = replaceRecord(snapshot.state, current.id, { + ...current, + confirmationPath, + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + current = requireRecordById(snapshot.state, current.id) + record = current + } + const procedureEvidence = [...current.evidence] + .reverse() + .find((entry) => entry.source === 'procedure') + const summary = procedureEvidence + ? `✅ Confirmed *${current.feature.name}* for manifest \`${shortRevision(current.manifestRevision)}\`. ${procedureEvidence.summary}` + : `✅ Confirmed *${current.feature.name}* for manifest \`${shortRevision(current.manifestRevision)}\` from ${turn.actorId}'s response.` + const replyTs = await postThreadReply( + createSlackClient, + current, + summary, + guardianReplyIdempotencyKey(current, turn.eventId, 'confirmed'), + ) + const terminal: GuardianConversationRecord = { + ...current, + status: 'confirmed', + pending: undefined, + seenEventIds: appendSeen(current.seenEventIds, turn.eventId), + lastProcessedEventOrder: turn.eventOrder, + finalReplyTs: replyTs, + updatedAt: now().toISOString(), + } + snapshot = await store.save(replaceRecord(snapshot.state, current.id, terminal), snapshot) + record = requireRecordById(snapshot.state, current.id) + } + + async function finishRemediation( + current: GuardianConversationRecord, + issue: GuardianIssueReceipt, + ): Promise { + if (!snapshot || !current.pending) return + const replyTs = await postThreadReply( + createSlackClient, + current, + `🔧 Verification established a ${current.pending.defectKind ?? 'feature'} defect for *${current.feature.name}*. Remediation: ${issue.url}`, + guardianReplyIdempotencyKey(current, current.pending.eventId, 'remediation'), + ) + const terminal: GuardianConversationRecord = { + ...current, + status: 'remediation-open', + pending: undefined, + seenEventIds: appendSeen(current.seenEventIds, current.pending.eventId), + lastProcessedEventOrder: current.pending.eventOrder, + finalReplyTs: replyTs, + updatedAt: now().toISOString(), + } + snapshot = await store.save(replaceRecord(snapshot.state, current.id, terminal), snapshot) + record = requireRecordById(snapshot.state, current.id) + } + + async function finishDeferred( + current: GuardianConversationRecord, + reason: string, + ): Promise { + if (!snapshot || !current.pending) return + const evidenceResult = current.pending.verification?.result ?? current.pending.evidence?.result + const outcome = evidenceResult === 'skip' ? 'SKIP' : 'MANUAL' + const replyTs = await postThreadReply( + createSlackClient, + current, + `🟡 *${current.feature.name}* remains unconfirmed: ${reason} Result: ${outcome}.`, + guardianReplyIdempotencyKey(current, current.pending.eventId, 'deferred'), + ) + const terminal: GuardianConversationRecord = { + ...current, + status: 'deferred', + pending: undefined, + seenEventIds: appendSeen(current.seenEventIds, current.pending.eventId), + lastProcessedEventOrder: current.pending.eventOrder, + finalReplyTs: replyTs, + updatedAt: now().toISOString(), + } + snapshot = await store.save(replaceRecord(snapshot.state, current.id, terminal), snapshot) + record = requireRecordById(snapshot.state, current.id) + } + + async function checkpointPending( + currentSnapshot: ConversationSnapshot, + current: GuardianConversationRecord, + turn: GuardianPendingTurn, + status: GuardianConversationStatus, + countTurn = false, + ): Promise { + const nextRecord: GuardianConversationRecord = { + ...current, + status, + turnCount: current.turnCount + (countTurn ? 1 : 0), + pending: turn, + lastProcessedEventOrder: turn.eventOrder, + evidence: appendEvidence(current.evidence, turn.evidence), + updatedAt: now().toISOString(), + } + return store.save(replaceRecord(currentSnapshot.state, current.id, nextRecord), currentSnapshot) + } +} + +/** Exact-revision store used by the reusable state machine. */ +export function createSdkConversationStore( + credentials: RelayfileCredentials, + options: { fetchImpl?: FetchLike; timeoutMs?: number } = {}, +): ConversationStore { + const client = relayfileClient(credentials, options.fetchImpl) + const timeoutMs = options.timeoutMs ?? GUARDIAN_STATE_TIMEOUT_MS + return exactStore({ + client, + credentials, + path: GUARDIAN_CONVERSATION_STATE_PATH, + timeoutMs, + parse: parseConversationState, + }) +} + +/** Exact create-only immutable confirmation store. */ +export function createSdkConfirmationStore( + credentials: RelayfileCredentials, + options: { fetchImpl?: FetchLike; timeoutMs?: number } = {}, +): ImmutableConfirmationStore { + const client = relayfileClient(credentials, options.fetchImpl) + const timeoutMs = options.timeoutMs ?? GUARDIAN_STATE_TIMEOUT_MS + return { + append: async (record) => { + const canonical = parseConfirmationRecord(record) + const path = guardianConfirmationPath(canonical) + const content = `${JSON.stringify(canonical)}\n` + assertBoundedContent(content, 'guardian confirmation') + return withDeadline('guardian confirmation append', timeoutMs, async (signal) => { + const existing = await readExactFile(client, credentials.workspaceId, path, signal) + if (existing) { + if (existing.content !== content) throw new Error('immutable guardian confirmation already differs') + return path + } + await writeExactFile(client, credentials.workspaceId, path, '0', content, signal) + const readBack = await readExactFile(client, credentials.workspaceId, path, signal) + if (!readBack || readBack.content !== content) { + throw new Error('guardian confirmation read-back did not match') + } + return path + }) + }, + } +} + +function createConversationStore(ctx: WorkforceCtx): ConversationStore { + const credentials = ctx.credentials.tryRequire() + if (credentials) return createSdkConversationStore(credentials.relayfile) + if (ctx.agent.id === 'sim-agent' && ctx.deployment.id === 'sim-deployment') { + return previewConversationStore(ctx) + } + throw new Error('exact Relayfile credentials are required for guardian conversations') +} + +function createConfirmationStore(ctx: WorkforceCtx): ImmutableConfirmationStore { + const credentials = ctx.credentials.tryRequire() + if (credentials) return createSdkConfirmationStore(credentials.relayfile) + if (ctx.agent.id === 'sim-agent' && ctx.deployment.id === 'sim-deployment') { + return { + append: async (record) => { + const canonical = parseConfirmationRecord(record) + const path = guardianConfirmationPath(canonical) + const content = `${JSON.stringify(canonical)}\n` + let existing: string | undefined + try { + existing = await ctx.files.read(path) + } catch (error) { + if (!String(error).includes('ENOENT')) throw error + } + if (existing !== undefined && existing !== content) { + throw new Error('immutable guardian confirmation already differs') + } + if (existing === undefined) await ctx.files.write(path, content) + return path + }, + } + } + throw new Error('exact Relayfile credentials are required for guardian confirmations') +} + +function previewConversationStore(ctx: WorkforceCtx): ConversationStore { + const load = async (): Promise => { + let content: string + try { + content = await ctx.files.read(GUARDIAN_CONVERSATION_STATE_PATH) + } catch (error) { + if (String(error).includes('ENOENT')) return null + throw error + } + assertBoundedContent(content, 'guardian conversation state') + const state = parseConversationState(JSON.parse(content) as unknown) + return { state, revision: previewRevision(state) } + } + return { + load, + save: async (state, expected) => { + const canonical = parseConversationState(state) + const current = await load() + if (current?.revision !== expected?.revision) throw new GuardianStateConflictError() + assertConversationTransition(expected?.state ?? null, canonical) + const content = `${JSON.stringify(canonical)}\n` + assertBoundedContent(content, 'guardian conversation state') + await ctx.files.write(GUARDIAN_CONVERSATION_STATE_PATH, content) + const readBack = await load() + if (!readBack || JSON.stringify(readBack.state) !== JSON.stringify(canonical)) { + throw new Error('guardian conversation read-back did not match') + } + return readBack + }, + } +} + +function exactStore(input: { + client: RelayFileClient + credentials: RelayfileCredentials + path: string + timeoutMs: number + parse: (value: unknown) => GuardianConversationState +}): ConversationStore { + const load = () => withDeadline('guardian conversation load', input.timeoutMs, async (signal) => { + const file = await readExactFile(input.client, input.credentials.workspaceId, input.path, signal) + if (!file) return null + assertBoundedContent(file.content, 'guardian conversation state') + return { + state: input.parse(JSON.parse(file.content) as unknown), + revision: file.revision, + } + }) + return { + load, + save: (state, expected) => { + const canonical = input.parse(state) + assertConversationTransition(expected?.state ?? null, canonical) + const content = `${JSON.stringify(canonical)}\n` + assertBoundedContent(content, 'guardian conversation state') + return withDeadline('guardian conversation save', input.timeoutMs, async (signal) => { + try { + await writeExactFile( + input.client, + input.credentials.workspaceId, + input.path, + expected?.revision ?? '0', + content, + signal, + ) + } catch (error) { + if ( + error instanceof RevisionConflictError || + (error instanceof RelayFileApiError && error.status === 409) + ) { + throw new GuardianStateConflictError() + } + throw error + } + const readBack = await readExactFile( + input.client, + input.credentials.workspaceId, + input.path, + signal, + ) + if (!readBack || readBack.content !== content) { + throw new Error('guardian conversation read-back did not match') + } + return { state: canonical, revision: readBack.revision } + }) + }, + } +} + +export function createGithubIssueWriter( + ctx: WorkforceCtx, + client: GithubClient = githubClient({ + writebackTimeoutMs: GUARDIAN_WRITEBACK_TIMEOUT_MS, + writebackPollMs: GUARDIAN_WRITEBACK_POLL_MS, + }), +): GuardianIssueWriter { + return { + upsert: async (policy) => { + const [owner, repo, extra] = policy.repository.split('/') + if (!owner || !repo || extra) throw new Error('guardian issue repository must be owner/repo') + const receiptPath = remediationReceiptPath(policy.dedupeKey) + const savedReceipt = await readImmutableJson(ctx, receiptPath) + const bodyRevision = guardianContentRevision(policy.body) + if (savedReceipt) { + const receipt = parseDurableIssueReceipt(savedReceipt, policy) + if (providerText(asRecord(savedReceipt)?.bodyRevision) !== bodyRevision) { + await updateExistingIssueEvidence(ctx, client, owner, repo, receipt.number, policy) + } + return receipt + } + const issues = await listGithubIssues(ctx, client, owner, repo) + const match = issues + .map(unwrapProviderRecord) + .find((issue) => providerText(issue.body).includes(remediationMarker(policy.dedupeKey))) + if (match) { + const number = providerNumber(match.number) ?? providerNumber(match.id) + const url = providerText(match.html_url) || providerText(match.url) + if (!number || !url) throw new Error('deduplicated guardian issue is missing number/url') + if (providerText(match.body) !== policy.body) { + await updateExistingIssueEvidence(ctx, client, owner, repo, number, policy) + } + const receipt = { repository: policy.repository, number, url, dedupeKey: policy.dedupeKey } + await appendImmutableJson(ctx, receiptPath, { ...receipt, bodyRevision }) + return receipt + } + const intentPath = remediationIntentPath(policy.dedupeKey) + const existingIntent = await readImmutableJson(ctx, intentPath) + if (existingIntent) { + throw new Error( + 'GitHub remediation submission is pending provider correlation; refusing a duplicate issue', + ) + } + await appendImmutableJson(ctx, intentPath, { + kind: 'feature-guardian:remediation-intent', + version: 1, + repository: policy.repository, + dedupeKey: policy.dedupeKey, + bodyRevision, + }) + const created = await client.createIssue({ + owner, + repo, + title: policy.title, + body: policy.body, + labels: policy.labels, + }) + if (created.status !== 'confirmed' || !created.url) { + throw new Error(`GitHub remediation issue create was ${created.status}, not provider-confirmed`) + } + const number = issueNumberFromCreated(created.id, created.url) + if (!number) throw new Error('GitHub remediation issue receipt is missing its issue number') + const receipt = { + repository: policy.repository, + number, + url: created.url, + dedupeKey: policy.dedupeKey, + } + await appendImmutableJson(ctx, receiptPath, { ...receipt, bodyRevision }) + return receipt + }, + } +} + +export function remediationMarker(dedupeKey: string): string { + return `` +} + +function remediationIntentPath(dedupeKey: string): string { + return `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`intent:${dedupeKey}`).digest('hex')}.json` +} + +function remediationReceiptPath(dedupeKey: string): string { + return `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`receipt:${dedupeKey}`).digest('hex')}.json` +} + +async function updateExistingIssueEvidence( + ctx: WorkforceCtx, + client: GithubClient, + owner: string, + repo: string, + issueNumber: number, + policy: GuardianIssuePolicy, +): Promise { + const bodyRevision = guardianContentRevision(policy.body) + const marker = `` + const ledgerKey = `${policy.dedupeKey}:${bodyRevision}` + const receiptPath = `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`update-receipt:${ledgerKey}`).digest('hex')}.json` + if (await readImmutableJson(ctx, receiptPath)) return + const comments = await client['issue-comments'].list>({ + owner, + repo, + issueNumber, + }) + if (comments.map(unwrapProviderRecord).some((comment) => providerText(comment.body).includes(marker))) { + await appendImmutableJson(ctx, receiptPath, { issueNumber, bodyRevision }) + return + } + const intentPath = `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`update-intent:${ledgerKey}`).digest('hex')}.json` + if (await readImmutableJson(ctx, intentPath)) { + throw new Error('GitHub remediation evidence update is pending; refusing a duplicate comment') + } + await appendImmutableJson(ctx, intentPath, { issueNumber, bodyRevision }) + const updated = await client.comment( + { owner, repo, number: issueNumber }, + `${marker}\n${policy.body}`, + ) + if (updated.status !== 'confirmed') { + throw new Error(`GitHub remediation evidence update was ${updated.status}, not provider-confirmed`) + } + await appendImmutableJson(ctx, receiptPath, { issueNumber, bodyRevision }) +} + +function parseDurableIssueReceipt( + value: unknown, + policy: GuardianIssuePolicy, +): GuardianIssueReceipt { + const receipt = parseIssueReceipt(value) + if (receipt.repository !== policy.repository || receipt.dedupeKey !== policy.dedupeKey) { + throw new Error('durable guardian issue receipt does not match its policy') + } + return receipt +} + +async function listGithubIssues( + ctx: WorkforceCtx, + client: GithubClient, + owner: string, + repo: string, +): Promise[]> { + const listed = await client.issues.list>({ owner, repo }) + const issues = listed.filter((value) => asRecord(value) !== undefined) + const root = `${ctx.sandbox.cwd}/github/repos/${owner}/${repo}/issues` + const rawIndex = await readOptionalSandboxFile(ctx, `${root}/_index.json`) + if (!rawIndex) return issues + const index = JSON.parse(rawIndex) as unknown + if (!Array.isArray(index)) throw new Error('guardian GitHub issue index is malformed') + for (const entry of index) { + const item = asRecord(entry) + const number = providerNumber(item?.number) ?? providerNumber(item?.id) + const title = providerText(item?.title) + if (!number || !title) continue + const slug = title + .toLowerCase() + .replace(/[^a-z0-9]+/gu, '-') + .replace(/^-|-$/gu, '') + const candidates = [ + `${root}/${number}__${slug}/meta.json`, + `${root}/${number}/meta.json`, + `${root}/${number}.json`, + ] + for (const path of candidates) { + const raw = await readOptionalSandboxFile(ctx, path) + if (!raw) continue + const record = asRecord(JSON.parse(raw) as unknown) + if (record) issues.push(record) + break + } + } + return issues +} + +async function readOptionalSandboxFile( + ctx: WorkforceCtx, + path: string, +): Promise { + try { + return await ctx.sandbox.readFile(path) + } catch (error) { + const message = String(error) + if (message.includes('ENOENT') || message.includes('not found') || message.includes('No such file')) { + return undefined + } + throw error + } +} + +async function readImmutableJson( + ctx: WorkforceCtx, + path: string, +): Promise { + const credentials = ctx.credentials.tryRequire() + let content: string | undefined + if (credentials) { + const client = relayfileClient(credentials.relayfile) + const file = await withDeadline('guardian immutable ledger read', GUARDIAN_STATE_TIMEOUT_MS, (signal) => + readExactFile(client, credentials.relayfile.workspaceId, path, signal), + ) + content = file?.content + } else if (ctx.agent.id === 'sim-agent' && ctx.deployment.id === 'sim-deployment') { + try { + content = await ctx.files.read(path) + } catch (error) { + if (!String(error).includes('ENOENT')) throw error + } + } else { + throw new Error('exact Relayfile credentials are required for guardian remediation ledger') + } + if (content === undefined) return null + assertBoundedContent(content, 'guardian immutable ledger') + return JSON.parse(content) as unknown +} + +async function appendImmutableJson( + ctx: WorkforceCtx, + path: string, + value: unknown, +): Promise { + const content = `${JSON.stringify(value)}\n` + assertBoundedContent(content, 'guardian immutable ledger') + const existing = await readImmutableJson(ctx, path) + if (existing !== null) { + if (`${JSON.stringify(existing)}\n` !== content) { + throw new Error('immutable guardian ledger entry already differs') + } + return + } + const credentials = ctx.credentials.tryRequire() + if (credentials) { + const client = relayfileClient(credentials.relayfile) + try { + await withDeadline('guardian immutable ledger append', GUARDIAN_STATE_TIMEOUT_MS, (signal) => + writeExactFile(client, credentials.relayfile.workspaceId, path, '0', content, signal), + ) + } catch (error) { + if ( + !(error instanceof RevisionConflictError) && + !(error instanceof RelayFileApiError && error.status === 409) + ) throw error + } + } else if (ctx.agent.id === 'sim-agent' && ctx.deployment.id === 'sim-deployment') { + await ctx.files.write(path, content) + } else { + throw new Error('exact Relayfile credentials are required for guardian remediation ledger') + } + const readBack = await readImmutableJson(ctx, path) + if (readBack === null || `${JSON.stringify(readBack)}\n` !== content) { + throw new Error('guardian immutable ledger read-back did not match') + } +} + +export function slackThreadBacklink(channelId: string, threadTs: string): string { + return `https://slack.com/archives/${channelId}/p${threadTs.replace('.', '')}` +} + +async function postThreadReply( + createSlackClient: typeof slackClient, + record: GuardianConversationRecord, + text: string, + idempotencyKey: string, +): Promise { + const client = createSlackClient({ + writebackTimeoutMs: GUARDIAN_WRITEBACK_TIMEOUT_MS, + writebackPollMs: GUARDIAN_WRITEBACK_POLL_MS, + }) + const result = await client.replies.write( + { channelId: record.channelId, messageTs: record.threadTs }, + { text, thread_ts: record.threadTs, idempotencyKey }, + ) + return requireSlackReceiptTs(result) +} + +function requireSlackReceiptTs(result: WritebackResult): string { + const receipt = asRecord(result.receipt) + const externalId = nonEmptyString(receipt?.externalId) + const fallbackTs = nonEmptyString(receipt?.ts) + const ts = externalId && /^\d+\.\d+$/u.test(externalId) ? externalId : fallbackTs + if (!ts || !/^\d+\.\d+$/u.test(ts)) { + throw new Error('guardian Slack reply did not receive a provider timestamp') + } + return ts +} + +function confirmationRecord( + record: GuardianConversationRecord, + turn: GuardianPendingTurn, + timestamp: string, +): GuardianConfirmationRecord { + const evidence = record.evidence + const commands = evidence.flatMap((entry) => entry.commands ?? []) + const tests = evidence.reduce( + (total, entry) => ({ + passed: total.passed + (entry.tests?.passed ?? 0), + failed: total.failed + (entry.tests?.failed ?? 0), + }), + { passed: 0, failed: 0 }, + ) + const automated = evidence.some( + (entry) => entry.source === 'procedure' && entry.result === 'positive', + ) + return { + kind: 'feature-guardian:confirmation', + version: 1, + featureId: record.feature.id, + manifestRevision: record.manifestRevision, + manifestVersion: record.manifestVersion, + procedureRevision: record.procedureRevision, + generation: record.generation, + result: 'confirmed', + ...(turn.actorId ? { actor: { id: turn.actorId } } : {}), + verifier: automated ? 'factory-feature-guardian:procedure-runner' : turn.actorId, + timestamp, + evidence, + commands, + tests, + slack: { + channelId: record.channelId, + threadTs: record.threadTs, + questionTs: record.threadTs, + }, + } +} + +function parseConversationState(value: unknown): GuardianConversationState { + const root = asRecord(value) + if (!root || root.kind !== 'feature-guardian:conversations' || root.version !== 1) { + throw new Error('guardian conversation state kind/version is invalid') + } + if (!Array.isArray(root.records) || root.records.length > MAX_CONVERSATIONS) { + throw new Error('guardian conversation state records are invalid') + } + const records = root.records.map(parseConversationRecord) + if (new Set(records.map((record) => record.id)).size !== records.length) { + throw new Error('guardian conversation state contains duplicate records') + } + return { kind: 'feature-guardian:conversations', version: 1, records } +} + +function parseConversationRecord(value: unknown): GuardianConversationRecord { + const record = asRecord(value) + if (!record) throw new Error('guardian conversation record must be an object') + const feature = parseFeatureSnapshot(record.feature) + const status = record.status + if (!isConversationStatus(status)) throw new Error('guardian conversation status is invalid') + const seenEventIds = stringArray(record.seenEventIds, MAX_SEEN_EVENTS, 'seen event ids') + const evidence = evidenceArray(record.evidence) + const turnCount = record.turnCount + if (!Number.isSafeInteger(turnCount) || (turnCount as number) < 0 || (turnCount as number) > MAX_TURNS) { + throw new Error('guardian conversation turn count is invalid') + } + const clarificationCount = record.clarificationCount + if ( + !Number.isSafeInteger(clarificationCount) || + (clarificationCount as number) < 0 || + (clarificationCount as number) > (turnCount as number) + ) { + throw new Error('guardian conversation clarification count is invalid') + } + const parsed: GuardianConversationRecord = { + id: requireString(record.id, 'conversation id'), + feature, + manifestRevision: requireString(record.manifestRevision, 'manifest revision'), + manifestVersion: requireString(record.manifestVersion, 'manifest version'), + procedureRevision: requireString(record.procedureRevision, 'procedure revision'), + generation: requirePositiveInteger(record.generation, 'generation'), + channelId: requireString(record.channelId, 'channel id'), + threadTs: requireSlackTs(record.threadTs, 'thread ts'), + askedAt: requireTimestamp(record.askedAt, 'askedAt'), + status, + turnCount: turnCount as number, + clarificationCount: clarificationCount as number, + seenEventIds, + evidence, + updatedAt: requireTimestamp(record.updatedAt, 'updatedAt'), + } + if (record.pending !== undefined) parsed.pending = parsePending(record.pending) + if (record.confirmationPath !== undefined) { + parsed.confirmationPath = requireString(record.confirmationPath, 'confirmation path') + } + if (record.issue !== undefined) parsed.issue = parseIssueReceipt(record.issue) + if (record.finalReplyTs !== undefined) { + parsed.finalReplyTs = requireSlackTs(record.finalReplyTs, 'final reply ts') + } + if (record.lastProcessedEventOrder !== undefined) { + parsed.lastProcessedEventOrder = requireEventOrder( + record.lastProcessedEventOrder, + 'last processed event order', + ) + } + if (guardianConversationId(parsed) !== parsed.id) { + throw new Error('guardian conversation record identity is invalid') + } + if (isTerminal(parsed.status) && parsed.pending) { + throw new Error('terminal guardian conversation cannot retain a pending turn') + } + if ((parsed.status === 'asked' || parsed.status === 'discussing') && parsed.pending) { + throw new Error(`guardian ${parsed.status} conversation cannot retain a pending turn`) + } + if (isTerminal(parsed.status) && !parsed.finalReplyTs) { + throw new Error('terminal guardian conversation requires a provider-confirmed final reply') + } + if ( + ['verifying', 'confirmation-recorded', 'remediation-recorded', 'deferred-recorded'].includes( + parsed.status, + ) && !parsed.pending + ) { + throw new Error(`guardian ${parsed.status} conversation requires a pending turn`) + } + if ( + (parsed.status === 'remediation-recorded' || parsed.status === 'remediation-open') && + !parsed.issue + ) { + throw new Error(`guardian ${parsed.status} conversation requires an issue receipt`) + } + if (parsed.status === 'confirmed' && !parsed.confirmationPath) { + throw new Error('confirmed guardian conversation requires a confirmation path') + } + if ( + parsed.confirmationPath && + parsed.status !== 'confirmation-recorded' && + parsed.status !== 'confirmed' + ) { + throw new Error('guardian confirmation path is invalid for the conversation status') + } + if ( + parsed.issue && + parsed.status !== 'remediation-recorded' && + parsed.status !== 'remediation-open' + ) { + throw new Error('guardian issue receipt is invalid for the conversation status') + } + return parsed +} + +function parseConfirmationRecord(value: unknown): GuardianConfirmationRecord { + const record = asRecord(value) + if (!record || record.kind !== 'feature-guardian:confirmation' || record.version !== 1) { + throw new Error('guardian confirmation kind/version is invalid') + } + const actor = asRecord(record.actor) + const slack = asRecord(record.slack) + const tests = asRecord(record.tests) + if (record.result !== 'confirmed' || !slack || !tests) { + throw new Error('guardian confirmation result/evidence is invalid') + } + return { + kind: 'feature-guardian:confirmation', + version: 1, + featureId: requireString(record.featureId, 'feature id'), + manifestRevision: requireString(record.manifestRevision, 'manifest revision'), + manifestVersion: requireString(record.manifestVersion, 'manifest version'), + procedureRevision: requireString(record.procedureRevision, 'procedure revision'), + generation: requirePositiveInteger(record.generation, 'generation'), + result: 'confirmed', + ...(actor ? { actor: { id: requireString(actor.id, 'actor id') } } : {}), + verifier: requireString(record.verifier, 'verifier'), + timestamp: requireTimestamp(record.timestamp, 'timestamp'), + evidence: evidenceArray(record.evidence), + commands: stringArray(record.commands, 128, 'commands'), + tests: { + passed: requireNonNegativeInteger(tests.passed, 'passed tests'), + failed: requireNonNegativeInteger(tests.failed, 'failed tests'), + }, + slack: { + channelId: requireString(slack.channelId, 'Slack channel id'), + threadTs: requireSlackTs(slack.threadTs, 'Slack thread ts'), + questionTs: requireSlackTs(slack.questionTs, 'Slack question ts'), + }, + } +} + +function assertConversationTransition( + previous: GuardianConversationState | null, + next: GuardianConversationState, +): void { + if (!previous) return + const nextById = new Map(next.records.map((record) => [record.id, record])) + const removed = previous.records.filter((record) => !nextById.has(record.id)) + if (removed.some((record) => !isTerminal(record.status))) { + throw new Error('guardian conversation records are append-only except bounded terminal pruning') + } + for (const prior of previous.records) { + const current = nextById.get(prior.id) + if (!current) continue + if (!sameQuestion(prior, current)) { + throw new Error('guardian conversation question identity is immutable') + } + if (isTerminal(prior.status) && JSON.stringify(prior) !== JSON.stringify(current)) { + throw new Error('terminal guardian conversation records are immutable') + } + if (!prior.seenEventIds.every((id) => current.seenEventIds.includes(id))) { + throw new Error('guardian conversation seen-event history cannot regress') + } + if ( + JSON.stringify(current.evidence.slice(0, prior.evidence.length)) !== + JSON.stringify(prior.evidence) + ) { + throw new Error('guardian conversation evidence is append-only') + } + if ( + prior.confirmationPath !== undefined && + current.confirmationPath !== prior.confirmationPath + ) { + throw new Error('guardian conversation confirmation path is immutable') + } + if (prior.issue !== undefined && JSON.stringify(current.issue) !== JSON.stringify(prior.issue)) { + throw new Error('guardian conversation issue receipt is immutable') + } + if (current.turnCount < prior.turnCount) { + throw new Error('guardian conversation turn count cannot regress') + } + if (current.clarificationCount < prior.clarificationCount) { + throw new Error('guardian conversation clarification count cannot regress') + } + if ( + prior.lastProcessedEventOrder && + (!current.lastProcessedEventOrder || + current.lastProcessedEventOrder < prior.lastProcessedEventOrder) + ) { + throw new Error('guardian conversation event-order watermark cannot regress') + } + if (!validStatusTransition(prior.status, current.status)) { + throw new Error(`invalid guardian conversation transition ${prior.status} -> ${current.status}`) + } + } +} + +function validStatusTransition( + previous: GuardianConversationStatus, + next: GuardianConversationStatus, +): boolean { + if (previous === next) return true + const allowed: Record = { + asked: ['discussing', 'verifying', 'confirmation-recorded', 'deferred-recorded'], + discussing: ['discussing', 'verifying', 'confirmation-recorded', 'deferred-recorded'], + verifying: ['discussing', 'confirmation-recorded', 'remediation-recorded', 'deferred-recorded'], + 'confirmation-recorded': ['confirmed'], + 'remediation-recorded': ['remediation-open'], + 'deferred-recorded': ['deferred'], + confirmed: [], + 'remediation-open': [], + deferred: [], + } + return allowed[previous].includes(next) +} + +function parseFeatureSnapshot(value: unknown): GuardianFeatureSnapshot { + const feature = asRecord(value) + if (!feature) throw new Error('guardian feature snapshot must be an object') + const tier = requirePositiveInteger(feature.tier, 'feature tier') + if (tier > 6) throw new Error('guardian feature tier is invalid') + return { + id: requireString(feature.id, 'feature id'), + name: requireString(feature.name, 'feature name'), + category: requireString(feature.category, 'feature category'), + ...(nonEmptyString(feature.cli) ? { cli: nonEmptyString(feature.cli) } : {}), + ...(nonEmptyString(feature.api) ? { api: nonEmptyString(feature.api) } : {}), + description: requireString(feature.description, 'feature description'), + locations: stringArray(feature.locations, 32, 'feature locations'), + procedure: requireString(feature.procedure, 'feature procedure'), + tier, + criticality: requireString(feature.criticality, 'feature criticality'), + } +} + +function parsePending(value: unknown): GuardianPendingTurn { + const pending = asRecord(value) + if (!pending || !isResponseKind(pending.response)) { + throw new Error('guardian pending turn is invalid') + } + return { + eventId: requireString(pending.eventId, 'event id'), + eventOrder: requireEventOrder(pending.eventOrder, 'pending event order'), + eventOccurredAt: requireTimestamp(pending.eventOccurredAt, 'event occurredAt'), + actorId: requireString(pending.actorId, 'actor id'), + response: pending.response, + text: typeof pending.text === 'string' ? bounded(pending.text, 4_000) : '', + ...(nonEmptyString(pending.reaction) ? { reaction: nonEmptyString(pending.reaction) } : {}), + ...(pending.evidence !== undefined ? { evidence: parseEvidence(pending.evidence) } : {}), + ...(pending.verification !== undefined + ? { verification: parseProcedureResult(pending.verification) } + : {}), + ...(isDefectKind(pending.defectKind) ? { defectKind: pending.defectKind } : {}), + } +} + +function parseProcedureResult(value: unknown): GuardianProcedureResult { + const result = asRecord(value) + const evidence = parseEvidence(value) + if ( + evidence.source !== 'procedure' || + !['passed', 'failed', 'skip', 'manual'].includes(String(result?.outcome)) + ) { + throw new Error('guardian procedure result is invalid') + } + const outcome = result?.outcome as GuardianProcedureResultKind + const expectedResult: Record = { + passed: 'positive', + failed: 'negative', + skip: 'skip', + manual: 'manual', + } + if (evidence.result !== expectedResult[outcome]) { + throw new Error('guardian procedure outcome/result is inconsistent') + } + return { + ...evidence, + source: 'procedure', + outcome, + verifier: requireString(result?.verifier, 'procedure verifier'), + } +} + +function parseIssueReceipt(value: unknown): GuardianIssueReceipt { + const issue = asRecord(value) + if (!issue) throw new Error('guardian issue receipt is invalid') + return { + repository: requireString(issue.repository, 'issue repository'), + number: requirePositiveInteger(issue.number, 'issue number'), + url: requireString(issue.url, 'issue url'), + dedupeKey: requireString(issue.dedupeKey, 'issue dedupe key'), + } +} + +function evidenceArray(value: unknown): GuardianEvidence[] { + if (!Array.isArray(value) || value.length > MAX_EVIDENCE_ITEMS) { + throw new Error('guardian evidence is invalid') + } + return value.map(parseEvidence) +} + +function parseEvidence(value: unknown): GuardianEvidence { + const evidence = asRecord(value) + if ( + !evidence || + !['actor', 'procedure', 'system'].includes(String(evidence.source)) || + !['positive', 'negative', 'skip', 'manual', 'unknown'].includes(String(evidence.result)) + ) { + throw new Error('guardian evidence item is invalid') + } + const tests = asRecord(evidence.tests) + return { + source: evidence.source as GuardianEvidence['source'], + result: evidence.result as GuardianEvidence['result'], + summary: requireString(evidence.summary, 'evidence summary'), + ...(evidence.commands !== undefined + ? { commands: stringArray(evidence.commands, 128, 'evidence commands') } + : {}), + ...(evidence.positiveAssertions !== undefined + ? { + positiveAssertions: stringArray( + evidence.positiveAssertions, + 128, + 'positive assertions', + ), + } + : {}), + ...(evidence.negativeAssertions !== undefined + ? { + negativeAssertions: stringArray( + evidence.negativeAssertions, + 128, + 'negative assertions', + ), + } + : {}), + ...(tests + ? { + tests: { + passed: requireNonNegativeInteger(tests.passed, 'passed tests'), + failed: requireNonNegativeInteger(tests.failed, 'failed tests'), + }, + } + : {}), + ...(evidence.cleanup !== undefined + ? { cleanup: stringArray(evidence.cleanup, 32, 'cleanup evidence') } + : {}), + } +} + +function findCorrelatedConversation( + state: GuardianConversationState, + event: ParsedSlackResponse, +): GuardianConversationRecord | undefined { + const candidates = state.records.filter( + (record) => + record.channelId === event.channelId && + record.threadTs === event.threadTs && + (!event.isReaction || event.messageTs === record.threadTs), + ) + return candidates.length === 1 ? candidates[0] : undefined +} + +function findSlackPayload(root: Record | undefined): Record | undefined { + if (!root) return undefined + const queue: Record[] = [root] + const seen = new Set>() + while (queue.length > 0) { + const current = queue.shift()! + if (seen.has(current)) continue + seen.add(current) + if ( + nonEmptyString(current.channel) || + nonEmptyString(asRecord(current.item)?.channel) || + nonEmptyString(current.thread_ts) + ) { + return current + } + for (const key of ['payload', 'event', 'message', 'data', 'resource', 'record']) { + const nested = asRecord(current[key]) + if (nested) queue.push(nested) + } + } + return undefined +} + +function channelFromPath(path: string | undefined): string | undefined { + const encoded = path?.match(/^\/slack\/channels\/([^/]+)/u)?.[1] + return encoded?.split('__')[0] +} + +function messageTsFromPath(path: string | undefined): string | undefined { + const match = path?.match(/\/messages\/([^/]+)(?:\/replies\/([^/.]+))?/u) + return (match?.[2] ?? match?.[1])?.replaceAll('_', '.') +} + +function pendingTurn( + parsed: ParsedSlackResponse, + response: GuardianResponseKind, + evidence: GuardianEvidence, +): GuardianPendingTurn { + return { + eventId: parsed.eventId, + eventOrder: parsed.eventOrder, + eventOccurredAt: parsed.occurredAt, + actorId: parsed.actorId, + response, + text: parsed.text, + ...(parsed.reaction ? { reaction: parsed.reaction } : {}), + evidence, + } +} + +function emptyConversationState(): GuardianConversationState { + return { kind: 'feature-guardian:conversations', version: 1, records: [] } +} + +function retainConversationCapacity( + records: GuardianConversationRecord[], + incoming: GuardianConversationRecord, +): GuardianConversationRecord[] { + let retained = [...records] + for (;;) { + const candidate = { + kind: 'feature-guardian:conversations' as const, + version: 1 as const, + records: [...retained, incoming], + } + const fitsCount = candidate.records.length <= MAX_CONVERSATIONS + const fitsBytes = UTF8_ENCODER.encode(`${JSON.stringify(candidate)}\n`).byteLength <= MAX_STATE_BYTES + if (fitsCount && fitsBytes) return retained + const removable = retained + .filter((record) => isTerminal(record.status)) + .sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))[0] + if (!removable) { + throw new Error('guardian conversation state reached its bounded active-record limit') + } + retained = retained.filter((record) => record.id !== removable.id) + } +} + +function replaceRecord( + state: GuardianConversationState, + id: string, + next: GuardianConversationRecord, +): GuardianConversationState { + return { + ...state, + records: state.records.map((record) => (record.id === id ? next : record)), + } +} + +function requireRecordById( + state: GuardianConversationState, + id: string, +): GuardianConversationRecord { + const record = state.records.find((candidate) => candidate.id === id) + if (!record) throw new Error('guardian conversation disappeared after checkpoint') + return record +} + +function sameQuestion( + left: GuardianQuestion, + right: GuardianQuestion, +): boolean { + return JSON.stringify({ + feature: left.feature, + manifestRevision: left.manifestRevision, + manifestVersion: left.manifestVersion, + procedureRevision: left.procedureRevision, + generation: left.generation, + channelId: left.channelId, + threadTs: left.threadTs, + askedAt: left.askedAt, + }) === JSON.stringify({ + feature: right.feature, + manifestRevision: right.manifestRevision, + manifestVersion: right.manifestVersion, + procedureRevision: right.procedureRevision, + generation: right.generation, + channelId: right.channelId, + threadTs: right.threadTs, + askedAt: right.askedAt, + }) +} + +function appendSeen(existing: string[], eventId: string): string[] { + if (existing.includes(eventId)) return existing + if (existing.length >= MAX_SEEN_EVENTS) { + throw new Error('guardian conversation seen-event history reached its bounded limit') + } + return [...existing, eventId] +} + +function appendEvidence( + existing: GuardianEvidence[], + evidence: GuardianEvidence | undefined, +): GuardianEvidence[] { + if (!evidence) return existing + if (existing.length >= MAX_EVIDENCE_ITEMS) { + throw new Error('guardian conversation evidence reached its bounded limit') + } + return [...existing, evidence] +} + +function previewRevision(state: GuardianConversationState): string { + return `preview:${guardianContentRevision(JSON.stringify(state))}` +} + +function isTerminal(status: GuardianConversationStatus): boolean { + return status === 'confirmed' || status === 'remediation-open' || status === 'deferred' +} + +function isConversationStatus(value: unknown): value is GuardianConversationStatus { + return [ + 'asked', + 'discussing', + 'verifying', + 'confirmation-recorded', + 'remediation-recorded', + 'deferred-recorded', + 'confirmed', + 'remediation-open', + 'deferred', + ].includes(String(value)) +} + +function isResponseKind(value: unknown): value is GuardianResponseKind { + return ['affirmative', 'failure', 'untested', 'ambiguous', 'deferred'].includes(String(value)) +} + +function isDefectKind(value: unknown): value is GuardianDefectKind { + return ['implementation', 'test', 'manifest', 'procedure', 'documentation'].includes(String(value)) +} + +function shortRevision(revision: string): string { + return revision.replace(/^sha256:/u, '').slice(0, 12) +} + +function resolvedInput(ctx: WorkforceCtx, name: string): string | undefined { + const spec = ctx.persona?.inputSpecs?.[name] + const value = process.env[spec?.env ?? name] ?? ctx.persona?.inputs?.[name] ?? spec?.default + return nonEmptyString(value) +} + +function remediationBodyHasMarker(body: string, key: string): boolean { + return body.includes(remediationMarker(key)) +} + +function unwrapProviderRecord(value: Record): Record { + return asRecord(value.payload) ?? asRecord(value.data) ?? value +} + +function providerText(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +function providerNumber(value: unknown): number | undefined { + const parsed = typeof value === 'number' ? value : Number(value) + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined +} + +function issueNumberFromCreated(id: string, url: string): number | undefined { + return providerNumber(id) ?? providerNumber(url.match(/\/issues\/(\d+)(?:$|[?#])/u)?.[1]) +} + +function relayfileClient(credentials: RelayfileCredentials, fetchImpl?: FetchLike): RelayFileClient { + return new RelayFileClient({ + baseUrl: credentials.url, + token: credentials.token, + fetchImpl, + readCache: false, + retry: { maxRetries: 0 }, + }) +} + +async function readExactFile( + client: RelayFileClient, + workspaceId: string, + path: string, + signal: AbortSignal, +): Promise<{ content: string; revision: string } | null> { + let file + try { + file = await client.readFile(workspaceId, path, `guardian-read-${randomUUID()}`, signal) + } catch (error) { + if (error instanceof RelayFileApiError && error.status === 404) return null + throw error + } + if (!file || file.path !== path || typeof file.content !== 'string') { + throw new Error('guardian exact read returned an invalid file') + } + const revision = nonEmptyString(file.revision) + if (!revision) throw new Error('guardian exact read returned an invalid revision') + return { content: file.content, revision } +} + +async function writeExactFile( + client: RelayFileClient, + workspaceId: string, + path: string, + baseRevision: string, + content: string, + signal: AbortSignal, +): Promise { + const correlationId = `guardian-write-${randomUUID()}` + const queued = await client.writeFile({ + workspaceId, + path, + baseRevision, + content, + contentType: 'application/json', + encoding: 'utf-8', + correlationId, + signal, + }) + if (!queued || !nonEmptyString(queued.opId)) { + throw new Error('guardian exact write did not return an operation ID') + } + await waitForOperation(client, workspaceId, queued.opId, signal, correlationId) +} + +async function waitForOperation( + client: RelayFileClient, + workspaceId: string, + opId: string, + signal: AbortSignal, + correlationId: string, +): Promise { + for (;;) { + const operation = await client.getOp(workspaceId, opId, correlationId, signal) + if (operation?.status === 'succeeded') return operation + if (['failed', 'dead_lettered', 'canceled'].includes(String(operation?.status))) { + throw new Error(`guardian exact write ${operation?.status}`) + } + if (!['pending', 'running'].includes(String(operation?.status))) { + throw new Error('guardian exact write returned an invalid operation status') + } + await abortableDelay(GUARDIAN_WRITEBACK_POLL_MS, signal) + } +} + +async function withDeadline( + label: string, + timeoutMs: number, + operation: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController() + let timeout: ReturnType | undefined + const expired = new Promise((_, reject) => { + timeout = setTimeout(() => { + controller.abort() + reject(new Error(`${label} timed out after ${timeoutMs}ms`)) + }, timeoutMs) + }) + try { + return await Promise.race([operation(controller.signal), expired]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } +} + +function abortableDelay(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timeout) + reject(signal.reason) + } + const timeout = setTimeout(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }, ms) + if (signal.aborted) onAbort() + else signal.addEventListener('abort', onAbort, { once: true }) + }) +} + +function assertBoundedContent(content: string, label: string): void { + if (UTF8_ENCODER.encode(content).byteLength > MAX_STATE_BYTES) { + throw new Error(`${label} exceeds size limit`) + } +} + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +function requireString(value: unknown, field: string): string { + const parsed = nonEmptyString(value) + if (!parsed || parsed.length > 8_192) throw new Error(`guardian ${field} is invalid`) + return parsed +} + +function stringArray(value: unknown, max: number, field: string): string[] { + if (!Array.isArray(value) || value.length > max) throw new Error(`guardian ${field} is invalid`) + const parsed = value.map((entry) => requireString(entry, field)) + if (new Set(parsed).size !== parsed.length && field === 'seen event ids') { + throw new Error(`guardian ${field} contains duplicates`) + } + return parsed +} + +function requirePositiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new Error(`guardian ${field} is invalid`) + } + return value as number +} + +function requireNonNegativeInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`guardian ${field} is invalid`) + } + return value as number +} + +function canonicalTimestamp(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined + const date = new Date(value) + return Number.isFinite(date.valueOf()) ? date.toISOString() : undefined +} + +function timestampFromSlackTs(value: string | undefined): string | undefined { + if (!value || !/^\d+\.\d+$/u.test(value)) return undefined + const seconds = Number(value) + if (!Number.isFinite(seconds) || seconds <= 0) return undefined + return new Date(seconds * 1_000).toISOString() +} + +function requireTimestamp(value: unknown, field: string): string { + const timestamp = canonicalTimestamp(value) + if (!timestamp || timestamp !== value) throw new Error(`guardian ${field} is invalid`) + return timestamp +} + +function requireSlackTs(value: unknown, field: string): string { + const ts = requireString(value, field) + if (!/^\d+\.\d+$/u.test(ts)) throw new Error(`guardian ${field} is invalid`) + return ts +} + +function requireEventOrder(value: unknown, field: string): string { + const order = requireString(value, field) + // Event order is internal and compared lexicographically. Its exact shape + // is ISO timestamp + Slack ts + stable response identity. + const match = order.match( + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z):\d+\.\d+:slack:[a-f0-9]{64}$/u, + ) + if (!match || canonicalTimestamp(match[1]) !== match[1]) { + throw new Error(`guardian ${field} is invalid`) + } + return order +} + +function bounded(value: string, max: number): string { + return value.length <= max ? value : `${value.slice(0, max - 1)}…` +} + +// Retain a named predicate for issue adapters/tests without exporting provider internals. +void remediationBodyHasMarker diff --git a/src/feature-guardian/index.ts b/src/feature-guardian/index.ts new file mode 100644 index 0000000..98b2cec --- /dev/null +++ b/src/feature-guardian/index.ts @@ -0,0 +1 @@ +export * from './conversation.js' diff --git a/src/index.ts b/src/index.ts index 5fd4cb3..7c40358 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ export type { TemplateRoute, } from './dispatch/templates' export * from './featuremap/index' +export * from './feature-guardian/index.js' export { createRelayflowPolicyRegistry, dispatchRelayflowForChangeEvent, From 0c55dfde5414af2260f888e22f815a7b013477bf Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 22 Jul 2026 16:43:18 +0200 Subject: [PATCH 2/2] fix: harden feature guardian remediation loop --- .../factory-feature-guardian/agent.test.ts | 150 ++++++- .../agents/factory-feature-guardian/agent.ts | 203 +++++++-- .../factory-feature-guardian/persona.json | 2 +- .agentworkforce/features/manifest.yaml | 2 +- .agentworkforce/features/verify/procedures.md | 44 +- package-lock.json | 1 + package.json | 1 + scripts/packed-e2e-scenario.mjs | 3 + src/__tests__/dist-entrypoints.test.ts | 3 + src/feature-guardian/conversation.test.ts | 241 ++++++++++- src/feature-guardian/conversation.ts | 402 +++++++++++++++--- 11 files changed, 930 insertions(+), 122 deletions(-) diff --git a/.agentworkforce/agents/factory-feature-guardian/agent.test.ts b/.agentworkforce/agents/factory-feature-guardian/agent.test.ts index c6f2ebe..f96d203 100644 --- a/.agentworkforce/agents/factory-feature-guardian/agent.test.ts +++ b/.agentworkforce/agents/factory-feature-guardian/agent.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; import type { WorkforceCtx } from '@agentworkforce/runtime'; +import { parsePersonaSpec } from '@agentworkforce/persona-kit'; import { bindPreviewTransport, slackClient, @@ -19,7 +20,9 @@ import guardian, { factoryFeatureGuardianAdapters, featurePostIdempotencyKey, gateFactoryGuardianTier, + loadFactoryGuardianCatalog, parseFactoryGuardianTestCounts, + parseManifestFeatures, resolveManifestPath, resolveFactoryGuardianProcedure, runGuardian, @@ -312,6 +315,7 @@ function guardianContext(failWriteCall: number): { sandbox: { cwd: '/home/daytona/workspace', readFile: vi.fn(async () => manifest), + exec: vi.fn(async () => ({ output: '', exitCode: 0 })), }, files: { read: vi.fn(async (path: string) => { @@ -358,6 +362,7 @@ function exactStateContext( sandbox: { cwd: '/home/daytona/workspace', readFile: vi.fn(async () => manifestText), + exec: vi.fn(async () => ({ output: '', exitCode: 0 })), }, files: { read: vi.fn(async (path: string) => { @@ -391,10 +396,30 @@ describe('factory-feature-guardian runtime paths', () => { ); }); + it('fails closed on missing or escaping manifest locations', async () => { + const missing = exactStateContext(JSON.stringify(progressState(1))); + missing.ctx.sandbox.exec = vi.fn(async () => ({ output: 'src/index.ts', exitCode: 1 })); + await expect(loadFactoryGuardianCatalog(missing.ctx)).rejects.toThrow( + 'missing location: src/index.ts', + ); + + const escaping = exactStateContext( + JSON.stringify(progressState(1)), + manifest.replace(' location: src/index.ts', ' location: ../outside'), + ); + await expect(loadFactoryGuardianCatalog(escaping.ctx)).rejects.toThrow( + 'path escapes the repository', + ); + }); + it('defaults delivery to the factory feature-check channel', () => { expect(persona.inputs.SLACK_CHANNEL.default).toBe('C0BHWJSF309'); }); + it('is a deployable relay-orchestrator persona', () => { + expect(() => parsePersonaSpec(persona, 'relay-orchestrator')).not.toThrow(); + }); + it('falls back with every declared Factory surface when quiz generation fails', async () => { const transport = new IdempotentSlackTransport(); const restore = bindPreviewTransport(transport); @@ -804,6 +829,22 @@ describe('factory-feature-guardian runtime paths', () => { receipt: { externalId: 'mountcmd-not-a-ts', ts: '1710000003.000300' }, }) ).toBe('1710000003.000300'); + expect( + deliveredSlackTs({ + path: '/pending.json', + absolutePath: '/pending.json', + deliveryStatus: 'pending', + receipt: { ts: '1710000004.000400' }, + }) + ).toBe(''); + expect( + deliveredSlackTs({ + path: '/dropped.json', + absolutePath: '/dropped.json', + deliveryStatus: 'dropped', + receipt: { ts: '1710000005.000500' }, + }) + ).toBe(''); }); }); @@ -1137,7 +1178,7 @@ describe('Factory feature guardian conversation adapters', () => { api: 'manifest.yaml#verification.categories', description: 'Routes features to exact procedures.', locations: ['.agentworkforce/features/manifest.yaml'], - procedure: 'cli-and-package', + procedure: 'release-verification', tier: 1, criticality: 'critical', }; @@ -1167,6 +1208,7 @@ describe('Factory feature guardian conversation adapters', () => { path: '.agentworkforce/features/verify/procedures.md', prerequisites: 'source checkout and npm ci', body: 'npm run build', + command: 'npm run build', }); expect(gate).toEqual({ outcome: 'skip', @@ -1174,6 +1216,43 @@ describe('Factory feature guardian conversation adapters', () => { }); }); + it('gates provider, fleet, cloud, and live-work procedures without turning absence into confirmation', async () => { + const procedure = { + name: 'provider-discovery', + path: '.agentworkforce/features/verify/procedures.md', + prerequisites: 'a disposable provider issue and config', + body: 'npm run build', + command: 'npm run build', + }; + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + ctx.sandbox.exec = vi.fn(async () => ({ output: '', exitCode: 0 })); + (ctx.persona.inputs as Record).GUARDIAN_LIVE_VERIFY_OPT_IN = + 'verification-procedure-routing,provider-discovery'; + + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 2 }, procedure)).resolves + .toMatchObject({ outcome: 'skip', reason: expect.stringContaining('provider issue or config') }); + + Object.assign(ctx.persona.inputs as Record, { + FACTORY_VERIFY_CANARY_ISSUE: 'AR-VERIFY', + FACTORY_VERIFY_CONFIG: '/tmp/factory-guardian.json', + }); + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 2 }, procedure)).resolves + .toMatchObject({ outcome: 'available' }); + + const fixtureProcedure = { ...procedure, name: 'fleet-execution' }; + (ctx.persona.inputs as Record).GUARDIAN_VERIFY_MAX_TIER = '6'; + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 4 }, fixtureProcedure)).resolves + .toMatchObject({ outcome: 'skip', reason: expect.stringContaining('disposable fleet') }); + (ctx.persona.inputs as Record).FACTORY_GUARDIAN_FLEET_OPT_IN = 'true'; + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 4 }, fixtureProcedure)).resolves + .toMatchObject({ outcome: 'manual', reason: expect.stringContaining('live fleet lifecycle') }); + + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 5 }, fixtureProcedure)).resolves + .toMatchObject({ outcome: 'skip', reason: expect.stringContaining('cloud credentials') }); + await expect(gateFactoryGuardianTier(ctx, { ...snapshot, tier: 6 }, fixtureProcedure)).resolves + .toMatchObject({ outcome: 'manual', reason: expect.stringContaining('tier 6') }); + }); + it('selects the complete documented command and runs it in a disposable checkout', async () => { const procedures = readFileSync( new URL('../../features/verify/procedures.md', import.meta.url), @@ -1193,13 +1272,78 @@ describe('Factory feature guardian conversation adapters', () => { outcome: 'passed', result: 'positive', tests: { passed: 42, failed: 0 }, - cleanup: ['Observed removal of the unique temporary checkout.'], + cleanup: ['Observed in-process and out-of-band removal of the unique temporary checkout.'], }); const script = String(exec.mock.calls[0]?.[0]); expect(script).toContain('tar --exclude=.git --exclude=node_modules') expect(script).toContain('cd "$TMP/repo"') expect(script).toContain('node bin/factory.mjs --help') - expect(script).toContain('node bin/factory.mjs featuremap check') + expect(script).toContain('npm run featuremap:check') + }); + + it('keeps every allowlisted command equal to the procedure first Bash block', async () => { + const procedures = readFileSync( + new URL('../../features/verify/procedures.md', import.meta.url), + 'utf8', + ); + const manifestText = readFileSync( + new URL('../../features/manifest.yaml', import.meta.url), + 'utf8', + ); + const features = parseManifestFeatures(manifestText); + const firstByProcedure = new Map( + features.map((feature) => [feature.procedure as string, feature]), + ); + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + ctx.sandbox.readFile = vi.fn(async () => procedures); + ctx.sandbox.exec = vi.fn(async () => ({ + output: ' Tests 42 passed (42)\n__FACTORY_GUARDIAN_CLEANUP_OK__\n', + exitCode: 0, + })); + + for (const feature of firstByProcedure.values()) { + const featureSnapshot = { + id: feature.id, + name: feature.name, + category: feature.category, + ...(feature.cli ? { cli: feature.cli } : {}), + ...(feature.api ? { api: feature.api } : {}), + description: feature.desc, + locations: feature.location.split(',').map((value) => value.trim()), + procedure: feature.procedure as string, + tier: feature.tier, + criticality: feature.criticality, + }; + const procedure = await resolveFactoryGuardianProcedure(ctx, featureSnapshot); + await expect( + runFactoryGuardianProcedure(ctx, featureSnapshot, procedure, `all-${feature.procedure}`), + ).resolves.toMatchObject({ outcome: 'passed' }); + } + }); + + it('fails closed when test output reports failures despite a zero shell exit', async () => { + const procedures = readFileSync( + new URL('../../features/verify/procedures.md', import.meta.url), + 'utf8', + ); + const { ctx } = exactStateContext(JSON.stringify(progressState(1))); + ctx.sandbox.readFile = vi.fn(async () => procedures); + ctx.sandbox.exec = vi.fn(async () => ({ + output: ' Tests 3 failed | 39 passed (42)\n__FACTORY_GUARDIAN_CLEANUP_OK__\n', + exitCode: 0, + })); + const procedure = await resolveFactoryGuardianProcedure(ctx, { + ...snapshot, + procedure: 'cli-and-package', + }); + await expect( + runFactoryGuardianProcedure(ctx, snapshot, procedure, 'failed-counts'), + ).resolves.toMatchObject({ + outcome: 'failed', + result: 'negative', + tests: { passed: 39, failed: 3 }, + negativeAssertions: expect.arrayContaining([expect.stringContaining('3 failed tests')]), + }); }); it('routes every remediation only to the Factory repository', () => { diff --git a/.agentworkforce/agents/factory-feature-guardian/agent.ts b/.agentworkforce/agents/factory-feature-guardian/agent.ts index dd486dd..c0d9dc1 100644 --- a/.agentworkforce/agents/factory-feature-guardian/agent.ts +++ b/.agentworkforce/agents/factory-feature-guardian/agent.ts @@ -115,6 +115,33 @@ export async function loadFactoryGuardianCatalog( ctx.sandbox.readFile(resolveProceduresPath(ctx.sandbox.cwd)), ]); const validated = validateFeatureManifest(raw); + if (validated.verificationDocument !== PROCEDURES_RELPATH) { + throw new Error( + `Factory guardian requires verification.document to be ${PROCEDURES_RELPATH}`, + ); + } + const repository = `${ctx.sandbox.cwd}/${FACTORY_REPO_RELPATH}`; + const declaredPaths = [ + validated.verificationDocument, + ...validated.features.flatMap(featureLocations), + ]; + for (const path of declaredPaths) { + if (!isSafeFactoryRepositoryPath(path)) { + throw new Error(`Factory guardian catalog path escapes the repository: ${path}`); + } + } + const uniquePaths = [...new Set(declaredPaths)]; + const pathProbe = await ctx.sandbox.exec( + ['set -eu', ...uniquePaths.map((path) => + `test -e ${shellQuote(path)} || { printf '%s\\n' ${shellQuote(path)}; exit 1; }`), + ].join('\n'), + { cwd: repository, timeoutMs: 30_000 }, + ); + if (pathProbe.exitCode !== 0) { + throw new Error( + `Factory guardian catalog contains a missing location: ${boundedText(pathProbe.output.trim(), 500)}`, + ); + } return { manifestRevision: guardianContentRevision(raw), manifestVersion: validated.version, @@ -123,6 +150,11 @@ export async function loadFactoryGuardianCatalog( }; } +function isSafeFactoryRepositoryPath(path: string): boolean { + if (!path || path.startsWith('/') || path.includes('\\')) return false; + return !path.split('/').some((segment) => segment === '' || segment === '.' || segment === '..'); +} + /** Compatibility loader retained for the scheduled feature ordering path. */ async function loadFeatures(ctx: WorkforceCtx): Promise { const catalog = await loadFactoryGuardianCatalog(ctx); @@ -612,6 +644,7 @@ export function featurePostIdempotencyKey( /** Extract and validate the delivered Slack timestamp from a writeback receipt. */ export function deliveredSlackTs(result: WritebackResult | null | undefined): string { + if (!result || (result.deliveryStatus && result.deliveryStatus !== 'confirmed')) return ''; const receipt = result?.receipt as { externalId?: unknown; ts?: unknown } | undefined; const externalId = typeof receipt?.externalId === 'string' ? receipt.externalId.trim() : ''; if (isSlackTs(externalId)) return externalId; @@ -994,9 +1027,10 @@ const FACTORY_PROCEDURE_COMMANDS: Record = { ' src/node/factory-node.test.ts', ].join('\n'), 'provider-discovery': [ - 'factory triage "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/triage.json"', - 'factory canary "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/canary.json"', - 'factory run-once --config "$CONFIG" --dry-run | tee "$TMP/discovery.json"', + 'npm run build', + 'node bin/factory.mjs triage "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/triage.json"', + 'node bin/factory.mjs canary "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/canary.json"', + 'node bin/factory.mjs run-once --config "$CONFIG" --dry-run | tee "$TMP/discovery.json"', "node -e 'const r=require(process.argv[1]); if (!r.ok) process.exit(1)' \"$TMP/canary.json\"", ].join('\n'), 'triage-and-configuration': [ @@ -1143,7 +1177,11 @@ export async function resolveFactoryGuardianProcedure( if (!prerequisites) { throw new Error(`Factory procedure ${feature.procedure} has no explicit prerequisites`); } - return { name: feature.procedure, path: PROCEDURES_RELPATH, prerequisites, body }; + const command = body.match(/```bash\s*\n([\s\S]*?)\n```/u)?.[1]?.trim(); + if (!command) { + throw new Error(`Factory procedure ${feature.procedure} has no executable Bash command`); + } + return { name: feature.procedure, path: PROCEDURES_RELPATH, prerequisites, body, command }; } /** Keep live/provider tiers opt-in and never turn missing prerequisites into confirmation. */ @@ -1177,7 +1215,8 @@ export async function gateFactoryGuardianTier( if (feature.tier >= 6) { return { outcome: 'manual' as const, reason: 'MANUAL: tier 6 requires a selected disposable live issue or pull request.' }; } - if (feature.tier >= 3) { + const requiresProvider = procedure.name === 'provider-discovery'; + if (feature.tier >= 3 || requiresProvider) { const optIn = new Set( (factoryGuardianInput(ctx, 'GUARDIAN_LIVE_VERIFY_OPT_IN') ?? '') .split(',') @@ -1192,25 +1231,50 @@ export async function gateFactoryGuardianTier( } } if ( - feature.tier === 3 && - (!process.env.FACTORY_VERIFY_CANARY_ISSUE || - !process.env.FACTORY_VERIFY_CONFIG || - (await ctx.sandbox.exec('command -v factory >/dev/null', { - cwd: repository, - timeoutMs: 5_000, - })).exitCode !== 0) + requiresProvider && + (!factoryGuardianInput(ctx, 'FACTORY_VERIFY_CANARY_ISSUE') || + !factoryGuardianInput(ctx, 'FACTORY_VERIFY_CONFIG')) ) { - return { outcome: 'skip' as const, reason: 'SKIP: the disposable provider issue/config or installed Factory CLI prerequisite is unavailable.' }; + return { outcome: 'skip' as const, reason: 'SKIP: the disposable provider issue or config prerequisite is unavailable.' }; + } + if (requiresProvider) { + const configuredPath = factoryGuardianInput(ctx, 'FACTORY_VERIFY_CONFIG') as string; + const configPath = configuredPath.startsWith('/') + ? configuredPath + : `${repository}/${configuredPath}`; + const configAvailable = await ctx.sandbox.exec(`test -f ${shellQuote(configPath)}`, { + cwd: repository, + timeoutMs: 5_000, + }); + if (configAvailable.exitCode !== 0) { + return { outcome: 'skip' as const, reason: 'SKIP: the selected disposable provider config is not readable.' }; + } } - if (feature.tier === 4 && process.env.FACTORY_GUARDIAN_FLEET_OPT_IN !== 'true') { + if (feature.tier === 3 && !requiresProvider) { + return { + outcome: 'manual' as const, + reason: `MANUAL: ${procedure.name} requires provider assertions beyond its bounded unattended command.`, + }; + } + if (feature.tier === 4 && factoryGuardianInput(ctx, 'FACTORY_GUARDIAN_FLEET_OPT_IN') !== 'true') { return { outcome: 'skip' as const, reason: 'SKIP: no explicitly authorized disposable fleet is available.' }; } - if ( - feature.tier === 5 && - (!ctx.credentials.tryRequire() || process.env.FACTORY_GUARDIAN_CLOUD_OPT_IN !== 'true') - ) { + if (feature.tier === 4) { + return { + outcome: 'manual' as const, + reason: `MANUAL: ${procedure.name} requires a live fleet lifecycle whose provider cleanup cannot be proven by the unattended runner.`, + }; + } + if (feature.tier === 5 && + (!ctx.credentials.tryRequire() || factoryGuardianInput(ctx, 'FACTORY_GUARDIAN_CLOUD_OPT_IN') !== 'true')) { return { outcome: 'skip' as const, reason: 'SKIP: cloud credentials and writable disposable mount opt-in are unavailable.' }; } + if (feature.tier === 5) { + return { + outcome: 'manual' as const, + reason: `MANUAL: ${procedure.name} requires provider readback and cleanup in a selected disposable cloud scope.`, + }; + } return { outcome: 'available' as const, reason: `Tier ${feature.tier} prerequisites are available.` }; } @@ -1222,7 +1286,7 @@ export async function runFactoryGuardianProcedure( runKey: string, ): Promise { const command = FACTORY_PROCEDURE_COMMANDS[procedure.name]; - if (!command || !normalizedWhitespace(procedure.body).includes(normalizedWhitespace(command))) { + if (!command || procedure.command.trim() !== command.trim()) { return { source: 'procedure', result: 'negative', @@ -1238,9 +1302,16 @@ export async function runFactoryGuardianProcedure( } const run = runKey.replace(/[^a-zA-Z0-9_.-]/gu, '-').slice(-80); const repository = `${ctx.sandbox.cwd}/${FACTORY_REPO_RELPATH}`; + const tempWorkspace = `/tmp/factory-feature-guardian-${run || randomUUID()}`; + const configuredPath = factoryGuardianInput(ctx, 'FACTORY_VERIFY_CONFIG'); + const configPath = configuredPath + ? (configuredPath.startsWith('/') ? configuredPath : `${repository}/${configuredPath}`) + : `${tempWorkspace}/factory.config.json`; const wrapped = [ 'set -Eeuo pipefail', - 'TMP="$(mktemp -d)"', + `TMP=${shellQuote(tempWorkspace)}`, + 'rm -rf -- "$TMP"', + 'mkdir -m 700 "$TMP"', 'cleanup() {', ' status=$?', ' cd /', @@ -1255,23 +1326,55 @@ export async function runFactoryGuardianProcedure( '}', 'trap cleanup EXIT', `RUN="${run}"`, - 'CONFIG="${FACTORY_VERIFY_CONFIG:-$TMP/factory.config.json}"', + `CONFIG=${shellQuote(configPath)}`, 'export TMP RUN CONFIG', `SOURCE=${shellQuote(repository)}`, 'mkdir "$TMP/repo"', '(cd "$SOURCE" && tar --exclude=.git --exclude=node_modules --exclude=dist --exclude=artifacts -cf - .) | tar -xf - -C "$TMP/repo"', - 'ln -s "$SOURCE/node_modules" "$TMP/repo/node_modules"', + 'mkdir "$TMP/repo/node_modules"', + 'find "$SOURCE/node_modules" -mindepth 1 -maxdepth 1 ! -name .vite ! -name .cache -exec ln -s {} "$TMP/repo/node_modules/" \\;', + 'export npm_config_cache="$TMP/npm-cache"', 'cd "$TMP/repo"', command, ].join('\n'); - const execution = await ctx.sandbox.exec(wrapped, { - cwd: repository, - timeoutMs: 240_000, - }); + let execution: Awaited>; + try { + execution = await ctx.sandbox.exec(wrapped, { + cwd: repository, + timeoutMs: 240_000, + }); + } catch (error) { + const cleanup = await ctx.sandbox.exec( + `rm -rf -- ${shellQuote(tempWorkspace)} && test ! -e ${shellQuote(tempWorkspace)}`, + { cwd: repository, timeoutMs: 10_000 }, + ); + return { + source: 'procedure', + result: 'negative', + outcome: 'failed', + verifier: 'factory-feature-guardian:procedure-runner', + summary: `${procedure.name} execution failed before a result was returned: ${boundedText(String(error), 1_000)}`, + commands: [command], + positiveAssertions: [], + negativeAssertions: ['The sandbox did not return a completed procedure result.'], + tests: { passed: 0, failed: 0 }, + cleanup: [cleanup.exitCode === 0 + ? 'Out-of-band cleanup removed the deterministic temporary checkout.' + : 'Out-of-band cleanup could not confirm temporary-checkout removal.'], + }; + } + const outOfBandCleanup = await ctx.sandbox.exec( + `rm -rf -- ${shellQuote(tempWorkspace)} && test ! -e ${shellQuote(tempWorkspace)}`, + { cwd: repository, timeoutMs: 10_000 }, + ); const counts = parseFactoryGuardianTestCounts(execution.output); const output = boundedText(execution.output.trim(), 2_000); - const cleaned = execution.output.includes('__FACTORY_GUARDIAN_CLEANUP_OK__'); - if (execution.exitCode !== 0 || !cleaned) { + const cleaned = execution.output.includes('__FACTORY_GUARDIAN_CLEANUP_OK__') && + outOfBandCleanup.exitCode === 0; + const expectsTests = /(?:^|\n)(?:npm test|npx vitest)\b/u.test(command); + const invalidTestEvidence = counts.failed > 0 || + (expectsTests && counts.passed === 0 && counts.failed === 0); + if (execution.exitCode !== 0 || !cleaned || invalidTestEvidence) { return { source: 'procedure', result: 'negative', @@ -1282,12 +1385,16 @@ export async function runFactoryGuardianProcedure( positiveAssertions: ['Exact tier-safe documented command and isolated cleanup trap were selected.'], negativeAssertions: [ `Command exited ${execution.exitCode}.`, + ...(counts.failed > 0 ? [`Observed ${counts.failed} failed tests despite the shell exit status.`] : []), + ...(expectsTests && counts.passed === 0 && counts.failed === 0 + ? ['The documented test command emitted no parseable test result.'] + : []), ...(!cleaned ? ['Isolated workspace cleanup was not positively observed.'] : []), ], tests: counts, cleanup: [cleaned - ? 'Observed removal of the unique temporary checkout.' - : 'Temporary-checkout removal was not confirmed.'], + ? 'Observed in-process and out-of-band removal of the unique temporary checkout.' + : 'Temporary-checkout removal was not confirmed by both cleanup paths.'], }; } return { @@ -1303,7 +1410,7 @@ export async function runFactoryGuardianProcedure( ], negativeAssertions: [], tests: counts, - cleanup: ['Observed removal of the unique temporary checkout.'], + cleanup: ['Observed in-process and out-of-band removal of the unique temporary checkout.'], }; } @@ -1326,14 +1433,24 @@ export const factoryFeatureGuardianAdapters: FeatureGuardianAdapters = { const focus = turnNumber === 1 ? 'What exact command or user-visible path did you try, and what did you observe?' : 'Please paste the positive and negative assertion results plus any cleanup evidence; if it was not run, say which prerequisite is unavailable.'; + const verification = turn.verification + ? [ + `Guardian check: ${turn.verification.summary}`, + ...(turn.verification.positiveAssertions?.map((value) => `PASS: ${value}`) ?? []), + ...(turn.verification.negativeAssertions?.map((value) => `FAIL: ${value}`) ?? []), + ...(turn.verification.cleanup?.map((value) => `Cleanup: ${value}`) ?? []), + ].join(' ') + : ''; return [ `I can't confirm *${feature.name}* from that response yet (clarification ${turnNumber}).`, `${surface}; source ${feature.locations.map((path) => `\`${path}\``).join(', ')}.`, `Use \`${procedure.name}\` at tier ${feature.tier}: ${procedure.prerequisites}`, + verification, focus, - ].join(' '); + ].filter(Boolean).join(' '); }, classifyDefect: (_feature, turn, evidence) => classifyFactoryDefect(turn.text, evidence), + isDefectEstablished: (_feature, turn) => hasConcreteFactoryDefectEvidence(turn.text), repositoryForFeature: () => 'AgentWorkforce/factory', issuePolicy: ({ feature, conversation, defectKind, slackBacklink }) => factoryGuardianIssuePolicy(feature, conversation, defectKind, slackBacklink), @@ -1351,9 +1468,13 @@ function factoryGuardianIssuePolicy( .filter((entry) => entry.result === 'negative') .map((entry) => `- ${entry.summary}`) .join('\n') || '- The scoped guardian response reported a defect.'; - const evidence = conversation.evidence.map((entry) => + const evidence = conversation.evidence.flatMap((entry) => [ `- ${entry.source}/${entry.result}: ${entry.summary}`, - ).join('\n'); + ...(entry.positiveAssertions?.map((assertion) => ` - PASS: ${assertion}`) ?? []), + ...(entry.negativeAssertions?.map((assertion) => ` - FAIL: ${assertion}`) ?? []), + ...(entry.tests ? [` - Tests: ${entry.tests.passed} passed, ${entry.tests.failed} failed`] : []), + ...(entry.cleanup?.map((cleanup) => ` - Cleanup: ${cleanup}`) ?? []), + ]).join('\n'); return { repository: 'AgentWorkforce/factory', defectKind, @@ -1420,16 +1541,22 @@ function classifyFactoryDefect( return 'implementation'; } +function hasConcreteFactoryDefectEvidence(text: string): boolean { + const normalized = text.trim(); + if (normalized.length < 32) return false; + const hasArtifact = /```[\s\S]+```/u.test(normalized) || + /`[^`\n]+`/u.test(normalized) || + /(?:^|\s)(?:src|test|tests|docs|\.agentworkforce)\/[\w./-]+/u.test(normalized); + const hasComparison = /\b(expected|actual|observed|outputs?|returns?|reports?|says?|but|instead)\b/iu.test(normalized); + return hasArtifact && hasComparison; +} + function factoryGuardianInput(ctx: WorkforceCtx, name: string): string | undefined { const spec = ctx.persona?.inputSpecs?.[name]; const value = process.env[spec?.env ?? name] ?? ctx.persona?.inputs?.[name] ?? spec?.default; return typeof value === 'string' && value.trim() ? value.trim() : undefined; } -function normalizedWhitespace(value: string): string { - return value.replace(/\s+/gu, ' ').trim(); -} - export function parseFactoryGuardianTestCounts(output: string): { passed: number; failed: number } { const summary = output .replace(/\u001b\[[0-9;]*m/gu, '') diff --git a/.agentworkforce/agents/factory-feature-guardian/persona.json b/.agentworkforce/agents/factory-feature-guardian/persona.json index 8d1bbec..cb19856 100644 --- a/.agentworkforce/agents/factory-feature-guardian/persona.json +++ b/.agentworkforce/agents/factory-feature-guardian/persona.json @@ -14,7 +14,7 @@ "github": { "scope": { "repo": "AgentWorkforce/factory", - "resources": ["issues"] + "resources": "issues" }, "relayfileMount": { "requiredReadPaths": [ diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 4c1ee18..d6150e5 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1718,7 +1718,7 @@ categories: - id: guardian-confirmation-evidence name: Immutable Guardian Confirmation Evidence - api: /memory/workspace/factory-feature-guardian/confirmations/*.json + api: /memory/workspace/feature-guardian/confirmations/*.json description: Append one exact-revision confirmation containing feature, generation, actor or verifier, provider event time, command and test evidence, and Slack thread identity before posting the final summary location: src/feature-guardian/conversation.ts, .agentworkforce/agents/factory-feature-guardian/agent.ts verify_tier: 5 diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index d5a95e7..d5a06bd 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -428,7 +428,11 @@ npx vitest run \ src/fleet/internal-fleet-client.test.ts \ src/fleet/relay-fleet-client.test.ts \ src/node/factory-node.test.ts +``` + +With the explicitly selected disposable internal fleet, run the live extension: +```bash NAME="factory-vf-$RUN" node bin/factory.mjs fleet roster --backend internal | tee "$TMP/roster-before.json" node bin/factory.mjs fleet spawn spawn:codex --backend internal \ @@ -461,9 +465,10 @@ connected integrations. Copy the real config to `$CONFIG` and change only test scope to records named with `$RUN`. ```bash -factory triage "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/triage.json" -factory canary "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/canary.json" -factory run-once --config "$CONFIG" --dry-run | tee "$TMP/discovery.json" +npm run build +node bin/factory.mjs triage "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/triage.json" +node bin/factory.mjs canary "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" | tee "$TMP/canary.json" +node bin/factory.mjs run-once --config "$CONFIG" --dry-run | tee "$TMP/discovery.json" node -e 'const r=require(process.argv[1]); if (!r.ok) process.exit(1)' "$TMP/canary.json" ``` @@ -518,6 +523,11 @@ npx vitest run \ src/git/agent-worktree.test.ts \ src/state/file-state-store.test.ts \ src/state/github-lifecycle-identity.test.ts +``` + +With the provider and fleet prerequisites selected, run the dry-run extension: + +```bash factory dispatch "$FACTORY_VERIFY_CANARY_ISSUE" --config "$CONFIG" --dry-run \ | tee "$TMP/dispatch-dry.json" ``` @@ -633,6 +643,11 @@ npx vitest run \ src/mount/relayfile-github-connection-write.test.ts \ src/mount/relayfile-integration-preflight.test.ts \ src/writeback/writeback.test.ts +``` + +With the disposable cloud/provider scope selected, run the live preflight extension: + +```bash factory run-once --config "$CONFIG" --dry-run | tee "$TMP/mount-preflight.json" ``` @@ -715,6 +730,11 @@ npx vitest run \ src/hosted/orchestrator.test.ts \ src/hosted/state-store.test.ts \ src/hosted/worker-safety.test.ts +``` + +With the disposable deployed control plane selected, run the live extension: + +```bash npm run verify:e2e ``` @@ -768,6 +788,11 @@ npm run build npm run featuremap:check npm test node bin/factory.mjs --help >/dev/null +``` + +With exact reviewed head/base SHAs available, run the PR-evidence extension: + +```bash FACTORY_E2E_HEAD_SHA="$(git rev-parse HEAD)" \ FACTORY_E2E_BASE_SHA="$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD^)" \ npm run verify:e2e @@ -830,8 +855,6 @@ state checkpoint advances. `$TMP`; a live manual check may start one exact process using `$CONFIG`. ```bash -node bin/factory.mjs loop --config "$CONFIG" --dry-run -node bin/factory.mjs loop-status --config "$CONFIG" | tee "$TMP/liveness.json" npx vitest run \ src/orchestrator/factory.test.ts \ src/orchestrator/process-identity.test.ts \ @@ -839,6 +862,13 @@ npx vitest run \ src/state/file-state-store.test.ts ``` +With the disposable fixture config prepared, run the process extension: + +```bash +node bin/factory.mjs loop --config "$CONFIG" --dry-run +node bin/factory.mjs loop-status --config "$CONFIG" | tee "$TMP/liveness.json" +``` + Assert iteration/failure limits, heartbeat/registry aliases and paths, stale threshold, circuit opening, idle completion, atomic registry recovery, PID identity, and protected infrastructure. For the manual signal check, launch @@ -865,7 +895,11 @@ npx vitest run \ src/environments/load-harness.test.ts \ src/environments/verification-pipeline.test.ts \ src/orchestrator/environment-reaper.test.ts +``` + +With a disposable Docker/kind environment selected, run the live extension: +```bash kind create cluster --name factory-gate-e2e npm run test:e2e:verification kind delete cluster --name factory-gate-e2e diff --git a/package-lock.json b/package-lock.json index 6646657..b295a62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "factory": "bin/factory.mjs" }, "devDependencies": { + "@agentworkforce/persona-kit": "^4.1.23", "@types/node": "^22.10.2", "@types/proper-lockfile": "^4.1.4", "esbuild": "^0.28.1", diff --git a/package.json b/package.json index faffda2..3ec80c2 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,7 @@ "listr2": "9.0.5" }, "devDependencies": { + "@agentworkforce/persona-kit": "^4.1.23", "@types/node": "^22.10.2", "@types/proper-lockfile": "^4.1.4", "esbuild": "^0.28.1", diff --git a/scripts/packed-e2e-scenario.mjs b/scripts/packed-e2e-scenario.mjs index 292a254..fc69e33 100644 --- a/scripts/packed-e2e-scenario.mjs +++ b/scripts/packed-e2e-scenario.mjs @@ -1,11 +1,14 @@ import assert from 'node:assert/strict' import { FactoryConfigSchema } from '@agent-relay/factory' +import { defineFeatureGuardianAgent } from '@agent-relay/factory/feature-guardian' import { createHostedFactory, InMemoryHostedFactoryStateStore, } from '@agent-relay/factory/hosted' const checks = ['public-exports-import'] +assert.equal(typeof defineFeatureGuardianAgent, 'function') +checks.push('feature-guardian-subpath-import') const workspaceId = 'factory-e2e-workspace' const issue = { uuid: 'factory-e2e-issue-1', diff --git a/src/__tests__/dist-entrypoints.test.ts b/src/__tests__/dist-entrypoints.test.ts index 7395c6a..d44f6b0 100644 --- a/src/__tests__/dist-entrypoints.test.ts +++ b/src/__tests__/dist-entrypoints.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from 'vitest' describe('published dist entrypoints', () => { it('are importable by Node ESM consumers', async () => { const featuremap = await import('../../dist/featuremap/index.js') + const featureGuardian = await import('../../dist/feature-guardian/index.js') const main = await import('../../dist/index.js') const hosted = await import('../../dist/hosted/index.js') const environments = await import('../../dist/environments/index.js') @@ -17,6 +18,8 @@ describe('published dist entrypoints', () => { expect(featuremap.checkFeatureMap).toBeTypeOf('function') expect(featuremap.generateFeatureMap).toBeTypeOf('function') expect(featuremap.validateFeatureManifestFile).toBeTypeOf('function') + expect(featureGuardian.defineFeatureGuardianAgent).toBeTypeOf('function') + expect(featureGuardian.runGuardianConversationTurn).toBeTypeOf('function') expect(main.FactoryConfigSchema).toBeDefined() expect(main.createFactory).toBeTypeOf('function') expect(main.createFleet).toBeTypeOf('function') diff --git a/src/feature-guardian/conversation.test.ts b/src/feature-guardian/conversation.test.ts index c252e0e..64761fa 100644 --- a/src/feature-guardian/conversation.test.ts +++ b/src/feature-guardian/conversation.test.ts @@ -165,6 +165,7 @@ function adapters(options: { path: '.agentworkforce/features/verify/procedures.md', prerequisites: 'source checkout with installed dependencies', body: 'npm run build\nnpm test', + command: 'npm run build\nnpm test', })), gateTier: vi.fn(async () => ({ outcome: 'available', reason: 'available' })), runProcedure: vi.fn(async () => options.result ?? procedureResult('passed')), @@ -178,6 +179,7 @@ function adapters(options: { if (/test/iu.test(turn.text)) return 'test' return 'implementation' }, + isDefectEstablished: (_snapshot, turn) => /concrete evidence/iu.test(turn.text), repositoryForFeature: () => 'AgentWorkforce/factory', issuePolicy: ({ conversation, defectKind, slackBacklink }) => { const dedupeKey = `factory:${feature.id}:${feature.procedure}` @@ -343,7 +345,7 @@ describe('feature guardian conversation ownership', () => { await runGuardianConversationTurn( f.ctx, - reaction('env-3', '1710000003.000100', 'white_check_mark'), + reply('env-3', '1710000003.000100', 'I tested it and confirmed it works as expected'), a, f.deps, ) @@ -396,7 +398,7 @@ describe('feature guardian conversation ownership', () => { expect(a.runProcedure).toHaveBeenCalledTimes(1) expect(f.store.record().status).toBe('confirmed') expect([...f.confirmations.records.values()][0]).toMatchObject({ - verifier: 'factory-feature-guardian:procedure-runner', + verifier: 'test-procedure-runner', commands: ['npm run build', 'npm test'], tests: { passed: 42, failed: 0 }, }) @@ -412,9 +414,21 @@ describe('feature guardian conversation ownership', () => { f.deps, ) expect(a.runProcedure).toHaveBeenCalledTimes(1) - expect([...f.confirmations.records.values()][0]?.verifier).toBe( - 'factory-feature-guardian:procedure-runner', + expect([...f.confirmations.records.values()][0]?.verifier).toBe('test-procedure-runner') + }) + + it('clarifies a bare checkmark instead of treating delivery as confirmation evidence', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn( + f.ctx, + reaction('bare-check', '1710000001.000100', 'white_check_mark'), + a, + f.deps, ) + expect(f.store.record()).toMatchObject({ status: 'discussing', clarificationCount: 1 }) + expect(f.confirmations.records).toHaveLength(0) + expect(a.runProcedure).not.toHaveBeenCalled() }) it('preserves exact SKIP and MANUAL outcomes without confirmation', async () => { @@ -447,6 +461,55 @@ describe('feature guardian conversation ownership', () => { expect(f.slack.posts.at(-1)?.text).toContain('different revision') }) + it('revalidates revision and tier gates before resuming a pending verification', async () => { + const revision = await fixture() + let currentCatalog = catalog + const revisionAdapters = adapters() + revisionAdapters.loadCatalog = vi.fn(async () => currentCatalog) + await runGuardianConversationTurn( + revision.ctx, + reply('revision-1', '1710000001.000100', 'untested'), + revisionAdapters, + revision.deps, + ) + revision.store.failSaveCalls.add(revision.store.saveCalls + 3) + const retry = reply('revision-2', '1710000002.000100', 'still untested') + await expect( + runGuardianConversationTurn(revision.ctx, retry, revisionAdapters, revision.deps), + ).rejects.toBeInstanceOf(GuardianStateConflictError) + expect(revision.store.record()).toMatchObject({ status: 'verifying' }) + currentCatalog = { ...catalog, procedureRevision: guardianContentRevision('procedures-b') } + await runGuardianConversationTurn(revision.ctx, retry, revisionAdapters, revision.deps) + expect(revision.store.record().status).toBe('deferred') + expect(revision.confirmations.records).toHaveLength(0) + expect(revisionAdapters.runProcedure).toHaveBeenCalledTimes(1) + + const gated = await fixture() + const gatedAdapters = adapters() + const originalRun = gatedAdapters.runProcedure + let runAttempts = 0 + const restartedRun = vi.fn(async (...args: Parameters) => { + runAttempts += 1 + if (runAttempts === 1) throw new Error('restart now') + return originalRun(...args) + }) + gatedAdapters.runProcedure = restartedRun + await runGuardianConversationTurn( + gated.ctx, + reply('gate-1', '1710000001.000100', 'untested'), + gatedAdapters, + gated.deps, + ) + const gatedRetry = reply('gate-2', '1710000002.000100', 'still untested') + await expect( + runGuardianConversationTurn(gated.ctx, gatedRetry, gatedAdapters, gated.deps), + ).rejects.toThrow('restart now') + gatedAdapters.gateTier = vi.fn(async () => ({ outcome: 'manual', reason: 'MANUAL: opt-in revoked' })) + await runGuardianConversationTurn(gated.ctx, gatedRetry, gatedAdapters, gated.deps) + expect(gated.store.record().status).toBe('deferred') + expect(restartedRun).toHaveBeenCalledTimes(1) + }) + it('ignores unrelated channels, threads, bots, and delayed older events', async () => { const f = await fixture() const a = adapters() @@ -471,6 +534,36 @@ describe('feature guardian conversation ownership', () => { expect(f.slack.posts).toHaveLength(1) }) + it('orders same-millisecond reactions by the full provider timestamp', async () => { + const f = await fixture({ result: procedureResult('failed') }) + const a = adapters({ result: procedureResult('failed') }) + await runGuardianConversationTurn( + f.ctx, + reaction('same-ms-1', '1710000001.000100', 'question'), + a, + f.deps, + ) + await runGuardianConversationTurn( + f.ctx, + reaction('same-ms-2', '1710000001.000200', 'wrench'), + a, + f.deps, + ) + expect(a.runProcedure).toHaveBeenCalledTimes(1) + expect(f.store.record().status).toBe('remediation-open') + }) + + it('rejects payload identities that disagree with the mounted resource path', async () => { + const f = await fixture() + const a = adapters() + const mismatched = reply('path-mismatch', '1710000001.000100', 'tested and works') as + WorkforceEvent & { resource: { path: string } } + mismatched.resource.path = `/slack/channels/${CHANNEL}/messages/1719999999_000100` + await runGuardianConversationTurn(f.ctx, mismatched, a, f.deps) + expect(f.store.record().status).toBe('asked') + expect(f.confirmations.records).toHaveLength(0) + }) + it('replays a clarification after a post-before-CAS conflict without duplicating Slack', async () => { const f = await fixture() const a = adapters() @@ -485,6 +578,29 @@ describe('feature guardian conversation ownership', () => { expect(f.store.record()).toMatchObject({ status: 'discussing', turnCount: 1 }) }) + it('rejects an explicitly pending Slack reply even when it carries a stale timestamp', async () => { + const f = await fixture() + const pendingSlack = (() => ({ + replies: { + write: vi.fn(async () => ({ + path: '/slack/pending.json', + absolutePath: '/slack/pending.json', + deliveryStatus: 'pending' as const, + receipt: { ts: '1710000099.000100' }, + })), + }, + })) as unknown as typeof slackClient + await expect( + runGuardianConversationTurn( + f.ctx, + reply('pending-slack', '1710000001.000100', 'maybe'), + adapters(), + { ...f.deps, createSlackClient: pendingSlack }, + ), + ).rejects.toThrow('not provider-confirmed') + expect(f.store.record().status).toBe('asked') + }) + it('resumes an issue-write failure without rerunning the procedure', async () => { let attempts = 0 const f = await fixture({ @@ -540,13 +656,36 @@ describe('feature guardian conversation ownership', () => { ) await runGuardianConversationTurn( f.ctx, - reply('docs-2', '1710000002.000100', 'the documentation is still wrong'), + reply( + 'docs-2', + '1710000002.000100', + 'the documentation is wrong; concrete evidence: `docs/guardian.md` says expected enabled, but the command outputs disabled', + ), a, f.deps, ) expect(f.store.record().status).toBe('remediation-open') expect(f.issueWrites[0]?.defectKind).toBe('documentation') }) + + it('does not open remediation from a repeated allegation without concrete evidence', async () => { + const f = await fixture() + const a = adapters() + await runGuardianConversationTurn( + f.ctx, + reply('unsupported-1', '1710000001.000100', 'the documentation is wrong'), + a, + f.deps, + ) + await runGuardianConversationTurn( + f.ctx, + reply('unsupported-2', '1710000002.000100', 'the documentation is still wrong'), + a, + f.deps, + ) + expect(f.store.record().status).toBe('discussing') + expect(f.issueWriter.upsert).not.toHaveBeenCalled() + }) }) describe('feature guardian malformed and bounded state', () => { @@ -575,7 +714,7 @@ describe('feature guardian malformed and bounded state', () => { }], } const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ - path: '/memory/workspace/factory-feature-guardian/conversations.json', + path: '/memory/workspace/feature-guardian/conversations.json', revision: 'rev-1', content: `${JSON.stringify(malformed)}\n`, encoding: 'utf-8', @@ -587,6 +726,47 @@ describe('feature guardian malformed and bounded state', () => { await expect(store.load()).rejects.toThrow('asked conversation cannot retain a pending turn') }) + it('rejects a resumable confirmation without a validated confirmation basis', async () => { + const id = guardianConversationId(question) + const eventId = `slack:${'b'.repeat(64)}` + const order = `${slackTime('1710000001.000100')}:1710000001.000100:${eventId}` + const malformed = { + kind: 'feature-guardian:conversations', + version: 1, + records: [{ + ...question, + id, + status: 'confirmation-recorded', + turnCount: 1, + clarificationCount: 0, + seenEventIds: [], + evidence: [{ source: 'actor', result: 'positive', summary: 'yes' }], + pending: { + eventId, + eventOrder: order, + eventOccurredAt: slackTime('1710000001.000100'), + actorId: 'U-UNTRUSTED', + response: 'affirmative', + text: 'yes', + evidence: { source: 'actor', result: 'positive', summary: 'yes' }, + }, + lastProcessedEventOrder: order, + updatedAt: slackTime('1710000001.000100'), + }], + } + const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ + path: '/memory/workspace/feature-guardian/conversations.json', + revision: 'rev-1', + content: `${JSON.stringify(malformed)}\n`, + encoding: 'utf-8', + }), { status: 200, headers: { 'content-type': 'application/json' } })) as typeof fetch + const store = createSdkConversationStore( + { url: 'https://relayfile.test', token: 'token', workspaceId: 'workspace' }, + { fetchImpl }, + ) + await expect(store.load()).rejects.toThrow('requires a confirmation basis') + }) + it('prunes oldest terminal records to satisfy count and byte bounds', async () => { const store = new MemoryConversationStore() const records = Array.from({ length: 512 }, (_, index): GuardianConversationRecord => { @@ -655,6 +835,25 @@ describe('feature guardian GitHub remediation durability', () => { expect(client.createIssue).toHaveBeenCalledTimes(1) }) + it('admits only one concurrent provider create', async () => { + const ctx = context() + const createIssue = vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 5)) + return { + status: 'confirmed' as const, + id: '1780', + url: 'https://github.com/AgentWorkforce/factory/issues/1780', + path: '/github/issues/draft.json', + receipt: { externalId: '1780' }, + } + }) + const writer = createGithubIssueWriter(ctx, github({ createIssue })) + const results = await Promise.allSettled([writer.upsert(policy), writer.upsert(policy)]) + expect(createIssue).toHaveBeenCalledTimes(1) + expect(results.filter((result) => result.status === 'fulfilled')).toHaveLength(1) + expect(results.filter((result) => result.status === 'rejected')).toHaveLength(1) + }) + it('appends changed evidence to the deduplicated issue exactly once', async () => { const ctx = context() const client = github({ @@ -690,6 +889,36 @@ describe('feature guardian GitHub remediation durability', () => { expect(client.createIssue).toHaveBeenCalledTimes(1) }) + it('retries after a provider positively drops the prior issue submission', async () => { + const ctx = context() + let attempt = 0 + const client = github({ + createIssue: vi.fn(async () => { + attempt += 1 + if (attempt === 1) { + return { + status: 'dropped' as const, + id: '/github/issues/draft.json', + url: '' as const, + path: '/github/issues/draft.json', + reason: 'provider rejected before admission', + } + } + return { + status: 'confirmed' as const, + id: '1779', + url: 'https://github.com/AgentWorkforce/factory/issues/1779', + path: '/github/issues/draft-2.json', + receipt: { externalId: '1779' }, + } + }), + }) + const writer = createGithubIssueWriter(ctx, client) + await expect(writer.upsert(policy)).rejects.toThrow('dropped before provider admission') + await expect(writer.upsert(policy)).resolves.toMatchObject({ number: 1779 }) + expect(client.createIssue).toHaveBeenCalledTimes(2) + }) + it('finds an existing marker in the canonical nested issue mount', async () => { const ctx = context() ctx.sandbox.readFile = vi.fn(async (path: string) => { diff --git a/src/feature-guardian/conversation.ts b/src/feature-guardian/conversation.ts index a0e0db2..235e69c 100644 --- a/src/feature-guardian/conversation.ts +++ b/src/feature-guardian/conversation.ts @@ -21,11 +21,11 @@ import { } from '@relayfile/relay-helpers' export const GUARDIAN_CONVERSATION_STATE_PATH = - '/memory/workspace/factory-feature-guardian/conversations.json' + '/memory/workspace/feature-guardian/conversations.json' export const GUARDIAN_CONFIRMATIONS_PATH = - '/memory/workspace/factory-feature-guardian/confirmations' + '/memory/workspace/feature-guardian/confirmations' export const GUARDIAN_REMEDIATIONS_PATH = - '/memory/workspace/factory-feature-guardian/remediations' + '/memory/workspace/feature-guardian/remediations' export const GUARDIAN_STATE_TIMEOUT_MS = 5_000 export const GUARDIAN_WRITEBACK_TIMEOUT_MS = 15_000 @@ -36,7 +36,9 @@ const MAX_CONVERSATIONS = 512 const MAX_SEEN_EVENTS = 128 const MAX_EVIDENCE_ITEMS = 32 const MAX_TURNS = 12 +const MAX_REMEDIATION_ATTEMPTS = 8 const UTF8_ENCODER = new TextEncoder() +const PREVIEW_IMMUTABLE_CLAIMS = new WeakMap>() export type GuardianConversationStatus = | 'asked' @@ -126,6 +128,11 @@ export interface GuardianIssueReceipt { dedupeKey: string } +export interface GuardianConfirmationBasis { + kind: 'actor' | 'procedure' + verifier: string +} + export interface GuardianConversationRecord extends GuardianQuestion { id: string status: GuardianConversationStatus @@ -134,6 +141,7 @@ export interface GuardianConversationRecord extends GuardianQuestion { seenEventIds: string[] evidence: GuardianEvidence[] pending?: GuardianPendingTurn + confirmationBasis?: GuardianConfirmationBasis confirmationPath?: string issue?: GuardianIssueReceipt finalReplyTs?: string @@ -181,6 +189,7 @@ export interface GuardianProcedure { path: string prerequisites: string body: string + command: string } export interface GuardianTierGate { @@ -226,6 +235,11 @@ export interface FeatureGuardianAdapters { turn: GuardianPendingTurn, evidence: readonly GuardianEvidence[], ): GuardianDefectKind + isDefectEstablished( + feature: GuardianFeatureSnapshot, + turn: GuardianPendingTurn, + evidence: readonly GuardianEvidence[], + ): boolean repositoryForFeature(feature: GuardianFeatureSnapshot): string issuePolicy(input: { feature: GuardianFeatureSnapshot @@ -433,12 +447,21 @@ export async function parseGuardianSlackEvent( nonEmptyString(event.summary?.actor?.id) if (!channelId || !messageTs || !threadTs || !actorId) return null if (!isReaction && messageTs === threadTs) return null + const providerEventTs = nonEmptyString(payload.event_ts) ?? (isReaction ? undefined : messageTs) const occurredAt = canonicalTimestamp(event.occurredAt) ?? - timestampFromSlackTs(nonEmptyString(payload.event_ts) ?? messageTs) + timestampFromSlackTs(providerEventTs ?? messageTs) if (!occurredAt) return null + const resourceChannelId = channelFromPath(event.resource?.path) + const resourceMessageTs = messageTsFromPath(event.resource?.path) + if (resourceChannelId && resourceChannelId !== channelId) return null + if ( + resourceMessageTs && + resourceMessageTs !== messageTs && + (!isReaction && resourceMessageTs !== threadTs) + ) return null const providerIdentity = isReaction - ? nonEmptyString(payload.event_ts) ?? envelopeId + ? providerEventTs ?? envelopeId : messageTs const eventId = `slack:${createHash('sha256') .update( @@ -452,7 +475,9 @@ export async function parseGuardianSlackEvent( ].join(':'), ) .digest('hex')}` - const eventOrder = [occurredAt, messageTs, eventId].join(':') + const eventOrder = [occurredAt, slackTsOrder(providerEventTs ?? messageTs), messageTs, eventId].join( + ':', + ) return { eventId, @@ -573,7 +598,7 @@ export async function runGuardianConversationTurn( if (pendingBeforeResume.eventId === parsed.eventId || record.seenEventIds.includes(parsed.eventId)) { return } - if (Date.parse(parsed.occurredAt) <= Date.parse(pendingBeforeResume.eventOccurredAt)) { + if (parsed.eventOrder <= pendingBeforeResume.eventOrder) { ctx.log('info', 'feature-guardian.event-ignored', { reason: 'superseded-out-of-order-event', eventId: parsed.eventId, @@ -639,7 +664,18 @@ export async function runGuardianConversationTurn( } if (response === 'affirmative' && adapters.isAuthorizedConfirmer(ctx, parsed.actorId)) { - snapshot = await checkpointPending(snapshot, record, turn, 'confirmation-recorded', true) + if (!hasActorConfirmationEvidence(parsed.text)) { + await clarify(turn) + return + } + snapshot = await checkpointPending( + snapshot, + record, + turn, + 'confirmation-recorded', + true, + { kind: 'actor', verifier: parsed.actorId }, + ) record = requireRecordById(snapshot.state, record.id) await finishConfirmation(record, turn) return @@ -669,9 +705,12 @@ export async function runGuardianConversationTurn( snapshot = await checkpointPending(snapshot, record, turn, 'verifying', true) record = requireRecordById(snapshot.state, record.id) - await resumePendingTurn(procedure) + await resumePendingTurn(procedure, true) - async function resumePendingTurn(preResolvedProcedure?: GuardianProcedure): Promise { + async function resumePendingTurn( + preResolvedProcedure?: GuardianProcedure, + prerequisitesValidated = false, + ): Promise { if (!snapshot || !record.pending) return const pending = record.pending if (record.status === 'confirmation-recorded') { @@ -689,7 +728,27 @@ export async function runGuardianConversationTurn( } if (record.status !== 'verifying') return + if (!prerequisitesValidated) { + const currentCatalog = await adapters.loadCatalog(ctx) + if ( + currentCatalog.manifestRevision !== record.manifestRevision || + currentCatalog.procedureRevision !== record.procedureRevision + ) { + await deferPendingVerification( + 'The manifest or procedure changed while verification was pending; refusing to run or confirm a different revision.', + 'manual', + ) + return + } + } const procedure = preResolvedProcedure ?? await adapters.resolveProcedure(ctx, record.feature) + if (!pending.verification && !prerequisitesValidated) { + const gate = await adapters.gateTier(ctx, record.feature, procedure) + if (gate.outcome !== 'available') { + await deferPendingVerification(gate.reason, gate.outcome) + return + } + } const result = pending.verification ?? await adapters.runProcedure( ctx, record.feature, @@ -712,7 +771,8 @@ export async function runGuardianConversationTurn( const reportedDefect = adapters.classifyDefect(record.feature, pending, evidence) if ( record.clarificationCount >= 1 && - ['test', 'manifest', 'procedure', 'documentation'].includes(reportedDefect) + ['test', 'manifest', 'procedure', 'documentation'].includes(reportedDefect) && + adapters.isDefectEstablished(record.feature, pending, evidence) ) { await recordRemediation(result, reportedDefect) return @@ -728,7 +788,7 @@ export async function runGuardianConversationTurn( createSlackClient, record, text, - guardianReplyIdempotencyKey(record, pending.eventId, `post-check-clarification-${clarificationNumber}`), + guardianReplyIdempotencyKey(record, pending.eventId, 'post-check-clarification'), ) const next = replaceRecord(snapshot.state, record.id, { ...record, @@ -747,6 +807,7 @@ export async function runGuardianConversationTurn( const next = replaceRecord(snapshot.state, record.id, { ...record, status: 'confirmation-recorded', + confirmationBasis: { kind: 'procedure', verifier: result.verifier }, updatedAt: now().toISOString(), }) snapshot = await store.save(next, snapshot) @@ -771,6 +832,28 @@ export async function runGuardianConversationTurn( await recordRemediation(result, defectKind) } + async function deferPendingVerification( + reason: string, + outcome: 'skip' | 'manual', + ): Promise { + if (!snapshot || !record.pending) return + const deferredPending: GuardianPendingTurn = { + ...record.pending, + response: 'deferred', + evidence: { source: 'system', result: outcome, summary: reason }, + } + const next = replaceRecord(snapshot.state, record.id, { + ...record, + status: 'deferred-recorded', + pending: deferredPending, + evidence: appendEvidence(record.evidence, deferredPending.evidence), + updatedAt: now().toISOString(), + }) + snapshot = await store.save(next, snapshot) + record = requireRecordById(snapshot.state, record.id) + await finishDeferred(record, reason) + } + async function recordRemediation( result: GuardianProcedureResult, defectKind: GuardianDefectKind, @@ -829,7 +912,7 @@ export async function runGuardianConversationTurn( createSlackClient, record, text, - guardianReplyIdempotencyKey(record, turn.eventId, `clarification-${clarificationNumber}`), + guardianReplyIdempotencyKey(record, turn.eventId, 'clarification'), ) const nextRecord: GuardianConversationRecord = { ...record, @@ -869,7 +952,7 @@ export async function runGuardianConversationTurn( .reverse() .find((entry) => entry.source === 'procedure') const summary = procedureEvidence - ? `✅ Confirmed *${current.feature.name}* for manifest \`${shortRevision(current.manifestRevision)}\`. ${procedureEvidence.summary}` + ? `✅ Confirmed *${current.feature.name}* for manifest \`${shortRevision(current.manifestRevision)}\`. ${formatEvidenceForSlack(procedureEvidence)}` : `✅ Confirmed *${current.feature.name}* for manifest \`${shortRevision(current.manifestRevision)}\` from ${turn.actorId}'s response.` const replyTs = await postThreadReply( createSlackClient, @@ -946,12 +1029,14 @@ export async function runGuardianConversationTurn( turn: GuardianPendingTurn, status: GuardianConversationStatus, countTurn = false, + confirmationBasis?: GuardianConfirmationBasis, ): Promise { const nextRecord: GuardianConversationRecord = { ...current, status, turnCount: current.turnCount + (countTurn ? 1 : 0), pending: turn, + ...(confirmationBasis ? { confirmationBasis } : {}), lastProcessedEventOrder: turn.eventOrder, evidence: appendEvidence(current.evidence, turn.evidence), updatedAt: now().toISOString(), @@ -1166,40 +1251,57 @@ export function createGithubIssueWriter( await appendImmutableJson(ctx, receiptPath, { ...receipt, bodyRevision }) return receipt } - const intentPath = remediationIntentPath(policy.dedupeKey) - const existingIntent = await readImmutableJson(ctx, intentPath) - if (existingIntent) { - throw new Error( - 'GitHub remediation submission is pending provider correlation; refusing a duplicate issue', - ) - } - await appendImmutableJson(ctx, intentPath, { - kind: 'feature-guardian:remediation-intent', - version: 1, - repository: policy.repository, - dedupeKey: policy.dedupeKey, - bodyRevision, - }) - const created = await client.createIssue({ - owner, - repo, - title: policy.title, - body: policy.body, - labels: policy.labels, - }) - if (created.status !== 'confirmed' || !created.url) { - throw new Error(`GitHub remediation issue create was ${created.status}, not provider-confirmed`) - } - const number = issueNumberFromCreated(created.id, created.url) - if (!number) throw new Error('GitHub remediation issue receipt is missing its issue number') - const receipt = { - repository: policy.repository, - number, - url: created.url, - dedupeKey: policy.dedupeKey, + for (let attempt = 1; attempt <= MAX_REMEDIATION_ATTEMPTS; attempt += 1) { + const intentPath = remediationIntentPath(policy.dedupeKey, attempt) + const droppedPath = remediationDroppedPath(policy.dedupeKey, attempt) + if (await readImmutableJson(ctx, droppedPath)) continue + const claimed = await claimImmutableJson(ctx, intentPath, { + kind: 'feature-guardian:remediation-intent', + version: 1, + repository: policy.repository, + dedupeKey: policy.dedupeKey, + bodyRevision, + ...(attempt === 1 ? {} : { attempt }), + }) + if (!claimed) { + if (await readImmutableJson(ctx, droppedPath)) continue + throw new Error( + 'GitHub remediation submission is pending provider correlation; refusing a duplicate issue', + ) + } + const created = await client.createIssue({ + owner, + repo, + title: policy.title, + body: policy.body, + labels: policy.labels, + }) + if (created.status === 'dropped') { + await appendImmutableJson(ctx, droppedPath, { + kind: 'feature-guardian:remediation-drop', + version: 1, + repository: policy.repository, + dedupeKey: policy.dedupeKey, + attempt, + reason: created.reason ?? 'provider dropped the remediation draft', + }) + throw new Error('GitHub remediation issue create was dropped before provider admission') + } + if (created.status !== 'confirmed' || !created.url) { + throw new Error(`GitHub remediation issue create was ${created.status}, not provider-confirmed`) + } + const number = issueNumberFromCreated(created.id, created.url) + if (!number) throw new Error('GitHub remediation issue receipt is missing its issue number') + const receipt = { + repository: policy.repository, + number, + url: created.url, + dedupeKey: policy.dedupeKey, + } + await appendImmutableJson(ctx, receiptPath, { ...receipt, bodyRevision }) + return receipt } - await appendImmutableJson(ctx, receiptPath, { ...receipt, bodyRevision }) - return receipt + throw new Error('GitHub remediation retry limit was exhausted after provider drops') }, } } @@ -1208,8 +1310,13 @@ export function remediationMarker(dedupeKey: string): string { return `` } -function remediationIntentPath(dedupeKey: string): string { - return `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`intent:${dedupeKey}`).digest('hex')}.json` +function remediationIntentPath(dedupeKey: string, attempt: number): string { + const identity = attempt === 1 ? `intent:${dedupeKey}` : `intent:${dedupeKey}:${attempt}` + return `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(identity).digest('hex')}.json` +} + +function remediationDroppedPath(dedupeKey: string, attempt: number): string { + return `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`drop:${dedupeKey}:${attempt}`).digest('hex')}.json` } function remediationReceiptPath(dedupeKey: string): string { @@ -1238,19 +1345,40 @@ async function updateExistingIssueEvidence( await appendImmutableJson(ctx, receiptPath, { issueNumber, bodyRevision }) return } - const intentPath = `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`update-intent:${ledgerKey}`).digest('hex')}.json` - if (await readImmutableJson(ctx, intentPath)) { - throw new Error('GitHub remediation evidence update is pending; refusing a duplicate comment') - } - await appendImmutableJson(ctx, intentPath, { issueNumber, bodyRevision }) - const updated = await client.comment( - { owner, repo, number: issueNumber }, - `${marker}\n${policy.body}`, - ) - if (updated.status !== 'confirmed') { - throw new Error(`GitHub remediation evidence update was ${updated.status}, not provider-confirmed`) + for (let attempt = 1; attempt <= MAX_REMEDIATION_ATTEMPTS; attempt += 1) { + const suffix = attempt === 1 ? '' : `:${attempt}` + const intentPath = `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`update-intent:${ledgerKey}${suffix}`).digest('hex')}.json` + const droppedPath = `${GUARDIAN_REMEDIATIONS_PATH}/${createHash('sha256').update(`update-drop:${ledgerKey}:${attempt}`).digest('hex')}.json` + if (await readImmutableJson(ctx, droppedPath)) continue + const claimed = await claimImmutableJson(ctx, intentPath, { + issueNumber, + bodyRevision, + ...(attempt === 1 ? {} : { attempt }), + }) + if (!claimed) { + if (await readImmutableJson(ctx, droppedPath)) continue + throw new Error('GitHub remediation evidence update is pending; refusing a duplicate comment') + } + const updated = await client.comment( + { owner, repo, number: issueNumber }, + `${marker}\n${policy.body}`, + ) + if (updated.status === 'dropped') { + await appendImmutableJson(ctx, droppedPath, { + issueNumber, + bodyRevision, + attempt, + reason: updated.reason ?? 'provider dropped the evidence update', + }) + throw new Error('GitHub remediation evidence update was dropped before provider admission') + } + if (updated.status !== 'confirmed') { + throw new Error(`GitHub remediation evidence update was ${updated.status}, not provider-confirmed`) + } + await appendImmutableJson(ctx, receiptPath, { issueNumber, bodyRevision }) + return } - await appendImmutableJson(ctx, receiptPath, { issueNumber, bodyRevision }) + throw new Error('GitHub remediation evidence retry limit was exhausted after provider drops') } function parseDurableIssueReceipt( @@ -1381,6 +1509,56 @@ async function appendImmutableJson( } } +/** Atomically claim one immutable provider submission. Only the CAS winner may write. */ +async function claimImmutableJson( + ctx: WorkforceCtx, + path: string, + value: unknown, +): Promise { + const content = `${JSON.stringify(value)}\n` + assertBoundedContent(content, 'guardian immutable claim') + const existing = await readImmutableJson(ctx, path) + if (existing !== null) { + if (`${JSON.stringify(existing)}\n` !== content) { + throw new Error('immutable guardian claim already differs') + } + return false + } + const credentials = ctx.credentials.tryRequire() + if (credentials) { + const client = relayfileClient(credentials.relayfile) + try { + await withDeadline('guardian immutable claim', GUARDIAN_STATE_TIMEOUT_MS, (signal) => + writeExactFile(client, credentials.relayfile.workspaceId, path, '0', content, signal), + ) + } catch (error) { + if ( + !(error instanceof RevisionConflictError) && + !(error instanceof RelayFileApiError && error.status === 409) + ) throw error + const winner = await readImmutableJson(ctx, path) + if (winner === null || `${JSON.stringify(winner)}\n` !== content) { + throw new Error('guardian immutable claim conflict did not match the winning intent') + } + return false + } + } else if (ctx.agent.id === 'sim-agent' && ctx.deployment.id === 'sim-deployment') { + const claimOwner = ctx.files as object + const claims = PREVIEW_IMMUTABLE_CLAIMS.get(claimOwner) ?? new Set() + if (claims.has(path)) return false + claims.add(path) + PREVIEW_IMMUTABLE_CLAIMS.set(claimOwner, claims) + await ctx.files.write(path, content) + } else { + throw new Error('exact Relayfile credentials are required for guardian remediation claims') + } + const readBack = await readImmutableJson(ctx, path) + if (readBack === null || `${JSON.stringify(readBack)}\n` !== content) { + throw new Error('guardian immutable claim read-back did not match') + } + return true +} + export function slackThreadBacklink(channelId: string, threadTs: string): string { return `https://slack.com/archives/${channelId}/p${threadTs.replace('.', '')}` } @@ -1403,6 +1581,9 @@ async function postThreadReply( } function requireSlackReceiptTs(result: WritebackResult): string { + if (result.deliveryStatus && result.deliveryStatus !== 'confirmed') { + throw new Error(`guardian Slack reply was ${result.deliveryStatus}, not provider-confirmed`) + } const receipt = asRecord(result.receipt) const externalId = nonEmptyString(receipt?.externalId) const fallbackTs = nonEmptyString(receipt?.ts) @@ -1418,6 +1599,9 @@ function confirmationRecord( turn: GuardianPendingTurn, timestamp: string, ): GuardianConfirmationRecord { + if (!record.confirmationBasis) { + throw new Error('guardian confirmation record is missing its confirmation basis') + } const evidence = record.evidence const commands = evidence.flatMap((entry) => entry.commands ?? []) const tests = evidence.reduce( @@ -1427,9 +1611,6 @@ function confirmationRecord( }), { passed: 0, failed: 0 }, ) - const automated = evidence.some( - (entry) => entry.source === 'procedure' && entry.result === 'positive', - ) return { kind: 'feature-guardian:confirmation', version: 1, @@ -1440,7 +1621,7 @@ function confirmationRecord( generation: record.generation, result: 'confirmed', ...(turn.actorId ? { actor: { id: turn.actorId } } : {}), - verifier: automated ? 'factory-feature-guardian:procedure-runner' : turn.actorId, + verifier: record.confirmationBasis.verifier, timestamp, evidence, commands, @@ -1506,6 +1687,9 @@ function parseConversationRecord(value: unknown): GuardianConversationRecord { updatedAt: requireTimestamp(record.updatedAt, 'updatedAt'), } if (record.pending !== undefined) parsed.pending = parsePending(record.pending) + if (record.confirmationBasis !== undefined) { + parsed.confirmationBasis = parseConfirmationBasis(record.confirmationBasis) + } if (record.confirmationPath !== undefined) { parsed.confirmationPath = requireString(record.confirmationPath, 'confirmation path') } @@ -1547,6 +1731,49 @@ function parseConversationRecord(value: unknown): GuardianConversationRecord { if (parsed.status === 'confirmed' && !parsed.confirmationPath) { throw new Error('confirmed guardian conversation requires a confirmation path') } + if ( + (parsed.status === 'confirmation-recorded' || parsed.status === 'confirmed') && + !parsed.confirmationBasis + ) { + throw new Error(`guardian ${parsed.status} conversation requires a confirmation basis`) + } + if ( + parsed.confirmationBasis && + parsed.status !== 'confirmation-recorded' && + parsed.status !== 'confirmed' + ) { + throw new Error('guardian confirmation basis is invalid for the conversation status') + } + if (parsed.status === 'confirmation-recorded' && parsed.pending && parsed.confirmationBasis) { + if ( + parsed.confirmationBasis.kind === 'actor' && + (parsed.pending.response !== 'affirmative' || + parsed.pending.actorId !== parsed.confirmationBasis.verifier || + parsed.pending.evidence?.result !== 'positive') + ) { + throw new Error('guardian actor confirmation basis is inconsistent with its pending turn') + } + if ( + parsed.confirmationBasis.kind === 'procedure' && + (parsed.pending.verification?.outcome !== 'passed' || + parsed.pending.verification.verifier !== parsed.confirmationBasis.verifier || + (parsed.pending.verification.tests?.failed ?? 0) !== 0) + ) { + throw new Error('guardian procedure confirmation basis is inconsistent with its pending turn') + } + } + if (parsed.status === 'confirmed' && parsed.confirmationBasis) { + const matchingEvidence = parsed.evidence.some((entry) => + parsed.confirmationBasis?.kind === 'actor' + ? entry.source === 'actor' && entry.result === 'positive' + : entry.source === 'procedure' && + entry.result === 'positive' && + (entry.tests?.failed ?? 0) === 0, + ) + if (!matchingEvidence) { + throw new Error('confirmed guardian conversation has no evidence for its confirmation basis') + } + } if ( parsed.confirmationPath && parsed.status !== 'confirmation-recorded' && @@ -1575,6 +1802,8 @@ function parseConfirmationRecord(value: unknown): GuardianConfirmationRecord { if (record.result !== 'confirmed' || !slack || !tests) { throw new Error('guardian confirmation result/evidence is invalid') } + const failed = requireNonNegativeInteger(tests.failed, 'failed tests') + if (failed !== 0) throw new Error('confirmed guardian evidence cannot contain failed tests') return { kind: 'feature-guardian:confirmation', version: 1, @@ -1591,7 +1820,7 @@ function parseConfirmationRecord(value: unknown): GuardianConfirmationRecord { commands: stringArray(record.commands, 128, 'commands'), tests: { passed: requireNonNegativeInteger(tests.passed, 'passed tests'), - failed: requireNonNegativeInteger(tests.failed, 'failed tests'), + failed, }, slack: { channelId: requireString(slack.channelId, 'Slack channel id'), @@ -1601,6 +1830,14 @@ function parseConfirmationRecord(value: unknown): GuardianConfirmationRecord { } } +function parseConfirmationBasis(value: unknown): GuardianConfirmationBasis { + const basis = asRecord(value) + if (!basis || (basis.kind !== 'actor' && basis.kind !== 'procedure')) { + throw new Error('guardian confirmation basis is invalid') + } + return { kind: basis.kind, verifier: requireString(basis.verifier, 'confirmation verifier') } +} + function assertConversationTransition( previous: GuardianConversationState | null, next: GuardianConversationState, @@ -1635,6 +1872,12 @@ function assertConversationTransition( ) { throw new Error('guardian conversation confirmation path is immutable') } + if ( + prior.confirmationBasis !== undefined && + JSON.stringify(current.confirmationBasis) !== JSON.stringify(prior.confirmationBasis) + ) { + throw new Error('guardian conversation confirmation basis is immutable') + } if (prior.issue !== undefined && JSON.stringify(current.issue) !== JSON.stringify(prior.issue)) { throw new Error('guardian conversation issue receipt is immutable') } @@ -1999,6 +2242,21 @@ function shortRevision(revision: string): string { return revision.replace(/^sha256:/u, '').slice(0, 12) } +function hasActorConfirmationEvidence(text: string): boolean { + return /\b(tested|verified|confirmed)\b/iu.test(text) && + /\b(works?|working|passes?|passed|expected|good)\b/iu.test(text) +} + +function formatEvidenceForSlack(evidence: GuardianEvidence): string { + return [ + evidence.summary, + ...(evidence.positiveAssertions?.map((value) => `PASS: ${value}`) ?? []), + ...(evidence.negativeAssertions?.map((value) => `FAIL: ${value}`) ?? []), + ...(evidence.tests ? [`Tests: ${evidence.tests.passed} passed, ${evidence.tests.failed} failed.`] : []), + ...(evidence.cleanup?.map((value) => `Cleanup: ${value}`) ?? []), + ].join(' ') +} + function resolvedInput(ctx: WorkforceCtx, name: string): string | undefined { const spec = ctx.persona?.inputSpecs?.[name] const value = process.env[spec?.env ?? name] ?? ctx.persona?.inputs?.[name] ?? spec?.default @@ -2195,6 +2453,12 @@ function timestampFromSlackTs(value: string | undefined): string | undefined { return new Date(seconds * 1_000).toISOString() } +function slackTsOrder(value: string): string { + const match = value.match(/^(\d+)\.(\d{1,9})$/u) + if (!match) return '00000000000000000000.000000000' + return `${match[1].padStart(20, '0')}.${match[2].padEnd(9, '0')}` +} + function requireTimestamp(value: unknown, field: string): string { const timestamp = canonicalTimestamp(value) if (!timestamp || timestamp !== value) throw new Error(`guardian ${field} is invalid`) @@ -2210,9 +2474,11 @@ function requireSlackTs(value: unknown, field: string): string { function requireEventOrder(value: unknown, field: string): string { const order = requireString(value, field) // Event order is internal and compared lexicographically. Its exact shape - // is ISO timestamp + Slack ts + stable response identity. + // is ISO timestamp + normalized provider ts + Slack message ts + stable + // response identity. Accept the prior three-part form so pre-upgrade state + // remains readable. const match = order.match( - /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z):\d+\.\d+:slack:[a-f0-9]{64}$/u, + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z):(?:(?:\d{20}\.\d{9}):)?\d+\.\d+:slack:[a-f0-9]{64}$/u, ) if (!match || canonicalTimestamp(match[1]) !== match[1]) { throw new Error(`guardian ${field} is invalid`)