Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion services/api/src/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,7 @@ async function checkReceiptIntegrity(db: Db, orgId: string): Promise<DoctorCheck
return fail(
"receipt_integrity",
"Agent receipt integrity",
`${report.invalidRunIds.length} invalid and ${report.unauditedRunIds.length} unaudited receipts found across ${report.checked} runs.`,
`${report.invalidRunIds.length} invalid, ${report.unauditedRunIds.length} unaudited, and ${report.missingReceiptRunIds.length} audited-but-missing receipts found across ${report.checked} runs.`,
"Stop outcome and learning jobs, preserve audit_events and runs, then investigate receipt mutation or an outdated runner.",
);
}
Expand Down
25 changes: 23 additions & 2 deletions services/api/src/receipt-integrity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ export type ReceiptIntegrityReport = {
checked: number;
invalidRunIds: string[];
unauditedRunIds: string[];
/** Audited run.finished digests whose run no longer carries a receipt —
* the receipt was destroyed or the run row deleted after the fact. Every
* terminal run writes its receipt and its audit event together, so this
* can only be an anomaly; it is reported distinctly from "never audited"
* so the two failure stories stay distinguishable. */
missingReceiptRunIds: string[];
};

export async function verifyStoredReceipts(
Expand All @@ -15,7 +21,13 @@ export async function verifyStoredReceipts(
runIds?: string[],
): Promise<ReceiptIntegrityReport> {
if (runIds?.length === 0) {
return { ok: true, checked: 0, invalidRunIds: [], unauditedRunIds: [] };
return {
ok: true,
checked: 0,
invalidRunIds: [],
unauditedRunIds: [],
missingReceiptRunIds: [],
};
}
const stored = await db
.select({ id: runs.id, receipt: runs.receipt })
Expand All @@ -41,6 +53,11 @@ export async function verifyStoredReceipts(
}
const invalidRunIds: string[] = [];
const unauditedRunIds: string[] = [];
const storedIds = new Set(stored.map((row) => row.id));
const scope = runIds ? new Set(runIds) : null;
const missingReceiptRunIds = [...auditedDigests.keys()].filter(
(id) => !storedIds.has(id) && (!scope || scope.has(id)),
);
for (const row of stored) {
const parsed = FacilityReceiptSchema.safeParse(row.receipt);
if (!parsed.success || !verifyFacilityReceipt(parsed.data)) {
Expand All @@ -52,10 +69,14 @@ export async function verifyStoredReceipts(
}
}
return {
ok: invalidRunIds.length === 0 && unauditedRunIds.length === 0,
ok:
invalidRunIds.length === 0 &&
unauditedRunIds.length === 0 &&
missingReceiptRunIds.length === 0,
checked: stored.length,
invalidRunIds,
unauditedRunIds,
missingReceiptRunIds,
};
}

Expand Down
34 changes: 34 additions & 0 deletions services/api/test/sandbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1609,6 +1609,40 @@ describe("sandbox api", async () => {
await expect(verifyStoredReceipts(db, orgId, [run.id])).resolves.toMatchObject({ ok: true });
});

it("receipt integrity notices a receipt destroyed after run.finished was audited", async () => {
// #226: verifyStoredReceipts only walked runs that still HAVE a receipt,
// so nulling one (or deleting the run) silently shrank `checked` while ok
// stayed true. The reverse question — audited digest, no receipt behind
// it — must go red.
const token = "frt_receipt_destroyed";
const run = await insertRunnerRun(token, "running");
const response = await app.inject({
method: "POST",
url: `/internal/runs/${run.id}/result`,
headers: { authorization: `Bearer ${token}` },
payload: { status: "succeeded" },
});
expect(response.statusCode).toBe(200);
await expect(verifyStoredReceipts(db, orgId, [run.id])).resolves.toMatchObject({ ok: true });

await db.update(runs).set({ receipt: null }).where(eq(runs.id, run.id));
const nulled = await verifyStoredReceipts(db, orgId, [run.id]);
expect(nulled.ok).toBe(false);
expect(nulled.missingReceiptRunIds).toEqual([run.id]);
expect(nulled.invalidRunIds).toEqual([]);

// Scope is respected: asking about other runs stays clean.
const outOfScope = await verifyStoredReceipts(db, orgId, ["run_someone_else"]);
expect(outOfScope.missingReceiptRunIds).toEqual([]);

// A deleted run row is the same anomaly — the audit trail outlives it.
await db.delete(runEvents).where(eq(runEvents.runId, run.id));
await db.delete(runs).where(eq(runs.id, run.id));
const deleted = await verifyStoredReceipts(db, orgId);
expect(deleted.ok).toBe(false);
expect(deleted.missingReceiptRunIds).toContain(run.id);
});

it("delivers run events over the NOTIFY-backed SSE path without safety polling", async () => {
const token = "frt_stream";
const run = await insertRunnerRun(token, "running");
Expand Down
Loading