diff --git a/docs/loop-manifest.md b/docs/loop-manifest.md index 0696991..c9055a0 100644 --- a/docs/loop-manifest.md +++ b/docs/loop-manifest.md @@ -115,6 +115,9 @@ Reports must not embed raw stdout, stderr, prompts, tokens, credentials, or secret-looking fixture values. Run and step status transitions, downstream blocking, approval-policy continuation, and lifecycle telemetry are handled by the transition layer rather than by the validation report runner. +Failed validation gates block downstream stages by default. Skipped required +gates also block downstream stages, even when other gates passed; skipped +optional gates remain inspectable in the report but do not block progression. Queued `validation_report` artifacts may carry `validationReportMetadataKind: validation_report_contract` with diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts new file mode 100644 index 0000000..6e8f32c --- /dev/null +++ b/src/lib/loops/development-run-transitions.ts @@ -0,0 +1,651 @@ +import { and, eq } from "drizzle-orm"; + +import type { db } from "@/db/client"; +import { artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import type { ValidationGateResultV1, ValidationReportV1 } from "@/lib/loops/validation-report"; +import { createValidationReportArtifactMetadata } from "@/lib/loops/validation-report"; +import type { LoopworksLogger } from "@/lib/observability/logger"; +import { + type DevelopmentLoopRunCompletedMetricInput, + type DevelopmentLoopRunDurationMetricInput, + type DevelopmentLoopStepDurationMetricInput, + type DevelopmentLoopStepRetryMetricInput, + type DevelopmentLoopValidationDurationMetricInput, + type DevelopmentLoopValidationOutcomeMetricInput, + recordDevelopmentLoopRunCompletedMetric, + recordDevelopmentLoopRunDurationMetric, + recordDevelopmentLoopStepDurationMetric, + recordDevelopmentLoopStepRetryMetric, + recordDevelopmentLoopValidationDurationMetric, + recordDevelopmentLoopValidationOutcomeMetric, +} from "@/lib/observability/metrics"; + +export type DevelopmentLoopTransitionDatabase = Pick; + +export type DevelopmentLoopValidationTransitionStatus = "advanced" | "blocked"; +export type DevelopmentLoopTerminalStatus = "succeeded" | "failed" | "canceled"; + +export type DevelopmentLoopTransitionMetrics = { + runCompleted?: (input: DevelopmentLoopRunCompletedMetricInput) => void; + runDuration?: (input: DevelopmentLoopRunDurationMetricInput) => void; + stepDuration?: (input: DevelopmentLoopStepDurationMetricInput) => void; + stepRetry?: (input: DevelopmentLoopStepRetryMetricInput) => void; + validationDuration?: (input: DevelopmentLoopValidationDurationMetricInput) => void; + validationOutcome?: (input: DevelopmentLoopValidationOutcomeMetricInput) => void; +}; + +export type ExpectedValidationGate = { + key: string; + required: boolean; +}; + +type RunMetadata = Record; + +type ValidationTransitionMetricInputs = { + stepDuration: DevelopmentLoopStepDurationMetricInput; + validationDurations: DevelopmentLoopValidationDurationMetricInput[]; + validationOutcomes: DevelopmentLoopValidationOutcomeMetricInput[]; +}; + +type ValidationTransitionResult = { + blockedReason?: string; + idempotent?: boolean; + runId: string; + stage: string; + status: DevelopmentLoopValidationTransitionStatus; + stepId: string; + traceId?: string; +}; + +export class DevelopmentLoopTransitionError extends Error { + constructor(message: string) { + super(message); + this.name = "DevelopmentLoopTransitionError"; + } +} + +function durationSecondsBetween(startedAt: Date, completedAt: Date): number { + return Math.max(0, (completedAt.getTime() - startedAt.getTime()) / 1000); +} + +function sumValidationDurationMs(report: ValidationReportV1): number { + return report.results.reduce((total, result) => total + Math.max(0, result.durationMs), 0); +} + +function requiredSkippedResults(report: ValidationReportV1): ValidationGateResultV1[] { + return report.results.filter((result) => result.required && result.outcome === "skipped"); +} + +function failedResults(report: ValidationReportV1): ValidationGateResultV1[] { + return report.results.filter((result) => result.outcome === "fail"); +} + +function missingRequiredGateKeys( + report: ValidationReportV1, + expectedValidationGates: readonly ExpectedValidationGate[] | undefined, +): string[] { + if (!expectedValidationGates) { + return []; + } + + const resultKeys = new Set(report.results.map((result) => result.key)); + return expectedValidationGates + .filter((gate) => gate.required && !resultKeys.has(gate.key)) + .map((gate) => gate.key); +} + +function getBlockedReason( + report: ValidationReportV1, + expectedValidationGates?: readonly ExpectedValidationGate[], +): string | undefined { + if (report.results.length === 0) { + return "Validation report contained no gate results."; + } + + if (failedResults(report).length > 0) { + return "Deterministic validation failed before review."; + } + + if (requiredSkippedResults(report).length > 0) { + return "Required validation gate skipped before review."; + } + + if (missingRequiredGateKeys(report, expectedValidationGates).length > 0) { + return "Required validation gate missing before review."; + } + + return undefined; +} + +function getStartedAtForDuration(input: { + completedAt: Date; + durationMs: number; + startedAt: Date | null; +}): Date { + if (input.startedAt) { + return input.startedAt; + } + + return new Date(input.completedAt.getTime() - Math.max(0, input.durationMs)); +} + +function metadataWithoutBlockedReason(metadata: RunMetadata | null | undefined): RunMetadata { + const { blockedReason: _blockedReason, ...rest } = metadata ?? {}; + return rest; +} + +function getPersistedBlockedReason(metadata: RunMetadata | null | undefined): string | undefined { + const blockedReason = metadata?.blockedReason; + return typeof blockedReason === "string" && blockedReason.length > 0 ? blockedReason : undefined; +} + +const safeReasonCodePattern = /^[a-z][a-z0-9_.:-]{0,79}$/; + +function normalizeReasonCode(reason: string | undefined): string | undefined { + const normalized = reason?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + + return safeReasonCodePattern.test(normalized) ? normalized : "unspecified"; +} + +function createValidationTransitionMetadata(input: { + metadata: RunMetadata | null | undefined; + report: ValidationReportV1; + blockedReason?: string; +}): RunMetadata { + return { + ...metadataWithoutBlockedReason(input.metadata), + ...(input.blockedReason ? { blockedReason: input.blockedReason } : {}), + validationCounts: input.report.counts, + validationOutcome: input.blockedReason ? "blocked" : input.report.overallOutcome, + validationReportSchemaId: input.report.schemaId, + }; +} + +function createStepValidationMetadata(input: { + metadata: RunMetadata | null | undefined; + report: ValidationReportV1; + requiredSkippedCount: number; +}): RunMetadata { + return { + ...(input.metadata ?? {}), + validationCounts: input.report.counts, + validationOutcome: input.report.overallOutcome, + validationReportSchemaId: input.report.schemaId, + validationRequiredSkippedGateCount: input.requiredSkippedCount, + }; +} + +function createValidationMetricInputs(input: { + loopKey: string; + report: ValidationReportV1; + stage: string; + stepStatus: "succeeded" | "failed"; + stepDurationSeconds: number; +}): ValidationTransitionMetricInputs { + const measurableResults = input.report.results.filter( + (result): result is ValidationGateResultV1 & { outcome: "pass" | "fail" } => + result.outcome === "pass" || result.outcome === "fail", + ); + + return { + stepDuration: { + durationSeconds: input.stepDurationSeconds, + loopKey: input.loopKey, + stage: input.stage, + status: input.stepStatus, + }, + validationDurations: measurableResults.map((result) => ({ + command: result.command, + durationSeconds: Math.max(0, result.durationMs) / 1000, + gate: result.key, + })), + validationOutcomes: measurableResults.map((result) => ({ + command: result.command, + gate: result.key, + status: result.outcome, + })), + }; +} + +function emitSafely(recorder: ((input: T) => void) | undefined, input: T): void { + try { + recorder?.(input); + } catch { + // Runtime state transitions must remain authoritative when telemetry sinks fail. + } +} + +function emitValidationTransitionMetrics( + metrics: DevelopmentLoopTransitionMetrics | undefined, + inputs: ValidationTransitionMetricInputs, +): void { + const recordStepDuration = metrics?.stepDuration ?? recordDevelopmentLoopStepDurationMetric; + const recordValidationDuration = + metrics?.validationDuration ?? recordDevelopmentLoopValidationDurationMetric; + const recordValidationOutcome = + metrics?.validationOutcome ?? recordDevelopmentLoopValidationOutcomeMetric; + + emitSafely(recordStepDuration, inputs.stepDuration); + for (const input of inputs.validationOutcomes) { + emitSafely(recordValidationOutcome, input); + } + for (const input of inputs.validationDurations) { + emitSafely(recordValidationDuration, input); + } +} + +export async function applyDevelopmentLoopValidationReport(input: { + database: DevelopmentLoopTransitionDatabase; + expectedValidationGates?: readonly ExpectedValidationGate[]; + logger?: LoopworksLogger; + metrics?: DevelopmentLoopTransitionMetrics; + occurredAt?: Date; + report: ValidationReportV1; + runId: string; +}): Promise { + const occurredAt = input.occurredAt ?? new Date(); + let metricInputs: ValidationTransitionMetricInputs | undefined; + + const result = await input.database.transaction(async (tx) => { + const [run] = await tx + .select({ + currentStage: loopRuns.currentStage, + id: loopRuns.id, + loopKey: loopRuns.loopKey, + metadata: loopRuns.metadata, + queuedAt: loopRuns.queuedAt, + repository: repositories.fullName, + startedAt: loopRuns.startedAt, + traceId: loopRuns.traceId, + }) + .from(loopRuns) + .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) + .where(eq(loopRuns.id, input.runId)) + .limit(1); + + if (!run) { + throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + } + + const [step] = await tx + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, input.runId), eq(runSteps.stage, "validation"))) + .limit(1); + + if (!step) { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} does not have a validation step.`, + ); + } + + const [artifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, step.id), + eq(artifacts.type, "validation_report"), + ), + ) + .limit(1); + + if (!artifact) { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} validation step ${step.id} does not have a validation_report artifact.`, + ); + } + + if (step.completedAt && (step.status === "succeeded" || step.status === "failed")) { + const traceId = step.traceId ?? run.traceId ?? undefined; + const persistedBlockedReason = getPersistedBlockedReason(run.metadata); + return { + ...(step.status === "failed" + ? { blockedReason: persistedBlockedReason ?? "Validation transition already failed." } + : {}), + idempotent: true, + runId: input.runId, + stage: step.stage, + status: step.status === "failed" ? "blocked" : "advanced", + stepId: step.id, + ...(traceId ? { traceId } : {}), + } satisfies ValidationTransitionResult; + } + + const blockedReason = getBlockedReason(input.report, input.expectedValidationGates); + const stepStatus = blockedReason ? "failed" : "succeeded"; + const stepDurationMs = sumValidationDurationMs(input.report); + const stepStartedAt = getStartedAtForDuration({ + completedAt: occurredAt, + durationMs: stepDurationMs, + startedAt: step.startedAt, + }); + const stepDurationSeconds = durationSecondsBetween(stepStartedAt, occurredAt); + const traceId = step.traceId ?? run.traceId; + const requiredSkippedCount = requiredSkippedResults(input.report).length; + + await tx + .update(artifacts) + .set({ + metadata: { + ...(artifact.metadata ?? {}), + ...createValidationReportArtifactMetadata(input.report), + }, + }) + .where(eq(artifacts.id, artifact.id)); + + await tx + .update(runSteps) + .set({ + completedAt: occurredAt, + metadata: createStepValidationMetadata({ + metadata: step.metadata, + report: input.report, + requiredSkippedCount, + }), + startedAt: stepStartedAt, + status: stepStatus, + traceId, + validationStatus: blockedReason ? "failed" : "passed", + }) + .where(eq(runSteps.id, step.id)); + + await tx + .update(loopRuns) + .set({ + currentStage: blockedReason ? "validation" : "code-review", + metadata: createValidationTransitionMetadata({ + blockedReason, + metadata: run.metadata, + report: input.report, + }), + startedAt: run.startedAt ?? run.queuedAt, + status: blockedReason ? "blocked" : "running", + }) + .where(eq(loopRuns.id, input.runId)); + + metricInputs = createValidationMetricInputs({ + loopKey: run.loopKey, + report: input.report, + stage: step.stage, + stepDurationSeconds, + stepStatus, + }); + + return { + ...(blockedReason ? { blockedReason } : {}), + runId: input.runId, + stage: step.stage, + status: blockedReason ? "blocked" : "advanced", + stepId: step.id, + ...(traceId ? { traceId } : {}), + } satisfies ValidationTransitionResult; + }); + + if (metricInputs) { + emitValidationTransitionMetrics(input.metrics, metricInputs); + } + + input.logger?.info( + { + blockedReason: result.blockedReason, + idempotent: "idempotent" in result ? result.idempotent : undefined, + runId: result.runId, + stage: result.stage, + status: result.status, + stepId: result.stepId, + traceId: result.traceId, + }, + "development_loop_validation_transition_persisted", + ); + + return result; +} + +export async function completeDevelopmentLoopRun(input: { + database: DevelopmentLoopTransitionDatabase; + logger?: LoopworksLogger; + metrics?: DevelopmentLoopTransitionMetrics; + occurredAt?: Date; + reason?: string; + runId: string; + status: DevelopmentLoopTerminalStatus; +}): Promise<{ + durationSeconds: number; + idempotent?: boolean; + runId: string; + status: DevelopmentLoopTerminalStatus; + traceId?: string; +}> { + const occurredAt = input.occurredAt ?? new Date(); + let runCompletedMetric: DevelopmentLoopRunCompletedMetricInput | undefined; + let runDurationMetric: DevelopmentLoopRunDurationMetricInput | undefined; + const terminalReason = normalizeReasonCode(input.reason); + + const result = await input.database.transaction(async (tx) => { + const [run] = await tx + .select({ + completedAt: loopRuns.completedAt, + currentStage: loopRuns.currentStage, + id: loopRuns.id, + loopKey: loopRuns.loopKey, + metadata: loopRuns.metadata, + queuedAt: loopRuns.queuedAt, + repository: repositories.fullName, + startedAt: loopRuns.startedAt, + status: loopRuns.status, + traceId: loopRuns.traceId, + }) + .from(loopRuns) + .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) + .where(eq(loopRuns.id, input.runId)) + .limit(1); + + if (!run) { + throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + } + + if ( + run.completedAt && + (run.status === "succeeded" || run.status === "failed" || run.status === "canceled") + ) { + return { + durationSeconds: durationSecondsBetween(run.startedAt ?? run.queuedAt, run.completedAt), + idempotent: true, + runId: input.runId, + status: run.status as DevelopmentLoopTerminalStatus, + ...(run.traceId ? { traceId: run.traceId } : {}), + }; + } + + const durationSeconds = durationSecondsBetween(run.startedAt ?? run.queuedAt, occurredAt); + await tx + .update(loopRuns) + .set({ + ...(input.status === "canceled" ? { canceledAt: occurredAt } : {}), + completedAt: occurredAt, + currentStage: input.status === "succeeded" ? "done" : run.currentStage, + metadata: { + ...(run.metadata ?? {}), + ...(terminalReason ? { terminalReason } : {}), + }, + status: input.status, + }) + .where(eq(loopRuns.id, input.runId)); + + runCompletedMetric = { + loopKey: run.loopKey, + repository: run.repository, + status: input.status, + }; + runDurationMetric = { + durationSeconds, + loopKey: run.loopKey, + status: input.status, + }; + + return { + durationSeconds, + runId: input.runId, + status: input.status, + ...(run.traceId ? { traceId: run.traceId } : {}), + }; + }); + + if (runCompletedMetric) { + emitSafely( + input.metrics?.runCompleted ?? recordDevelopmentLoopRunCompletedMetric, + runCompletedMetric, + ); + } + if (runDurationMetric) { + emitSafely( + input.metrics?.runDuration ?? recordDevelopmentLoopRunDurationMetric, + runDurationMetric, + ); + } + + input.logger?.info( + { + durationSeconds: result.durationSeconds, + idempotent: "idempotent" in result ? result.idempotent : undefined, + runId: result.runId, + status: result.status, + traceId: "traceId" in result ? result.traceId : undefined, + }, + "development_loop_run_completed", + ); + + return result; +} + +export async function retryDevelopmentLoopStep(input: { + database: DevelopmentLoopTransitionDatabase; + logger?: LoopworksLogger; + metrics?: DevelopmentLoopTransitionMetrics; + occurredAt?: Date; + reason: string; + runId: string; + stage: string; +}): Promise<{ + attempt: number; + idempotent?: boolean; + runId: string; + stage: string; + stepId: string; + traceId?: string; +}> { + const occurredAt = input.occurredAt ?? new Date(); + let retryMetric: DevelopmentLoopStepRetryMetricInput | undefined; + const reason = normalizeReasonCode(input.reason) ?? "unspecified"; + + const result = await input.database.transaction(async (tx) => { + const [run] = await tx + .select({ + id: loopRuns.id, + loopKey: loopRuns.loopKey, + metadata: loopRuns.metadata, + traceId: loopRuns.traceId, + }) + .from(loopRuns) + .where(eq(loopRuns.id, input.runId)) + .limit(1); + + if (!run) { + throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + } + + const [step] = await tx + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, input.runId), eq(runSteps.stage, input.stage))) + .limit(1); + + if (!step) { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} does not have a ${input.stage} step.`, + ); + } + + if (step.status !== "failed") { + const traceId = step.traceId ?? run.traceId ?? undefined; + return { + attempt: step.attempt, + idempotent: true, + runId: input.runId, + stage: input.stage, + stepId: step.id, + ...(traceId ? { traceId } : {}), + }; + } + + const attempt = step.attempt + 1; + await tx + .update(runSteps) + .set({ + attempt, + completedAt: null, + metadata: { + ...(step.metadata ?? {}), + lastRetryReason: reason, + retriedAt: occurredAt.toISOString(), + }, + queuedAt: occurredAt, + startedAt: null, + status: "queued", + traceId: step.traceId ?? run.traceId, + }) + .where(eq(runSteps.id, step.id)); + + await tx + .update(loopRuns) + .set({ + currentStage: input.stage, + metadata: { + ...metadataWithoutBlockedReason(run.metadata), + lastRetryReason: reason, + retryStage: input.stage, + }, + status: "queued", + }) + .where(eq(loopRuns.id, input.runId)); + + retryMetric = { + loopKey: run.loopKey, + reason, + stage: input.stage, + }; + + return { + attempt, + runId: input.runId, + stage: input.stage, + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + }; + }); + + if (retryMetric) { + emitSafely(input.metrics?.stepRetry ?? recordDevelopmentLoopStepRetryMetric, retryMetric); + } + + input.logger?.info( + { + attempt: result.attempt, + idempotent: "idempotent" in result ? result.idempotent : undefined, + runId: result.runId, + stage: result.stage, + stepId: result.stepId, + traceId: "traceId" in result ? result.traceId : undefined, + }, + "development_loop_step_retry_queued", + ); + + return result; +} diff --git a/src/lib/observability/metrics.ts b/src/lib/observability/metrics.ts index 742510d..14c23dc 100644 --- a/src/lib/observability/metrics.ts +++ b/src/lib/observability/metrics.ts @@ -185,7 +185,15 @@ const lockContentionCounters = new WeakMap>(); const approvalPendingGauges = new WeakMap>(); const queueDepthGauges = new WeakMap>(); const controlPlaneGaugeSourcesByMeter = new WeakMap(); +const runCompletedCounters = new WeakMap>(); +const runDurationHistograms = new WeakMap>(); +const stepDurationHistograms = new WeakMap>(); +const stepRetryCounters = new WeakMap>(); +const validationOutcomeCounters = new WeakMap>(); +const validationDurationHistograms = new WeakMap>(); const supportedGithubWebhookMetricEvents = new Set(["issues", "unknown", "unsupported"]); +const sensitiveMetricCommandPattern = + /\b(token|secret|password|authorization|credential|api[-_]?key|prompt)\b|Bearer\s+|gh[pousr]_|sk-[A-Za-z0-9_-]+/i; export type RunStartedMeter = Pick; export type CounterMeter = Pick; @@ -215,6 +223,45 @@ export type LockContentionMetricInput = { scope: string; }; +export type DevelopmentLoopRunMetricStatus = "succeeded" | "failed" | "canceled" | "cancelled"; + +export type DevelopmentLoopRunCompletedMetricInput = { + loopKey: string; + repository: string; + status: DevelopmentLoopRunMetricStatus; +}; + +export type DevelopmentLoopRunDurationMetricInput = { + durationSeconds: number; + loopKey: string; + status: DevelopmentLoopRunMetricStatus; +}; + +export type DevelopmentLoopStepDurationMetricInput = { + durationSeconds: number; + loopKey: string; + stage: string; + status: "succeeded" | "failed" | "skipped"; +}; + +export type DevelopmentLoopStepRetryMetricInput = { + loopKey: string; + reason: string; + stage: string; +}; + +export type DevelopmentLoopValidationOutcomeMetricInput = { + command: string; + gate: string; + status: "pass" | "fail"; +}; + +export type DevelopmentLoopValidationDurationMetricInput = { + command: string; + durationSeconds: number; + gate: string; +}; + export type ControlPlanePendingApprovalMeasurement = { gate: string; value: number; @@ -268,6 +315,96 @@ function getRunStartedCounter(meter: RunStartedMeter): Counter return counter; } +function getRunCompletedCounter(meter: CounterMeter): Counter { + const cached = runCompletedCounters.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.run.completed"); + const counter = meter.createCounter(metric.name, { + description: "Development and workflow runs completed by Loopworks.", + unit: metric.unit, + }); + runCompletedCounters.set(meter, counter); + return counter; +} + +function getRunDurationHistogram(meter: HistogramMeter): Histogram { + const cached = runDurationHistograms.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.run.duration"); + const histogram = meter.createHistogram(metric.name, { + description: "Elapsed seconds for completed Loopworks runs.", + unit: metric.unit, + }); + runDurationHistograms.set(meter, histogram); + return histogram; +} + +function getStepDurationHistogram(meter: HistogramMeter): Histogram { + const cached = stepDurationHistograms.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.step.duration"); + const histogram = meter.createHistogram(metric.name, { + description: "Elapsed seconds for completed Loopworks run steps.", + unit: metric.unit, + }); + stepDurationHistograms.set(meter, histogram); + return histogram; +} + +function getStepRetryCounter(meter: CounterMeter): Counter { + const cached = stepRetryCounters.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.step.retries"); + const counter = meter.createCounter(metric.name, { + description: "Retry attempts for Loopworks run steps.", + unit: metric.unit, + }); + stepRetryCounters.set(meter, counter); + return counter; +} + +function getValidationOutcomeCounter(meter: CounterMeter): Counter { + const cached = validationOutcomeCounters.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.validation.outcome"); + const counter = meter.createCounter(metric.name, { + description: "Deterministic validation gate outcomes.", + unit: metric.unit, + }); + validationOutcomeCounters.set(meter, counter); + return counter; +} + +function getValidationDurationHistogram(meter: HistogramMeter): Histogram { + const cached = validationDurationHistograms.get(meter); + if (cached) { + return cached; + } + + const metric = resolveObservabilityMetricDefinition("loopworks.validation.duration"); + const histogram = meter.createHistogram(metric.name, { + description: "Elapsed seconds for deterministic validation gates.", + unit: metric.unit, + }); + validationDurationHistograms.set(meter, histogram); + return histogram; +} + function getWebhookOutcomeCounter(meter: CounterMeter): Counter { const cached = webhookOutcomeCounters.get(meter); if (cached) { @@ -348,6 +485,25 @@ function normalizeMetricAttribute(value: string | null | undefined, fallback: st return normalized || fallback; } +function normalizeRunMetricStatus( + status: DevelopmentLoopRunMetricStatus, +): Exclude { + return status === "canceled" ? "cancelled" : status; +} + +function sanitizeMetricCommandAttribute(command: string): string { + const trimmed = command.trim(); + if (!trimmed) { + return "unknown"; + } + + if (sensitiveMetricCommandPattern.test(trimmed)) { + return "[redacted]"; + } + + return trimmed.slice(0, 200); +} + function normalizeGithubWebhookEventAttribute(value: string): string { const normalized = normalizeMetricAttribute(value, "unknown"); return supportedGithubWebhookMetricEvents.has(normalized) ? normalized : "unsupported"; @@ -383,6 +539,94 @@ export function recordGithubWebhookOutcomeMetric( } } +export function recordDevelopmentLoopRunCompletedMetric( + input: DevelopmentLoopRunCompletedMetricInput, + meter: CounterMeter = getLoopworksMeter(), +): void { + try { + getRunCompletedCounter(meter).add(1, { + "loop.key": input.loopKey, + repository: input.repository, + status: normalizeRunMetricStatus(input.status), + }); + } catch { + // OTel emission must never affect run lifecycle persistence. + } +} + +export function recordDevelopmentLoopRunDurationMetric( + input: DevelopmentLoopRunDurationMetricInput, + meter: HistogramMeter = getLoopworksMeter(), +): void { + try { + getRunDurationHistogram(meter).record(Math.max(0, input.durationSeconds), { + "loop.key": input.loopKey, + status: normalizeRunMetricStatus(input.status), + }); + } catch { + // OTel emission must never affect run lifecycle persistence. + } +} + +export function recordDevelopmentLoopStepDurationMetric( + input: DevelopmentLoopStepDurationMetricInput, + meter: HistogramMeter = getLoopworksMeter(), +): void { + try { + getStepDurationHistogram(meter).record(Math.max(0, input.durationSeconds), { + "loop.key": input.loopKey, + stage: normalizeMetricAttribute(input.stage, "unknown"), + status: input.status, + }); + } catch { + // OTel emission must never affect step lifecycle persistence. + } +} + +export function recordDevelopmentLoopStepRetryMetric( + input: DevelopmentLoopStepRetryMetricInput, + meter: CounterMeter = getLoopworksMeter(), +): void { + try { + getStepRetryCounter(meter).add(1, { + "loop.key": input.loopKey, + reason: normalizeMetricAttribute(input.reason, "unknown"), + stage: normalizeMetricAttribute(input.stage, "unknown"), + }); + } catch { + // OTel emission must never affect step retry persistence. + } +} + +export function recordDevelopmentLoopValidationOutcomeMetric( + input: DevelopmentLoopValidationOutcomeMetricInput, + meter: CounterMeter = getLoopworksMeter(), +): void { + try { + getValidationOutcomeCounter(meter).add(1, { + command: sanitizeMetricCommandAttribute(input.command), + gate: normalizeMetricAttribute(input.gate, "unknown"), + status: input.status, + }); + } catch { + // OTel emission must never affect validation transition persistence. + } +} + +export function recordDevelopmentLoopValidationDurationMetric( + input: DevelopmentLoopValidationDurationMetricInput, + meter: HistogramMeter = getLoopworksMeter(), +): void { + try { + getValidationDurationHistogram(meter).record(Math.max(0, input.durationSeconds), { + command: sanitizeMetricCommandAttribute(input.command), + gate: normalizeMetricAttribute(input.gate, "unknown"), + }); + } catch { + // OTel emission must never affect validation transition persistence. + } +} + export function recordApprovalWaitTimeMetric( input: ApprovalWaitTimeMetricInput, meter: HistogramMeter = getLoopworksMeter(), diff --git a/tests/unit/loops/development-run-transitions.test.ts b/tests/unit/loops/development-run-transitions.test.ts new file mode 100644 index 0000000..b08d296 --- /dev/null +++ b/tests/unit/loops/development-run-transitions.test.ts @@ -0,0 +1,712 @@ +/** @vitest-environment node */ +import { and, eq } from "drizzle-orm"; + +import { artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import { + createDevelopmentLoopRun, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + applyDevelopmentLoopValidationReport, + completeDevelopmentLoopRun, + type DevelopmentLoopTransitionDatabase, + retryDevelopmentLoopStep, +} from "@/lib/loops/development-run-transitions"; +import { + type ValidationGateResultV1, + type ValidationReportV1, + validationReportSchemaId, + validationReportV1Schema, +} from "@/lib/loops/validation-report"; +import type { LoopworksLogger } from "@/lib/observability/logger"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const issueTrigger = { + body: "## Acceptance Criteria\n- Lifecycle telemetry records deterministic validation outcomes.", + deliveryId: "issue-73-delivery", + issueNumber: 73, + issueUrl: "https://github.com/ncolesummers/loopworks/issues/73", + labels: ["agent-ready", "area:loops", "area:validation", "area:observability"], + milestone: "M4 Validation + PR Path + MVP Security Review", + repositoryFullName: "ncolesummers/loopworks", + title: "Add lifecycle telemetry to deterministic validation and run transitions", +}; + +function testRunDatabase(context: PgliteTestDatabase): DevelopmentLoopRunDatabase { + return context.db as unknown as DevelopmentLoopRunDatabase; +} + +function transitionDatabase(context: PgliteTestDatabase): DevelopmentLoopTransitionDatabase { + return context.db as unknown as DevelopmentLoopTransitionDatabase; +} + +async function insertRepository(context: PgliteTestDatabase) { + await context.db.insert(repositories).values({ + githubRepoId: 73_000_001, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Agent-ready development loop"], + validationGates: ["Focused tests", "Aggregate validation"], + }); +} + +async function createRun(context: PgliteTestDatabase) { + await insertRepository(context); + + const run = await createDevelopmentLoopRun({ + database: testRunDatabase(context), + now: () => new Date("2026-07-08T16:00:00.000Z"), + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + trigger: issueTrigger, + }); + + if (run.mode !== "created") { + throw new Error("Expected test run creation."); + } + + return run.runId; +} + +function gateResult(input: { + durationMs: number; + exitCode: number | null; + key: string; + outcome: "pass" | "fail" | "skipped"; + required: boolean; + skipReason?: string; +}): ValidationGateResultV1 { + return { + command: `bun run ${input.key}`, + durationMs: input.durationMs, + exitCode: input.exitCode, + key: input.key, + name: input.key + .split("-") + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "), + outcome: input.outcome, + phase: "before_review", + produces: "validation_report", + required: input.required, + ...(input.skipReason ? { skipReason: input.skipReason } : {}), + }; +} + +function validationReport(results: ValidationGateResultV1[]): ValidationReportV1 { + const counts = { + failed: results.filter((result) => result.outcome === "fail").length, + passed: results.filter((result) => result.outcome === "pass").length, + skipped: results.filter((result) => result.outcome === "skipped").length, + total: results.length, + }; + + return validationReportV1Schema.parse({ + counts, + generatedAt: "2026-07-08T16:05:00.000Z", + overallOutcome: counts.failed > 0 ? "fail" : counts.passed > 0 ? "pass" : "skipped", + results, + schemaId: validationReportSchemaId, + version: 1, + }); +} + +function createMetricRecorder() { + return { + runCompleted: vi.fn(), + runDuration: vi.fn(), + stepDuration: vi.fn(), + stepRetry: vi.fn(), + validationDuration: vi.fn(), + validationOutcome: vi.fn(), + }; +} + +describe("development-loop run transitions", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + }); + + afterEach(async () => { + await context.close(); + }); + + it("persists a passing validation report, advances to review, and skips optional telemetry", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + gateResult({ + durationMs: 0, + exitCode: null, + key: "playwright", + outcome: "skipped", + required: false, + skipReason: "No UI change in this issue.", + }), + ]); + + const validationStepBefore = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))) + .limit(1); + const validationArtifactBefore = await context.db + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, runId), + eq(artifacts.stepId, validationStepBefore[0]?.id ?? ""), + eq(artifacts.type, "validation_report"), + ), + ) + .limit(1); + + await expect( + applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }), + ).resolves.toMatchObject({ + runId, + status: "advanced", + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + const validationArtifacts = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, runId), eq(artifacts.type, "validation_report"))); + const [validationArtifact] = validationArtifacts.filter( + (artifact) => artifact.stepId === validationStep?.id, + ); + + expect(run).toMatchObject({ + currentStage: "code-review", + status: "running", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + expect(run.metadata).not.toMatchObject({ + blockedReason: expect.any(String), + }); + expect(validationStep).toMatchObject({ + id: validationStepBefore[0]?.id, + status: "succeeded", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + validationStatus: "passed", + }); + expect(validationArtifacts).toHaveLength(2); + expect(validationArtifact).toMatchObject({ + id: validationArtifactBefore[0]?.id, + runId, + stepId: validationStep?.id, + type: "validation_report", + }); + expect(validationArtifact?.metadata).toMatchObject({ + validationReportMetadataKind: "validation_report_result", + validationReport: report, + validationReportSchemaId, + validationReportVersion: 1, + }); + expect(metrics.validationOutcome).toHaveBeenCalledTimes(1); + expect(metrics.validationOutcome).toHaveBeenCalledWith({ + command: "bun run focused-tests", + gate: "focused-tests", + status: "pass", + }); + expect(metrics.validationDuration).toHaveBeenCalledTimes(1); + expect(metrics.stepDuration).toHaveBeenCalledWith({ + durationSeconds: 1, + loopKey: "development-loop", + stage: "validation", + status: "succeeded", + }); + }); + + it("blocks downstream stages when validation fails while preserving inspectable artifacts", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + gateResult({ + durationMs: 2000, + exitCode: 1, + key: "aggregate-validation", + outcome: "fail", + required: true, + }), + ]); + + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + const [validationArtifact] = await context.db + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, runId), + eq(artifacts.stepId, validationStep?.id ?? ""), + eq(artifacts.type, "validation_report"), + ), + ); + + expect(run).toMatchObject({ + currentStage: "validation", + status: "blocked", + }); + expect(run.metadata).toMatchObject({ + blockedReason: "Deterministic validation failed before review.", + }); + expect(validationStep).toMatchObject({ + status: "failed", + validationStatus: "failed", + }); + expect(validationArtifact.metadata).toMatchObject({ + validationReportMetadataKind: "validation_report_result", + validationReport: report, + }); + expect(metrics.validationOutcome).toHaveBeenCalledWith({ + command: "bun run aggregate-validation", + gate: "aggregate-validation", + status: "fail", + }); + expect(metrics.stepDuration).toHaveBeenCalledWith({ + durationSeconds: 3, + loopKey: "development-loop", + stage: "validation", + status: "failed", + }); + }); + + it("returns the persisted blocked reason when a failed validation transition is replayed", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 3000, + exitCode: 1, + key: "aggregate-validation", + outcome: "fail", + required: true, + }), + ]); + + const firstResult = await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + const replayResult = await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + expect(firstResult).toMatchObject({ + blockedReason: "Deterministic validation failed before review.", + status: "blocked", + }); + expect(replayResult).toMatchObject({ + blockedReason: "Deterministic validation failed before review.", + idempotent: true, + status: "blocked", + }); + expect(metrics.validationOutcome).toHaveBeenCalledTimes(1); + expect(metrics.validationDuration).toHaveBeenCalledTimes(1); + expect(metrics.stepDuration).toHaveBeenCalledTimes(1); + }); + + it("blocks required skipped gates without emitting skipped validation metrics", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + gateResult({ + durationMs: 0, + exitCode: null, + key: "aggregate-validation", + outcome: "skipped", + required: true, + skipReason: "Required gate was unavailable.", + }), + ]); + + expect(report.overallOutcome).toBe("pass"); + + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + expect(run).toMatchObject({ + currentStage: "validation", + status: "blocked", + }); + expect(run.metadata).toMatchObject({ + blockedReason: "Required validation gate skipped before review.", + }); + expect(validationStep).toMatchObject({ + status: "failed", + validationStatus: "failed", + }); + expect(metrics.validationOutcome).toHaveBeenCalledTimes(1); + expect(metrics.validationOutcome).not.toHaveBeenCalledWith( + expect.objectContaining({ + status: "skipped", + }), + ); + }); + + it("blocks empty validation reports with a dedicated blocked reason", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([]); + + const result = await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + expect(result).toMatchObject({ + blockedReason: "Validation report contained no gate results.", + status: "blocked", + }); + expect(run).toMatchObject({ + currentStage: "validation", + status: "blocked", + }); + expect(run.metadata).toMatchObject({ + blockedReason: "Validation report contained no gate results.", + }); + expect(validationStep).toMatchObject({ + status: "failed", + validationStatus: "failed", + }); + expect(metrics.validationOutcome).not.toHaveBeenCalled(); + }); + + it("fails closed when required validation gates are missing from the report", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + ]); + + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + expectedValidationGates: [ + { key: "focused-tests", required: true }, + { key: "aggregate-validation", required: true }, + ], + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + expect(run).toMatchObject({ + currentStage: "validation", + status: "blocked", + }); + expect(run.metadata).toMatchObject({ + blockedReason: "Required validation gate missing before review.", + }); + expect(validationStep).toMatchObject({ + status: "failed", + validationStatus: "failed", + }); + expect(metrics.validationOutcome).toHaveBeenCalledTimes(1); + }); + + it("does not emit duplicate validation metrics when the same report transition is replayed", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + ]); + + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + expect(metrics.validationOutcome).toHaveBeenCalledTimes(1); + expect(metrics.validationDuration).toHaveBeenCalledTimes(1); + expect(metrics.stepDuration).toHaveBeenCalledTimes(1); + }); + + it("includes persisted trace ids in transition logs without active trace context", async () => { + const runId = await createRun(context); + const loggerInfo = vi.fn(); + const logger = { + info: loggerInfo, + } as unknown as LoopworksLogger; + const report = validationReport([ + gateResult({ + durationMs: 1000, + exitCode: 0, + key: "focused-tests", + outcome: "pass", + required: true, + }), + ]); + + await applyDevelopmentLoopValidationReport({ + database: transitionDatabase(context), + logger, + occurredAt: new Date("2026-07-08T16:05:00.000Z"), + report, + runId, + }); + + expect(loggerInfo).toHaveBeenCalledWith( + expect.objectContaining({ + runId, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }), + "development_loop_validation_transition_persisted", + ); + }); + + it.each([ + { expectedMetricStatus: "succeeded", status: "succeeded" }, + { expectedMetricStatus: "failed", status: "failed" }, + { expectedMetricStatus: "cancelled", status: "canceled" }, + ] as const)("completes a run as $status and emits run lifecycle metrics", async ({ + expectedMetricStatus, + status, + }) => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + await context.db + .update(loopRuns) + .set({ + currentStage: "done", + startedAt: new Date("2026-07-08T16:00:00.000Z"), + status: "running", + }) + .where(eq(loopRuns.id, runId)); + + await completeDevelopmentLoopRun({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:10:00.000Z"), + runId, + status, + }); + await completeDevelopmentLoopRun({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:10:00.000Z"), + runId, + status, + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + + expect(run).toMatchObject({ + status, + completedAt: new Date("2026-07-08T16:10:00.000Z"), + }); + if (status === "canceled") { + expect(run.canceledAt).toEqual(new Date("2026-07-08T16:10:00.000Z")); + } + expect(metrics.runCompleted).toHaveBeenCalledWith({ + loopKey: "development-loop", + repository: "ncolesummers/loopworks", + status, + }); + expect(metrics.runDuration).toHaveBeenCalledWith({ + durationSeconds: 600, + loopKey: "development-loop", + status, + }); + expect(metrics.runCompleted).toHaveBeenCalledTimes(1); + expect(metrics.runDuration).toHaveBeenCalledTimes(1); + expect(expectedMetricStatus).toBe(status === "canceled" ? "cancelled" : status); + }); + + it("queues a retry for an implemented transition branch and records retry telemetry", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + await context.db + .update(runSteps) + .set({ + completedAt: new Date("2026-07-08T16:05:00.000Z"), + startedAt: new Date("2026-07-08T16:04:00.000Z"), + status: "failed", + }) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + await retryDevelopmentLoopStep({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:06:00.000Z"), + reason: "validation_failed", + runId, + stage: "validation", + }); + + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + expect(run).toMatchObject({ + currentStage: "validation", + status: "queued", + }); + expect(validationStep).toMatchObject({ + attempt: 2, + completedAt: null, + queuedAt: new Date("2026-07-08T16:06:00.000Z"), + startedAt: null, + status: "queued", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + expect(metrics.stepRetry).toHaveBeenCalledWith({ + loopKey: "development-loop", + reason: "validation_failed", + stage: "validation", + }); + }); + + it("does not duplicate retry telemetry and sanitizes unsafe retry reasons", async () => { + const runId = await createRun(context); + const metrics = createMetricRecorder(); + await context.db + .update(runSteps) + .set({ + completedAt: new Date("2026-07-08T16:05:00.000Z"), + startedAt: new Date("2026-07-08T16:04:00.000Z"), + status: "failed", + }) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + await retryDevelopmentLoopStep({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:06:00.000Z"), + reason: "validation failed token=secret", + runId, + stage: "validation", + }); + await retryDevelopmentLoopStep({ + database: transitionDatabase(context), + metrics, + occurredAt: new Date("2026-07-08T16:06:00.000Z"), + reason: "validation failed token=secret", + runId, + stage: "validation", + }); + + const [validationStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, "validation"))); + + expect(validationStep).toMatchObject({ + attempt: 2, + status: "queued", + }); + expect(validationStep.metadata).toMatchObject({ + lastRetryReason: "unspecified", + }); + expect(metrics.stepRetry).toHaveBeenCalledTimes(1); + expect(metrics.stepRetry).toHaveBeenCalledWith({ + loopKey: "development-loop", + reason: "unspecified", + stage: "validation", + }); + }); +}); diff --git a/tests/unit/observability/metrics.test.ts b/tests/unit/observability/metrics.test.ts index 43dfccb..d242e21 100644 --- a/tests/unit/observability/metrics.test.ts +++ b/tests/unit/observability/metrics.test.ts @@ -8,9 +8,15 @@ import { collectControlPlaneGaugeMeasurements, developmentLoopRunCreatedDurableMetricName, observabilityMetricNames, + recordDevelopmentLoopRunCompletedMetric, recordApprovalWaitTimeMetric, recordDevelopmentLoopRunCreatedObservability, + recordDevelopmentLoopRunDurationMetric, recordDevelopmentLoopRunStartedMetric, + recordDevelopmentLoopStepDurationMetric, + recordDevelopmentLoopStepRetryMetric, + recordDevelopmentLoopValidationDurationMetric, + recordDevelopmentLoopValidationOutcomeMetric, recordGithubWebhookOutcomeMetric, recordLockContentionMetric, registerControlPlaneGaugeMetrics, @@ -188,6 +194,212 @@ describe("ADR 0012 observability metric contract", () => { ]); }); + it("records lifecycle metrics with ADR attributes and cancellation spelling", () => { + const recordings: { + attributes: Record | undefined; + name: string; + type: "counter" | "histogram"; + value: number; + }[] = []; + const meter = { + createCounter(name: string) { + return { + add(value: number, attributes?: Record) { + recordings.push({ attributes, name, type: "counter", value }); + }, + }; + }, + createHistogram(name: string) { + return { + record(value: number, attributes?: Record) { + recordings.push({ attributes, name, type: "histogram", value }); + }, + }; + }, + }; + + recordDevelopmentLoopRunCompletedMetric( + { + loopKey: "development-loop", + repository: "ncolesummers/loopworks", + status: "canceled", + }, + meter, + ); + recordDevelopmentLoopRunDurationMetric( + { + durationSeconds: 600, + loopKey: "development-loop", + status: "canceled", + }, + meter, + ); + recordDevelopmentLoopStepDurationMetric( + { + durationSeconds: 12, + loopKey: "development-loop", + stage: "validation", + status: "failed", + }, + meter, + ); + recordDevelopmentLoopStepRetryMetric( + { + loopKey: "development-loop", + reason: "validation_failed", + stage: "validation", + }, + meter, + ); + recordDevelopmentLoopValidationOutcomeMetric( + { + command: "bun run test", + gate: "unit-tests", + status: "fail", + }, + meter, + ); + recordDevelopmentLoopValidationDurationMetric( + { + command: "bun run test", + durationSeconds: 4, + gate: "unit-tests", + }, + meter, + ); + + expect(recordings).toEqual([ + { + attributes: { + "loop.key": "development-loop", + repository: "ncolesummers/loopworks", + status: "cancelled", + }, + name: "loopworks.run.completed", + type: "counter", + value: 1, + }, + { + attributes: { + "loop.key": "development-loop", + status: "cancelled", + }, + name: "loopworks.run.duration", + type: "histogram", + value: 600, + }, + { + attributes: { + "loop.key": "development-loop", + stage: "validation", + status: "failed", + }, + name: "loopworks.step.duration", + type: "histogram", + value: 12, + }, + { + attributes: { + "loop.key": "development-loop", + reason: "validation_failed", + stage: "validation", + }, + name: "loopworks.step.retries", + type: "counter", + value: 1, + }, + { + attributes: { + command: "bun run test", + gate: "unit-tests", + status: "fail", + }, + name: "loopworks.validation.outcome", + type: "counter", + value: 1, + }, + { + attributes: { + command: "bun run test", + gate: "unit-tests", + }, + name: "loopworks.validation.duration", + type: "histogram", + value: 4, + }, + ]); + }); + + it("keeps lifecycle persistence independent from telemetry sink failures", () => { + const meter = { + createCounter() { + throw new Error("counter unavailable"); + }, + createHistogram() { + throw new Error("histogram unavailable"); + }, + }; + + expect(() => + recordDevelopmentLoopRunCompletedMetric( + { + loopKey: "development-loop", + repository: "ncolesummers/loopworks", + status: "succeeded", + }, + meter, + ), + ).not.toThrow(); + expect(() => + recordDevelopmentLoopValidationDurationMetric( + { + command: "bun run validate", + durationSeconds: 1, + gate: "aggregate-validation", + }, + meter, + ), + ).not.toThrow(); + }); + + it("redacts sensitive validation command metric attributes", () => { + const recordings: { + attributes: Record | undefined; + name: string; + value: number; + }[] = []; + const meter = { + createCounter(name: string) { + return { + add(value: number, attributes?: Record) { + recordings.push({ attributes, name, value }); + }, + }; + }, + }; + + recordDevelopmentLoopValidationOutcomeMetric( + { + command: "bun run test --token ghp_secret", + gate: "unit-tests", + status: "fail", + }, + meter, + ); + + expect(recordings).toEqual([ + { + attributes: { + command: "[redacted]", + gate: "unit-tests", + status: "fail", + }, + name: "loopworks.validation.outcome", + value: 1, + }, + ]); + }); + it("registers pending approval and queue-depth observable gauges", async () => { const callbacks: { callback: (result: {