diff --git a/cli/README.md b/cli/README.md index b95eb5648..2a41a5df6 100644 --- a/cli/README.md +++ b/cli/README.md @@ -76,6 +76,16 @@ so it prints that snapshot as the matching data restore point (`aws rds restore-db-instance-from-db-snapshot`). Pre-deploy snapshots are pruned to a bounded count; `aws.predeployDbSnapshot: false` opts out. +AWS `plan` performs metadata and static validation only and never starts an ECS +task. The CLI runner uses Secrets Manager metadata and does not directly retrieve +plaintext secret values during `plan`, `up`, or `check --live`. For `up` and live +checks, ECS injects required values into a stack-scoped validation task, which +returns only a bounded status. This limits accidental exposure through the CLI +transport; it is not a defense against a malicious or compromised deployment +principal. A principal that can deploy arbitrary runtime code and pass a role +authorized to read runtime secrets can exfiltrate those secrets, so workflow +review and least-privilege control of that principal remain required. + `sandbox build` is a local validation build. `sandbox publish` pushes through the configured OCI registry, resolves the image and base digests, records the base pin in the config and the image pin in the config (docker/fly) or the durable AWS deployment diff --git a/cli/src/aws-private-core-request.ts b/cli/src/aws-private-core-request.ts new file mode 100644 index 000000000..2f9ab988f --- /dev/null +++ b/cli/src/aws-private-core-request.ts @@ -0,0 +1,135 @@ +export const AWS_PRIVATE_CORE_REQUEST_BODY_MAX_BYTES = 1_000_000; +export const AWS_PRIVATE_CORE_RESPONSE_MAX_BYTES = 1_100_000; +export const AWS_PRIVATE_CORE_OBJECT_MAX_BYTES = 2_300_000; +export const AWS_PRIVATE_CORE_OVERRIDES_MAX_BYTES = 8_192; + +const FAILURE_CODES = new Set([ + "input_unavailable", + "invalid_input", + "missing_signing_secret", + "core_unavailable", + "core_response_too_large", +]); + +export type AwsPrivateCoreFailureCode = + "input_unavailable" | "invalid_input" | "missing_signing_secret" | "core_unavailable" | "core_response_too_large"; + +export type AwsPrivateCoreResponse = + { version: 1; ok: true; status: number; body: string } | { version: 1; ok: false; code: AwsPrivateCoreFailureCode }; + +export interface AwsPrivateSecretValidation { + command: string[]; + environment: Array<{ name: string; value: string }>; + invalidSecret: (exitCode: number | undefined) => string | undefined; +} + +export function awsPrivateCoreKeys(requestId: string): { request: string; response: string } { + const prefix = `deployment/core-requests/${requestId}`; + return { request: `${prefix}/request.json`, response: `${prefix}/response.json` }; +} + +export function awsPrivateCoreRequestBody(method: "GET" | "PUT", body: string): string { + if (Buffer.byteLength(body) > AWS_PRIVATE_CORE_REQUEST_BODY_MAX_BYTES || (method === "GET" && body !== "")) { + throw new Error("AWS private core request body is invalid"); + } + const encoded = JSON.stringify({ version: 1, method, body }); + if (Buffer.byteLength(encoded) > AWS_PRIVATE_CORE_OBJECT_MAX_BYTES) { + throw new Error("AWS private core request object is too large"); + } + return encoded; +} + +export function parseAwsPrivateCoreResponse(body: string): AwsPrivateCoreResponse { + if (Buffer.byteLength(body) > AWS_PRIVATE_CORE_OBJECT_MAX_BYTES) { + throw new Error("AWS private core response object is too large"); + } + let value: unknown; + try { + value = JSON.parse(body) as unknown; + } catch { + throw new Error("AWS private core response object is invalid"); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("AWS private core response object is invalid"); + } + const result = value as Record; + const keys = Object.keys(result).sort().join("\0"); + if (result.version !== 1 || typeof result.ok !== "boolean") { + throw new Error("AWS private core response object is invalid"); + } + if (result.ok) { + if ( + keys !== ["body", "ok", "status", "version"].sort().join("\0") || + !Number.isInteger(result.status) || + (result.status as number) < 200 || + (result.status as number) > 599 || + typeof result.body !== "string" || + Buffer.byteLength(result.body) > AWS_PRIVATE_CORE_RESPONSE_MAX_BYTES || + (((result.status as number) < 200 || (result.status as number) >= 300) && result.body !== "") + ) { + throw new Error("AWS private core response object is invalid"); + } + return { version: 1, ok: true, status: result.status as number, body: result.body }; + } + if ( + keys !== ["code", "ok", "version"].sort().join("\0") || + typeof result.code !== "string" || + !FAILURE_CODES.has(result.code as AwsPrivateCoreFailureCode) + ) { + throw new Error("AWS private core response object is invalid"); + } + return { version: 1, ok: false, code: result.code as AwsPrivateCoreFailureCode }; +} + +export function awsPrivateSecretValidation(names: string[], expectedPublicApiUrl: string): AwsPrivateSecretValidation { + if ( + names.length === 0 || + names.length > 245 || + names.some((name) => !/^[A-Z][A-Z0-9_]*$/.test(name)) || + names.some((name, index) => names.indexOf(name) !== index) + ) { + throw new Error("AWS private secret validation specification is invalid"); + } + const expected = new URL(expectedPublicApiUrl); + if (expected.protocol !== "https:") throw new Error("AWS private secret validation URL must use HTTPS"); + const ordered = [...names].sort(); + const script = `command -v printenv >/dev/null && command -v sed >/dev/null && command -v tr >/dev/null && command -v awk >/dev/null || exit 2 +i=10 +while IFS= read -r name; do + value="$(printenv "$name" 2>/dev/null || true)" + trimmed="$(printf '%s' "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + lower="$(printf '%s' "$trimmed" | tr '[:upper:]' '[:lower:]')" + invalid=0 + case "$lower" in ''|replace-me|placeholder|changeme|todo) invalid=1;; esac + case "$name" in + CONNECTOR_SECRET_KEY|CORE_SIGNING_SECRET|SKILL_SIGNING_SECRET) [ "\${#trimmed}" -ge 32 ] || invalid=1;; + ADMIN_GRANTS) printf '%s\n' "$trimmed" | awk -F, '{if(NF<1)exit 1;for(i=1;i<=NF;i++){e=$i;gsub(/^[ \t]+|[ \t]+$/, "", e);if(!match(e,/:[^:]*$/))exit 1;p=substr(e,1,RSTART-1);r=substr(e,RSTART+1);gsub(/^[ \t]+|[ \t]+$/, "", p);gsub(/^[ \t]+|[ \t]+$/, "", r);if(p==""||r!="org_admin")exit 1}}' || invalid=1;; + PUBLIC_API_URL) [ "\${trimmed%/}" = "$QM_EXPECTED_PUBLIC_API_URL" ] || invalid=1;; + esac + [ "$invalid" -eq 0 ] || exit "$i" + i=$((i+1)) +done < + exitCode !== undefined && exitCode >= 10 && exitCode < 10 + ordered.length ? ordered[exitCode - 10] : undefined, + }; +} + +export function awsPrivateCoreTaskScript(): string { + return `(async()=>{const{createHmac}=require("node:crypto"),{S3Client,GetObjectCommand,PutObjectCommand,DeleteObjectCommand}=require("@aws-sdk/client-s3"); +const maxRequest=${AWS_PRIVATE_CORE_REQUEST_BODY_MAX_BYTES},maxResponse=${AWS_PRIVATE_CORE_RESPONSE_MAX_BYTES},maxObject=${AWS_PRIVATE_CORE_OBJECT_MAX_BYTES},id=process.env.QM_AWS_CORE_REQUEST_ID||"",bucket=process.env.QM_AWS_CORE_REQUEST_BUCKET||"",core=process.env.QM_AWS_CORE_REQUEST_URL||"",secret=process.env.CORE_SIGNING_SECRET||"",prefix="deployment/core-requests/"+id,inputKey=prefix+"/request.json",outputKey=prefix+"/response.json",s3=new S3Client(process.env.S3_REGION?{region:process.env.S3_REGION}:{}); +class F extends Error{constructor(code){super(code);this.code=code}} +const exact=(o,n)=>Object.keys(o).sort().join("\\0")===n.sort().join("\\0"),bounded=async r=>{const declared=Number(r.headers.get("content-length"));if(Number.isFinite(declared)&&declared>maxResponse)throw new F("core_response_too_large");if(!r.body)return"";const reader=r.body.getReader(),chunks=[];let size=0;for(;;){const{done,value}=await reader.read();if(done)break;size+=value.byteLength;if(size>maxResponse){await reader.cancel().catch(()=>{});throw new F("core_response_too_large")}chunks.push(Buffer.from(value))}return Buffer.concat(chunks).toString("utf8")}; +const fail=e=>({version:1,ok:false,code:e instanceof F?e.code:"core_unavailable"}); +let result; +try{if(!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(id)||!bucket)throw new F("invalid_input");let origin;try{origin=new URL(core)}catch{throw new F("invalid_input")}if(origin.protocol!=="http:"||!origin.hostname||origin.username||origin.password||origin.pathname!=="/"||origin.search||origin.hash)throw new F("invalid_input");let got;try{got=await s3.send(new GetObjectCommand({Bucket:bucket,Key:inputKey}))}catch{throw new F("input_unavailable")}if(!got.Body||!Number.isInteger(got.ContentLength)||got.ContentLength<1||got.ContentLength>maxObject)throw new F("input_unavailable");let bytes;try{bytes=Buffer.from(await got.Body.transformToByteArray())}catch{throw new F("input_unavailable")}if(bytes.length!==got.ContentLength)throw new F("input_unavailable");let request;try{request=JSON.parse(bytes.toString("utf8"))}catch{throw new F("invalid_input")}if(!request||typeof request!=="object"||!exact(request,["version","method","body"])||request.version!==1||(request.method!=="GET"&&request.method!=="PUT")||typeof request.body!=="string"||Buffer.byteLength(request.body)>maxRequest||(request.method==="GET"&&request.body!==""))throw new F("invalid_input");if(!secret)throw new F("missing_signing_secret");const path="/v1/deployment-layer",timestamp=Math.floor(Date.now()/1000),canonical=request.method+"\\n"+path+"\\n"+request.body,signature="v0="+createHmac("sha256",secret).update("v0:"+timestamp+":"+canonical).digest("hex");let response;try{response=await fetch(origin.toString().replace(/\\/$/,"")+path,{method:request.method,headers:{"content-type":"application/json","x-timestamp":String(timestamp),"x-signature":signature},...(request.method==="PUT"?{body:request.body}:{}),redirect:"manual",signal:AbortSignal.timeout(60000)})}catch{throw new F("core_unavailable")}if(!response.ok){await response.body?.cancel().catch(()=>{});result={version:1,ok:true,status:response.status,body:""}}else result={version:1,ok:true,status:response.status,body:await bounded(response)}}catch(error){result=fail(error)} +let published=false;try{await s3.send(new PutObjectCommand({Bucket:bucket,Key:outputKey,Body:JSON.stringify(result),ContentType:"application/json",CacheControl:"no-store"}));published=true}catch{console.error("AWS private core request could not publish its result")}finally{await s3.send(new DeleteObjectCommand({Bucket:bucket,Key:inputKey})).catch(()=>{})}if(!published||!result.ok)process.exitCode=1})().catch(()=>{console.error("AWS private core request failed unexpectedly");process.exitCode=1});`; +} diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.ts index 82bb73596..3c764bf9b 100644 --- a/cli/src/backends/aws.ts +++ b/cli/src/backends/aws.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; import { spawnSync } from "node:child_process"; import { lookup, resolveCname } from "node:dns/promises"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { @@ -21,6 +21,15 @@ import { type QmConfig, } from "../config.ts"; import { manifestRef } from "../manifest.ts"; +import { + AWS_PRIVATE_CORE_OBJECT_MAX_BYTES, + AWS_PRIVATE_CORE_OVERRIDES_MAX_BYTES, + awsPrivateCoreKeys, + awsPrivateCoreRequestBody, + awsPrivateCoreTaskScript, + awsPrivateSecretValidation, + parseAwsPrivateCoreResponse, +} from "../aws-private-core-request.ts"; import { computedSecrets, runtimeSecretNames, secretsForService, type ComputedSecret } from "../secrets.ts"; import { brokerWiring, @@ -52,36 +61,13 @@ import { import { doctorCommon } from "./doctor.ts"; import { awsObjectStoreBucket, declaredVariables, terraformVarsDrift } from "../terraform.ts"; import { + CoreUnreachableError, currentDeploymentLayerState, deploymentLayerBody, syncDeploymentLayerBody, type DeploymentLayerSyncResult, - httpDeploymentLayerTransport, type DeploymentLayerTransport, } from "../deployment-layer.ts"; - -/** - * Deployment-layer transport for AWS: signed HTTP to the public core URL, - * with a Secrets Manager fallback for CORE_SIGNING_SECRET and a 60s timeout. - */ -export const awsDeploymentLayerTransport: DeploymentLayerTransport = httpDeploymentLayerTransport({ - secretFallback: (config) => - config.aws - ? capture(process.env.AWS_BIN ?? "aws", [ - "secretsmanager", - "get-secret-value", - "--secret-id", - `${config.aws.secretsPrefix}CORE_SIGNING_SECRET`, - "--query", - "SecretString", - "--output", - "text", - "--region", - config.aws.region, - ]).trim() - : undefined, - timeoutMs: 60_000, -}); export interface AwsUpOpts { dryRun?: boolean; yes?: boolean; @@ -586,15 +572,16 @@ function secretArns(config: QmConfig): Record { const pairs = computedSecrets(config).flatMap((secret) => { const id = `${aws.secretsPrefix}${secret.name}`; try { - const value = awsJson<{ ARN?: string; SecretString?: string }>(aws, [ + const value = awsJson<{ ARN?: string; VersionIdsToStages?: Record }>(aws, [ "secretsmanager", - "get-secret-value", + "describe-secret", "--secret-id", id, ]); - if (!value.ARN || isInvalidSecret(secret.name, value.SecretString)) { + const current = Object.values(value.VersionIdsToStages ?? {}).some((stages) => stages.includes("AWSCURRENT")); + if (!value.ARN || !current) { if (!secret.required) return []; - throw new CliError(`required AWS secret ${secret.name} has no usable, non-placeholder AWSCURRENT value`); + throw new CliError(`required AWS secret ${secret.name} has no AWSCURRENT value`); } return [[secret.name, value.ARN] as const]; } catch (error) { @@ -605,32 +592,6 @@ function secretArns(config: QmConfig): Record { return Object.fromEntries(pairs); } -function assertAwsPublicApiUrl(config: QmConfig): void { - if (!computedSecrets(config).some((secret) => secret.name === "PUBLIC_API_URL")) return; - const aws = requireAws(config); - const value = awsText(aws, [ - "secretsmanager", - "get-secret-value", - "--secret-id", - `${aws.secretsPrefix}PUBLIC_API_URL`, - "--query", - "SecretString", - ]); - const bound = config.apiUrl ? ("apiUrl" as const) : ("publicUrl" as const); - const expected = new URL(config.apiUrl ?? config.publicUrl).toString().replace(/\/$/, ""); - let normalized: string; - try { - const parsed = new URL(value); - if (parsed.protocol !== "https:") throw new Error("not HTTPS"); - normalized = parsed.toString().replace(/\/$/, ""); - } catch { - throw new CliError(`required AWS secret PUBLIC_API_URL must be a valid HTTPS URL equal to the configured ${bound}`); - } - if (normalized !== expected) { - throw new CliError(`required AWS secret PUBLIC_API_URL must equal the configured HTTPS ${bound} (${expected})`); - } -} - function liveTask(config: QmConfig, service: string): Record | null { const aws = requireAws(config); const spec = aws.services[service]!; @@ -785,49 +746,96 @@ interface EcsServiceState { tags?: Array<{ key?: string; value?: string }>; } -function awsLiveSession(config: QmConfig, core: EcsServiceState): void { +interface EcsTaskResult { + exitCode?: number; + stoppedReason?: string; + containerReason?: string; +} + +function sanitizedEcsDiagnostic(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const sanitized = value + .replace(/[\r\n\t]+/g, " ") + .replace(/:\/\/[^@\s]+@/g, "://[redacted]@") + .replace(/\b(authorization|password|secretstring|token)\s*[=:]\s*\S+/gi, "$1=[redacted]") + .replace(/[^\x20-\x7e]/g, "") + .trim() + .slice(0, 240); + return sanitized || undefined; +} + +function ecsTaskDiagnostic(result: EcsTaskResult): string { + const details = [ + result.exitCode === undefined ? undefined : `exit ${result.exitCode}`, + result.stoppedReason ? `stopped: ${result.stoppedReason}` : undefined, + result.containerReason ? `container: ${result.containerReason}` : undefined, + ].filter(Boolean); + return details.length ? ` (${details.join("; ")})` : ""; +} + +function runEcsTask( + config: QmConfig, + options: { + taskDefinition: string; + networkConfiguration: NonNullable; + containerName: string; + command: string[]; + environment?: Array<{ name: string; value: string }>; + }, +): EcsTaskResult { const aws = requireAws(config); - if (!core.taskDefinition) throw new Error("core service has no live task definition"); - if (!core.networkConfiguration?.awsvpcConfiguration) throw new Error("core service has no VPC network configuration"); - const started = awsJson<{ - tasks?: Array<{ taskArn?: string }>; - failures?: Array<{ arn?: string; reason?: string; detail?: string }>; - }>(aws, [ - "ecs", - "run-task", - "--cluster", - aws.cluster, - "--task-definition", - core.taskDefinition, - "--launch-type", - "FARGATE", - "--network-configuration", - JSON.stringify(core.networkConfiguration), - "--overrides", - JSON.stringify({ - containerOverrides: [ - { - name: "core", - command: [ - "node", - "src/deployment/postdeploy-smoke.ts", - "session", - `http://core.${aws.networking.cloudMapNamespace}:8080`, - ], - }, - ], - }), - "--count", - "1", - ]); + if (!options.networkConfiguration.awsvpcConfiguration) + throw new Error("ECS service has no VPC network configuration"); + const overrides = JSON.stringify({ + containerOverrides: [ + { + name: options.containerName, + command: options.command, + environment: options.environment ?? [], + }, + ], + }); + if (Buffer.byteLength(overrides) > AWS_PRIVATE_CORE_OVERRIDES_MAX_BYTES) { + throw new Error("ECS task override exceeds the size limit"); + } + let started: { tasks?: Array<{ taskArn?: string }>; failures?: Array<{ reason?: string; detail?: string }> }; + try { + started = awsJson(aws, [ + "ecs", + "run-task", + "--cluster", + aws.cluster, + "--task-definition", + options.taskDefinition, + "--launch-type", + "FARGATE", + "--network-configuration", + JSON.stringify(options.networkConfiguration), + "--overrides", + overrides, + "--count", + "1", + ]); + } catch (error) { + throw new Error(`ECS RunTask failed: ${sanitizedEcsDiagnostic(errMessage(error)) ?? "unknown error"}`, { + cause: error, + }); + } const taskArn = started.tasks?.[0]?.taskArn; if (!taskArn) { const failure = started.failures?.[0]; - throw new Error( - `could not start canary task: ${failure?.reason ?? failure?.detail ?? failure?.arn ?? "no task returned"}`, - ); + const detail = [sanitizedEcsDiagnostic(failure?.reason), sanitizedEcsDiagnostic(failure?.detail)] + .filter(Boolean) + .join(": "); + throw new Error(`ECS RunTask returned no task${detail ? `: ${detail}` : ""}`); + } + try { + awsText(aws, ["ecs", "wait", "tasks-stopped", "--cluster", aws.cluster, "--tasks", taskArn]); + } catch (error) { + throw new Error(`ECS task did not stop: ${sanitizedEcsDiagnostic(errMessage(error)) ?? "unknown error"}`, { + cause: error, + }); } - awsText(aws, ["ecs", "wait", "tasks-stopped", "--cluster", aws.cluster, "--tasks", taskArn]); const stopped = awsJson<{ tasks?: Array<{ stoppedReason?: string; @@ -835,14 +843,195 @@ function awsLiveSession(config: QmConfig, core: EcsServiceState): void { }>; }>(aws, ["ecs", "describe-tasks", "--cluster", aws.cluster, "--tasks", taskArn]); const task = stopped.tasks?.[0]; - const coreContainer = task?.containers?.find((container) => container.name === "core"); - if (coreContainer?.exitCode !== 0) { - throw new Error( - `canary task exited ${coreContainer?.exitCode ?? "without a code"}: ${coreContainer?.reason ?? task?.stoppedReason ?? "unknown reason"}`, + if (!task) throw new Error("ECS DescribeTasks returned no stopped task"); + const container = task.containers?.find((item) => item.name === options.containerName); + if (!container) throw new Error(`ECS stopped task omitted the ${options.containerName} container`); + return { + exitCode: container.exitCode, + stoppedReason: sanitizedEcsDiagnostic(task.stoppedReason), + containerReason: sanitizedEcsDiagnostic(container.reason), + }; +} + +function runCoreTask( + config: QmConfig, + core: EcsServiceState, + command: string[], + environment: Array<{ name: string; value: string }> = [], +): EcsTaskResult { + if (!core.taskDefinition) throw new Error("core service has no live task definition"); + if (!core.networkConfiguration) throw new Error("core service has no VPC network configuration"); + return runEcsTask(config, { + taskDefinition: core.taskDefinition, + networkConfiguration: core.networkConfiguration, + containerName: "core", + command, + environment, + }); +} + +function awsSecretValidationTask(config: QmConfig): { + taskDefinition: string; + validation: ReturnType; +} { + const aws = requireAws(config); + const required = computedSecrets(config) + .filter((secret) => secret.required) + .map((secret) => secret.name) + .sort(); + const validation = awsPrivateSecretValidation(required, config.apiUrl ?? config.publicUrl); + const described = awsJson<{ + taskDefinition?: { + taskDefinitionArn?: string; + status?: string; + containerDefinitions?: Array<{ name?: string; secrets?: Array<{ name?: string }> }>; + }; + }>(aws, ["ecs", "describe-task-definition", "--task-definition", `${aws.cluster}-secret-validation`]); + const task = described.taskDefinition; + const container = task?.containerDefinitions?.find((item) => item.name === "secret-validation"); + const injected = (container?.secrets ?? []).flatMap((secret) => (secret.name ? [secret.name] : [])).sort(); + if ( + !task?.taskDefinitionArn || + task.status !== "ACTIVE" || + !container || + canonicalJson(injected) !== canonicalJson(required) + ) { + throw new CliError( + "AWS secret validator task is missing or stale; rerender and apply the current Terraform scaffold", ); } + return { taskDefinition: task.taskDefinitionArn, validation }; } +function assertAwsSecretValues(config: QmConfig, core: EcsServiceState): void { + if (!core.networkConfiguration) throw new CliError("core service has no VPC network configuration"); + const { taskDefinition, validation } = awsSecretValidationTask(config); + const result = runEcsTask(config, { + taskDefinition, + networkConfiguration: core.networkConfiguration, + containerName: "secret-validation", + command: validation.command, + environment: validation.environment, + }); + const invalid = validation.invalidSecret(result.exitCode); + if (invalid) { + throw new CliError(`required AWS secret ${invalid} has a missing, placeholder, weak, or mismatched value`); + } + if (result.exitCode !== 0) { + throw new CliError(`AWS private secret validation task failed${ecsTaskDiagnostic(result)}`); + } +} + +function awsLiveSession(config: QmConfig, core: EcsServiceState): void { + const aws = requireAws(config); + const result = runCoreTask(config, core, [ + "node", + "src/deployment/postdeploy-smoke.ts", + "session", + `http://core.${aws.networking.cloudMapNamespace}:8080`, + ]); + if (result.exitCode !== 0) + throw new Error(`canary task exited ${result.exitCode ?? "without a code"}${ecsTaskDiagnostic(result)}`); +} + +export const awsDeploymentLayerTransport: DeploymentLayerTransport = async (opts) => { + const aws = requireAws(opts.config); + const requestId = randomUUID(); + const bucket = awsObjectStoreBucket(opts.config); + const keys = awsPrivateCoreKeys(requestId); + const dir = mkdtempSync(join(tmpdir(), "qm-private-core-")); + const requestFile = join(dir, "request.json"); + const responseFile = join(dir, "response.json"); + const removeObjects = (): void => { + for (const key of [keys.request, keys.response]) { + try { + awsText(aws, ["s3api", "delete-object", "--bucket", bucket, "--key", key]); + } catch { + warn("could not remove a temporary AWS private core object"); + } + } + }; + try { + writeFileSync(requestFile, awsPrivateCoreRequestBody(opts.method, opts.body)); + awsText(aws, [ + "s3api", + "put-object", + "--bucket", + bucket, + "--key", + keys.request, + "--body", + requestFile, + "--content-type", + "application/json", + "--cache-control", + "no-store", + ]); + const core = describedServices(opts.config, ["core"]).get("core"); + if (!core) throw new CoreUnreachableError("AWS private core request could not resolve the core service"); + let taskResult: EcsTaskResult; + try { + taskResult = runCoreTask( + opts.config, + core, + ["node", "-e", awsPrivateCoreTaskScript()], + [ + { name: "QM_AWS_CORE_REQUEST_BUCKET", value: bucket }, + { name: "QM_AWS_CORE_REQUEST_ID", value: requestId }, + { name: "QM_AWS_CORE_REQUEST_URL", value: `http://core.${aws.networking.cloudMapNamespace}:8080` }, + ], + ); + } catch (error) { + throw new CoreUnreachableError(`AWS private core request task did not complete: ${errMessage(error)}`); + } + let metadata: { ContentLength?: number }; + try { + metadata = awsJson(aws, ["s3api", "head-object", "--bucket", bucket, "--key", keys.response]); + if ( + !Number.isInteger(metadata.ContentLength) || + metadata.ContentLength! < 1 || + metadata.ContentLength! > AWS_PRIVATE_CORE_OBJECT_MAX_BYTES + ) { + throw new Error("invalid response size"); + } + awsText(aws, [ + "s3api", + "get-object", + "--bucket", + bucket, + "--key", + keys.response, + responseFile, + "--range", + `bytes=0-${AWS_PRIVATE_CORE_OBJECT_MAX_BYTES - 1}`, + ]); + if (statSync(responseFile).size !== metadata.ContentLength) throw new Error("response size changed"); + } catch { + throw new CoreUnreachableError( + `AWS private core request returned no bounded result${ecsTaskDiagnostic(taskResult)}`, + ); + } + let response: ReturnType; + try { + response = parseAwsPrivateCoreResponse(readFileSync(responseFile, "utf8")); + } catch { + throw new CoreUnreachableError("AWS private core request returned an invalid result"); + } + if (!response.ok) { + if (response.code === "core_unavailable") { + throw new CoreUnreachableError("AWS private core request could not reach core"); + } + throw new CliError(`AWS private core request failed (${response.code})`); + } + if (taskResult.exitCode !== 0) + throw new CoreUnreachableError(`AWS private core request task failed${ecsTaskDiagnostic(taskResult)}`); + return { status: response.status, body: response.body }; + } finally { + removeObjects(); + rmSync(dir, { recursive: true, force: true }); + } +}; + type DeploymentImageProvenance = | { kind: "configured"; source: string } | { kind: "source-build"; source?: "plugin" | "checkout"; gitCommit?: string; dirty?: boolean }; @@ -1658,12 +1847,14 @@ export async function awsUp(config: QmConfig, _configDir: string, opts: AwsUpOpt assertAwsCallerAccount(aws); assertAwsPublicFrontDoor(config); if (!opts.dryRun) await assertAwsPublicNetwork(config); - assertAwsPublicApiUrl(config); assertAwsDeployImage(config); header(`qm ${opts.dryRun ? "plan" : "up"} — ${config.orgId} (aws)`); const allServices = Object.keys(aws.services); - assertOwnedServices(config, describedServices(config, allServices), allServices); + const liveServices = describedServices(config, allServices); + assertOwnedServices(config, liveServices, allServices); const arns = secretArns(config); + if (opts.dryRun) awsSecretValidationTask(config); + else assertAwsSecretValues(config, liveServices.get("core")!); if (opts.dryRun) { if (usesFlySandboxes(config) && services.includes("core")) { const pin = resolveAwsSandboxPin(config, () => currentDeploymentManifest(aws)); @@ -2921,7 +3112,6 @@ export async function awsDoctor(config: QmConfig, configDir: string): Promise; }>; }>(aws, ["rds", "describe-db-instances", "--db-instance-identifier", rdsInstanceIdentifier(aws)]).DBInstances?.[0]; @@ -2959,16 +3149,6 @@ export async function awsDoctor(config: QmConfig, configDir: string): Promise pair.GroupId && coreGroups.includes(pair.GroupId)), ); if (!reachable) throw new Error("database security groups do not allow the core ECS service on port 5432"); - const databaseUrl = awsText(aws, [ - "secretsmanager", - "get-secret-value", - "--secret-id", - `${aws.secretsPrefix}DATABASE_URL`, - "--query", - "SecretString", - ]); - if (!database.Endpoint?.Address || new URL(databaseUrl).hostname !== database.Endpoint.Address) - throw new Error("DATABASE_URL does not point at the configured RDS endpoint"); }); const ecsServices = new Map(); for (const service of Object.keys(aws.services)) { @@ -3038,38 +3218,14 @@ export async function awsDoctor(config: QmConfig, configDir: string): Promise assertAwsPublicRouting(config, ecsServices)); await checkAsync("public URL DNS and TLS", () => assertAwsPublicNetwork(config)); - const probe = probeAwsSecretStore( - computedSecrets(config), - (name) => - awsText(aws, [ - "secretsmanager", - "get-secret-value", - "--secret-id", - `${aws.secretsPrefix}${name}`, - "--query", - "SecretString", - ]), - () => assertAwsPublicApiUrl(config), - ); - failures.push(...probe.failures); - const runtimeSecrets = probe.values; + check("AWS secret values", () => { + secretArns(config); + const core = describedServices(config, ["core"]).get("core"); + if (!core) throw new Error("core service is missing"); + assertAwsSecretValues(config, core); + }); if (failures.length) throw new CliError(`doctor failed:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`); - const runtimeNames = ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"] as const; - const priorRuntime = new Map(runtimeNames.map((name) => [name, process.env[name]])); - for (const name of runtimeNames) { - const stored = runtimeSecrets.get(name); - if (stored !== undefined) process.env[name] = stored; - else delete process.env[name]; - } - try { - await doctorCommon(config, runtimeSecrets, { configDir, requiredSecretValues: probe.pending.length === 0 }); - } finally { - for (const name of runtimeNames) { - const prior = priorRuntime.get(name); - if (prior === undefined) delete process.env[name]; - else process.env[name] = prior; - } - } + await doctorCommon(config, new Map(), { configDir }); ok("all AWS deployment prerequisites are ready"); } @@ -3089,6 +3245,11 @@ async function checkLive( } const arns = secretArns(config); const states = describedServices(config, services); + try { + assertAwsSecretValues(config, states.get("core")!); + } catch (error) { + failures.push(`secret values: ${errMessage(error)}`); + } const manifest = currentDeploymentManifest(aws); if (!manifest) throw new CliError("live drift detected: no current AWS deployment manifest", { clause: "aws.live-drift" }); diff --git a/cli/src/terraform.ts b/cli/src/terraform.ts index 67d35f8ed..01660fb70 100644 --- a/cli/src/terraform.ts +++ b/cli/src/terraform.ts @@ -23,6 +23,7 @@ const DERIVED_VARS = new Set([ "deploy_microvm_execution_role_arn", "services", "secret_names", + "required_secret_names", ]); const OPERATOR_DEFAULTS: Record = { @@ -119,6 +120,9 @@ function derivedValues( json: { services, secret_names: secrets.map((secret) => secret.name), + ...(declared.includes("required_secret_names") + ? { required_secret_names: secrets.filter((secret) => secret.required).map((secret) => secret.name) } + : {}), }, }; } diff --git a/cli/templates/aws/main.tf b/cli/templates/aws/main.tf index 792d950b3..860fc7a93 100644 --- a/cli/templates/aws/main.tf +++ b/cli/templates/aws/main.tf @@ -349,7 +349,10 @@ resource "aws_iam_role_policy" "github_deploy" { Sid = "RunDeploymentCanaries" Effect = "Allow" Action = ["ecs:RunTask"] - Resource = ["arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task-definition/${var.services["core"].ecs_service}:*"] + Resource = [ + "arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task-definition/${var.services["core"].ecs_service}:*", + "arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:task-definition/${var.cluster_name}-secret-validation:*" + ] Condition = { ArnEquals = { "ecs:cluster" = aws_ecs_cluster.this.arn } } @@ -419,7 +422,6 @@ resource "aws_iam_role_policy" "github_deploy" { Effect = "Allow" Action = [ "secretsmanager:DescribeSecret", - "secretsmanager:GetSecretValue", "secretsmanager:PutSecretValue" ] Resource = [for secret in aws_secretsmanager_secret.contract : secret.arn] @@ -451,6 +453,12 @@ resource "aws_iam_role_policy" "github_deploy" { Action = ["s3:GetObject", "s3:PutObject"] Resource = ["${aws_s3_bucket.objects.arn}/deployment/layers/*"] }, + { + Sid = "ManagePrivateCoreRequests" + Effect = "Allow" + Action = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"] + Resource = ["${aws_s3_bucket.objects.arn}/deployment/core-requests/*"] + }, { Sid = "ManageMicrovmBuildArtifacts" Effect = "Allow" @@ -807,6 +815,39 @@ resource "aws_ecs_task_definition" "bootstrap" { tags = local.tags } +resource "aws_ecs_task_definition" "secret_validation" { + family = "${var.cluster_name}-secret-validation" + cpu = 256 + memory = 512 + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = local.default_execution_role_arn + task_role_arn = local.default_task_role_arn + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + container_definitions = jsonencode([{ + name = "secret-validation" + image = "public.ecr.aws/docker/library/alpine:3.20" + essential = true + command = ["sh", "-c", "exit 1"] + secrets = [for name in sort(tolist(var.required_secret_names)) : { + name = name + valueFrom = aws_secretsmanager_secret.contract[name].arn + }] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.service["core"].name + awslogs-region = var.region + awslogs-stream-prefix = "secret-validation" + } + } + }]) + tags = local.tags +} + resource "aws_ecs_service" "service" { for_each = var.services depends_on = [aws_lb_listener.public, aws_lb_listener_rule.paths] diff --git a/cli/templates/aws/variables.tf b/cli/templates/aws/variables.tf index 2a06202f6..ab1b245c9 100644 --- a/cli/templates/aws/variables.tf +++ b/cli/templates/aws/variables.tf @@ -105,3 +105,10 @@ variable "services" { })) } variable "secret_names" { type = set(string) } +variable "required_secret_names" { + type = set(string) + validation { + condition = length(setsubtract(var.required_secret_names, var.secret_names)) == 0 + error_message = "required_secret_names must be a subset of secret_names" + } +} diff --git a/cli/test/aws-private-core-request.test.ts b/cli/test/aws-private-core-request.test.ts new file mode 100644 index 000000000..992e7d039 --- /dev/null +++ b/cli/test/aws-private-core-request.test.ts @@ -0,0 +1,224 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { createHmac } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + AWS_PRIVATE_CORE_OBJECT_MAX_BYTES, + AWS_PRIVATE_CORE_OVERRIDES_MAX_BYTES, + AWS_PRIVATE_CORE_REQUEST_BODY_MAX_BYTES, + AWS_PRIVATE_CORE_RESPONSE_MAX_BYTES, + awsPrivateCoreKeys, + awsPrivateCoreRequestBody, + awsPrivateCoreTaskScript, + awsPrivateSecretValidation, + parseAwsPrivateCoreResponse, +} from "../src/aws-private-core-request.ts"; + +test("private secret validation is silent and maps invalid values without exposing them", () => { + const validation = awsPrivateSecretValidation( + ["PUBLIC_API_URL", "CORE_SIGNING_SECRET", "ADMIN_GRANTS"], + "https://agent.acme.example/", + ); + const run = (env: Record) => + spawnSync(validation.command[0]!, validation.command.slice(1), { + encoding: "utf8", + env: { + ...process.env, + ...env, + ...Object.fromEntries(validation.environment.map((item) => [item.name, item.value])), + }, + }); + const valid = { + ADMIN_GRANTS: "ops@example.com:org_admin", + CORE_SIGNING_SECRET: "a".repeat(32), + PUBLIC_API_URL: "https://agent.acme.example/", + }; + const passed = run(valid); + assert.equal(passed.status, 0); + assert.equal(passed.stdout, ""); + assert.equal(passed.stderr, ""); + for (const [name, value] of [ + ["ADMIN_GRANTS", "ops@example.com:viewer"], + ["CORE_SIGNING_SECRET", "short"], + ["PUBLIC_API_URL", "https://wrong.example"], + ] as const) { + const failed = run({ ...valid, [name]: value }); + assert.equal(validation.invalidSecret(failed.status ?? undefined), name); + assert.equal(failed.stdout, ""); + assert.equal(failed.stderr, ""); + } + assert.doesNotMatch(JSON.stringify(validation), /ops@example|a{32}/); +}); + +test("private core request envelopes are deterministic and bounded", () => { + const id = "123e4567-e89b-42d3-a456-426614174000"; + assert.deepEqual(awsPrivateCoreKeys(id), { + request: `deployment/core-requests/${id}/request.json`, + response: `deployment/core-requests/${id}/response.json`, + }); + assert.deepEqual(JSON.parse(awsPrivateCoreRequestBody("GET", "")), { version: 1, method: "GET", body: "" }); + assert.deepEqual(JSON.parse(awsPrivateCoreRequestBody("PUT", "payload")), { + version: 1, + method: "PUT", + body: "payload", + }); + assert.throws(() => awsPrivateCoreRequestBody("GET", "payload"), /body is invalid/); + assert.throws( + () => awsPrivateCoreRequestBody("PUT", "x".repeat(AWS_PRIVATE_CORE_REQUEST_BODY_MAX_BYTES + 1)), + /body is invalid/, + ); +}); + +test("private core response envelopes expose only typed bounded terminal results", () => { + assert.deepEqual( + parseAwsPrivateCoreResponse(JSON.stringify({ version: 1, ok: true, status: 200, body: "payload" })), + { version: 1, ok: true, status: 200, body: "payload" }, + ); + assert.deepEqual(parseAwsPrivateCoreResponse(JSON.stringify({ version: 1, ok: true, status: 503, body: "" })), { + version: 1, + ok: true, + status: 503, + body: "", + }); + assert.deepEqual(parseAwsPrivateCoreResponse(JSON.stringify({ version: 1, ok: false, code: "core_unavailable" })), { + version: 1, + ok: false, + code: "core_unavailable", + }); + for (const value of [ + { version: 1, ok: true, status: 199, body: "" }, + { version: 1, ok: true, status: 503, body: "private failure" }, + { version: 1, ok: true, status: 200, body: "", extra: true }, + { version: 1, ok: false, code: "arbitrary_failure" }, + ]) { + assert.throws(() => parseAwsPrivateCoreResponse(JSON.stringify(value)), /response object is invalid/); + } + assert.throws( + () => + parseAwsPrivateCoreResponse( + JSON.stringify({ + version: 1, + ok: true, + status: 200, + body: "x".repeat(AWS_PRIVATE_CORE_RESPONSE_MAX_BYTES + 1), + }), + ), + /response object is (?:invalid|too large)/, + ); + assert.throws( + () => parseAwsPrivateCoreResponse("x".repeat(AWS_PRIVATE_CORE_OBJECT_MAX_BYTES + 1)), + /response object is too large/, + ); +}); + +test("private core task signing stays inside a bounded sanitized ECS override", () => { + const script = awsPrivateCoreTaskScript(); + const overrides = JSON.stringify({ + containerOverrides: [ + { + name: "core", + command: ["node", "-e", script], + environment: [ + { name: "QM_AWS_CORE_REQUEST_BUCKET", value: "qm-object-store-123456789012-us-west-2" }, + { name: "QM_AWS_CORE_REQUEST_ID", value: "123e4567-e89b-42d3-a456-426614174000" }, + { name: "QM_AWS_CORE_REQUEST_URL", value: "http://core.acme.internal:8080" }, + ], + }, + ], + }); + assert.ok(Buffer.byteLength(overrides) <= AWS_PRIVATE_CORE_OVERRIDES_MAX_BYTES); + assert.match(script, /CORE_SIGNING_SECRET/); + assert.match(script, /createHmac/); + assert.match(script, /\/v1\/deployment-layer/); + assert.doesNotMatch( + script, + /get-secret-value|SecretString|console\.log|error\.message|String\(error\)|JSON\.stringify\(error\)/, + ); +}); + +test("private core task signs in-process and publishes a sanitized terminal response", async () => { + const dir = mkdtempSync(join(tmpdir(), "qm-private-core-task-")); + const moduleDir = join(dir, "node_modules", "@aws-sdk", "client-s3"); + const requestFile = join(dir, "request.json"); + const responseFile = join(dir, "response.json"); + const deletedFile = join(dir, "deleted.txt"); + const signingSecret = "private-signing-value"; + const serverDetail = "do-not-expose-server-detail"; + mkdirSync(moduleDir, { recursive: true }); + writeFileSync( + join(moduleDir, "index.js"), + `const fs=require("node:fs"); +class GetObjectCommand{constructor(input){this.input=input}} +class PutObjectCommand{constructor(input){this.input=input}} +class DeleteObjectCommand{constructor(input){this.input=input}} +class S3Client{async send(command){if(command instanceof GetObjectCommand){const body=fs.readFileSync(process.env.QM_TEST_REQUEST_FILE);return{ContentLength:body.length,Body:{transformToByteArray:async()=>body}}}if(command instanceof PutObjectCommand){fs.writeFileSync(process.env.QM_TEST_RESPONSE_FILE,String(command.input.Body));return{}}if(command instanceof DeleteObjectCommand){fs.writeFileSync(process.env.QM_TEST_DELETED_FILE,command.input.Key);return{}}throw new Error("unexpected command")}} +module.exports={S3Client,GetObjectCommand,PutObjectCommand,DeleteObjectCommand}; +`, + ); + writeFileSync(requestFile, awsPrivateCoreRequestBody("PUT", "payload")); + let captured: { body: string; path: string; signature: string; timestamp: string } | undefined; + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + request.on("end", () => { + captured = { + body: Buffer.concat(chunks).toString("utf8"), + path: request.url ?? "", + signature: String(request.headers["x-signature"] ?? ""), + timestamp: String(request.headers["x-timestamp"] ?? ""), + }; + response.statusCode = 503; + response.end(serverDetail); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const port = (server.address() as AddressInfo).port; + const result = await new Promise<{ code: number | null; stderr: string; stdout: string }>((resolve) => { + const child = spawn(process.execPath, ["-e", awsPrivateCoreTaskScript()], { + cwd: dir, + env: { + ...process.env, + CORE_SIGNING_SECRET: signingSecret, + QM_AWS_CORE_REQUEST_BUCKET: "bucket", + QM_AWS_CORE_REQUEST_ID: "123e4567-e89b-42d3-a456-426614174000", + QM_AWS_CORE_REQUEST_URL: `http://127.0.0.1:${port}`, + QM_TEST_DELETED_FILE: deletedFile, + QM_TEST_REQUEST_FILE: requestFile, + QM_TEST_RESPONSE_FILE: responseFile, + }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += String(chunk))); + child.stderr.on("data", (chunk) => (stderr += String(chunk))); + child.on("close", (code) => resolve({ code, stderr, stdout })); + }); + assert.equal(result.code, 0, result.stderr); + assert.ok(captured); + assert.equal(captured.path, "/v1/deployment-layer"); + assert.equal(captured.body, "payload"); + const canonical = `PUT\n/v1/deployment-layer\npayload`; + const expected = createHmac("sha256", signingSecret).update(`v0:${captured.timestamp}:${canonical}`).digest("hex"); + assert.equal(captured.signature, `v0=${expected}`); + assert.deepEqual(JSON.parse(readFileSync(responseFile, "utf8")), { + version: 1, + ok: true, + status: 503, + body: "", + }); + assert.match(readFileSync(deletedFile, "utf8"), /request\.json$/); + assert.doesNotMatch( + `${result.stdout}\n${result.stderr}\n${readFileSync(responseFile, "utf8")}`, + new RegExp(`${signingSecret}|${serverDetail}`), + ); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/cli/test/aws.test.ts b/cli/test/aws.test.ts index cdab4e818..1613aa09f 100644 --- a/cli/test/aws.test.ts +++ b/cli/test/aws.test.ts @@ -48,7 +48,11 @@ function fakeAws( dir: string, script: string, frontService: "core" | "portal" = "portal", - ingress: { coreHosts?: string[]; targetGroups?: Partial> } = {}, + ingress: { + coreHosts?: string[]; + targetGroups?: Partial>; + currentSecret?: boolean; + } = {}, ): { log: string; restore: () => void } { const bin = join(dir, "aws-fake"); const log = join(dir, "aws.log"); @@ -83,6 +87,7 @@ function fakeAws( bin, `#!/usr/bin/env node const fs = require("node:fs"); +const { createHash } = require("node:crypto"); const a = process.argv.slice(2).join(" "); fs.appendFileSync(${JSON.stringify(log)}, a + "\\n"); if (a.includes("sts get-caller-identity")) console.log(process.env.AWS_FAKE_ACCOUNT || "123456789012"); @@ -113,6 +118,7 @@ else if (a.includes("elbv2 describe-rules")) { console.log(JSON.stringify({ Rules: rules })); } else if (a.includes("elbv2 describe-target-health")) console.log(JSON.stringify({ TargetHealthDescriptions: [{ TargetHealth: { State: process.env.AWS_FAKE_UNHEALTHY_TARGET === "1" ? "unhealthy" : "healthy" } }] })); +else if (a.includes("secretsmanager describe-secret")) console.log(JSON.stringify({ ARN: "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf", VersionIdsToStages: ${JSON.stringify(ingress.currentSecret === false ? { old: ["AWSPREVIOUS"] } : { current: ["AWSCURRENT"] })} })); else { ${script} } @@ -181,6 +187,10 @@ function statefulAws( return pinned ? [[name, pinned]] : []; }), ); + const requiredSecretNames = computedSecrets(configured) + .filter((secret) => secret.required) + .map((secret) => secret.name) + .sort(); const tgName = (name: "core" | "portal"): string => targetGroups[name] ?? `acme-qm-${name.replaceAll("-", "").slice(0, 4)}-${createHash("sha1").update(`acme-qm:${name}`).digest("hex").slice(0, 6)}`; @@ -269,12 +279,41 @@ else if (a.includes("ecs describe-services")) { return [{ serviceName: name, status: "ACTIVE", desiredCount: service.desiredCount, runningCount: ${JSON.stringify(opts.drainRollout ?? false)} ? service.desiredCount + 1 : service.desiredCount, taskDefinition: service.taskDefinition, networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-test"], securityGroups: ["sg-test"], assignPublicIp: "DISABLED" } }, deployments, loadBalancers: service.workload === ${JSON.stringify(frontService)} ? [{ targetGroupArn: ${JSON.stringify(frontTargetArn)} }] : (service.workload === "core" && ${JSON.stringify(coreHosts.length > 0)} ? [{ targetGroupArn: ${JSON.stringify(coreTargetArn)} }] : []), tags: [{ key: "Deployment", value: ${JSON.stringify(opts.foreignServiceTags ? "other" : configured.orgId)} }, { key: "ManagedBy", value: "terraform" }] }]; }), failures: names.filter((name) => !s.services[name]).map((name) => ({ arn: name, reason: "MISSING" })) })); } -else if (a.includes("ecs run-task")) console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/canary" }] })); +else if (a.includes("ecs run-task")) { + const override = JSON.parse(after("--overrides")); + const containerOverride = override.containerOverrides?.[0] || {}; + const environment = Object.fromEntries((containerOverride.environment || []).map((item) => [item.name, item.value])); + const requestId = environment.QM_AWS_CORE_REQUEST_ID; + if (requestId) { + const requestKey = "deployment/core-requests/" + requestId + "/request.json"; + const responseKey = "deployment/core-requests/" + requestId + "/response.json"; + const request = JSON.parse(s.objects[requestKey]); + s.privateCoreMethods = [...(s.privateCoreMethods || []), request.method]; + if (request.method === "PUT") s.liveLayerBody = request.body; + const body = request.method === "PUT" + ? JSON.stringify({ version: 1, contentHash: createHash("sha256").update(request.body).digest("hex"), durable: true, status: "applied" }) + : (() => { + const layer = s.liveLayerBody || ${JSON.stringify(EMPTY_LAYER_BODY)}; + const hash = createHash("sha256").update(layer).digest("hex"); + return JSON.stringify({ bundle: JSON.parse(layer), contentHash: hash, runtimeContentHash: hash, status: "applied" }); + })(); + s.objects[responseKey] = JSON.stringify({ version: 1, ok: true, status: 200, body }); + delete s.objects[requestKey]; + } + s.lastTaskContainer = containerOverride.name; + s.lastTaskWasPrivate = Boolean(requestId); + save(); + console.log(JSON.stringify({ tasks: [{ taskArn: "arn:aws:ecs:us-west-2:123456789012:task/canary" }] })); +} else if (a.includes("ecs wait tasks-stopped")) console.log(""); -else if (a.includes("ecs describe-tasks")) console.log(JSON.stringify({ tasks: [{ stoppedReason: "Essential container exited", containers: [{ name: "core", exitCode: Number(process.env.AWS_FAKE_CANARY_EXIT || "0"), reason: process.env.AWS_FAKE_CANARY_REASON }] }] })); +else if (a.includes("ecs describe-tasks")) { + const secretValidation = s.lastTaskContainer === "secret-validation"; + console.log(JSON.stringify({ tasks: [{ stoppedReason: process.env.AWS_FAKE_STOPPED_REASON || "Essential container exited", containers: [{ name: s.lastTaskContainer || "core", exitCode: secretValidation ? Number(process.env.AWS_FAKE_SECRET_VALIDATION_EXIT || "0") : (s.lastTaskWasPrivate ? 0 : Number(process.env.AWS_FAKE_CANARY_EXIT || "0")), reason: process.env.AWS_FAKE_CANARY_REASON }] }] })); +} else if (a.includes("ecs describe-task-definition")) { const id = after("--task-definition"); - console.log(JSON.stringify({ taskDefinition: s.definitions[id] })); + if (id === ${JSON.stringify(`${configured.aws!.cluster}-secret-validation`)}) console.log(JSON.stringify({ taskDefinition: { taskDefinitionArn: "arn:aws:ecs:us-west-2:123456789012:task-definition/${configured.aws!.cluster}-secret-validation:1", status: "ACTIVE", containerDefinitions: [{ name: "secret-validation", secrets: ${JSON.stringify(requiredSecretNames.map((name) => ({ name })))} }] } })); + else console.log(JSON.stringify({ taskDefinition: s.definitions[id] })); } else if (a.includes("ecs register-task-definition")) { const file = after("--cli-input-json").slice("file://".length); @@ -335,6 +374,16 @@ else if (a.includes("s3api get-object")) { fs.writeFileSync(output, body); console.log(""); } +else if (a.includes("s3api head-object")) { + const body = s.objects[after("--key")]; + if (body === undefined) process.exit(5); + console.log(JSON.stringify({ ContentLength: Buffer.byteLength(body) })); +} +else if (a.includes("s3api delete-object")) { + delete s.objects[after("--key")]; + save(); + console.log(""); +} else if (a.includes("rds describe-db-instances")) console.log(JSON.stringify({ DBInstances: [{ DBInstanceStatus: process.env.AWS_FAKE_DB_STATUS ?? "available", BackupRetentionPeriod: Number(process.env.AWS_FAKE_DB_RETENTION ?? "7") }] })); else if (a.includes("rds create-db-snapshot")) { s.rdsSnapshots = s.rdsSnapshots || []; @@ -951,12 +1000,7 @@ test("AWS portal ALB adopts pinned target groups and requires exactly the env-de const bothHosts = { apiUrl: "https://api.agent.acme.example", appsDomain: "apps.agent.acme.example" }; const run = async ( configured: QmConfig, - env?: - | "AWS_FAKE_NO_CORE_RULE" - | "AWS_FAKE_EXTRA_RULE" - | "AWS_FAKE_WRONG_RULE_TARGET" - | "AWS_FAKE_WRONG_RULE_HOST" - | "AWS_FAKE_PUBLIC_API_URL", + env?: "AWS_FAKE_NO_CORE_RULE" | "AWS_FAKE_EXTRA_RULE" | "AWS_FAKE_WRONG_RULE_TARGET" | "AWS_FAKE_WRONG_RULE_HOST", expected?: RegExp, envValue = "1", ): Promise => { @@ -966,8 +1010,7 @@ test("AWS portal ALB adopts pinned target groups and requires exactly the env-de try { if (expected) await assert.rejects(() => awsUp(configured, dir, { dryRun: true }), expected); else await awsUp(configured, dir, { dryRun: true }); - if (!expected || (env && env !== "AWS_FAKE_PUBLIC_API_URL")) - assert.match(readFileSync(fake.log, "utf8"), /elbv2 describe-rules/); + if (!expected) assert.match(readFileSync(fake.log, "utf8"), /elbv2 describe-rules/); } finally { if (env) { if (prior === undefined) delete process.env[env]; @@ -999,12 +1042,6 @@ test("AWS portal ALB adopts pinned target groups and requires exactly the env-de undefined, /env\.core\.DEPLOY_APPS_DOMAIN or AWS_DEPLOY_APPS_DOMAIN.* does not derive a valid ALB host-header hostname/, ); - await run( - hostSplitConfig(bothHosts), - "AWS_FAKE_PUBLIC_API_URL", - /PUBLIC_API_URL must equal the configured HTTPS apiUrl/, - config.publicUrl, - ); } finally { process.env.PATH = priorPath; rmSync(dir, { recursive: true, force: true }); @@ -1467,22 +1504,60 @@ console.log("");`, } }); -test("AWS deploy requires PUBLIC_API_URL to equal the declared HTTPS public URL", async () => { - const dir = mkdtempSync(join(tmpdir(), "qm-aws-public-api-url-")); - const fake = statefulAws(dir, oneServiceConfig()); - const prior = process.env.AWS_FAKE_PUBLIC_API_URL; - process.env.AWS_FAKE_PUBLIC_API_URL = "http://agent.acme.example"; +test("AWS plan, up, and live check avoid direct plaintext secret retrieval by the CLI runner", async () => { + const dir = mkdtempSync(join(tmpdir(), "qm-aws-secret-metadata-")); + const dockerBin = join(dir, "docker"); + writeFileSync(dockerBin, "#!/usr/bin/env node\n"); + chmodSync(dockerBin, 0o755); + const single = oneServiceConfig(); + const fake = statefulAws(dir, single); + const priorPath = process.env.PATH; + const priorValidationExit = process.env.AWS_FAKE_SECRET_VALIDATION_EXIT; + process.env.PATH = `${dir}:${priorPath}`; try { - await assert.rejects( - () => awsUp(oneServiceConfig(), dir, { yes: true }), - /PUBLIC_API_URL must be a valid HTTPS URL equal to the configured publicUrl/, + await awsUp(single, dir, { dryRun: true }); + const planCalls = readFileSync(fake.log, "utf8"); + assert.match(planCalls, /ecs describe-task-definition .*acme-qm-secret-validation/); + assert.doesNotMatch( + planCalls, + /ecs (?:run-task|register-task-definition|update-service)|s3api put-object|dynamodb put-item|rds create-db-snapshot|ecr put-image/, ); + await awsUp(single, dir, { yes: true }); + await awsCheckLive(single, { report: false }); const calls = readFileSync(fake.log, "utf8"); - assert.match(calls, /get-secret-value .*PUBLIC_API_URL .*--query SecretString/); - assert.doesNotMatch(calls, /dynamodb put-item|ecr get-login-password|ecs update-service/); + assert.match(calls, /secretsmanager describe-secret/); + assert.doesNotMatch(calls, /secretsmanager get-secret-value/); + assert.match(calls, /s3api put-object .*deployment\/core-requests\//); + assert.match(calls, /s3api head-object .*deployment\/core-requests\//); + assert.match(calls, /s3api get-object .*deployment\/core-requests\//); + assert.match(calls, /s3api delete-object .*deployment\/core-requests\//); + assert.match(calls, /ecs run-task .*QM_AWS_CORE_REQUEST_ID/); + const state = JSON.parse(readFileSync(fake.state, "utf8")); + assert.deepEqual( + Object.keys(state.objects).filter((key) => key.startsWith("deployment/core-requests/")), + [], + ); + const required = computedSecrets(single) + .filter((secret) => secret.required) + .map((secret) => secret.name) + .sort(); + process.env.AWS_FAKE_SECRET_VALIDATION_EXIT = String(10 + required.indexOf("CORE_SIGNING_SECRET")); + const beforePlan = readFileSync(fake.log, "utf8").split("\n").length; + await assert.doesNotReject(() => awsUp(single, dir, { dryRun: true })); + const badValuePlanCalls = readFileSync(fake.log, "utf8").split("\n").slice(beforePlan).join("\n"); + assert.doesNotMatch(badValuePlanCalls, /ecs run-task/); + await assert.rejects( + () => awsUp(single, dir, { yes: true }), + /required AWS secret CORE_SIGNING_SECRET has a missing, placeholder, weak, or mismatched value/, + ); + await assert.rejects( + () => awsCheckLive(single, { report: false }), + /secret values: required AWS secret CORE_SIGNING_SECRET has a missing, placeholder, weak, or mismatched value/, + ); } finally { - if (prior === undefined) delete process.env.AWS_FAKE_PUBLIC_API_URL; - else process.env.AWS_FAKE_PUBLIC_API_URL = prior; + if (priorValidationExit === undefined) delete process.env.AWS_FAKE_SECRET_VALIDATION_EXIT; + else process.env.AWS_FAKE_SECRET_VALIDATION_EXIT = priorValidationExit; + process.env.PATH = priorPath; fake.restore(); rmSync(dir, { recursive: true, force: true }); } @@ -1518,38 +1593,30 @@ test("AWS deploy rejects required secret containers without an AWSCURRENT value const fake = fakeAws( dir, ` -if (a.includes("ecs describe-services")) console.log(JSON.stringify({ services: ${JSON.stringify(Object.entries(config.aws!.services).map(([name, spec]) => ({ serviceName: spec.ecsService, loadBalancers: name === "portal" ? [{ targetGroupArn: targetArn }] : [] })))} })); -else if (a.includes("secretsmanager get-secret-value")) { - console.error("ResourceNotFoundException: Secrets Manager can't find the specified secret value"); - process.exit(1); -} +if (a.includes("ecs describe-services")) console.log(JSON.stringify({ services: ${JSON.stringify( + Object.entries(config.aws!.services).map(([name, spec]) => ({ + serviceName: spec.ecsService, + loadBalancers: name === "portal" ? [{ targetGroupArn: targetArn }] : [], + tags: [ + { key: "Deployment", value: config.orgId }, + { key: "ManagedBy", value: "terraform" }, + ], + })), + )} })); console.log("");`, + "portal", + { currentSecret: false }, ); - try { - await assert.rejects(() => awsUp(config, process.cwd(), { yes: true }), /ResourceNotFoundException/); - const calls = readFileSync(fake.log, "utf8"); - assert.match(calls, /secretsmanager get-secret-value/); - assert.doesNotMatch(calls, /dynamodb put-item|ecr get-login-password|ecr describe-images|ecs update-service/); - } finally { - fake.restore(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test("AWS deploy rejects weak signing keys before mutation", async () => { - const dir = mkdtempSync(join(tmpdir(), "qm-aws-weak-secret-")); - const fake = statefulAws(dir, oneServiceConfig()); - const prior = process.env.AWS_FAKE_SECRET_VALUE; - process.env.AWS_FAKE_SECRET_VALUE = "short"; try { await assert.rejects( - () => awsUp(oneServiceConfig(), dir, { yes: true }), - /required AWS secret CORE_SIGNING_SECRET has no usable/, + () => awsUp(config, process.cwd(), { yes: true }), + /required AWS secret .* has no AWSCURRENT value/, ); - assert.doesNotMatch(readFileSync(fake.log, "utf8"), /dynamodb put-item|ecr describe-images|ecs update-service/); + const calls = readFileSync(fake.log, "utf8"); + assert.match(calls, /secretsmanager describe-secret/); + assert.doesNotMatch(calls, /secretsmanager get-secret-value/); + assert.doesNotMatch(calls, /dynamodb put-item|ecr get-login-password|ecr describe-images|ecs update-service/); } finally { - if (prior === undefined) delete process.env.AWS_FAKE_SECRET_VALUE; - else process.env.AWS_FAKE_SECRET_VALUE = prior; fake.restore(); rmSync(dir, { recursive: true, force: true }); } @@ -2601,34 +2668,14 @@ test("AWS rollback restores the recorded layer without reading the broken curren state.objects[oldLayer.key] = oldBody; state.objects[currentLayer.key] = currentBody; writeFileSync(fake.state, JSON.stringify(state)); - const priorFetch = globalThis.fetch; - const priorSecret = process.env.CORE_SIGNING_SECRET; - const bodies: string[] = []; - process.env.CORE_SIGNING_SECRET = "test-signing-secret"; - globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { - if (init?.method !== "PUT") { - return new Response("current release is broken", { status: 503 }); - } - const body = String(init.body ?? ""); - bodies.push(body); - return new Response( - JSON.stringify({ - version: 3, - contentHash: createHash("sha256").update(body).digest("hex"), - durable: true, - status: "applied", - }), - { status: 200 }, - ); - }) as typeof fetch; try { await awsRollback(single, undefined, { configDir: dir }); - assert.deepEqual(bodies, [oldBody]); - assert.equal(JSON.parse(readFileSync(fake.state, "utf8")).dynamo["deployment/current"].manifestId.S, "old"); + const after = JSON.parse(readFileSync(fake.state, "utf8")); + assert.equal(after.liveLayerBody, oldBody); + assert.deepEqual(after.privateCoreMethods, ["PUT"]); + assert.equal(after.dynamo["deployment/current"].manifestId.S, "old"); + assert.doesNotMatch(readFileSync(fake.log, "utf8"), /secretsmanager get-secret-value/); } finally { - globalThis.fetch = priorFetch; - if (priorSecret === undefined) delete process.env.CORE_SIGNING_SECRET; - else process.env.CORE_SIGNING_SECRET = priorSecret; fake.restore(); rmSync(dir, { recursive: true, force: true }); } @@ -2879,7 +2926,7 @@ console.log("");`, } }); -test("AWS live check rejects downed services and classifies probe errors as live drift", async () => { +test("AWS live check rejects downed services and classifies the failure as live drift", async () => { const dir = mkdtempSync(join(tmpdir(), "qm-aws-live-runtime-")); const single = oneServiceConfig(); const taskArn = "arn:aws:ecs:us-west-2:123456789012:task-definition/acme-core:1"; @@ -2897,21 +2944,6 @@ test("AWS live check rejects downed services and classifies probe errors as live ); } finally { fake.restore(); - } - const denied = fakeAws( - dir, - `if (a.includes("get-secret-value")) { console.error("AccessDeniedException"); process.exit(1); } console.log("");`, - ); - try { - await assert.rejects( - () => awsCheckLive(single), - (error: unknown) => - error instanceof Error && - /AccessDeniedException/.test(error.message) && - (error as { clause?: string }).clause === "aws.live-drift", - ); - } finally { - denied.restore(); rmSync(dir, { recursive: true, force: true }); } }); @@ -3020,7 +3052,6 @@ test("AWS live check uses the package-pinned source image without consulting mut ); const priorImageState = process.env.AWS_FAKE_IMAGE_STATE; const priorAlbDns = process.env.AWS_FAKE_ALB_DNS; - const priorSecretValue = process.env.AWS_FAKE_SECRET_VALUE; const state = JSON.parse(readFileSync(fake.state, "utf8")); const secretArn = "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-AbCdEf"; const arns = Object.fromEntries(computedSecrets(single).map((secret) => [secret.name, secretArn])); @@ -3046,10 +3077,7 @@ test("AWS live check uses the package-pinned source image without consulting mut process.env.PATH = `${dir}:${priorPath}`; try { await assert.doesNotReject(() => awsCheckLive(single, { report: false })); - process.env.AWS_FAKE_SECRET_VALUE = "short"; - await assert.rejects(() => awsCheckLive(single, { report: false }), /secret CORE_SIGNING_SECRET/); - if (priorSecretValue === undefined) delete process.env.AWS_FAKE_SECRET_VALUE; - else process.env.AWS_FAKE_SECRET_VALUE = priorSecretValue; + assert.doesNotMatch(readFileSync(fake.log, "utf8"), /secretsmanager get-secret-value/); assert.doesNotMatch(readFileSync(fake.log, "utf8"), /ecr describe-images/); assert.doesNotMatch(readFileSync(dockerLog, "utf8"), /buildx imagetools inspect/); const overridden: QmConfig = { @@ -3092,8 +3120,6 @@ test("AWS live check uses the package-pinned source image without consulting mut /manifest label release does not match configured release other-release/, ); } finally { - if (priorSecretValue === undefined) delete process.env.AWS_FAKE_SECRET_VALUE; - else process.env.AWS_FAKE_SECRET_VALUE = priorSecretValue; process.env.PATH = priorPath; if (priorImageState === undefined) delete process.env.AWS_FAKE_IMAGE_STATE; else process.env.AWS_FAKE_IMAGE_STATE = priorImageState; diff --git a/cli/test/cli-dispatch.test.ts b/cli/test/cli-dispatch.test.ts index d5b2bf131..e4d46b5f7 100644 --- a/cli/test/cli-dispatch.test.ts +++ b/cli/test/cli-dispatch.test.ts @@ -186,7 +186,7 @@ test("check --live on aws runs live drift checks in plain and JSON modes", async `#!/usr/bin/env node const args = process.argv.slice(2).join(" "); if (args.includes("sts get-caller-identity")) console.log("123456789012"); -else if (args.includes("get-secret-value")) console.log(JSON.stringify({ ARN: "arn", SecretString: "secret-value".repeat(3) })); +else if (args.includes("describe-secret")) console.log(JSON.stringify({ ARN: "arn", VersionIdsToStages: { current: ["AWSCURRENT"] } })); else if (args.includes("describe-services")) console.log(JSON.stringify({ services: [{ serviceName: "s" }] })); else console.log("{}"); `, @@ -258,6 +258,9 @@ test("successful check --json --live reports the live-drift clause", async () => writeFileSync(configPath, JSON.stringify(raw)); const config = loadConfigAt(configPath).config; const arns = Object.fromEntries(computedSecrets(config).map((secret) => [secret.name, "arn"])); + const requiredSecretNames = computedSecrets(config) + .filter((secret) => secret.required) + .map((secret) => secret.name); const image = `123456789012.dkr.ecr.us-west-2.amazonaws.com/repo@${digest}`; const task = renderTaskDefinition(config, "core", image, arns); const layerBody = JSON.stringify({ contract: 1, tools: [], skills: [] }); @@ -270,6 +273,7 @@ test("successful check --json --live reports the live-drift clause", async () => tasks: { core: "task" }, layer: { key: `deployment/layers/${layerHash}.json`, sha256: layerHash }, }; + const privateResponse = join(dir, "private-response.json"); writeFileSync( aws, `#!/usr/bin/env node @@ -279,15 +283,21 @@ const args = argv.join(" "); if (args.includes("sts get-caller-identity")) console.log("123456789012"); else if (args.includes("lambda-microvms get-microvm-image")) console.log(JSON.stringify({ imageArn: "arn:aws:lambda:us-west-2:123456789012:microvm-image:acme-microvm-app" })); else if (args.includes("lambda-microvms list-microvm-image-versions")) console.log(JSON.stringify({ items: [{ imageVersion: "1", state: "SUCCESSFUL", status: "ACTIVE" }] })); -else if (args.includes("get-secret-value") && args.includes("--query SecretString")) console.log("signing-secret".repeat(3)); -else if (args.includes("get-secret-value")) console.log(JSON.stringify({ ARN: "arn", SecretString: "secret-value".repeat(3) })); +else if (args.includes("describe-secret")) console.log(JSON.stringify({ ARN: "arn", VersionIdsToStages: { current: ["AWSCURRENT"] } })); else if (args.includes("describe-services")) console.log(JSON.stringify({ services: [{ serviceName: "s", status: "ACTIVE", desiredCount: 1, runningCount: 1, taskDefinition: "task", networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet"], securityGroups: ["sg"], assignPublicIp: "DISABLED" } }, deployments: [{ status: "PRIMARY", rolloutState: "COMPLETED", taskDefinition: "task" }], loadBalancers: [{ targetGroupArn: "tg" }] }] })); +else if (args.includes("describe-task-definition") && args.includes("c-secret-validation")) console.log(${JSON.stringify(JSON.stringify({ taskDefinition: { taskDefinitionArn: "secret-validation-task", status: "ACTIVE", containerDefinitions: [{ name: "secret-validation", secrets: requiredSecretNames.map((name) => ({ name })) }] } }))}); else if (args.includes("describe-task-definition")) console.log(${JSON.stringify(JSON.stringify({ taskDefinition: task }))}); +else if (args.includes("s3api put-object") && args.includes("deployment/core-requests/")) { + const request = JSON.parse(fs.readFileSync(argv[argv.indexOf("--body") + 1], "utf8")); + const body = JSON.stringify({ bundle: JSON.parse(${JSON.stringify(layerBody)}), contentHash: ${JSON.stringify(layerHash)}, status: "applied", runtimeContentHash: ${JSON.stringify(layerHash)} }); + fs.writeFileSync(${JSON.stringify(privateResponse)}, JSON.stringify({ version: 1, ok: true, status: 200, body })); +} +else if (args.includes("s3api head-object") && args.includes("deployment/core-requests/")) console.log(JSON.stringify({ ContentLength: fs.statSync(${JSON.stringify(privateResponse)}).size })); else if (args.includes("run-task")) console.log(JSON.stringify({ tasks: [{ taskArn: "canary" }] })); -else if (args.includes("describe-tasks")) console.log(JSON.stringify({ tasks: [{ containers: [{ name: "core", exitCode: 0 }] }] })); +else if (args.includes("describe-tasks")) console.log(JSON.stringify({ tasks: [{ containers: [{ name: "core", exitCode: 0 }, { name: "secret-validation", exitCode: 0 }] }] })); else if (args.includes("dynamodb get-item") && args.includes("deployment/current")) console.log(JSON.stringify({ Item: { manifestId: { S: "manifest" } } })); else if (args.includes("dynamodb get-item") && args.includes("deployment/manifest/manifest")) console.log(JSON.stringify({ Item: { manifest: { S: ${JSON.stringify(JSON.stringify(manifest))} } } })); -else if (args.includes("s3api get-object")) fs.writeFileSync(argv[argv.indexOf("--key") + 2], ${JSON.stringify(layerBody)}); +else if (args.includes("s3api get-object")) fs.writeFileSync(argv[argv.indexOf("--key") + 2], args.includes("deployment/core-requests/") ? fs.readFileSync(${JSON.stringify(privateResponse)}, "utf8") : ${JSON.stringify(layerBody)}); else if (args.includes("elbv2 describe-load-balancers")) console.log(JSON.stringify({ LoadBalancers: [{ LoadBalancerArn: "lb", DNSName: "acme.example.com", State: { Code: "active" } }] })); else if (args.includes("elbv2 describe-listeners")) console.log(JSON.stringify({ Listeners: [{ ListenerArn: "listener", Protocol: "HTTPS", Port: 443, Certificates: [{ CertificateArn: "certificate" }], DefaultActions: [{ Type: "fixed-response", FixedResponseConfig: { StatusCode: "404" } }] }] })); else if (args.includes("elbv2 describe-target-groups")) console.log(JSON.stringify({ TargetGroups: [{ TargetGroupArn: "tg", TargetGroupName: ${JSON.stringify(targetGroupName)} }] })); diff --git a/cli/test/deployment-layer.test.ts b/cli/test/deployment-layer.test.ts index 3c599f80d..82584ff22 100644 --- a/cli/test/deployment-layer.test.ts +++ b/cli/test/deployment-layer.test.ts @@ -5,10 +5,14 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, w import { tmpdir } from "node:os"; import { join } from "node:path"; import { CONFIG_FILENAME, loadConfigInDir, type QmConfig } from "../src/config.ts"; -import { currentDeploymentLayerState, deploymentLayerBundle, syncDeploymentLayer } from "../src/deployment-layer.ts"; +import { + currentDeploymentLayerState, + deploymentLayerBundle, + httpDeploymentLayerTransport, + syncDeploymentLayer, +} from "../src/deployment-layer.ts"; import { dockerDeploymentLayerTransport } from "../src/backends/docker.ts"; import { flyDeploymentLayerTransport } from "../src/backends/fly.ts"; -import { awsDeploymentLayerTransport } from "../src/backends/aws.ts"; import { expectedDescriptors, runConformance } from "../src/commands/conformance.ts"; const SECRET = "conformance-test-secret"; @@ -438,7 +442,7 @@ test("a publicUrl with a base path keeps it in the request path and the signed c await withEnv({ CORE_SIGNING_SECRET: SECRET }, () => syncDeploymentLayer({ config: makeConfig(`http://127.0.0.1:${port}/base`), - transport: awsDeploymentLayerTransport, + transport: httpDeploymentLayerTransport(), configDir: dir, sandboxDir: join(dir, "sandbox"), }), diff --git a/cli/test/package.test.ts b/cli/test/package.test.ts index 461171622..946740fb5 100644 --- a/cli/test/package.test.ts +++ b/cli/test/package.test.ts @@ -220,6 +220,10 @@ test( ), ); const cluster = "acme-aws-qm"; + const tfvars = readFileSync(join(awsDeployment, "infra", "terraform.tfvars"), "utf8"); + const requiredSecretNames = JSON.parse( + tfvars.match(/^required_secret_names\s*=\s*(\[[\s\S]*?\n\])/m)?.[1] ?? "[]", + ) as string[]; const portalTarget = `arn:aws:elasticloadbalancing:us-west-2:000000000000:targetgroup/${cluster}-port-${createHash("sha1").update(`${cluster}:portal`).digest("hex").slice(0, 6)}/1`; const fakeAws = join(dir, "aws"); writeFileSync( @@ -242,20 +246,23 @@ else if (command === "ecs describe-services") { status: "ACTIVE", desiredCount: 0, taskDefinition: "arn:task/" + serviceName + ":1", + networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet"], securityGroups: ["sg"], assignPublicIp: "DISABLED" } }, loadBalancers: serviceName.endsWith("-portal") ? [{ targetGroupArn: ${JSON.stringify(portalTarget)} }] : [], tags: [{ key: "Deployment", value: "acme-aws" }, { key: "ManagedBy", value: "terraform" }], })) }); -} else if (command === "ecs describe-task-definition") json({ taskDefinition: { family: "legacy", containerDefinitions: [] } }); +} else if (command === "ecs describe-task-definition" && option("--task-definition") === ${JSON.stringify(`${cluster}-secret-validation`)}) json({ taskDefinition: { taskDefinitionArn: "arn:task/secret-validation:1", status: "ACTIVE", containerDefinitions: [{ name: "secret-validation", secrets: ${JSON.stringify(requiredSecretNames.map((name) => ({ name })))} }] } }); +else if (command === "ecs describe-task-definition") json({ taskDefinition: { family: "legacy", containerDefinitions: [] } }); +else if (command === "ecs run-task") json({ tasks: [{ taskArn: "arn:task/canary" }] }); +else if (command === "ecs wait") process.stdout.write(""); +else if (command === "ecs describe-tasks") json({ tasks: [{ containers: [{ name: "secret-validation", exitCode: 0 }] }] }); else if (command === "lambda-microvms get-microvm-image") json({ imageArn: "arn:aws:lambda:us-west-2:000000000000:microvm-image:acme-aws-qm-sandbox" }); else if (command === "lambda-microvms list-microvm-image-versions") json({ items: [{ imageVersion: "1", state: "SUCCESSFUL", status: "ACTIVE" }] }); -else if (command === "secretsmanager get-secret-value") { +else if (command === "secretsmanager describe-secret") { const name = option("--secret-id").split("/").at(-1); - const value = name === "ADMIN_GRANTS" ? "admin@example.com:org_admin" - : name === "PUBLIC_API_URL" ? "https://acme-aws.example.com" - : name.endsWith("SIGNING_SECRET") || name === "CONNECTOR_SECRET_KEY" ? "a".repeat(64) - : "fixture-" + name.toLowerCase(); - if (args.includes("--query")) process.stdout.write(value + "\\n"); - else json({ ARN: "arn:secret/" + name, SecretString: value }); + json({ ARN: "arn:secret/" + name, VersionIdsToStages: { current: ["AWSCURRENT"] } }); +} else if (command === "secretsmanager get-secret-value") { + process.stderr.write("CLI runner must not directly retrieve plaintext secret values\\n"); + process.exit(19); } else json({}); `, ); diff --git a/cli/test/terraform.test.ts b/cli/test/terraform.test.ts index 5ff16169d..519b5a825 100644 --- a/cli/test/terraform.test.ts +++ b/cli/test/terraform.test.ts @@ -48,6 +48,7 @@ test("declaredVariables reads the scaffolded variables.tf", () => { "deploy_microvm_execution_role_arn", "services", "secret_names", + "required_secret_names", ]) { assert.ok(declared.includes(name), `variables.tf declares ${name}`); } @@ -60,6 +61,16 @@ test("the ECS execution role can read every managed contract secret", () => { assert.doesNotMatch(policy, /secret:\$\{var\.secrets_prefix\}\*/); }); +test("the deploy role has no direct plaintext retrieval permission but retains runtime deployment authority", () => { + const policy = mainTf.match(/resource "aws_iam_role_policy" "github_deploy" \{([\s\S]*?)\n\}/)?.[1] ?? ""; + const secrets = policy.match(/Sid\s*= "ManageContractSecrets"([\s\S]*?)\n\s*\},/)?.[1] ?? ""; + assert.match(secrets, /secretsmanager:DescribeSecret/); + assert.match(secrets, /secretsmanager:PutSecretValue/); + assert.doesNotMatch(policy, /secretsmanager:GetSecretValue/); + assert.match(policy, /ecs:RegisterTaskDefinition/); + assert.match(policy, /iam:PassRole/); +}); + test("new S3 buckets use AWS's default public-access block without a separate mutation", () => { assert.ok(!declared.includes("manage_object_store_public_access_block")); assert.doesNotMatch(mainTf, /aws_s3_bucket_public_access_block/); @@ -88,6 +99,7 @@ test("the deploy role can run and inspect only stack-scoped deployment canaries" const run = policy.match(/Sid\s*= "RunDeploymentCanaries"([\s\S]*?)\n\s*\},/)?.[1] ?? ""; assert.match(run, /ecs:RunTask/); assert.match(run, /task-definition\/\$\{var\.services\["core"\]\.ecs_service\}:\*/); + assert.match(run, /task-definition\/\$\{var\.cluster_name\}-secret-validation:\*/); assert.match(run, /"ecs:cluster"\s*=\s*aws_ecs_cluster\.this\.arn/); assert.doesNotMatch(run, /Resource\s*= "\*"/); @@ -97,6 +109,16 @@ test("the deploy role can run and inspect only stack-scoped deployment canaries" assert.doesNotMatch(inspect, /Resource\s*= "\*"/); }); +test("the deploy role can exchange and clean up only private core request objects", () => { + const policy = mainTf.match(/resource "aws_iam_role_policy" "github_deploy" \{([\s\S]*?)\n\}/)?.[1] ?? ""; + const requests = policy.match(/Sid\s*= "ManagePrivateCoreRequests"([\s\S]*?)\n\s*\},/)?.[1] ?? ""; + for (const action of ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"]) { + assert.match(requests, new RegExp(action)); + } + assert.match(requests, /deployment\/core-requests\/\*/); + assert.doesNotMatch(requests, /Resource\s*= "\*"|DeleteObjectVersion/); +}); + test("AWS deployments retain recovery history and can create scoped predeploy snapshots", () => { const variables = readFileSync(new URL("../templates/aws/variables.tf", import.meta.url), "utf8"); assert.match(variables, /variable "db_backup_retention_days" \{[\s\S]*default = 35/); @@ -259,13 +281,24 @@ test("terraform derives the transfer lifecycle prefix from the same core S3 pref test("terraform owns secret containers but never creates operator-secret placeholder values", () => { const rendered = terraformVars(config, "", declared); - const all = rendered.match(/secret_names\s*=\s*(\[[\s\S]*?\])\s*$/)?.[1] ?? ""; + const all = rendered.match(/^secret_names\s*=\s*(\[[\s\S]*?\n\])/m)?.[1] ?? ""; + const required = rendered.match(/^required_secret_names\s*=\s*(\[[\s\S]*?\n\])/m)?.[1] ?? ""; assert.match(all, /CORE_SIGNING_SECRET/); + assert.match(required, /CORE_SIGNING_SECRET/); assert.doesNotMatch(mainTf, /aws_secretsmanager_secret_version" "placeholder/); assert.doesNotMatch(mainTf, /secret_string\s*=\s*"replace-me"/); assert.match(mainTf, /aws_secretsmanager_secret_version" "database/); }); +test("secret values are validated only inside an ECS task with the execution role", () => { + const validator = mainTf.match(/resource "aws_ecs_task_definition" "secret_validation" \{([\s\S]*?)\n\}/)?.[1] ?? ""; + assert.match(validator, /family\s*= "\$\{var\.cluster_name\}-secret-validation"/); + assert.match(validator, /execution_role_arn\s*= local\.default_execution_role_arn/); + assert.match(validator, /task_role_arn\s*= local\.default_task_role_arn/); + assert.match(validator, /var\.required_secret_names/); + assert.match(validator, /aws_secretsmanager_secret\.contract\[name\]\.arn/); +}); + test("AWS module keeps portal as the sole front door and preserves CLI-owned ECS state", () => { assert.match(mainTf, /public_service_names\s*= local\.has_portal \? \["portal"\] : \["core"\]/); assert.match(mainTf, /direct_path_services\s*= local\.has_portal \? \{\} : \{ core = \["\/v1\/\*"\] \}/);