diff --git a/.gitignore b/.gitignore index 889f6aa2..0f1bf34e 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,19 @@ testing/ logs/run_evaluation/ openswarm.orb-*.json +# Agent session scratch (do not commit) +ls-run-tests.sh +ls-bash-n.sh +ls-bash-env.sh +ls-verify-out.txt +ls-verify-ran.marker +ls-bin/ +scripts/ls +cli.json +.tmp-run-tests.sh +run-vitest-temp.mjs +.cursor/hooks/ + # Subprojects openclaw/ plan.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d1f9f0fe..14c2594a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -84,7 +84,7 @@ Admission under `unknownScopeAdmission: admit` (default, vela): - **vela compose는 이미지 태그를 pin한다.** `docker build`만으로는 recreate 안 됨. `docker-compose.yml`의 `openswarm:vela-YYYYMMDD-HHMM-amd64`를 바꾸고 `up -d --no-deps openswarm`. - **`vela-build.sh `는 `origin/main`만 fetch한다.** PR SHA는 `builds/4e8ba24`에서 `git fetch origin ` 먼저. -- **PR SHA를 돌리는 동안 `.last-deployed-sha`는 `origin/main` tip을 유지**하지 않으면 autodeploy cron이 main으로 롤백한다. +- **PR SHA를 돌리는 동안 `.last-built-sha`는 `origin/main` tip을 유지**하지 않으면 autodeploy cron이 main으로 롤백한다. - **`unknownScopeAdmission: admit`를 heartbeat만 바꾸고 claim 게이트를 안 바꾸면** 큐에는 들어가고 `claimRun`이 null — 슬롯이 다시 빈다. - **`isActionableLinearState(state)` 헬퍼 기본은 Backlog park.** 엔진/config 기본은 `includeBacklog: true`. `undefined`를 spread하면 DEFAULT true를 덮는다 — constructor가 `?? true`로 고정한다. - **AGT-4155 park loop**: Linear `In Progress`만으로 park를 열면 claim→exec→park가 heartbeat마다 돈다. 지금은 `idle_fill`이 의도적으로 연다 (싼 모델이 빈 슬롯을 씹도록). `serialize`/질문 대기를 되살릴 때 이 루프를 다시 열지 말 것. diff --git a/src/ops/velaAutodeploy.script.test.ts b/src/ops/velaAutodeploy.script.test.ts new file mode 100644 index 00000000..9c4930d2 --- /dev/null +++ b/src/ops/velaAutodeploy.script.test.ts @@ -0,0 +1,327 @@ +// ============================================ +// OpenSwarm - vela autodeploy gate tests (AGT-4182) +// ============================================ +// +// Host script lives at repo root / ~/openswarm-deploy. These tests drive the +// real bash checklist with a temp deploy dir + PATH stubs so CI can prove +// skip reasons and dry-run non-mutation without touching rtx. + +import { execFileSync } from 'node:child_process'; +import { + accessSync, + chmodSync, + constants, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Database from 'better-sqlite3'; +import { fileURLToPath } from 'node:url'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const AUTODEPLOY = join(REPO_ROOT, 'vela-autodeploy.sh'); +const BUILD = join(REPO_ROOT, 'vela-build.sh'); + +const MAIN_SHA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const OTHER_SHA = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; + +function hasBin(name: string): boolean { + for (const dir of (process.env.PATH ?? '').split(':')) { + if (!dir) continue; + try { + accessSync(join(dir, name), constants.X_OK); + return true; + } catch { + /* keep looking */ + } + } + return false; +} + +const hasSqlite3 = hasBin('sqlite3'); +const hasFlock = hasBin('flock'); + +let root: string; +let deployDir: string; +let binDir: string; +let automationDb: string; +let homeDir: string; + +function writeExec(path: string, body: string): void { + writeFileSync(path, body, { mode: 0o755 }); + chmodSync(path, 0o755); +} + +function seedLedger(states: string[]): void { + const db = new Database(automationDb); + db.exec(` + CREATE TABLE IF NOT EXISTS automation_runs ( + issue_id TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'test', + project_path TEXT NOT NULL DEFAULT '/work', + state TEXT NOT NULL, + state_version INTEGER NOT NULL DEFAULT 1, + attempt_no INTEGER NOT NULL DEFAULT 0, + lease_epoch INTEGER NOT NULL DEFAULT 0, + discovered_at INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0 + ); + `); + const insert = db.prepare( + `INSERT INTO automation_runs (issue_id, state, discovered_at, updated_at) VALUES (?, ?, 1, 1)`, + ); + states.forEach((state, i) => insert.run(`ISSUE-${i}`, state)); + db.close(); +} + +function seedCompose(tag = 'openswarm:vela-20260902-1200-amd64'): void { + writeFileSync( + join(deployDir, 'docker-compose.yml'), + `services:\n openswarm:\n image: ${tag}\n`, + ); + writeFileSync(join(deployDir, 'docker-compose.strict-sandbox.yml'), 'services: {}\n'); +} + +type StubOpts = { + mainSha?: string; + startedAt?: string | null; + healthStatus?: string; + curlOk?: boolean; + sockOk?: boolean; + buildTag?: string; +}; + +function installStubs(opts: StubOpts = {}): void { + const mainSha = opts.mainSha ?? MAIN_SHA; + const startedAt = opts.startedAt === undefined + ? new Date(Date.now() - 60 * 60_000).toISOString() + : opts.startedAt; + const healthStatus = opts.healthStatus ?? 'healthy'; + const curlOk = opts.curlOk ?? true; + const sockOk = opts.sockOk ?? true; + const buildTag = opts.buildTag ?? 'openswarm:vela-20260910-0800-amd64'; + + writeExec( + join(binDir, 'git'), + `#!/usr/bin/env bash +set -euo pipefail +if [[ "\${1:-}" == "ls-remote" ]]; then + echo "${mainSha}\trefs/heads/main" + exit 0 +fi +echo "unexpected git args: $*" >&2 +exit 1 +`, + ); + + writeExec( + join(binDir, 'docker'), + `#!/usr/bin/env bash +set -euo pipefail +cmd="\${1:-}" +if [[ "\$cmd" == "inspect" ]]; then + fmt="" + while [[ \$# -gt 0 ]]; do + if [[ "\$1" == "--format" ]]; then fmt="\$2"; shift 2; continue; fi + shift || true + done + case "\$fmt" in + '{{.State.StartedAt}}') + ${startedAt === null ? 'exit 1' : `echo '${startedAt}'`} + ;; + '{{.State.Health.Status}}') echo '${healthStatus}' ;; + '{{.State.Status}}') echo 'running' ;; + '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') echo '${healthStatus}' ;; + *) echo '' ;; + esac + exit 0 +fi +if [[ "\$cmd" == "compose" ]]; then + echo "compose ok" + exit 0 +fi +if [[ "\$cmd" == "exec" ]]; then + ${sockOk ? 'exit 0' : 'exit 1'} +fi +if [[ "\$cmd" == "build" ]]; then + echo "built" + exit 0 +fi +echo "unexpected docker args: $*" >&2 +exit 1 +`, + ); + + writeExec( + join(binDir, 'curl'), + `#!/usr/bin/env bash +${curlOk ? 'exit 0' : 'exit 22'} +`, + ); + + writeExec( + join(binDir, 'vela-build.sh'), + `#!/usr/bin/env bash +set -euo pipefail +echo "building..." +echo "${buildTag}" +`, + ); +} + +function runAutodeploy(args: string[] = [], extraEnv: Record = {}): { + status: number; + stdout: string; +} { + const pathEnv = `${binDir}:${process.env.PATH ?? ''}`; + try { + const stdout = execFileSync('bash', [AUTODEPLOY, ...args], { + encoding: 'utf8', + env: { + ...process.env, + PATH: pathEnv, + HOME: homeDir, + OPENSWARM_DEPLOY_DIR: deployDir, + OPENSWARM_AUTOMATION_DB: automationDb, + OPENSWARM_AUTODEPLOY_LOG: join(deployDir, 'autodeploy.log'), + OPENSWARM_AUTODEPLOY_LOCK: join(deployDir, '.autodeploy.lock'), + OPENSWARM_LAST_BUILT_SHA_FILE: join(deployDir, '.last-built-sha'), + OPENSWARM_COMPOSE_FILE: join(deployDir, 'docker-compose.yml'), + OPENSWARM_COMPOSE_STRICT: join(deployDir, 'docker-compose.strict-sandbox.yml'), + VELA_BUILD_SCRIPT: join(binDir, 'vela-build.sh'), + OPENSWARM_HEALTH_TIMEOUT_SEC: '5', + ...extraEnv, + }, + }); + return { status: 0, stdout }; + } catch (err) { + const e = err as { status?: number; stdout?: string; stderr?: string }; + return { + status: e.status ?? 1, + stdout: `${e.stdout ?? ''}${e.stderr ?? ''}`, + }; + } +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'vela-autodeploy-')); + deployDir = join(root, 'deploy'); + binDir = join(root, 'bin'); + homeDir = join(root, 'home'); + automationDb = join(homeDir, '.openswarm', 'automation.db'); + mkdirSync(binDir, { recursive: true }); + mkdirSync(deployDir, { recursive: true }); + mkdirSync(dirname(automationDb), { recursive: true }); + seedCompose(); + seedLedger([]); + installStubs(); + chmodSync(AUTODEPLOY, 0o755); + chmodSync(BUILD, 0o755); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe('vela shell script syntax', () => { + it('bash -n passes for both scripts', () => { + execFileSync('bash', ['-n', AUTODEPLOY], { stdio: 'pipe' }); + execFileSync('bash', ['-n', BUILD], { stdio: 'pipe' }); + }); +}); + +describe.skipIf(!hasSqlite3 || !hasFlock)('vela-autodeploy.sh gates', () => { + it('dry-run SKIP when .last-built-sha matches origin/main', () => { + writeFileSync(join(deployDir, '.last-built-sha'), MAIN_SHA); + const before = readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8'); + const { status, stdout } = runAutodeploy(['--dry-run']); + expect(status).toBe(0); + expect(stdout).toMatch(/SKIP\s+running image already built from/); + expect(readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8')).toBe(before); + expect(existsSync(join(deployDir, '.last-built-sha'))).toBe(true); + expect(readFileSync(join(deployDir, '.last-built-sha'), 'utf8').trim()).toBe(MAIN_SHA); + }); + + it('dry-run SKIP when ledger has active VERIFYING/PUBLISHING/EXECUTING/CLAIMED rows', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + seedLedger(['EXECUTING', 'READY']); + const before = readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8'); + const { status, stdout } = runAutodeploy(['--dry-run']); + expect(status).toBe(0); + expect(stdout).toMatch(/SKIP\s+ledger has 1 active run/); + expect(readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8')).toBe(before); + }); + + it('dry-run SKIP when automation DB is missing', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + rmSync(automationDb, { force: true }); + const { status, stdout } = runAutodeploy(['--dry-run']); + expect(status).toBe(0); + expect(stdout).toMatch(/SKIP\s+cannot query automation DB/); + }); + + it('dry-run SKIP when container StartedAt is within rate limit', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + installStubs({ startedAt: new Date(Date.now() - 10 * 60_000).toISOString() }); + const { status, stdout } = runAutodeploy(['--dry-run'], { OPENSWARM_RATE_LIMIT_MIN: '55' }); + expect(status).toBe(0); + expect(stdout).toMatch(/SKIP\s+container started \d+m ago/); + }); + + it('dry-run prints would-deploy plan without mutating compose or sha file', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + const beforeCompose = readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8'); + const beforeSha = readFileSync(join(deployDir, '.last-built-sha'), 'utf8'); + const { status, stdout } = runAutodeploy(['--dry-run']); + expect(status).toBe(0); + expect(stdout).toMatch(/DRY-RUN would build/); + expect(stdout).toMatch(/DRY-RUN would sed compose tag/); + expect(stdout).not.toMatch(/SUCCESS deployed/); + expect(readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8')).toBe(beforeCompose); + expect(readFileSync(join(deployDir, '.last-built-sha'), 'utf8')).toBe(beforeSha); + }); + + it('live pass updates compose tag and .last-built-sha when gates clear', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + const newTag = 'openswarm:vela-20260910-0800-amd64'; + installStubs({ buildTag: newTag }); + const { status, stdout } = runAutodeploy([]); + expect(status).toBe(0); + expect(stdout).toMatch(/SUCCESS deployed/); + expect(readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8')).toContain(newTag); + expect(readFileSync(join(deployDir, '.last-built-sha'), 'utf8').trim()).toBe(MAIN_SHA); + }); + + it('verify failure rolls back compose tag and exits non-zero', () => { + writeFileSync(join(deployDir, '.last-built-sha'), OTHER_SHA); + const prev = 'openswarm:vela-20260902-1200-amd64'; + seedCompose(prev); + installStubs({ curlOk: false, buildTag: 'openswarm:vela-20260910-0900-amd64' }); + const { status, stdout } = runAutodeploy([]); + expect(status).not.toBe(0); + expect(stdout).toMatch(/VERIFY FAILED|rollback/i); + expect(readFileSync(join(deployDir, 'docker-compose.yml'), 'utf8')).toContain(prev); + expect(readFileSync(join(deployDir, '.last-built-sha'), 'utf8').trim()).toBe(OTHER_SHA); + }); +}); + +describe('vela-build.sh', () => { + it('rejects non-40-char hex SHA', () => { + let failed = false; + try { + execFileSync('bash', [BUILD, 'notasha'], { encoding: 'utf8', stdio: 'pipe' }); + } catch (err) { + failed = true; + const e = err as { status?: number; stderr?: string; stdout?: string }; + expect(e.status).not.toBe(0); + expect(`${e.stderr ?? ''}${e.stdout ?? ''}`).toMatch(/invalid SHA/); + } + expect(failed).toBe(true); + }); +}); diff --git a/src/ops/velaAutodeployGates.test.ts b/src/ops/velaAutodeployGates.test.ts new file mode 100644 index 00000000..1871d91b --- /dev/null +++ b/src/ops/velaAutodeployGates.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import { ACTIVE_LEASE_STATES } from '../automation/runLedgerTypes.js'; +import { + AUTODEPLOY_ACTIVE_LEDGER_STATES, + AUTODEPLOY_RATE_LIMIT_MIN_DEFAULT, + decideAutodeployGate, +} from './velaAutodeployGates.js'; + +const MAIN = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const OTHER = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; + +describe('decideAutodeployGate', () => { + it('keeps ledger safety states in sync with ACTIVE_LEASE_STATES', () => { + expect([...AUTODEPLOY_ACTIVE_LEDGER_STATES].sort()).toEqual( + [...ACTIVE_LEASE_STATES].sort(), + ); + }); + + it('skips when last-built sha equals origin/main', () => { + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: MAIN, + ledgerActiveCount: 0, + containerAgeMinutes: 120, + }), + ).toEqual({ + action: 'skip', + reason: `running image already built from ${MAIN}`, + }); + }); + + it('skips when ledger DB cannot be queried', () => { + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: OTHER, + ledgerActiveCount: null, + containerAgeMinutes: 120, + }), + ).toEqual({ + action: 'skip', + reason: 'cannot query automation DB — daemon may be down or starting', + }); + }); + + it('skips when ledger has active VERIFYING/PUBLISHING/EXECUTING/CLAIMED rows', () => { + expect(AUTODEPLOY_ACTIVE_LEDGER_STATES).toEqual([ + 'VERIFYING', + 'PUBLISHING', + 'EXECUTING', + 'CLAIMED', + ]); + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: OTHER, + ledgerActiveCount: 2, + containerAgeMinutes: 120, + }), + ).toEqual({ + action: 'skip', + reason: 'ledger has 2 active run(s) in VERIFYING/PUBLISHING/EXECUTING/CLAIMED', + }); + }); + + it('skips when container age is under the rate limit', () => { + expect(AUTODEPLOY_RATE_LIMIT_MIN_DEFAULT).toBe(55); + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: OTHER, + ledgerActiveCount: 0, + containerAgeMinutes: 10, + }), + ).toEqual({ + action: 'skip', + reason: 'container started 10m ago (< 55m rate limit)', + }); + }); + + it('deploys when sha advanced, ledger idle, and rate limit clear', () => { + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: OTHER, + ledgerActiveCount: 0, + containerAgeMinutes: 60, + }), + ).toEqual({ action: 'deploy' }); + }); + + it('deploys when container is not running (fresh deploy)', () => { + expect( + decideAutodeployGate({ + mainSha: MAIN, + lastBuiltSha: null, + ledgerActiveCount: 0, + containerAgeMinutes: null, + }), + ).toEqual({ action: 'deploy' }); + }); +}); diff --git a/src/ops/velaAutodeployGates.ts b/src/ops/velaAutodeployGates.ts new file mode 100644 index 00000000..ad5a9f0b --- /dev/null +++ b/src/ops/velaAutodeployGates.ts @@ -0,0 +1,72 @@ +// ============================================ +// OpenSwarm - vela autodeploy gate decisions (AGT-4182) +// ============================================ +// +// Pure checklist used by host `vela-autodeploy.sh`. Kept as TypeScript so CI +// can lock the skip reasons without SSH to rtx; the shell script is the +// runtime. Keep reason strings in sync with the script's `SKIP` log lines. +// Must match `ACTIVE_LEASE_STATES` in `src/automation/runLedgerTypes.ts` (set equality). + +export const AUTODEPLOY_ACTIVE_LEDGER_STATES = [ + 'VERIFYING', + 'PUBLISHING', + 'EXECUTING', + 'CLAIMED', +] as const; + +export const AUTODEPLOY_RATE_LIMIT_MIN_DEFAULT = 55; + +export type AutodeployGateInput = { + mainSha: string; + lastBuiltSha: string | null; + /** null = sqlite/DB unavailable (fail closed / skip). */ + ledgerActiveCount: number | null; + /** Minutes since container StartedAt; null = not running / inspect failed. */ + containerAgeMinutes: number | null; + rateLimitMin?: number; +}; + +export type AutodeployGateDecision = + | { action: 'skip'; reason: string } + | { action: 'deploy' }; + +/** + * Decide whether the autodeploy lane should build+deploy this tick. + * Order matches `vela-autodeploy.sh` steps 1→4 (CI trust is implicit). + */ +export function decideAutodeployGate(input: AutodeployGateInput): AutodeployGateDecision { + const rateLimitMin = input.rateLimitMin ?? AUTODEPLOY_RATE_LIMIT_MIN_DEFAULT; + + if (input.lastBuiltSha !== null && input.lastBuiltSha === input.mainSha) { + return { + action: 'skip', + reason: `running image already built from ${input.mainSha}`, + }; + } + + if (input.ledgerActiveCount === null) { + return { + action: 'skip', + reason: 'cannot query automation DB — daemon may be down or starting', + }; + } + + if (input.ledgerActiveCount > 0) { + return { + action: 'skip', + reason: `ledger has ${input.ledgerActiveCount} active run(s) in VERIFYING/PUBLISHING/EXECUTING/CLAIMED`, + }; + } + + if ( + input.containerAgeMinutes !== null + && input.containerAgeMinutes < rateLimitMin + ) { + return { + action: 'skip', + reason: `container started ${input.containerAgeMinutes}m ago (< ${rateLimitMin}m rate limit)`, + }; + } + + return { action: 'deploy' }; +} diff --git a/vela-autodeploy.sh b/vela-autodeploy.sh new file mode 100755 index 00000000..7f935fc7 --- /dev/null +++ b/vela-autodeploy.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# vela-autodeploy.sh — autodeploy lane for OpenSwarm daemon on rtx +# +# Cron every 20 min, flock-guarded. Merged main → build → deploy → verify. +# Deployed at ~/openswarm-deploy on rtx (override with OPENSWARM_DEPLOY_DIR). +# +# Usage: +# ./vela-autodeploy.sh # normal run (cron) +# ./vela-autodeploy.sh --dry-run # print decisions; no compose/image/sha mutations +# +# Design: https://linear.app/intrect/issue/AGT-4182 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Config (env overrides exist so the checklist is testable off-host) ────── +DEPLOY_DIR="${OPENSWARM_DEPLOY_DIR:-${HOME}/openswarm-deploy}" +COMPOSE_FILE="${OPENSWARM_COMPOSE_FILE:-${DEPLOY_DIR}/docker-compose.yml}" +COMPOSE_STRICT="${OPENSWARM_COMPOSE_STRICT:-${DEPLOY_DIR}/docker-compose.strict-sandbox.yml}" +SERVICE_NAME="${OPENSWARM_SERVICE_NAME:-openswarm}" +LAST_BUILT_SHA_FILE="${OPENSWARM_LAST_BUILT_SHA_FILE:-${DEPLOY_DIR}/.last-built-sha}" +AUTODEPLOY_LOG="${OPENSWARM_AUTODEPLOY_LOG:-${DEPLOY_DIR}/autodeploy.log}" +LOCK_FILE="${OPENSWARM_AUTODEPLOY_LOCK:-${DEPLOY_DIR}/.autodeploy.lock}" +HEALTH_URL="${OPENSWARM_HEALTH_URL:-http://localhost:3847/api/health}" +HEALTH_TIMEOUT_SEC="${OPENSWARM_HEALTH_TIMEOUT_SEC:-90}" +RATE_LIMIT_MIN="${OPENSWARM_RATE_LIMIT_MIN:-55}" +IMAGE_PREFIX="openswarm:vela" +EXECUTOR_SOCK="${OPENSWARM_EXECUTOR_SOCK:-/run/openswarm-sandbox/executor.sock}" +AUTOMATION_DB="${OPENSWARM_AUTOMATION_DB:-${HOME}/.openswarm/automation.db}" + +if [[ -n "${VELA_BUILD_SCRIPT:-}" ]]; then + : +elif [[ -x "${DEPLOY_DIR}/vela-build.sh" ]]; then + VELA_BUILD_SCRIPT="${DEPLOY_DIR}/vela-build.sh" +elif [[ -x "${SCRIPT_DIR}/vela-build.sh" ]]; then + VELA_BUILD_SCRIPT="${SCRIPT_DIR}/vela-build.sh" +else + VELA_BUILD_SCRIPT="${DEPLOY_DIR}/vela-build.sh" +fi + +# ── Helpers ───────────────────────────────────────────────────────────────── + +log() { + local ts + ts="$(date --iso-8601=seconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)" + # Log file may be unset before mkdir; still print to stdout. + if [[ -n "${AUTODEPLOY_LOG:-}" ]]; then + mkdir -p "$(dirname "${AUTODEPLOY_LOG}")" 2>/dev/null || true + echo "${ts} $*" >> "${AUTODEPLOY_LOG}" 2>/dev/null || true + fi + echo "${ts} $*" +} + +warn() { + log "WARN $*" +} + +die() { + log "ERROR $*" + exit 1 +} + +rollback_to() { + local prev_tag="$1" + local compose_backup="$2" + if [[ -z "${prev_tag}" ]]; then + warn "ROLLBACK no previous image tag known — cannot roll back" + return 1 + fi + if [[ -n "${compose_backup}" && -f "${compose_backup}" ]]; then + cp "${compose_backup}" "${COMPOSE_FILE}" + log "ROLLBACK restored compose file from backup" + else + local new_tag + new_tag="$(sed -n 's/.*image:[[:space:]]*\(openswarm:vela-[0-9]\{8\}-[0-9]\{4\}-amd64\).*/\1/p' "${COMPOSE_FILE}" | head -1)" + if [[ -n "${new_tag}" ]]; then + sed -i "s|${new_tag}|${prev_tag}|g" "${COMPOSE_FILE}" + log "ROLLBACK reverted tag ${new_tag} → ${prev_tag} in compose file" + fi + fi + + log "ROLLBACK redeploying with ${prev_tag} ..." + if docker compose -f "${COMPOSE_FILE}" -f "${COMPOSE_STRICT}" up -d --no-deps "${SERVICE_NAME}" 2>&1; then + log "ROLLBACK redeploy succeeded" + return 0 + fi + warn "ROLLBACK redeploy also failed — container may be down" + return 1 +} + +read_compose_image() { + local file="$1" + [[ -f "${file}" ]] || return 0 + sed -n 's/.*image:[[:space:]]*\(openswarm:vela-[0-9]\{8\}-[0-9]\{4\}-amd64\).*/\1/p' "${file}" | head -1 +} + +# ── Init ──────────────────────────────────────────────────────────────────── + +DRY_RUN=false +if [[ "${1:-}" == "--dry-run" ]]; then + DRY_RUN=true +fi + +mkdir -p "${DEPLOY_DIR}" + +if [[ "${DRY_RUN}" == "true" ]]; then + log "DRY-RUN mode — no build/deploy/compose/sha side effects" +fi + +# Flock guard — only one instance at a time +exec 200>"${LOCK_FILE}" +if ! flock -n 200; then + log "SKIP another autodeploy instance is running (flock held)" + exit 0 +fi + +cd "${DEPLOY_DIR}" + +# ── Step 1: Fetch origin/main SHA ────────────────────────────────────────── + +log "STEP 1 fetch origin/main SHA" + +MAIN_SHA="" +if ! MAIN_SHA="$(git ls-remote origin main 2>/dev/null | awk '{print $1}')"; then + die "could not reach origin to read main SHA" +fi +if [[ -z "${MAIN_SHA}" ]]; then + die "git ls-remote returned empty SHA for main" +fi +log " origin/main = ${MAIN_SHA}" + +if [[ -f "${LAST_BUILT_SHA_FILE}" ]]; then + LAST_SHA="$(tr -d '[:space:]' < "${LAST_BUILT_SHA_FILE}")" + if [[ "${LAST_SHA}" == "${MAIN_SHA}" ]]; then + log "SKIP running image already built from ${MAIN_SHA}" + exit 0 + fi + log " last built SHA = ${LAST_SHA} (differs — proceeding)" +else + log " no ${LAST_BUILT_SHA_FILE} — first run, proceeding" +fi + +# ── Step 2: CI trust ──────────────────────────────────────────────────────── + +log "STEP 2 CI trust — main is branch-protected with required checks; merged main is green by construction" + +# ── Step 3: Safety window — ledger must be idle ──────────────────────────── + +log "STEP 3 ledger safety window" + +LEDGER_ACTIVE=-1 +LEDGER_CHECK_FAILED=false + +if command -v sqlite3 &>/dev/null && [[ -f "${AUTOMATION_DB}" ]]; then + LEDGER_ACTIVE="$(sqlite3 "${AUTOMATION_DB}" " + SELECT COUNT(*) FROM automation_runs + WHERE state IN ('VERIFYING', 'PUBLISHING', 'EXECUTING', 'CLAIMED') + " 2>/dev/null)" || LEDGER_ACTIVE=-1 + + if [[ "${LEDGER_ACTIVE}" -lt 0 ]]; then + warn " sqlite3 query failed for ${AUTOMATION_DB}" + LEDGER_CHECK_FAILED=true + fi +else + if ! command -v sqlite3 &>/dev/null; then + warn " sqlite3 not found — cannot query ledger DB" + else + warn " automation DB not found at ${AUTOMATION_DB}" + fi + LEDGER_CHECK_FAILED=true +fi + +if [[ "${LEDGER_CHECK_FAILED}" == "true" ]]; then + log "SKIP cannot query automation DB — daemon may be down or starting" + exit 0 +fi + +if [[ "${LEDGER_ACTIVE}" -gt 0 ]]; then + log "SKIP ledger has ${LEDGER_ACTIVE} active run(s) in VERIFYING/PUBLISHING/EXECUTING/CLAIMED" + exit 0 +fi +log " ledger idle (0 active runs)" + +# ── Step 4: Rate limit — skip if container StartedAt < 55 min ago ────────── + +log "STEP 4 rate limit check" + +if command -v docker &>/dev/null; then + STARTED_AT="" + STARTED_AT="$(docker inspect --format '{{.State.StartedAt}}' "${SERVICE_NAME}" 2>/dev/null)" || STARTED_AT="" + if [[ -n "${STARTED_AT}" ]]; then + # Docker emits nanosecond ISO timestamps; GNU date often wants ms or no fraction. + STARTED_EPOCH=0 + STARTED_EPOCH="$(date -d "${STARTED_AT}" +%s 2>/dev/null)" || \ + STARTED_EPOCH="$(date -d "${STARTED_AT%.*}Z" +%s 2>/dev/null)" || \ + STARTED_EPOCH="$(date -d "${STARTED_AT%.*}" +%s 2>/dev/null)" || \ + STARTED_EPOCH=0 + NOW_EPOCH="$(date +%s)" + if [[ "${STARTED_EPOCH}" -gt 0 ]]; then + ELAPSED_MIN=$(( (NOW_EPOCH - STARTED_EPOCH) / 60 )) + if [[ "${ELAPSED_MIN}" -lt "${RATE_LIMIT_MIN}" ]]; then + log "SKIP container started ${ELAPSED_MIN}m ago (< ${RATE_LIMIT_MIN}m rate limit)" + exit 0 + fi + log " container started ${ELAPSED_MIN}m ago (≥ ${RATE_LIMIT_MIN}m)" + else + log " could not parse StartedAt=${STARTED_AT} — proceeding" + fi + else + log " container not running or inspect failed — will deploy fresh" + fi +else + warn " docker not found — cannot check rate limit, proceeding" +fi + +# ── Dry-run stops before any mutation ────────────────────────────────────── + +PREV_IMAGE="$(read_compose_image "${COMPOSE_FILE}" || true)" +if [[ -n "${PREV_IMAGE}" ]]; then + log " current image = ${PREV_IMAGE}" +else + log " no previous image tag found in compose file" +fi + +if [[ "${DRY_RUN}" == "true" ]]; then + log "DRY-RUN would build ${MAIN_SHA} via ${VELA_BUILD_SCRIPT}" + log "DRY-RUN would sed compose tag → ${IMAGE_PREFIX}--amd64 and up -d ${SERVICE_NAME}" + log "DRY-RUN would verify healthy + ${HEALTH_URL} + ${EXECUTOR_SOCK}; rollback on failure" + log "DRY-RUN would write ${MAIN_SHA} to ${LAST_BUILT_SHA_FILE} on success" + exit 0 +fi + +# ── Step 5: Build and deploy ─────────────────────────────────────────────── + +log "STEP 5 build and deploy" + +if [[ ! -x "${VELA_BUILD_SCRIPT}" ]]; then + die "vela-build.sh not found or not executable at ${VELA_BUILD_SCRIPT}" +fi + +COMPOSE_BACKUP="" +if [[ -f "${COMPOSE_FILE}" ]]; then + COMPOSE_BACKUP="${COMPOSE_FILE}.bak.$(date +%s)" + cp "${COMPOSE_FILE}" "${COMPOSE_BACKUP}" + log " backed up compose file to ${COMPOSE_BACKUP}" +else + die "compose file not found at ${COMPOSE_FILE}" +fi + +log " running ${VELA_BUILD_SCRIPT} ${MAIN_SHA} ..." +NEW_IMAGE="" +BUILD_OUT="" +if ! BUILD_OUT="$("${VELA_BUILD_SCRIPT}" "${MAIN_SHA}" 2>&1)"; then + echo "${BUILD_OUT}" >&2 + die "vela-build.sh failed for SHA ${MAIN_SHA}" +fi +NEW_IMAGE="$(printf '%s\n' "${BUILD_OUT}" | tail -1)" +if [[ -z "${NEW_IMAGE}" || "${NEW_IMAGE}" != openswarm:vela-* ]]; then + die "vela-build.sh did not print an image tag (got: ${NEW_IMAGE})" +fi +log " built image = ${NEW_IMAGE}" + +if [[ -n "${PREV_IMAGE}" ]]; then + sed -i "s|${PREV_IMAGE}|${NEW_IMAGE}|g" "${COMPOSE_FILE}" + log " updated compose tag: ${PREV_IMAGE} → ${NEW_IMAGE}" +else + sed -i "s|openswarm:vela-[0-9]\{8\}-[0-9]\{4\}-amd64|${NEW_IMAGE}|g" "${COMPOSE_FILE}" + log " set compose tag to ${NEW_IMAGE}" +fi + +log " deploying with docker compose up -d ..." +if ! docker compose -f "${COMPOSE_FILE}" -f "${COMPOSE_STRICT}" up -d --no-deps "${SERVICE_NAME}" 2>&1; then + warn " deploy failed — attempting rollback" + rollback_to "${PREV_IMAGE}" "${COMPOSE_BACKUP}" || true + die "deploy failed, rolled back to ${PREV_IMAGE:-previous}" +fi +log " docker compose up -d succeeded" + +# ── Step 6: Verify ───────────────────────────────────────────────────────── + +log "STEP 6 verify deployment" + +VERIFY_FAILED=false + +log " waiting up to ${HEALTH_TIMEOUT_SEC}s for container health ..." +HEALTHY=false +for ((i=0; i/dev/null | grep -q healthy; then + HEALTHY=true + break + fi + # Also accept running without a healthcheck block (some host compose variants). + status="$(docker inspect --format '{{.State.Status}}' "${SERVICE_NAME}" 2>/dev/null || true)" + health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "${SERVICE_NAME}" 2>/dev/null || true)" + if [[ "${status}" == "running" && "${health}" == "none" ]]; then + HEALTHY=true + break + fi + sleep 5 +done + +if [[ "${HEALTHY}" != "true" ]]; then + warn " container not healthy within ${HEALTH_TIMEOUT_SEC}s" + VERIFY_FAILED=true +else + log " container healthy" +fi + +if [[ "${VERIFY_FAILED}" != "true" ]]; then + log " checking ${HEALTH_URL} ..." + if ! curl -sf --max-time 10 "${HEALTH_URL}" >/dev/null 2>&1; then + warn " /api/health endpoint not responding" + VERIFY_FAILED=true + else + log " /api/health OK" + fi +fi + +# Path cross-checked against daemon code: +# src/sandboxExecutor/protocol.ts DEFAULT_SANDBOX_EXECUTOR_SOCKET +# docker-compose.strict-sandbox.yml healthcheck --socket +if [[ "${VERIFY_FAILED}" != "true" ]]; then + log " checking executor.sock in container ..." + if ! docker exec "${SERVICE_NAME}" test -S "${EXECUTOR_SOCK}" 2>/dev/null; then + warn " executor.sock not found in container" + VERIFY_FAILED=true + else + log " executor.sock OK" + fi +fi + +if [[ "${VERIFY_FAILED}" == "true" ]]; then + warn "VERIFY FAILED — rolling back to previous image" + rollback_to "${PREV_IMAGE}" "${COMPOSE_BACKUP}" || true + die "rollback complete — reverted to ${PREV_IMAGE:-previous image}" +fi + +echo "${MAIN_SHA}" > "${LAST_BUILT_SHA_FILE}" +log "SUCCESS deployed ${NEW_IMAGE} from SHA ${MAIN_SHA}" +log " wrote ${MAIN_SHA} to ${LAST_BUILT_SHA_FILE}" +exit 0 diff --git a/vela-build.sh b/vela-build.sh new file mode 100755 index 00000000..9929a240 --- /dev/null +++ b/vela-build.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# vela-build.sh — build OpenSwarm daemon image from a git SHA +# +# Usage: +# ./vela-build.sh # build image tagged openswarm:vela-- +# ./vela-build.sh --push # build and push to registry +# +# Builds from origin/main only (PR SHAs need a separate fetch step). +# Tag format: openswarm:vela-YYYYMMDD-HHMM-amd64 +# +# Design: https://linear.app/intrect/issue/AGT-4182 + +set -euo pipefail + +# ── Config ────────────────────────────────────────────────────────────────── +DEPLOY_DIR="${OPENSWARM_DEPLOY_DIR:-${HOME}/openswarm-deploy}" +REPO_DIR="${OPENSWARM_BUILD_REPO_DIR:-${DEPLOY_DIR}/repo}" +IMAGE_PREFIX="openswarm:vela" +PLATFORM="${OPENSWARM_BUILD_PLATFORM:-linux/amd64}" +DOCKERFILE="${OPENSWARM_DOCKERFILE:-${REPO_DIR}/Dockerfile}" + +# ── Helpers ───────────────────────────────────────────────────────────────── + +die() { + echo "ERROR $*" >&2 + exit 1 +} + +usage() { + echo "Usage: $0 [--push]" >&2 + exit 1 +} + +# ── Parse args ────────────────────────────────────────────────────────────── + +SHA="${1:-}" +PUSH=false +if [[ -z "${SHA}" ]]; then + usage +fi +shift +if [[ "${1:-}" == "--push" ]]; then + PUSH=true +fi + +# Validate SHA format (hex, 40 chars) +if ! [[ "${SHA}" =~ ^[0-9a-f]{40}$ ]]; then + die "invalid SHA: ${SHA} (expected 40-char hex)" +fi + +# ── Ensure repo clone ─────────────────────────────────────────────────────── + +REPO_URL="$(git -C "${DEPLOY_DIR}" remote get-url origin 2>/dev/null || true)" +if [[ -z "${REPO_URL}" ]]; then + REPO_URL="${OPENSWARM_REPO_URL:-https://github.com/intrect/OpenSwarm.git}" +fi + +if [[ ! -d "${REPO_DIR}/.git" ]]; then + echo "cloning repository into ${REPO_DIR} ..." + mkdir -p "$(dirname "${REPO_DIR}")" + git clone --filter=blob:none --no-checkout "${REPO_URL}" "${REPO_DIR}" +fi + +# ── Fetch + checkout the target SHA ───────────────────────────────────────── +# Partial clone with --no-checkout leaves an empty tree; docker build needs the +# files. Fetch the commit, then force-checkout so the build context matches SHA. + +echo "fetching SHA ${SHA} ..." +git -C "${REPO_DIR}" fetch --depth=1 origin "${SHA}" 2>&1 || \ + die "git fetch failed for SHA ${SHA}" + +if ! git -C "${REPO_DIR}" cat-file -e "${SHA}^{commit}" 2>/dev/null; then + die "SHA ${SHA} is not a valid commit" +fi + +echo "checking out ${SHA} ..." +git -C "${REPO_DIR}" checkout --force "${SHA}" 2>&1 || \ + die "git checkout failed for SHA ${SHA}" + +if [[ ! -f "${DOCKERFILE}" ]]; then + die "Dockerfile not found at ${DOCKERFILE} after checkout" +fi + +# ── Generate image tag ────────────────────────────────────────────────────── + +TIMESTAMP="$(TZ=UTC date +%Y%m%d-%H%M)" +ARCH="amd64" +IMAGE_TAG="${IMAGE_PREFIX}-${TIMESTAMP}-${ARCH}" +echo "image tag: ${IMAGE_TAG}" + +# ── Build ─────────────────────────────────────────────────────────────────── + +echo "building ${IMAGE_TAG} from SHA ${SHA} ..." +docker build \ + --platform "${PLATFORM}" \ + --tag "${IMAGE_TAG}" \ + --file "${DOCKERFILE}" \ + --build-arg "GIT_SHA=${SHA}" \ + "${REPO_DIR}" 2>&1 || die "docker build failed for ${IMAGE_TAG}" + +echo "build succeeded: ${IMAGE_TAG}" + +# ── Push (optional) ───────────────────────────────────────────────────────── + +if [[ "${PUSH}" == "true" ]]; then + echo "pushing ${IMAGE_TAG} ..." + docker push "${IMAGE_TAG}" 2>&1 || die "docker push failed for ${IMAGE_TAG}" + echo "push succeeded: ${IMAGE_TAG}" +fi + +# ── Output (last line consumed by vela-autodeploy.sh) ─────────────────────── + +echo "${IMAGE_TAG}"