fix(control-plane): STALE_PHYSICAL_LEASE for overlapping stale leases; sidecar refuses closed file with held lease (#415, #516)

scripts/ops/m5-cutover/m5-cutover-lib.mjs

@@ -72,7 +72,16 @@ export function recoveryFor(file, liveIsOld, liveIsTarget) {
  * Decide the sidecar mode from persisted state + live identity (read-only).
  * Returns "full" | "resume_drain" | "resume_restart" | "resume_finish"; throws Refuse otherwise.
  */
-export function selectMode({ file, contract, liveIsOld, liveIsTarget }) {
+export function selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }) {
+  // `closedLease` is the state-root lease row of a CLOSED cutover ({leaseId, version, terminalState}) or null/undefined
+  // when unknown/absent. The file and the SQLite lease are separate stores: a crash between file close and lease
+  // finalization leaves a closed file with a still-held lease, and a new attempt would only fail on overlap (#516).
+  if (file && file.phase === "closed" && closedLease && !closedLease.terminalState) {
+    const id = file.cutoverId;
+    throw new Refuse(`cutover ${id} is closed but its state-root lease ${closedLease.leaseId} is not terminal; a new attempt would fail with an overlapping resource scope`,
+      `finish lease replay, do NOT start a new attempt: ${RUNBOOK_CLI} cutover release-terminal-lease --cutover-id ${id} --lease-id ${closedLease.leaseId} --lease-version ${closedLease.version} --carrier <revoked-root-carrier-id> --carrier-version <revoked-version> --terminal-record-hash <cutoverTerminalRecordHash> --confirm ${id}`,
+      { activePhase: "closed", cutoverId: id, leaseId: closedLease.leaseId });
+  }
   if (!file || file.phase === "closed") return "full"; // Branch a: brand-new attempt.
   const target = contract.cutover.expectedIdentity;
   const ours = sameDeep(file.oldServerIdentity, contract.cutover.currentIdentity)

scripts/ops/m5-cutover/m5-cutover-sidecar.mjs

@@ -98,7 +98,16 @@ try {
   const liveIsTarget = matchesTarget(before, target)
     && before.mcp.serverInstanceId !== contract.cutover.currentIdentity.serverInstanceId;
   if (file && file.phase !== "closed") cutoverId = file.cutoverId;
-  const mode = selectMode({ file, contract, liveIsOld, liveIsTarget }); // "full" | "resume_drain" | "resume_restart" | "resume_finish"
+  // Closed file but state-root lease not yet terminal (crash between file close and lease finalization): read-only probe.
+  let closedLease;
+  if (file?.phase === "closed" && file.coordinationBinding?.leaseId) {
+    const probeBindings = new CarrierBindingStore(config.stateDir);
+    try {
+      const row = probeBindings.ownership.get(file.coordinationBinding.leaseId);
+      if (row) closedLease = { leaseId: row.leaseId, version: row.version, terminalState: row.terminalState ?? null };
+    } finally { probeBindings.close(); }
+  }
+  const mode = selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }); // "full" | "resume_drain" | "resume_restart" | "resume_finish"
   log("phase_detected", { mode, activePhase: file?.phase ?? null, cutoverId: file?.cutoverId ?? null, liveIsOld, liveIsTarget });
 
   if (mode !== "resume_finish" && Date.parse(contract.cutover.expiresAt) <= Date.now()) {

scripts/ops/m5-cutover/m5-cutover.test.mjs

@@ -54,6 +54,19 @@ test("no active generation or a closed one selects full", () => {
   assert.equal(select(record({ phase: "closed" }), false, true), "full");
 });
 
+test("closed file whose state-root lease is still held is refused with a release-terminal-lease hint (#516)", () => {
+  const closed = record({ phase: "closed", coordinationBinding: { leaseId: "lease_held" } });
+  const held = { leaseId: "lease_held", version: 3, terminalState: null };
+  const run = () => selectMode({ file: closed, contract, liveIsOld: false, liveIsTarget: true, closedLease: held });
+  assert.throws(run, (error) => error instanceof Refuse
+    && /cutover release-terminal-lease --cutover-id cut_1 --lease-id lease_held --lease-version 3 .* --confirm cut_1$/.test(error.recovery) && error.extra.cutoverId === "cut_1"
+    && error.extra.activePhase === "closed" && error.extra.leaseId === "lease_held");
+  // Liveness only: a terminal lease, an absent lease row, or no lease info keep the brand-new-attempt path.
+  for (const closedLease of [{ leaseId: "lease_held", version: 4, terminalState: "released" }, null, undefined]) {
+    assert.equal(selectMode({ file: closed, contract, liveIsOld: true, liveIsTarget: false, closedLease }), "full");
+  }
+});
+
 test("prepared without progress and old live resumes at drain", () => {
   assert.equal(select(record(), true, false), "resume_drain");
 });

src/carrier-binding.test.ts

@@ -2419,3 +2419,57 @@ test("revoked expired prepare-only cutover lease cleanup requires exact no-effec
     assert.deepEqual(replay.lease,result.lease);
   } finally {f.close();}
 });
+
+function idleCutoverScenario(f:ReturnType<typeof fixture>,key:string,opts:{revoke?:boolean;advanceMs?:number}={}) {
+  const context={clientId:"shared-oauth",sessionId:`idle-${key}`};
+  const pairing=f.store.requestPairing(context);
+  const cutover={
+    stateRoot:f.root,
+    attemptKey:key,
+    currentIdentity:{serverInstanceId:"original",sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)},
+    expectedIdentity:{sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)},
+    expiresAt:new Date(f.clock()+30_000).toISOString(),
+    restart:{buildReady:{verifiedBy:"fixture",verifiedAt:new Date(f.clock()).toISOString(),evidence:"prepare-only canary"},actuator:"launchd-self" as const,serviceLabel:"test.service",launchdTarget:"gui/501/test.service"},
+    finish:{workspaceId:"ws_idle",agentId:"agt_idle"},
+  };
+  const contract={...f.contract,scope:[f.root],operations:["cutover_start" as const],cutover,expiresAt:new Date(f.clock()+60_000).toISOString()};
+  const approved=f.store.approveLocal(pairing.pendingId,contract);
+  f.store.redeem(context,pairing.credential);
+  const plan=planCutoverStart(f.root,cutover);
+  const lease=f.store.prepareEffect(context,plan.subject);
+  if(opts.revoke!==false) f.store.revokeLocal(approved.id,approved.version);
+  f.advance(opts.advanceMs ?? 120_000);
+  const input={leaseId:lease.leaseId,expectedLeaseVersion:lease.version,carrierId:approved.id,expectedCarrierVersion:opts.revoke===false?approved.version:2,expectedValidityVersion:1,expectedOperationId:plan.operationId,confirmLeaseId:lease.leaseId};
+  return {input,lease,plan,approved};
+}
+
+test("idle orphan lease cleanup fails closed for non-revoked carrier, live lease, pinned lease, durable evidence and wrong identity (#415)", () => {
+  const leaseRow=(f:ReturnType<typeof fixture>,id:string)=>f.db.sqlite.prepare("select terminal_state,version from control_plane_resource_leases where lease_id=?").get(id) as {terminal_state:string|null;version:number};
+  const expectDenied=(f:ReturnType<typeof fixture>,s:ReturnType<typeof idleCutoverScenario>,override:Partial<typeof s.input>,pattern:RegExp)=>{
+    const before=f.snapshot();
+    assert.throws(()=>f.store.releaseRevokedIdleCutoverLeaseLocal({...s.input,...override}),pattern);
+    assert.equal(f.snapshot(),before,"a denied cleanup must not mutate state");
+    assert.equal(leaseRow(f,s.lease.leaseId).terminal_state,null);
+  };
+  // Active (not revoked) root carrier.
+  { const f=fixture();try { const s=idleCutoverScenario(f,"active",{revoke:false}); expectDenied(f,s,{expectedCarrierVersion:2},/exact revoked root carrier/); } finally {f.close();} }
+  // Revoked carrier but not expired lease/validity.
+  { const f=fixture();try { const s=idleCutoverScenario(f,"live",{advanceMs:1_000}); expectDenied(f,s,{},/expired/); } finally {f.close();} }
+  // Wrong confirmation, carrier id, carrier version, lease id.
+  { const f=fixture();try { const s=idleCutoverScenario(f,"wrong");
+    expectDenied(f,s,{confirmLeaseId:"lease_other"},/confirmation must equal the exact lease id/);
+    expectDenied(f,s,{carrierId:"carrier_nope"},/exact revoked root carrier/);
+    expectDenied(f,s,{expectedCarrierVersion:s.input.expectedCarrierVersion+1},/exact revoked root carrier/);
+    expectDenied(f,s,{leaseId:"lease_nope",confirmLeaseId:"lease_nope"},/effect binding changed/);
+  } finally {f.close();} }
+  // Pinned lease (active operation handle) must go through reconciliation, never hygiene.
+  { const f=fixture();try { const s=idleCutoverScenario(f,"pinned");
+    f.db.sqlite.prepare("update control_plane_resource_leases set active_operation_handle='pinned-op', operation_state='active' where lease_id=?").run(s.lease.leaseId);
+    expectDenied(f,s,{},/exact expired unpinned prepare-only lease/);
+  } finally {f.close();} }
+  // Terminal witness present.
+  { const f=fixture();try { const s=idleCutoverScenario(f,"witness");
+    f.db.sqlite.prepare("insert into dependency_terminal_witnesses(operation_id,request_hash,lease_id,exit_code,frozen_inputs_unchanged) values(?,?,?,0,1)").run(s.plan.operationId,s.plan.requestHash,s.lease.leaseId);
+    expectDenied(f,s,{},/refuses any durable or terminal effect evidence/);
+  } finally {f.close();} }
+});

src/control-plane-ownership.test.ts

@@ -483,3 +483,23 @@ test("non-expired missing physical lease still fails closed for unrelated acquis
     assert.throws(()=>store.acquire(context("two"),request(b,"second")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="INVALID_INPUT"&&/resource identity cannot be resolved/.test(error.message));
   } finally {sqlite.close();rmSync(root,{recursive:true,force:true});}
 });
+
+test("expired stale lease with a removed path that overlaps a request blocks with a bounded stale-physical-lease code naming the lease and hygiene command (#415)", () => {
+  const root=realpathSync.native(mkdtempSync(join(tmpdir(),"devspace-stale-overlap-"))).replaceAll("\\","/");
+  const a=join(root,"a"),b=join(root,"b");mkdirSync(a);mkdirSync(b);
+  const sqlite=db();let now=Date.now();
+  const store=new ControlPlaneOwnershipStore(sqlite,{resolveOwnerContext:options.resolveOwnerContext,verifyGrantEvidence:options.verifyGrantEvidence,now:()=>now});
+  const request=(path:string,key:string)=>({...input([path]),resourceKind:"filesystem",resourceId:path,resource:path,idempotencyKey:key,expiresAt:new Date(now+60_000).toISOString()});
+  try {
+    const stale=store.acquire(context("one"),{...request(a,"first"),expiresAt:new Date(now+1_000).toISOString()});
+    now+=2_000;
+    rmSync(a,{recursive:true});
+    const before=sqlite.prepare("select * from control_plane_resource_leases").all();
+    // The parent directory covers the removed path: the stale lease may overlap and must block, not be ignored.
+    assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&new RegExp(`carrier release-revoked-idle-cutover-lease ${stale.leaseId} --lease-version .* --confirm ${stale.leaseId}$`).test(error.message)&&error.message.length<600);
+    assert.deepEqual(sqlite.prepare("select * from control_plane_resource_leases").all(),before);
+    // A disjoint request is still unaffected, and the stale lease is neither released nor altered.
+    assert.equal(store.acquire(context("three"),request(b,"disjoint")).resource,b);
+    assert.equal(store.get(stale.leaseId)?.terminalState,undefined);
+  } finally {sqlite.close();rmSync(root,{recursive:true,force:true});}
+});

src/control-plane-ownership.ts

@@ -109,7 +109,7 @@ export function resolvePhysicalResource(input: ResourceLeaseInput): Pick<Resourc
 }
 
 export class ControlPlaneOwnershipError extends Error {
-  constructor(readonly code: "INVALID_INPUT" | "AUTHORITY_REQUIRED" | "OWNERSHIP_CONFLICT" | "CAS_CONFLICT" | "EXPIRED" | "MALFORMED", message: string) {
+  constructor(readonly code: "INVALID_INPUT" | "AUTHORITY_REQUIRED" | "OWNERSHIP_CONFLICT" | "CAS_CONFLICT" | "EXPIRED" | "MALFORMED" | "STALE_PHYSICAL_LEASE", message: string) {
     super(message);
     this.name = "ControlPlaneOwnershipError";
   }
@@ -318,6 +318,14 @@ export class ControlPlaneOwnershipStore {
         (other.resourceKind !== "filesystem" && other.resourceKind !== "checkout" &&
           other.resourceKind === resource.resourceKind && other.resourceId === resource.resourceId && other.resource === resource.resource);
       if (storedConflict) {
+        // An expired unresolved lease whose physical identity is gone still fences the scope (never auto-released);
+        // name it and the Owner-local hygiene path so the blocker is actionable.
+        if (Date.parse(other.expiresAt) <= this.now()) {
+          let unresolved = false;
+          try { this.assertPhysicalBinding(other); }
+          catch (error) { unresolved = error instanceof ControlPlaneOwnershipError && error.code === "INVALID_INPUT" && error.message === "resource identity cannot be resolved"; }
+          if (unresolved) throw new ControlPlaneOwnershipError("STALE_PHYSICAL_LEASE", `expired lease ${other.leaseId} has an unresolvable path and may overlap this scope; Owner-local hygiene (revoked, expired, unpinned, no durable effect only): devspace carrier release-revoked-idle-cutover-lease ${other.leaseId} --lease-version <n> --carrier <id> --carrier-version <n> --validity-version <n> --operation-id <id> --confirm ${other.leaseId}`);
+        }
         throw new ControlPlaneOwnershipError("OWNERSHIP_CONFLICT", "overlapping resource scope is already leased");
       }
       try {