fix(desktop): wait out a held engine lock in the launch and setup installs

desktop/main/engine-busy-retry.ts

@@ -0,0 +1,48 @@
+/**
+ * The install the app runs by itself, tried again while the engine is taken.
+ *
+ * Why: `models update` never waits for the engine operation lock
+ * (src/local-llm/core/core-backend.ts `installCoreBackend`, `waitMs: 0`), which
+ * is right for someone who asked for it and is told at once. At launch and in
+ * setup nobody asked: Studio's first call can be putting Core in place under
+ * that same lock (studio-core-start.ts `ensureStudioCore`), and the install
+ * then failed in seconds and left a red "Download failed" until Retry.
+ */
+
+/** How long the automatic install waits; the same patience as `withStudioOperation`. */
+export const ENGINE_BUSY_WAIT_MS = 5 * 60_000;
+const ENGINE_BUSY_POLL_MS = 3_000;
+
+/** The agent's two refusals for a held lock (src/local-llm/core/core-operation.ts). */
+const ENGINE_BUSY = /Another engine operation is (running|starting)/;
+
+export function engineBusy(res: { ok: boolean; error?: string | null; stderr?: string | null }): boolean {
+  return !res.ok && ENGINE_BUSY.test(`${res.error ?? ""}\n${res.stderr ?? ""}`);
+}
+
+function pause(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal.aborted) { resolve(); return; }
+    const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); };
+    const timer = setTimeout(done, ms);
+    signal.addEventListener("abort", done, { once: true });
+  });
+}
+
+/**
+ * Run `attempt` until it is answered by anything but a held lock: success, a
+ * real failure, a stop (`signal`) or the end of the wait, whose last refusal
+ * is returned as it came.
+ */
+export async function retryWhileEngineBusy<T extends { ok: boolean; error?: string | null; stderr?: string | null }>(
+  attempt: () => Promise<T>,
+  opts: { signal: AbortSignal; waitMs?: number; pollMs?: number },
+): Promise<T> {
+  const deadline = Date.now() + (opts.waitMs ?? ENGINE_BUSY_WAIT_MS);
+  for (;;) {
+    const res = await attempt();
+    if (!engineBusy(res) || opts.signal.aborted || Date.now() >= deadline) return res;
+    await pause(opts.pollMs ?? ENGINE_BUSY_POLL_MS, opts.signal);
+    if (opts.signal.aborted) return res;
+  }
+}

desktop/main/main.ts

@@ -11,6 +11,7 @@ import { NOTIFY_DEFAULTS, readNotifyConfig, TurnNotifier } from "./turn-notify.j
 import { workspaceChangeBusy } from "./workspace-busy.js";
 import { detectSubscriptionCli, isSubscriptionCli } from "./subscription-cli-detect.js";
 import { resetCoreForRedownload, seedBundledCore } from "./core-seed.js";
+import { engineBusy, retryWhileEngineBusy } from "./engine-busy-retry.js";
 import { coreUpdateNoticeVersion, readDismissedCoreUpdate, writeDismissedCoreUpdate } from "./core-update-notice.js";
 import { readChatFolders, writeChatFolders } from "./chat-folders.js";
 import { cpuFallbackOf, engineFailureOf, type EngineFailure } from "./engine-failure.js";
@@ -669,11 +670,18 @@ function engineInstallStream(
         console.error(`[desktop] local-llm: Download again: Atomic Core ${reset.version}${reset.removedRecord ? ", its install record removed" : ""}${reset.removedBinary ? ", its binary (it did not run) removed" : ""}`);
       }
       if (own.signal.aborted) return stopped(UPDATE_STOPPED);
-      const started = modelsUpdateStream((line) =>
-        pullFrame(slot, { id, line, ...parsePullProgress(line, "runtime") }),
-      );
-      own.signal.addEventListener("abort", () => started.cancel(), { once: true });
-      return started.done.then(updateSwept);
+      const attempt = (): Promise<InstallEnded> => {
+        const started = modelsUpdateStream((line) =>
+          pullFrame(slot, { id, line, ...parsePullProgress(line, "runtime") }),
+        );
+        const cancel = () => started.cancel();
+        own.signal.addEventListener("abort", cancel, { once: true });
+        return started.done.finally(() => own.signal.removeEventListener("abort", cancel));
+      };
+      // Settings' install hears at once that the engine is taken. The launch's and the setup's belong to a queue that was going to run anyway, so they wait.
+      const ended = await (via === "settings" ? attempt() : retryWhileEngineBusy(attempt, { signal: own.signal }));
+      if (own.signal.aborted && engineBusy(ended)) return stopped(UPDATE_STOPPED);
+      return updateSwept(ended);
     },
   );
   pullUpdate = slot = { done, cancel: () => own.abort(), kind: "runtime", id, last: null };

desktop/test/engine-busy-retry.test.mjs

@@ -0,0 +1,88 @@
+// Unit tests for main/engine-busy-retry.ts, against the built output.
+// Run: npm run build && npm run test:unit
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+const { engineBusy, retryWhileEngineBusy } = require("../out/main/engine-busy-retry.js");
+
+const BUSY = { ok: false, error: "Another engine operation is running. Wait for it to finish." };
+const STARTING = { ok: false, stderr: "Another engine operation is starting. Retry shortly." };
+const FAILED = { ok: false, error: "Download failed: no network" };
+const OK = { ok: true };
+
+function script(...answers) {
+  const calls = { n: 0 };
+  return { calls, attempt: async () => answers[Math.min(calls.n++, answers.length - 1)] };
+}
+const live = () => new AbortController().signal;
+
+test("both of the agent's lock refusals read as busy, nothing else does", () => {
+  assert.equal(engineBusy(BUSY), true);
+  assert.equal(engineBusy(STARTING), true);
+  assert.equal(engineBusy(FAILED), false);
+  assert.equal(engineBusy(OK), false);
+  assert.equal(engineBusy({ ok: true, stderr: "Another engine operation is running." }), false);
+});
+
+test("a held lock is tried again until the install goes through", async () => {
+  const { calls, attempt } = script(BUSY, STARTING, OK);
+  const res = await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 });
+  assert.deepEqual(res, OK);
+  assert.equal(calls.n, 3);
+});
+
+test("a real failure is returned at once, not retried", async () => {
+  const { calls, attempt } = script(FAILED, OK);
+  assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 }), FAILED);
+  assert.equal(calls.n, 1);
+});
+
+test("a failure after a wait is returned as it came", async () => {
+  const { calls, attempt } = script(BUSY, FAILED);
+  assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 }), FAILED);
+  assert.equal(calls.n, 2);
+});
+
+test("the wait ends at its limit with the last refusal", async () => {
+  const { calls, attempt } = script(BUSY);
+  const res = await retryWhileEngineBusy(attempt, { signal: live(), waitMs: 30, pollMs: 5 });
+  assert.deepEqual(res, BUSY);
+  assert.ok(calls.n > 1, "it tried more than once");
+});
+
+test("a stop during the wait ends it at once, with no further attempt", async () => {
+  const stop = new AbortController();
+  const { calls, attempt } = script(BUSY, OK);
+  const began = Date.now();
+  const running = retryWhileEngineBusy(attempt, { signal: stop.signal, pollMs: 60_000 });
+  setTimeout(() => stop.abort(), 10);
+  assert.deepEqual(await running, BUSY);
+  assert.equal(calls.n, 1);
+  assert.ok(Date.now() - began < 5_000, "it did not sit out the pause");
+});
+
+test("every lock refusal the agent can give is recognised", () => {
+  const source = readFileSync(new URL("../../src/local-llm/core/core-operation.ts", import.meta.url), "utf8");
+  const refusals = [...source.matchAll(/new CoreOperationBusyError\("([^"]+)"\)/g)].map((m) => m[1]);
+  assert.equal(refusals.length, 2, "the agent's refusals were found");
+  for (const error of refusals) assert.equal(engineBusy({ ok: false, error }), true, error);
+});
+
+test("the launch and setup installs wait, Settings' does not, and a stop during the wait reads as stopped", () => {
+  const main = readFileSync(new URL("../main/main.ts", import.meta.url), "utf8");
+  const body = main.slice(main.indexOf("function engineInstallStream("), main.indexOf("pullUpdate = slot = { done"));
+  assert.ok(body.length > 0, "the install was found");
+  assert.match(body, /via === "settings" \? attempt\(\) : retryWhileEngineBusy\(attempt, \{ signal: own\.signal \}\)/);
+  assert.match(body, /own\.signal\.aborted && engineBusy\(ended\)\) return stopped\(UPDATE_STOPPED\)/);
+});
+
+test("a stop made before the refusal comes back is not retried", async () => {
+  const stop = new AbortController();
+  stop.abort();
+  const { calls, attempt } = script(BUSY, OK);
+  assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: stop.signal, pollMs: 1 }), BUSY);
+  assert.equal(calls.n, 1);
+});