From 05a6c2bae56c81393f4cc4ee3ff083165a54dcb5 Mon Sep 17 00:00:00 2001 From: Nadia Rodionova <273119990+sosidudku1@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:26:45 +0300 Subject: [PATCH] fix(desktop): wait out a held engine lock in the launch and setup installs models update never waits for the engine operation lock. At launch the Studio shell's first call can hold it while starting or installing Core, so the launch install failed within seconds and left a red Download failed card until Retry, with the model not started. The launch's and the setup's install now retry while the refusal is the lock, for up to five minutes. Settings still answers at once. --- desktop/main/engine-busy-retry.ts | 48 ++++++++++++++ desktop/main/main.ts | 18 +++-- desktop/test/engine-busy-retry.test.mjs | 88 +++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 desktop/main/engine-busy-retry.ts create mode 100644 desktop/test/engine-busy-retry.test.mjs diff --git a/desktop/main/engine-busy-retry.ts b/desktop/main/engine-busy-retry.ts new file mode 100644 index 000000000..51229dec2 --- /dev/null +++ b/desktop/main/engine-busy-retry.ts @@ -0,0 +1,48 @@ +/** + * The install the app runs by itself, tried again while the engine is taken. + * + * Why: `models update` never waits for the engine operation lock + * (src/local-llm/core/core-backend.ts `installCoreBackend`, `waitMs: 0`), which + * is right for someone who asked for it and is told at once. At launch and in + * setup nobody asked: Studio's first call can be putting Core in place under + * that same lock (studio-core-start.ts `ensureStudioCore`), and the install + * then failed in seconds and left a red "Download failed" until Retry. + */ + +/** How long the automatic install waits; the same patience as `withStudioOperation`. */ +export const ENGINE_BUSY_WAIT_MS = 5 * 60_000; +const ENGINE_BUSY_POLL_MS = 3_000; + +/** The agent's two refusals for a held lock (src/local-llm/core/core-operation.ts). */ +const ENGINE_BUSY = /Another engine operation is (running|starting)/; + +export function engineBusy(res: { ok: boolean; error?: string | null; stderr?: string | null }): boolean { + return !res.ok && ENGINE_BUSY.test(`${res.error ?? ""}\n${res.stderr ?? ""}`); +} + +function pause(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) { resolve(); return; } + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, ms); + signal.addEventListener("abort", done, { once: true }); + }); +} + +/** + * Run `attempt` until it is answered by anything but a held lock: success, a + * real failure, a stop (`signal`) or the end of the wait, whose last refusal + * is returned as it came. + */ +export async function retryWhileEngineBusy( + attempt: () => Promise, + opts: { signal: AbortSignal; waitMs?: number; pollMs?: number }, +): Promise { + const deadline = Date.now() + (opts.waitMs ?? ENGINE_BUSY_WAIT_MS); + for (;;) { + const res = await attempt(); + if (!engineBusy(res) || opts.signal.aborted || Date.now() >= deadline) return res; + await pause(opts.pollMs ?? ENGINE_BUSY_POLL_MS, opts.signal); + if (opts.signal.aborted) return res; + } +} diff --git a/desktop/main/main.ts b/desktop/main/main.ts index 6df45db78..30703694a 100644 --- a/desktop/main/main.ts +++ b/desktop/main/main.ts @@ -11,6 +11,7 @@ import { NOTIFY_DEFAULTS, readNotifyConfig, TurnNotifier } from "./turn-notify.j import { workspaceChangeBusy } from "./workspace-busy.js"; import { detectSubscriptionCli, isSubscriptionCli } from "./subscription-cli-detect.js"; import { resetCoreForRedownload, seedBundledCore } from "./core-seed.js"; +import { engineBusy, retryWhileEngineBusy } from "./engine-busy-retry.js"; import { coreUpdateNoticeVersion, readDismissedCoreUpdate, writeDismissedCoreUpdate } from "./core-update-notice.js"; import { readChatFolders, writeChatFolders } from "./chat-folders.js"; import { cpuFallbackOf, engineFailureOf, type EngineFailure } from "./engine-failure.js"; @@ -669,11 +670,18 @@ function engineInstallStream( console.error(`[desktop] local-llm: Download again: Atomic Core ${reset.version}${reset.removedRecord ? ", its install record removed" : ""}${reset.removedBinary ? ", its binary (it did not run) removed" : ""}`); } if (own.signal.aborted) return stopped(UPDATE_STOPPED); - const started = modelsUpdateStream((line) => - pullFrame(slot, { id, line, ...parsePullProgress(line, "runtime") }), - ); - own.signal.addEventListener("abort", () => started.cancel(), { once: true }); - return started.done.then(updateSwept); + const attempt = (): Promise => { + const started = modelsUpdateStream((line) => + pullFrame(slot, { id, line, ...parsePullProgress(line, "runtime") }), + ); + const cancel = () => started.cancel(); + own.signal.addEventListener("abort", cancel, { once: true }); + return started.done.finally(() => own.signal.removeEventListener("abort", cancel)); + }; + // Settings' install hears at once that the engine is taken. The launch's and the setup's belong to a queue that was going to run anyway, so they wait. + const ended = await (via === "settings" ? attempt() : retryWhileEngineBusy(attempt, { signal: own.signal })); + if (own.signal.aborted && engineBusy(ended)) return stopped(UPDATE_STOPPED); + return updateSwept(ended); }, ); pullUpdate = slot = { done, cancel: () => own.abort(), kind: "runtime", id, last: null }; diff --git a/desktop/test/engine-busy-retry.test.mjs b/desktop/test/engine-busy-retry.test.mjs new file mode 100644 index 000000000..6c3222d11 --- /dev/null +++ b/desktop/test/engine-busy-retry.test.mjs @@ -0,0 +1,88 @@ +// Unit tests for main/engine-busy-retry.ts, against the built output. +// Run: npm run build && npm run test:unit +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { engineBusy, retryWhileEngineBusy } = require("../out/main/engine-busy-retry.js"); + +const BUSY = { ok: false, error: "Another engine operation is running. Wait for it to finish." }; +const STARTING = { ok: false, stderr: "Another engine operation is starting. Retry shortly." }; +const FAILED = { ok: false, error: "Download failed: no network" }; +const OK = { ok: true }; + +function script(...answers) { + const calls = { n: 0 }; + return { calls, attempt: async () => answers[Math.min(calls.n++, answers.length - 1)] }; +} +const live = () => new AbortController().signal; + +test("both of the agent's lock refusals read as busy, nothing else does", () => { + assert.equal(engineBusy(BUSY), true); + assert.equal(engineBusy(STARTING), true); + assert.equal(engineBusy(FAILED), false); + assert.equal(engineBusy(OK), false); + assert.equal(engineBusy({ ok: true, stderr: "Another engine operation is running." }), false); +}); + +test("a held lock is tried again until the install goes through", async () => { + const { calls, attempt } = script(BUSY, STARTING, OK); + const res = await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 }); + assert.deepEqual(res, OK); + assert.equal(calls.n, 3); +}); + +test("a real failure is returned at once, not retried", async () => { + const { calls, attempt } = script(FAILED, OK); + assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 }), FAILED); + assert.equal(calls.n, 1); +}); + +test("a failure after a wait is returned as it came", async () => { + const { calls, attempt } = script(BUSY, FAILED); + assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: live(), pollMs: 1 }), FAILED); + assert.equal(calls.n, 2); +}); + +test("the wait ends at its limit with the last refusal", async () => { + const { calls, attempt } = script(BUSY); + const res = await retryWhileEngineBusy(attempt, { signal: live(), waitMs: 30, pollMs: 5 }); + assert.deepEqual(res, BUSY); + assert.ok(calls.n > 1, "it tried more than once"); +}); + +test("a stop during the wait ends it at once, with no further attempt", async () => { + const stop = new AbortController(); + const { calls, attempt } = script(BUSY, OK); + const began = Date.now(); + const running = retryWhileEngineBusy(attempt, { signal: stop.signal, pollMs: 60_000 }); + setTimeout(() => stop.abort(), 10); + assert.deepEqual(await running, BUSY); + assert.equal(calls.n, 1); + assert.ok(Date.now() - began < 5_000, "it did not sit out the pause"); +}); + +test("every lock refusal the agent can give is recognised", () => { + const source = readFileSync(new URL("../../src/local-llm/core/core-operation.ts", import.meta.url), "utf8"); + const refusals = [...source.matchAll(/new CoreOperationBusyError\("([^"]+)"\)/g)].map((m) => m[1]); + assert.equal(refusals.length, 2, "the agent's refusals were found"); + for (const error of refusals) assert.equal(engineBusy({ ok: false, error }), true, error); +}); + +test("the launch and setup installs wait, Settings' does not, and a stop during the wait reads as stopped", () => { + const main = readFileSync(new URL("../main/main.ts", import.meta.url), "utf8"); + const body = main.slice(main.indexOf("function engineInstallStream("), main.indexOf("pullUpdate = slot = { done")); + assert.ok(body.length > 0, "the install was found"); + assert.match(body, /via === "settings" \? attempt\(\) : retryWhileEngineBusy\(attempt, \{ signal: own\.signal \}\)/); + assert.match(body, /own\.signal\.aborted && engineBusy\(ended\)\) return stopped\(UPDATE_STOPPED\)/); +}); + +test("a stop made before the refusal comes back is not retried", async () => { + const stop = new AbortController(); + stop.abort(); + const { calls, attempt } = script(BUSY, OK); + assert.deepEqual(await retryWhileEngineBusy(attempt, { signal: stop.signal, pollMs: 1 }), BUSY); + assert.equal(calls.n, 1); +});