From aa3a06bd4bb5a3190cdceb0d87eae0faefb42b38 Mon Sep 17 00:00:00 2001 From: Frost <90551822+frostybittn@users.noreply.github.com> Date: Fri, 22 May 2026 13:17:19 -0600 Subject: [PATCH 1/7] Add LM Studio backend (Ollama -> LM Studio) with dynamic model picker Adds LM Studio as a switchable local LLM backend alongside Ollama, using LM Studio's native /api/v0 REST API. The avatar's brain, the command-center model picker, and the raw-LLM lab all run on LM Studio; the dashboard auto-discovers the entire LM Studio model library. Selected via LLM_BACKEND (lmstudio is the new default; ollama reverts to the original path). TTS stays Kokoro -- LM Studio has no audio API. Orchestrator: - providers/lmstudio.py: native /api/v0/chat/completions client (streaming + tool_calls verified), a thin subclass of OllamaProvider (identical wire format) - system/lmstudio.py: model discovery + residency via /api/v0/models - catalog kind=lmstudio + sync_dynamic_brains(); dispatcher model_override - config LLM_BACKEND/LMSTUDIO_URL/LMSTUDIO_DEFAULT_MODEL; backend-aware routes Frontend: - dashboard loaded-state chip; /api/lmstudio/ proxy (nginx + vite) - Lab 1 + shared helpers converted to LM Studio native API; lab labels updated Docs/config: - docker-compose default LLM_BACKEND=lmstudio; docs/lmstudio-runbook.md - CLAUDE.md, README, wiki swap-model FAQ Tests: - tests/test_lmstudio.py (16 new); fixed 3 pre-existing stale qwen3-4b tests - full suite 171 passing Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 13 +- CLAUDE.md | 28 +- README.md | 6 +- configs/catalog.yml | 20 ++ docker-compose.yml | 8 + docs/lmstudio-runbook.md | 138 ++++++++++ frontend/nginx.conf | 13 + frontend/public/lab/01-llm.html | 38 ++- frontend/public/lab/04-orchestrator.html | 2 +- frontend/public/lab/06-pipeline.html | 2 +- frontend/public/lab/_shared.js | 19 +- frontend/public/lab/index.html | 6 +- .../src/dashboard/components/ControlsPanel.js | 21 ++ frontend/vite.config.js | 6 + services/orchestrator/orchestrator/catalog.py | 33 ++- services/orchestrator/orchestrator/config.py | 13 + services/orchestrator/orchestrator/main.py | 42 ++- .../orchestrator/providers/dispatcher.py | 20 +- .../orchestrator/providers/lmstudio.py | 24 ++ .../orchestrator/providers/ollama.py | 14 +- .../orchestrator/routes/catalog.py | 41 ++- .../orchestrator/orchestrator/routes/chat.py | 31 ++- .../orchestrator/routes/models.py | 10 +- .../orchestrator/system/lmstudio.py | 117 ++++++++ services/orchestrator/tests/test_lmstudio.py | 250 ++++++++++++++++++ .../orchestrator/tests/test_routes_catalog.py | 8 +- .../orchestrator/tests/test_routes_state.py | 2 +- wiki/faqs/swap-model.md | 8 +- 28 files changed, 886 insertions(+), 47 deletions(-) create mode 100644 docs/lmstudio-runbook.md create mode 100644 services/orchestrator/orchestrator/providers/lmstudio.py create mode 100644 services/orchestrator/orchestrator/system/lmstudio.py create mode 100644 services/orchestrator/tests/test_lmstudio.py diff --git a/.env.example b/.env.example index 00cbe7c..b0354d0 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,18 @@ STT_PORT=8080 TTS_PORT=8880 LLM_PORT=8081 -# LLM settings +# LLM backend (Plan #11): "lmstudio" (default) or "ollama". +# lmstudio → orchestrator talks LM Studio's native /api/v0 API + auto-discovers +# your whole LM Studio model library for the dashboard picker. +# ollama → original host-Ollama path. +LLM_BACKEND=lmstudio +# LM Studio server URL as seen from inside the orchestrator container. LM Studio +# must be "Served on Local Network" (binds 0.0.0.0) for host.docker.internal to work. +LMSTUDIO_URL=http://host.docker.internal:1234 +# Model the "auto" brain falls back to when nothing is loaded (must exist in LM Studio). +LMSTUDIO_DEFAULT_MODEL=qwen/qwen3-4b-2507 + +# LLM settings (Ollama path) LLM_MODEL_FILE=Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf LLM_CTX_SIZE=4096 diff --git a/CLAUDE.md b/CLAUDE.md index 8541665..4372bdc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,11 +11,13 @@ Browser (localhost:3000) └── nginx ──┬── /api/stt/ ──► whisper.cpp (port 8080) [Vulkan] ├── /api/tts/ ──► Kokoro-FastAPI (port 8880) [CUDA/ROCm] └── /api/llm/ ──► Orchestrator (port 8082) - └─► Ollama on host (port 11434) [GPU] + ├─► LM Studio on host (:1234, native /api/v0) [default, Plan #11] + └─► Ollama on host (:11434) [LLM_BACKEND=ollama] ``` - **Frontend**: Vite + Three.js + TalkingHead + VAD-web (browser-based orchestrator) -- **LLM**: Qwen3-4B via Ollama on host (thinking model with `` tags) +- **LLM**: LM Studio on host (native `/api/v0` API; whole library auto-discovered) OR + Qwen3-4B via Ollama — selected by `LLM_BACKEND` (see `docs/lmstudio-runbook.md`) - **TTS**: Kokoro-82M via Kokoro-FastAPI (returns PCM + word timestamps) - **STT**: Whisper base.en via whisper.cpp (Vulkan) - **Orchestrator**: OpenAI-compatible proxy + agentic tool loop + command center backend @@ -182,3 +184,25 @@ Docker Desktop on macOS runs a Linux VM — no GPU passthrough to Metal/MPS. Doc - Tool toggles (web_search / wiki) now live in `state.tools` rather than browser localStorage; frontend's ControlPanel POSTs to /v1/swap on change - Interactive teaching scripts in `scripts/demos/` (test-llm, test-tts, test-stt, test-pipeline, test-orchestrator, list-models) back slides 13-24 of the workshop deck - Setup: `bash scripts/setup-linux.sh` (Linux/WSL2) or `bash scripts/setup-mac.sh` (macOS) installs Ollama and pulls default models + +## Plan #11 — LM Studio backend (native API + dynamic discovery) + +Full guide: **`docs/lmstudio-runbook.md`**. Selected by `LLM_BACKEND` env (`lmstudio` default | `ollama`). + +- **New brain kind `lmstudio`** → `LMStudioProvider` (`providers/lmstudio.py`), a 3-line subclass of + `OllamaProvider` that targets LM Studio's NATIVE endpoint `POST /api/v0/chat/completions`. The wire + format is identical OpenAI-shaped SSE — streaming + `tool_calls` both verified — so the parser is reused. + The native response also carries `stats`/`model_info`/`runtime` (TTFT, tok/s). +- **Discovery + residency** in `system/lmstudio.py` (`LMStudioBackend`): `GET /api/v0/models` → + `list_models()` (filters to `llm`/`vlm`, flags `loaded` + `thinks`) and `query()` (residency snapshot in + the same shape as `OllamaResidency`). Never raises. +- **Dynamic catalog**: `/v1/catalog` calls `Catalog.sync_dynamic_brains(...)` to merge every LM Studio model + as an `lmstudio:` brain (loaded-first). The dashboard Brain dropdown auto-lists the whole library. +- **`lmstudio-auto` brain** (`model: auto`) resolves at request time to the loaded model, else + `LMSTUDIO_DEFAULT_MODEL`. `main.py` promotes it to the default brain when `LLM_BACKEND=lmstudio` and + resets a stale non-LM-Studio active brain on boot. +- **Backend-aware routes**: `/v1/models`, `/v1/state`, `/v1/swap`, `/v1/catalog`, `/v1/chat/completions` + all follow `llm_backend`. `nginx.conf` + `vite.config.js` add `/api/lmstudio/` (raw passthrough, Lab 1). +- **TTS unchanged**: LM Studio has no audio API (tested — see runbook §5). Kokoro remains the TTS engine. +- Tests: `tests/test_lmstudio.py` (config, provider, discovery/residency, dispatcher, dynamic merge, and + route-level catalog + chat-auto). Whole suite: 171 passing. diff --git a/README.md b/README.md index b8afd2c..3971a69 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ NodeAva ships with a default avatar (sourced from the [TalkingHead](https://gith ## Prerequisites -**Ollama (required on all platforms):** -- Install on the host (not in Docker): `curl -fsSL https://ollama.com/install.sh | sh` (Linux/WSL2) or `brew install ollama` (macOS) -- The workshop setup scripts download default models: `bash scripts/setup-linux.sh` (Linux/WSL2) or `bash scripts/setup-mac.sh` (macOS) +**LLM backend — Ollama _or_ LM Studio:** +- **Ollama** (host install): `curl -fsSL https://ollama.com/install.sh | sh` (Linux/WSL2) or `brew install ollama` (macOS). Setup scripts pull defaults: `bash scripts/setup-linux.sh` / `bash scripts/setup-mac.sh`. +- **LM Studio** (default, `LLM_BACKEND=lmstudio`): run LM Studio with "Serve on Local Network" enabled. The orchestrator talks its native `/api/v0` API and **auto-discovers your whole model library** into the dashboard's Brain picker. See **`docs/lmstudio-runbook.md`**. _(LM Studio serves the LLM only; TTS stays Kokoro.)_ **Docker + Compose (Linux/Windows only):** - Docker Engine with Compose V2 diff --git a/configs/catalog.yml b/configs/catalog.yml index 146a254..09e2dc7 100644 --- a/configs/catalog.yml +++ b/configs/catalog.yml @@ -2,6 +2,26 @@ brains: # `thinks: true` flags models that emit ... reasoning blocks. # The frontend gates token-streaming detection on this flag — thinking models # buffer until appears, non-thinking models stream immediately. + # + # ── LM Studio brains (Plan #11) ────────────────────────────────────────── + # kind: lmstudio talks LM Studio's NATIVE REST API at /api/v0. + # When LLM_BACKEND=lmstudio (set in docker-compose.yml), the orchestrator + # ALSO auto-discovers every model in your LM Studio library at runtime and + # adds each as a selectable `lmstudio:` brain — so this static list + # is just the stable defaults. `lmstudio-auto` follows whatever model you + # have loaded in LM Studio; it is promoted to the default brain at startup + # when LM Studio is the active backend. + - id: lmstudio-auto + label: "LM Studio (auto — uses your loaded model)" + kind: lmstudio + model: auto + thinks: false + - id: lmstudio-qwen3-4b + label: "LM Studio · Qwen3 4B 2507 (fast default)" + kind: lmstudio + model: qwen/qwen3-4b-2507 + thinks: false + # ── Ollama brains (used when LLM_BACKEND=ollama, the upstream default) ──── - id: qwen3-4b-instruct label: "Qwen3 4B Instruct (default — fast, no reasoning)" kind: ollama diff --git a/docker-compose.yml b/docker-compose.yml index 3147ca1..fc4d9e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,14 @@ services: # Plan #7: catalog.yml must be visible inside the container. - ./configs:/app/configs:ro environment: + # Plan #11: LM Studio is the active local LLM backend by default. Flip + # LLM_BACKEND=ollama (in .env) to fall back to the host Ollama install. + # LM Studio must be serving on the local network (Developer ▸ Settings ▸ + # "Serve on Local Network") so the container can reach it via + # host.docker.internal:1234. + - LLM_BACKEND=${LLM_BACKEND:-lmstudio} + - LMSTUDIO_URL=${LMSTUDIO_URL:-http://host.docker.internal:1234} + - LMSTUDIO_DEFAULT_MODEL=${LMSTUDIO_DEFAULT_MODEL:-qwen/qwen3-4b-2507} - OLLAMA_URL=http://host.docker.internal:11434 - REQUEST_TIMEOUT=300 - BIND_HOST=0.0.0.0 diff --git a/docs/lmstudio-runbook.md b/docs/lmstudio-runbook.md new file mode 100644 index 0000000..12658f4 --- /dev/null +++ b/docs/lmstudio-runbook.md @@ -0,0 +1,138 @@ +# LM Studio Backend — Runbook (Plan #11) + +Switch NodeAva's brain from **Ollama** to **LM Studio**, with a live, auto-discovered +picker for *every* model in your LM Studio library. The avatar's LLM now runs through +LM Studio's **native** REST API (`/api/v0`); TTS (Kokoro) and STT (Whisper) are unchanged. + +> **Verified end-to-end** against a live LM Studio (115-model library): backend switch, +> 112-model auto-discovery, residency, per-model swap, and streaming + non-streaming chat +> all working (`[happy] Welcome to NodeAva!` in ~155 ms on `qwen/qwen3-4b-2507`). + +--- + +## 0. Prerequisites (one-time) + +1. **LM Studio is running** with the local server started (Developer tab → **Start Server**, + or `lms server start`). Default port **1234**. +2. **"Serve on Local Network" is ON** (Developer ▸ Server Settings). This binds the server to + `0.0.0.0` so the orchestrator *container* can reach it via `host.docker.internal:1234`. + Without it, LM Studio only listens on `127.0.0.1` and Docker can't see it. +3. At least one chat model downloaded (e.g. `qwen/qwen3-4b-2507` — small & fast, good default). + Models JIT-load on first use; you don't need to pre-load. + +Quick reachability check from the host: + +```bash +curl http://localhost:1234/api/v0/models | head # should list your models +``` + +--- + +## 1. Activate + +LM Studio is the **default** backend now (`LLM_BACKEND=lmstudio` in `docker-compose.yml`). +Rebuild + restart so the new orchestrator code, env, and dashboard take effect: + +```bash +docker compose up -d --build orchestrator frontend +# or rebuild the whole stack: docker compose up -d --build +``` + +- **orchestrator** rebuild → new provider/discovery code + `LLM_BACKEND=lmstudio` env. +- **frontend** rebuild → dashboard brain-picker chips + the `/api/lmstudio/` nginx route. +- `configs/catalog.yml` is bind-mounted, so brain edits there only need a restart, not a rebuild. + +--- + +## 2. Verify + +```bash +# Active brain + LM Studio residency (which model is loaded right now) +curl -s http://localhost:8082/v1/state | python3 -m json.tool + +# The full picker: your LM Studio library merged as lmstudio: brains +curl -s http://localhost:8082/v1/catalog | python3 -c \ + "import sys,json;b=json.load(sys.stdin)['brains'];d=[x for x in b if x.get('dynamic')];\ +print('dynamic LM Studio brains:',len(d));print('loaded first:',[x['id'] for x in d[:3]])" + +# A real chat through LM Studio's native API +curl -s -X POST http://localhost:8082/v1/chat/completions -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Hi in five words"}],"stream":false}' +``` + +Then open the dashboard (**http://localhost:3000**, drawer ▸ Controls). The **Brain** +dropdown now lists every LM Studio model — loaded ones first, with a green **loaded** chip. + +--- + +## 3. Using it + +- **Pick any model live** from the Brain dropdown. Selecting one and chatting JIT-loads it in + LM Studio. (Big 27B–35B models take a few seconds + VRAM to load the first time.) +- **`LM Studio (auto)`** brain = "use whatever I have loaded in LM Studio right now". It resolves + at request time to your loaded model, else falls back to `LMSTUDIO_DEFAULT_MODEL`. +- **Thinking models** (names containing `thinking`/`reasoning`/`r1`/`magistral`/`qwq`) are auto-flagged + `thinks: true` so the frontend buffers the `` block instead of speaking it. +- **Native API bonus**: chat runs on `/api/v0/chat/completions`, which also returns + `stats` (tokens/sec, TTFT), `model_info`, and `runtime` — available for a future telemetry hook. + +### Raw model playground (Lab 1) +**Lab 1 ("The Brain")** now talks directly to LM Studio's native API: pick any model from +your library (loaded ones first) and measure raw TTFT / tokens-per-sec — it even logs LM +Studio's own *server-measured* stats. Powered by the `/api/lmstudio/` passthrough, available +in both Docker (nginx) and `npm run dev` (Vite proxy). Labs 4 & 6 already exercise LM Studio +via the orchestrator. + +--- + +## 4. Switch back to Ollama + +```bash +echo "LLM_BACKEND=ollama" >> .env # (or edit .env) +docker compose up -d --build orchestrator frontend +``` + +Everything reverts to the host-Ollama path. Both backends coexist in the catalog; only the +*default* and the discovery source change. + +Per-deploy overrides (in `.env`): + +| Var | Default | Meaning | +|---|---|---| +| `LLM_BACKEND` | `lmstudio` | `lmstudio` or `ollama` | +| `LMSTUDIO_URL` | `http://host.docker.internal:1234` | LM Studio server as seen from the container | +| `LMSTUDIO_DEFAULT_MODEL` | `qwen/qwen3-4b-2507` | fallback for the `auto` brain | + +--- + +## 5. TTS reality check ⚠️ + +**LM Studio cannot do text-to-speech — through any API.** This was tested, not assumed: + +| Probe | Result | +|---|---| +| `POST /v1/audio/speech` (OpenAI-compat) | `{"error":"Unexpected endpoint or method"}` | +| `POST /v1/audio/transcriptions`, `/v1/audio/generations`, `/api/v0/audio/speech` | none exist | +| Official docs (`lmstudio.ai/docs/developer/rest`) | "no audio endpoints" | +| `voxtral-4b-tts-2603` prompted directly | `arch: llama`, `type: llm`; returned **empty** — it's a text model, not a vocoder | + +The same is true of **Ollama** — both are *LLM inference engines* (text / embeddings / vision-input). +Text→speech is a different model class needing a dedicated server, which is why this kit uses +**Kokoro-FastAPI**. So TTS stays Kokoro: + +- **Change voice** in the dashboard (Voice dropdown: Bella / Nova / Fenrir / Emma / George), or +- **Swap the TTS engine** entirely (Piper / XTTS / Orpheus) by replacing the `tts` service in + `docker-compose.yml`. (A future *experimental* path: an Orpheus-style model that emits SNAC audio + tokens via LM Studio chat + an external SNAC decoder — that's a separate service, not wired here.) + +--- + +## 6. Troubleshooting + +| Symptom | Fix | +|---|---| +| Brain dropdown shows no LM Studio models | LM Studio not reachable from the container. Confirm **Serve on Local Network** is ON and `curl http://localhost:1234/api/v0/models` works on the host. | +| `LM Studio unreachable` in `/v1/catalog` | Server stopped, or port ≠ 1234. Set `LMSTUDIO_URL` in `.env`. | +| Chat errors / long pause on first message | Model JIT-loading in LM Studio (esp. 27B+). Pick a smaller model or pre-load it in LM Studio. | +| Avatar speaks the `` reasoning | You picked a thinking model not caught by the name heuristic. Use an `-instruct` model, or mark it `thinks: true`. | +| Want it to "just use whatever I loaded" | Select the **LM Studio (auto)** brain. | diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 0359498..de44f2b 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -81,6 +81,19 @@ server { chunked_transfer_encoding on; } + # Plan #11: direct passthrough to LM Studio's native API on the host + # (bypasses the orchestrator) for Lab 1's raw model-latency playground. + # Same rationale as /api/ollama/ above. LM Studio must be "Served on Local + # Network"; requires host.docker.internal:host-gateway on this service. + location /api/lmstudio/ { + proxy_pass http://host.docker.internal:1234/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 120s; + proxy_buffering off; + chunked_transfer_encoding on; + } + # Health check location /health { return 200 'ok'; diff --git a/frontend/public/lab/01-llm.html b/frontend/public/lab/01-llm.html index 1af39f6..410657a 100644 --- a/frontend/public/lab/01-llm.html +++ b/frontend/public/lab/01-llm.html @@ -11,7 +11,7 @@
← Workshop Lab

Lab 1 · The Brain

- POST /api/ollama/v1/chat/completions + POST /api/lmstudio/api/v0/chat/completions
@@ -19,11 +19,11 @@

Lab 1 · The Brain

Checking services…
- What you're testing: Ollama running on your host machine, - serving qwen3:4b-instruct directly (this lab bypasses - the orchestrator so you measure raw LLM latency — Lab 4 demos the - orchestrator). This is the cognition layer — the part that decides - what to say. + What you're testing: LM Studio running on your host machine, + serving your selected model directly via its native API (this lab + bypasses the orchestrator so you measure raw LLM latency — Lab 4 demos the + orchestrator). Pick any model from your LM Studio library below. This is the + cognition layer — the part that decides what to say. What you'll see: tokens streamed one chunk at a time (not a wall of text). Watch TTFT (time to first token — the lag before the @@ -59,7 +59,7 @@

Try the model

- +
@@ -121,14 +121,18 @@

Console

const out = $("out"); const con = $("console"); -NA.preflight("preflight", ["ollama"]); +NA.preflight("preflight", ["lmstudio"]); -// Populate model dropdown from the host's Ollama +// Populate model dropdown from LM Studio (native /api/v0) — loaded models first. (async () => { - const models = await NA.listOllamaModels(); + const models = await NA.listLmStudioModels(); if (!models.length) return; + const firstLoaded = models.find(m => m.loaded); + const defaultName = firstLoaded ? firstLoaded.name : "qwen/qwen3-4b-2507"; NA.fillSelect($("model"), models.map(m => ({ - value: m.name, label: m.name, selected: m.name === "qwen3:4b-instruct", + value: m.name, + label: m.name + (m.loaded ? " (loaded)" : ""), + selected: m.name === defaultName, }))); })(); @@ -156,7 +160,7 @@

Console

NA.clearChips("chip-ttft", "chip-total", "chip-tps", "chip-tokens"); $("go").disabled = true; - NA.logLine(con, "→ POST /api/ollama/v1/chat/completions (stream=" + streaming + ")", "event"); + NA.logLine(con, "→ POST /api/lmstudio/api/v0/chat/completions (stream=" + streaming + ")", "event"); NA.logLine(con, " body.messages = [{role: 'user', content: }]", ""); const t0 = NA.now(); @@ -171,7 +175,7 @@

Console

if (sysPrompt) messages.push({ role: "system", content: sysPrompt }); messages.push({ role: "user", content: prompt }); - const resp = await fetch("/api/ollama/v1/chat/completions", { + const resp = await fetch("/api/lmstudio/api/v0/chat/completions", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -209,9 +213,15 @@

Console

const json = await resp.json(); const text = json.choices?.[0]?.message?.content || ""; out.textContent = text; - tokenCount = Math.ceil(text.length / 4); // rough estimate + tokenCount = json.usage?.completion_tokens || Math.ceil(text.length / 4); charCount = text.length; NA.logLine(con, "← non-streaming response (" + text.length + " chars)", "ok"); + // LM Studio's native API reports server-measured stats (Plan #11). + if (json.stats) { + NA.logLine(con, "← LM Studio: " + (json.stats.tokens_per_second || 0).toFixed(1) + + " tok/s, TTFT " + Math.round((json.stats.time_to_first_token || 0) * 1000) + + " ms (server-measured)", "ok"); + } } const tTotal = NA.now() - t0; diff --git a/frontend/public/lab/04-orchestrator.html b/frontend/public/lab/04-orchestrator.html index 9c2461f..a9be3b7 100644 --- a/frontend/public/lab/04-orchestrator.html +++ b/frontend/public/lab/04-orchestrator.html @@ -45,7 +45,7 @@

Lab 4 · The Nervous System

What you're testing: the FastAPI orchestrator that sits between - the browser and Ollama. It holds the personality — a system + the browser and the LLM backend (LM Studio). It holds the personality — a system prompt that gets prepended to every chat — and the brain selection (which model to use). The same model under two different personalities produces dramatically different answers; the personality is the diff --git a/frontend/public/lab/06-pipeline.html b/frontend/public/lab/06-pipeline.html index 2da8791..b4f31fb 100644 --- a/frontend/public/lab/06-pipeline.html +++ b/frontend/public/lab/06-pipeline.html @@ -112,7 +112,7 @@

Speak, then listen

STT (Whisper)
—
-
LLM (Ollama)
—
+
LLM (LM Studio)
—
TTS (Kokoro)
—
Total wall-clock
—
diff --git a/frontend/public/lab/_shared.js b/frontend/public/lab/_shared.js index a7c4ffd..a5fe752 100644 --- a/frontend/public/lab/_shared.js +++ b/frontend/public/lab/_shared.js @@ -144,6 +144,9 @@ async function preflight(elId, services) { // Lab 1 uses this so attendees measure raw LLM TTFT without the // orchestrator's personality prefill or agentic tool loop in the way. ollama: () => fetch("/api/ollama/v1/models").then(r => r.ok), + // 'lmstudio' = bypasses the orchestrator, hits LM Studio's native API on + // the host directly (Plan #11). Lab 1 uses this for raw model-latency tests. + lmstudio: () => fetch("/api/lmstudio/api/v0/models").then(r => r.ok), tts: () => fetch("/api/tts/v1/audio/voices").then(r => r.ok), stt: () => fetch("/api/stt/").then(r => r.ok || r.status === 404 || r.status === 405), orch: () => fetch("/api/orch/v1/state").then(r => r.ok), @@ -183,6 +186,20 @@ async function listOllamaModels() { } catch { return []; } } +// Fetch chat-capable models from LM Studio via its native /api/v0/models +// (Plan #11). Returns [{name, loaded}], loaded models first; embeddings excluded. +async function listLmStudioModels() { + try { + const r = await fetch("/api/lmstudio/api/v0/models"); + if (!r.ok) return []; + const j = await r.json(); + return (j.data || []) + .filter((m) => m.type === "llm" || m.type === "vlm") + .map((m) => ({ name: m.id, loaded: m.state === "loaded" })) + .sort((a, b) => (b.loaded - a.loaded) || a.name.localeCompare(b.name)); + } catch { return []; } +} + // Populate a at the top of the controls grid; Ollama selected by default - BACKENDS map parameterizes the 4 things that actually differ (endpoint, model-list source, preflight key, default model); everything else (SSE parser, sampling knobs, chip timing, native-stats logging) stays backend-agnostic because both backends speak the same OpenAI-shaped wire - refreshModels() updates the step-tag textContent + model dropdown reactively when the backend changes - syncBackend() is async + awaits refreshModels() so a rapid toggle + Run click can't race ahead of the model-list fetch (caught by adversarial review) - $(go).onclick captures the backend at click time so a mid-request toggle doesn't move the fetch goalposts Drive-by fix discovered while running pytest on this branch: - services/orchestrator/orchestrator/system/lmstudio.py: corrected over-indented block in LMStudioBackend.query() (lines 80-94) that caused an IndentationError on import, blocking pytest collection of test_lmstudio.py and test_routes_personality.py. Introduced by bbeec6d "Refactor model loading logic for clarity"; logic was correct, only the indentation was off by one space. Full suite 172/172 passes after the fix. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 13 +-- CLAUDE.md | 10 +-- README.md | 6 +- configs/catalog.yml | 39 ++++----- docker-compose.yml | 13 +-- docs/lmstudio-runbook.md | 21 ++--- frontend/public/lab/01-llm.html | 81 +++++++++++++++---- frontend/public/lab/04-orchestrator.html | 3 +- frontend/public/lab/06-pipeline.html | 2 +- frontend/public/lab/index.html | 7 +- services/orchestrator/orchestrator/config.py | 7 +- .../orchestrator/system/lmstudio.py | 30 +++---- wiki/faqs/swap-model.md | 4 +- 13 files changed, 146 insertions(+), 90 deletions(-) diff --git a/.env.example b/.env.example index b0354d0..7c64294 100644 --- a/.env.example +++ b/.env.example @@ -7,15 +7,16 @@ STT_PORT=8080 TTS_PORT=8880 LLM_PORT=8081 -# LLM backend (Plan #11): "lmstudio" (default) or "ollama". +# LLM backend (Plan #11): "ollama" (default — the upstream workshop path) or "lmstudio". +# ollama → host-installed Ollama; setup scripts pull qwen3:4b-instruct + smollm2. # lmstudio → orchestrator talks LM Studio's native /api/v0 API + auto-discovers # your whole LM Studio model library for the dashboard picker. -# ollama → original host-Ollama path. -LLM_BACKEND=lmstudio -# LM Studio server URL as seen from inside the orchestrator container. LM Studio -# must be "Served on Local Network" (binds 0.0.0.0) for host.docker.internal to work. +LLM_BACKEND=ollama +# (Used only when LLM_BACKEND=lmstudio.) LM Studio server URL as seen from inside the +# orchestrator container. LM Studio must be "Served on Local Network" (binds 0.0.0.0) +# so host.docker.internal resolves. LMSTUDIO_URL=http://host.docker.internal:1234 -# Model the "auto" brain falls back to when nothing is loaded (must exist in LM Studio). +# Model the LM Studio "auto" brain falls back to when nothing is loaded (must exist in LM Studio). LMSTUDIO_DEFAULT_MODEL=qwen/qwen3-4b-2507 # LLM settings (Ollama path) diff --git a/CLAUDE.md b/CLAUDE.md index 4372bdc..60b9a63 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,13 +11,13 @@ Browser (localhost:3000) └── nginx ──┬── /api/stt/ ──► whisper.cpp (port 8080) [Vulkan] ├── /api/tts/ ──► Kokoro-FastAPI (port 8880) [CUDA/ROCm] └── /api/llm/ ──► Orchestrator (port 8082) - ├─► LM Studio on host (:1234, native /api/v0) [default, Plan #11] - └─► Ollama on host (:11434) [LLM_BACKEND=ollama] + ├─► Ollama on host (:11434) [default] + └─► LM Studio on host (:1234, native /api/v0) [LLM_BACKEND=lmstudio, Plan #11] ``` - **Frontend**: Vite + Three.js + TalkingHead + VAD-web (browser-based orchestrator) -- **LLM**: LM Studio on host (native `/api/v0` API; whole library auto-discovered) OR - Qwen3-4B via Ollama — selected by `LLM_BACKEND` (see `docs/lmstudio-runbook.md`) +- **LLM**: Qwen3-4B via Ollama on host (default), OR LM Studio (native `/api/v0` API; whole + library auto-discovered) — selected by `LLM_BACKEND` (see `docs/lmstudio-runbook.md`) - **TTS**: Kokoro-82M via Kokoro-FastAPI (returns PCM + word timestamps) - **STT**: Whisper base.en via whisper.cpp (Vulkan) - **Orchestrator**: OpenAI-compatible proxy + agentic tool loop + command center backend @@ -187,7 +187,7 @@ Docker Desktop on macOS runs a Linux VM — no GPU passthrough to Metal/MPS. Doc ## Plan #11 — LM Studio backend (native API + dynamic discovery) -Full guide: **`docs/lmstudio-runbook.md`**. Selected by `LLM_BACKEND` env (`lmstudio` default | `ollama`). +Full guide: **`docs/lmstudio-runbook.md`**. Selected by `LLM_BACKEND` env (`ollama` default | `lmstudio` opt-in). - **New brain kind `lmstudio`** → `LMStudioProvider` (`providers/lmstudio.py`), a 3-line subclass of `OllamaProvider` that targets LM Studio's NATIVE endpoint `POST /api/v0/chat/completions`. The wire diff --git a/README.md b/README.md index 3971a69..1cd4154 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ NodeAva ships with a default avatar (sourced from the [TalkingHead](https://gith ## Prerequisites -**LLM backend — Ollama _or_ LM Studio:** -- **Ollama** (host install): `curl -fsSL https://ollama.com/install.sh | sh` (Linux/WSL2) or `brew install ollama` (macOS). Setup scripts pull defaults: `bash scripts/setup-linux.sh` / `bash scripts/setup-mac.sh`. -- **LM Studio** (default, `LLM_BACKEND=lmstudio`): run LM Studio with "Serve on Local Network" enabled. The orchestrator talks its native `/api/v0` API and **auto-discovers your whole model library** into the dashboard's Brain picker. See **`docs/lmstudio-runbook.md`**. _(LM Studio serves the LLM only; TTS stays Kokoro.)_ +**LLM backend — Ollama (default) _or_ LM Studio (opt-in):** +- **Ollama** (default, host install): `curl -fsSL https://ollama.com/install.sh | sh` (Linux/WSL2) or `brew install ollama` (macOS). Setup scripts pull defaults: `bash scripts/setup-linux.sh` / `bash scripts/setup-mac.sh`. +- **LM Studio** (opt-in, set `LLM_BACKEND=lmstudio` in `.env`): run LM Studio with "Serve on Local Network" enabled. The orchestrator talks its native `/api/v0` API and **auto-discovers your whole model library** into the dashboard's Brain picker. See **`docs/lmstudio-runbook.md`**. _(LM Studio serves the LLM only; TTS stays Kokoro.)_ **Docker + Compose (Linux/Windows only):** - Docker Engine with Compose V2 diff --git a/configs/catalog.yml b/configs/catalog.yml index 09e2dc7..d5be228 100644 --- a/configs/catalog.yml +++ b/configs/catalog.yml @@ -3,25 +3,7 @@ brains: # The frontend gates token-streaming detection on this flag — thinking models # buffer until appears, non-thinking models stream immediately. # - # ── LM Studio brains (Plan #11) ────────────────────────────────────────── - # kind: lmstudio talks LM Studio's NATIVE REST API at /api/v0. - # When LLM_BACKEND=lmstudio (set in docker-compose.yml), the orchestrator - # ALSO auto-discovers every model in your LM Studio library at runtime and - # adds each as a selectable `lmstudio:` brain — so this static list - # is just the stable defaults. `lmstudio-auto` follows whatever model you - # have loaded in LM Studio; it is promoted to the default brain at startup - # when LM Studio is the active backend. - - id: lmstudio-auto - label: "LM Studio (auto — uses your loaded model)" - kind: lmstudio - model: auto - thinks: false - - id: lmstudio-qwen3-4b - label: "LM Studio · Qwen3 4B 2507 (fast default)" - kind: lmstudio - model: qwen/qwen3-4b-2507 - thinks: false - # ── Ollama brains (used when LLM_BACKEND=ollama, the upstream default) ──── + # ── Ollama brains (default — used when LLM_BACKEND=ollama) ─────────────── - id: qwen3-4b-instruct label: "Qwen3 4B Instruct (default — fast, no reasoning)" kind: ollama @@ -43,6 +25,25 @@ brains: kind: ollama model: deepseek-r1:8b thinks: true + # ── LM Studio brains (Plan #11, opt-in via LLM_BACKEND=lmstudio) ───────── + # kind: lmstudio talks LM Studio's NATIVE REST API at /api/v0. + # When LLM_BACKEND=lmstudio, the orchestrator ALSO auto-discovers every model + # in your LM Studio library at runtime and adds each as a selectable + # `lmstudio:` brain — so this static list is just the stable + # defaults. `lmstudio-auto` follows whatever model you have loaded in LM + # Studio; it is promoted to the default brain at startup only when LM Studio + # is the active backend. + - id: lmstudio-auto + label: "LM Studio (auto — uses your loaded model)" + kind: lmstudio + model: auto + thinks: false + - id: lmstudio-qwen3-4b + label: "LM Studio · Qwen3 4B 2507 (small + fast)" + kind: lmstudio + model: qwen/qwen3-4b-2507 + thinks: false + # ── Cloud brains (require API keys) ────────────────────────────────────── - id: claude-sonnet label: "Claude Sonnet 4.6 (cloud)" kind: cloud-litellm diff --git a/docker-compose.yml b/docker-compose.yml index fc4d9e8..105ee12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,12 +35,13 @@ services: # Plan #7: catalog.yml must be visible inside the container. - ./configs:/app/configs:ro environment: - # Plan #11: LM Studio is the active local LLM backend by default. Flip - # LLM_BACKEND=ollama (in .env) to fall back to the host Ollama install. - # LM Studio must be serving on the local network (Developer ▸ Settings ▸ - # "Serve on Local Network") so the container can reach it via - # host.docker.internal:1234. - - LLM_BACKEND=${LLM_BACKEND:-lmstudio} + # Plan #11: local LLM backend. Defaults to "ollama" (the upstream + # workshop path). Set LLM_BACKEND=lmstudio in .env to opt in to LM Studio + # — the orchestrator then talks LM Studio's native /api/v0 API and the + # dashboard auto-discovers your whole LM Studio model library. + # LM Studio must be served on the local network (Developer ▸ Settings ▸ + # "Serve on Local Network") so the container reaches host.docker.internal:1234. + - LLM_BACKEND=${LLM_BACKEND:-ollama} - LMSTUDIO_URL=${LMSTUDIO_URL:-http://host.docker.internal:1234} - LMSTUDIO_DEFAULT_MODEL=${LMSTUDIO_DEFAULT_MODEL:-qwen/qwen3-4b-2507} - OLLAMA_URL=http://host.docker.internal:11434 diff --git a/docs/lmstudio-runbook.md b/docs/lmstudio-runbook.md index 12658f4..06b7811 100644 --- a/docs/lmstudio-runbook.md +++ b/docs/lmstudio-runbook.md @@ -1,8 +1,9 @@ # LM Studio Backend — Runbook (Plan #11) -Switch NodeAva's brain from **Ollama** to **LM Studio**, with a live, auto-discovered -picker for *every* model in your LM Studio library. The avatar's LLM now runs through -LM Studio's **native** REST API (`/api/v0`); TTS (Kokoro) and STT (Whisper) are unchanged. +**Opt-in alternative** to the default Ollama backend. Switch NodeAva's brain from Ollama to +**LM Studio** by setting `LLM_BACKEND=lmstudio`, with a live, auto-discovered picker for +*every* model in your LM Studio library. The avatar's LLM then runs through LM Studio's +**native** REST API (`/api/v0`); TTS (Kokoro) and STT (Whisper) are unchanged. > **Verified end-to-end** against a live LM Studio (115-model library): backend switch, > 112-model auto-discovery, residency, per-model swap, and streaming + non-streaming chat @@ -30,8 +31,8 @@ curl http://localhost:1234/api/v0/models | head # should list your models ## 1. Activate -LM Studio is the **default** backend now (`LLM_BACKEND=lmstudio` in `docker-compose.yml`). -Rebuild + restart so the new orchestrator code, env, and dashboard take effect: +Set `LLM_BACKEND=lmstudio` in your `.env` (the docker-compose default is `ollama`). +Then rebuild + restart so the new orchestrator code, env, and dashboard take effect: ```bash docker compose up -d --build orchestrator frontend @@ -87,19 +88,21 @@ via the orchestrator. ## 4. Switch back to Ollama +Ollama is the docker-compose default, so just remove the override (or set it explicitly): + ```bash -echo "LLM_BACKEND=ollama" >> .env # (or edit .env) +sed -i '/^LLM_BACKEND=/d' .env # or: echo "LLM_BACKEND=ollama" >> .env docker compose up -d --build orchestrator frontend ``` -Everything reverts to the host-Ollama path. Both backends coexist in the catalog; only the -*default* and the discovery source change. +The host-Ollama path takes over. Both backends coexist in the catalog; only the *active* +backend and the discovery source change. Per-deploy overrides (in `.env`): | Var | Default | Meaning | |---|---|---| -| `LLM_BACKEND` | `lmstudio` | `lmstudio` or `ollama` | +| `LLM_BACKEND` | `ollama` | `ollama` (default) or `lmstudio` (opt-in) | | `LMSTUDIO_URL` | `http://host.docker.internal:1234` | LM Studio server as seen from the container | | `LMSTUDIO_DEFAULT_MODEL` | `qwen/qwen3-4b-2507` | fallback for the `auto` brain | diff --git a/frontend/public/lab/01-llm.html b/frontend/public/lab/01-llm.html index 410657a..bd2dfcf 100644 --- a/frontend/public/lab/01-llm.html +++ b/frontend/public/lab/01-llm.html @@ -11,7 +11,7 @@
← Workshop Lab

Lab 1 · The Brain

- POST /api/lmstudio/api/v0/chat/completions + POST /api/ollama/v1/chat/completions
@@ -19,11 +19,14 @@

Lab 1 · The Brain

Checking services…
- What you're testing: LM Studio running on your host machine, - serving your selected model directly via its native API (this lab - bypasses the orchestrator so you measure raw LLM latency — Lab 4 demos the - orchestrator). Pick any model from your LM Studio library below. This is the - cognition layer — the part that decides what to say. + What you're testing: a local LLM running on your host machine, + serving your selected model directly (this lab bypasses the orchestrator + so you measure raw LLM latency — Lab 4 demos the orchestrator). The Backend + knob below lets you point at Ollama (default — serving + qwen3:4b-instruct) or LM Studio (native /api/v0 — + picks from your whole library, even logs server-measured tok/s + TTFT). Same + OpenAI-shaped wire on both. This is the cognition layer — the part that decides + what to say. What you'll see: tokens streamed one chunk at a time (not a wall of text). Watch TTFT (time to first token — the lag before the @@ -57,9 +60,16 @@

Try the model

⚙ Experiment knobs
+
+ + +
- +
@@ -121,20 +131,55 @@

Console

const out = $("out"); const con = $("console"); -NA.preflight("preflight", ["lmstudio"]); +// Plan #11 — two-backend toggle. Both backends speak the same OpenAI-shaped wire, +// so the BACKENDS map only parameterizes the 4 things that actually differ: +// the endpoint, the model-list source, the preflight key, and the default model. +// The SSE parser, sampling controls, and chip timing below are backend-agnostic. +const BACKENDS = { + ollama: { + endpoint: "/api/ollama/v1/chat/completions", + listModels: () => NA.listOllamaModels(), + preflight: "ollama", + defaultModel: "qwen3:4b-instruct", + }, + lmstudio: { + endpoint: "/api/lmstudio/api/v0/chat/completions", + listModels: () => NA.listLmStudioModels(), + preflight: "lmstudio", + defaultModel: "qwen/qwen3-4b-2507", + }, +}; +const currentBackend = () => BACKENDS[$("backend").value]; -// Populate model dropdown from LM Studio (native /api/v0) — loaded models first. -(async () => { - const models = await NA.listLmStudioModels(); - if (!models.length) return; +async function refreshModels() { + const b = currentBackend(); + $("step-tag").textContent = "POST " + b.endpoint; + const models = await b.listModels(); + if (!models.length) { + NA.fillSelect($("model"), [{ + value: b.defaultModel, + label: b.defaultModel + " (backend not reachable)", + }]); + return; + } + // listLmStudioModels reports m.loaded; listOllamaModels does not — that branch is harmless. const firstLoaded = models.find(m => m.loaded); - const defaultName = firstLoaded ? firstLoaded.name : "qwen/qwen3-4b-2507"; + const defaultName = firstLoaded ? firstLoaded.name : b.defaultModel; NA.fillSelect($("model"), models.map(m => ({ value: m.name, label: m.name + (m.loaded ? " (loaded)" : ""), selected: m.name === defaultName, }))); -})(); +} + +// Initial preflight + populate. Re-run both when the Backend toggle changes. +// async/await so a rapid toggle + Run can't race ahead of the model-list fetch. +async function syncBackend() { + NA.preflight("preflight", [currentBackend().preflight]); + await refreshModels(); +} +syncBackend(); +$("backend").addEventListener("change", syncBackend); // Live value displays for sliders NA.bindSlider($("temp"), $("temp-val"), (v) => Number(v).toFixed(2)); @@ -154,13 +199,15 @@

Console

$("go").onclick = async () => { const prompt = $("prompt").value.trim(); if (!prompt) return; + // Capture backend at click time so a mid-request toggle doesn't move the goalposts. + const b = currentBackend(); const streaming = $("stream").checked; out.textContent = ""; NA.clearConsole(con); NA.clearChips("chip-ttft", "chip-total", "chip-tps", "chip-tokens"); $("go").disabled = true; - NA.logLine(con, "→ POST /api/lmstudio/api/v0/chat/completions (stream=" + streaming + ")", "event"); + NA.logLine(con, "→ POST " + b.endpoint + " (stream=" + streaming + ")", "event"); NA.logLine(con, " body.messages = [{role: 'user', content: }]", ""); const t0 = NA.now(); @@ -175,7 +222,7 @@

Console

if (sysPrompt) messages.push({ role: "system", content: sysPrompt }); messages.push({ role: "user", content: prompt }); - const resp = await fetch("/api/lmstudio/api/v0/chat/completions", { + const resp = await fetch(b.endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -216,7 +263,7 @@

Console

tokenCount = json.usage?.completion_tokens || Math.ceil(text.length / 4); charCount = text.length; NA.logLine(con, "← non-streaming response (" + text.length + " chars)", "ok"); - // LM Studio's native API reports server-measured stats (Plan #11). + // LM Studio's native API reports server-measured stats (Plan #11; Ollama does not). if (json.stats) { NA.logLine(con, "← LM Studio: " + (json.stats.tokens_per_second || 0).toFixed(1) + " tok/s, TTFT " + Math.round((json.stats.time_to_first_token || 0) * 1000) diff --git a/frontend/public/lab/04-orchestrator.html b/frontend/public/lab/04-orchestrator.html index a9be3b7..bd78be6 100644 --- a/frontend/public/lab/04-orchestrator.html +++ b/frontend/public/lab/04-orchestrator.html @@ -45,7 +45,8 @@

Lab 4 · The Nervous System

What you're testing: the FastAPI orchestrator that sits between - the browser and the LLM backend (LM Studio). It holds the personality — a system + the browser and the LLM backend (Ollama by default; switchable to LM Studio + via LLM_BACKEND). It holds the personality — a system prompt that gets prepended to every chat — and the brain selection (which model to use). The same model under two different personalities produces dramatically different answers; the personality is the diff --git a/frontend/public/lab/06-pipeline.html b/frontend/public/lab/06-pipeline.html index b4f31fb..1776efc 100644 --- a/frontend/public/lab/06-pipeline.html +++ b/frontend/public/lab/06-pipeline.html @@ -112,7 +112,7 @@

Speak, then listen

STT (Whisper)
—
-
LLM (LM Studio)
—
+
LLM (Ollama / LM Studio)
—
TTS (Kokoro)
—
Total wall-clock
—
diff --git a/frontend/public/lab/index.html b/frontend/public/lab/index.html index 1018fd3..bd159ba 100644 --- a/frontend/public/lab/index.html +++ b/frontend/public/lab/index.html @@ -30,9 +30,10 @@

Build a digital human
one organ at a time.
Lab 1 · ~20 min

The Brain — local LLM

-

LM Studio on the host (native API). Stream tokens from any model in - your library, watch time-to-first-token, measure tokens-per-second. This - is the cognition layer; nothing else works until it does.

+

Ollama on the host. Stream tokens from qwen3:4b-instruct, + watch time-to-first-token, measure tokens-per-second. Toggle the lab's + Backend dropdown to test LM Studio (native API) instead. This is the + cognition layer; nothing else works until it does.

diff --git a/services/orchestrator/orchestrator/config.py b/services/orchestrator/orchestrator/config.py index 0065e2a..1839197 100644 --- a/services/orchestrator/orchestrator/config.py +++ b/services/orchestrator/orchestrator/config.py @@ -25,9 +25,10 @@ class Settings(BaseSettings): # LM Studio backend (Plan #11). LM Studio's native REST API lives at # /api/v0/* and is OpenAI-shaped for chat (streaming + # tool_calls verified) while additionally returning stats/model_info. - # `llm_backend` selects the active LOCAL backend: "ollama" or "lmstudio". - # Default stays "ollama" (preserves the upstream workshop + test suite); - # docker-compose.yml sets LLM_BACKEND=lmstudio for the LM Studio deployment. + # `llm_backend` selects the active LOCAL backend: "ollama" (default) or + # "lmstudio". docker-compose.yml passes the same default through + # `LLM_BACKEND=${LLM_BACKEND:-ollama}`; set `LLM_BACKEND=lmstudio` in `.env` + # to opt into the LM Studio path. lmstudio_url: str = "http://host.docker.internal:1234" llm_backend: str = "ollama" # Concrete model the LM Studio "auto" brain falls back to when nothing is diff --git a/services/orchestrator/orchestrator/system/lmstudio.py b/services/orchestrator/orchestrator/system/lmstudio.py index b895bf4..28e40e5 100644 --- a/services/orchestrator/orchestrator/system/lmstudio.py +++ b/services/orchestrator/orchestrator/system/lmstudio.py @@ -77,21 +77,21 @@ async def query(self) -> dict[str, Any]: data = await self._get_models() if data is None: return {"reachable": False, "loaded": []} - loaded: list[dict[str, Any]] = [] - for m in data: - if m.get("state") != "loaded": - continue - mid = m.get("id") or m.get("key") or "" - if not mid: - continue - loaded.append( - { - "model": mid, - "size_bytes": 0, - "size_vram_bytes": 0, - "residency": "loaded", - } - ) + loaded: list[dict[str, Any]] = [] + for m in data: + if m.get("state") != "loaded": + continue + mid = m.get("id") or m.get("key") or "" + if not mid: + continue + loaded.append( + { + "model": mid, + "size_bytes": 0, + "size_vram_bytes": 0, + "residency": "loaded", + } + ) return {"reachable": True, "loaded": loaded} async def pick_model(self, *, fallback: str) -> str: diff --git a/wiki/faqs/swap-model.md b/wiki/faqs/swap-model.md index e6e6532..10c6724 100644 --- a/wiki/faqs/swap-model.md +++ b/wiki/faqs/swap-model.md @@ -2,9 +2,9 @@ NodeAva's [[orchestrator]] sits at port 8082 and selects the model from server-side state, so swapping the LLM can be done live from the dashboard, at the deploy level, or by changing which backend you point at. -## Easiest: pick from the dashboard (LM Studio) +## Easiest: pick from the dashboard (LM Studio opt-in) - When `LLM_BACKEND=lmstudio` (the docker-compose default), the orchestrator auto-discovers **every model in your LM Studio library** and lists them in the dashboard's **Brain** dropdown (loaded models first, with a green "loaded" chip). Just pick one — selecting a model and chatting JIT-loads it in LM Studio. The **LM Studio (auto)** brain always follows whatever model you currently have loaded. +When you set `LLM_BACKEND=lmstudio` (the docker-compose default is `ollama`), the orchestrator auto-discovers **every model in your LM Studio library** and lists them in the dashboard's **Brain** dropdown (loaded models first, with a green "loaded" chip). Just pick one — selecting a model and chatting JIT-loads it in LM Studio. The **LM Studio (auto)** brain always follows whatever model you currently have loaded. This uses LM Studio's native API (`/api/v0`) and needs LM Studio running with "Serve on Local Network" enabled. Full guide: `docs/lmstudio-runbook.md`. TTS is unaffected — LM Studio cannot do text-to-speech, so [[kokoro-tts]] still handles the voice.