Lab 1 · The Brain
Checking services…
qwen3:4b-instruct directly (this lab bypasses
- the orchestrator so you measure raw LLM latency — Lab 4 demos the
- orchestrator). This is the cognition layer — the part that decides
+ What you're testing: a local LLM running on your host machine,
+ serving your selected model directly (this lab bypasses the orchestrator
+ so you measure raw LLM latency — Lab 4 demos the orchestrator). The Backend
+ knob below lets you point at Ollama (default — serving
+ qwen3:4b-instruct) or LM Studio (native /api/v0 —
+ picks from your whole library, even logs server-measured tok/s + TTFT). Same
+ OpenAI-shaped wire on both. This is the cognition layer — the part that decides
what to say.
What you'll see: tokens streamed one chunk at a time (not a wall
@@ -57,9 +60,16 @@ Try the model
+ /api/v0/* and is OpenAI-shaped for chat (streaming +
+ # tool_calls verified) while additionally returning stats/model_info.
+ # `llm_backend` selects the active LOCAL backend: "ollama" (default) or
+ # "lmstudio". docker-compose.yml passes the same default through
+ # `LLM_BACKEND=${LLM_BACKEND:-ollama}`; set `LLM_BACKEND=lmstudio` in `.env`
+ # to opt into the LM Studio path.
+ lmstudio_url: str = "http://host.docker.internal:1234"
+ llm_backend: str = "ollama"
+ # Concrete model the LM Studio "auto" brain falls back to when nothing is
+ # loaded. Must exist in the user's LM Studio library (JIT-loads on first use).
+ lmstudio_default_model: str = "qwen/qwen3-4b-2507"
+
request_timeout: float = 300.0
bind_host: str = "127.0.0.1"
bind_port: int = 8082
diff --git a/services/orchestrator/orchestrator/main.py b/services/orchestrator/orchestrator/main.py
index 0e9e239..e925561 100644
--- a/services/orchestrator/orchestrator/main.py
+++ b/services/orchestrator/orchestrator/main.py
@@ -13,6 +13,7 @@
from orchestrator.routes import state as state_route
from orchestrator.routes import swap as swap_route
from orchestrator.state import StateStore
+from orchestrator.system.lmstudio import LMStudioBackend
from orchestrator.system.residency import OllamaResidency
from orchestrator import tools as tool_registry
from orchestrator.tools.browser import BrowserFind, BrowserOpen, BrowserSearch
@@ -59,6 +60,18 @@ def _resolve_catalog_path() -> Path:
_REPO_CATALOG = _resolve_catalog_path()
+def _promote_brain_default(catalog, brain_id: str) -> bool:
+ """Plan #11 — make `brain_id` the sole default brain (used when the active
+ backend is LM Studio so the avatar boots on an LM Studio model). Returns
+ True if the brain exists in the catalog."""
+ if not any(b.id == brain_id for b in catalog.brains):
+ log.warning("cannot promote default brain '%s' (not in catalog)", brain_id)
+ return False
+ for b in catalog.brains:
+ b.default = (b.id == brain_id)
+ return True
+
+
def create_app(settings: Settings | None = None) -> FastAPI:
"""Application factory.
@@ -103,8 +116,40 @@ def create_app(settings: Settings | None = None) -> FastAPI:
)],
)
app.state.catalog = catalog
+
+ # Plan #11: LM Studio backend (discovery + residency via the native /api/v0
+ # API). Always constructed — it's cheap and the catalog route + chat "auto"
+ # resolution use it whenever llm_backend == "lmstudio". Residency and the
+ # boot-default brain follow the active backend.
+ lmstudio = LMStudioBackend(base_url=settings.lmstudio_url)
+ app.state.lmstudio = lmstudio
+ lmstudio_default_ok = False
+ if settings.llm_backend == "lmstudio":
+ lmstudio_default_ok = _promote_brain_default(catalog, "lmstudio-auto")
+ app.state.residency = lmstudio # duck-typed: exposes async query()
+ else:
+ app.state.residency = OllamaResidency(base_url=settings.ollama_url)
+
app.state.state_store = StateStore(path=settings.state_path, catalog=catalog)
- app.state.residency = OllamaResidency(base_url=settings.ollama_url)
+
+ # If LM Studio is active but the persisted brain is a stale non-LM-Studio
+ # selection (e.g. an Ollama brain from a previous run), reset it so chat
+ # doesn't route to a possibly-offline backend. Reset to "lmstudio-auto" only
+ # when it actually exists in the catalog (a user may have edited
+ # configs/catalog.yml and removed it); otherwise fall back to the validated
+ # catalog default. We never write an id the catalog doesn't know.
+ if settings.llm_backend == "lmstudio":
+ target = "lmstudio-auto" if lmstudio_default_ok else catalog.default_brain().id
+ try:
+ current_id = app.state.state_store.get_state()["brain"]
+ needs_reset = catalog.brain(current_id).kind != "lmstudio"
+ except CatalogError:
+ current_id, needs_reset = None, True
+ if needs_reset and current_id != target:
+ try:
+ app.state.state_store.set_state("brain", target)
+ except (ValueError, CatalogError) as e: # pragma: no cover - defensive
+ log.warning("could not set LM Studio default brain '%s': %s", target, e)
_register_builtin_tools(settings)
app.include_router(health.router)
diff --git a/services/orchestrator/orchestrator/providers/dispatcher.py b/services/orchestrator/orchestrator/providers/dispatcher.py
index 2525a9b..f52571e 100644
--- a/services/orchestrator/orchestrator/providers/dispatcher.py
+++ b/services/orchestrator/orchestrator/providers/dispatcher.py
@@ -12,6 +12,7 @@
from orchestrator.events import ErrorEvent, Event, FinalDoneEvent
from orchestrator.providers.base import Provider
from orchestrator.providers.litellm_provider import LiteLLMProvider
+from orchestrator.providers.lmstudio import LMStudioProvider
from orchestrator.providers.ollama import OllamaProvider
@@ -39,15 +40,28 @@ def dispatch_for_brain(
ollama_url: str,
request_timeout: float,
api_key: str | None,
+ lmstudio_url: str = "http://host.docker.internal:1234",
+ model_override: str | None = None,
) -> Provider:
- """Construct a Provider for this brain."""
+ """Construct a Provider for this brain.
+
+ `model_override` lets the chat route resolve a brain whose model is "auto"
+ (the LM Studio default brain) to a concrete, currently-loaded model id.
+ """
+ model = model_override or brain.model
if brain.kind == "ollama":
return OllamaProvider(
- base_url=ollama_url, model=brain.model, timeout=request_timeout,
+ base_url=ollama_url, model=model, timeout=request_timeout,
)
if brain.kind == "openai-compatible":
return OllamaProvider(
- base_url=brain.url, model=brain.model, timeout=request_timeout,
+ base_url=brain.url, model=model, timeout=request_timeout,
+ )
+ if brain.kind == "lmstudio":
+ # Native /api/v0 client. Brains may pin their own url; otherwise the
+ # deploy-wide LM Studio url is used.
+ return LMStudioProvider(
+ base_url=(brain.url or lmstudio_url), model=model, timeout=request_timeout,
)
if brain.kind == "cloud-litellm":
if not api_key:
diff --git a/services/orchestrator/orchestrator/providers/lmstudio.py b/services/orchestrator/orchestrator/providers/lmstudio.py
new file mode 100644
index 0000000..47692bb
--- /dev/null
+++ b/services/orchestrator/orchestrator/providers/lmstudio.py
@@ -0,0 +1,24 @@
+"""LMStudioProvider — chat client for LM Studio's NATIVE REST API.
+
+LM Studio exposes two HTTP surfaces:
+ - an OpenAI-compatible API at brain
+(see Catalog.sync_dynamic_brains). This is what makes the dashboard's brain
+dropdown list every model the user has in LM Studio, with loaded ones first.
"""
from __future__ import annotations
@@ -24,6 +31,24 @@ async def get_catalog(request: Request) -> dict:
settings = request.app.state.settings
ollama_tags = await _fetch_ollama_tags(settings.ollama_url)
+ # Plan #11: discover the LM Studio library and merge it as dynamic brains.
+ # Best-effort — a down/absent LM Studio just yields no dynamic brains and
+ # marks the static lmstudio brains unavailable (no exception bubbles up).
+ lmstudio_info = None
+ if settings.llm_backend == "lmstudio":
+ lmstudio = getattr(request.app.state, "lmstudio", None)
+ reachable, loaded = False, set()
+ if lmstudio is not None:
+ models = await lmstudio.list_models()
+ catalog.sync_dynamic_brains(models)
+ if models:
+ reachable = True
+ loaded = {m["id"] for m in models if m.get("loaded")}
+ else:
+ snap = await lmstudio.query()
+ reachable = bool(snap.get("reachable"))
+ lmstudio_info = {"reachable": reachable, "loaded": loaded}
+
brains_out = []
for b in catalog.brains:
entry = {
@@ -34,7 +59,9 @@ async def get_catalog(request: Request) -> dict:
entry["requires_key"] = b.requires_key
if b.url:
entry["url"] = b.url
- entry.update(await _brain_availability(b, ollama_tags))
+ if b.dynamic:
+ entry["dynamic"] = True
+ entry.update(await _brain_availability(b, ollama_tags, lmstudio_info))
brains_out.append(entry)
voices_out = [
@@ -75,7 +102,7 @@ async def _fetch_ollama_tags(ollama_url: str) -> set[str]:
return set()
-async def _brain_availability(brain, ollama_tags: set[str]) -> dict:
+async def _brain_availability(brain, ollama_tags: set[str], lmstudio_info: dict | None) -> dict:
if brain.kind == "ollama":
if brain.model in ollama_tags:
return {"available": True}
@@ -93,4 +120,14 @@ async def _brain_availability(brain, ollama_tags: set[str]) -> dict:
return {"available": False, "reason": f"server HTTP {resp.status_code}"}
except httpx.HTTPError:
return {"available": False, "reason": f"unreachable at {brain.url}"}
+ if brain.kind == "lmstudio":
+ if lmstudio_info is None:
+ return {"available": False, "reason": "LM Studio backend not active"}
+ if not lmstudio_info["reachable"]:
+ return {"available": False, "reason": "LM Studio unreachable"}
+ # Reachable → available (LM Studio JIT-loads the model on first request).
+ # "auto" follows whatever is loaded; concrete models report their state.
+ if brain.model in ("auto", ""):
+ return {"available": True, "loaded": False}
+ return {"available": True, "loaded": brain.model in lmstudio_info["loaded"]}
return {"available": False, "reason": "unknown brain kind"}
diff --git a/services/orchestrator/orchestrator/routes/chat.py b/services/orchestrator/orchestrator/routes/chat.py
index 2c8316c..137302b 100644
--- a/services/orchestrator/orchestrator/routes/chat.py
+++ b/services/orchestrator/orchestrator/routes/chat.py
@@ -24,6 +24,7 @@
from fastapi.responses import JSONResponse, StreamingResponse
from orchestrator.agentic import agentic_loop
+from orchestrator.catalog import CatalogError
from orchestrator.events import (
ErrorEvent,
FinalDoneEvent,
@@ -67,10 +68,31 @@ async def chat_completions(request: Request):
state_store = request.app.state.state_store
catalog = request.app.state.catalog
+ settings = request.app.state.settings
state = state_store.get_state()
- # Brain selection from state
- brain = catalog.brain(state["brain"])
+ # Brain selection from state. Fall back to the catalog default if the
+ # persisted brain id is gone (e.g. a dynamic LM Studio brain selected
+ # before a restart, before the next /v1/catalog refresh re-adds it).
+ try:
+ brain = catalog.brain(state["brain"])
+ except CatalogError:
+ brain = catalog.default_brain()
+ state_store.set_state("brain", brain.id)
+
+ # Plan #11: the LM Studio "auto" brain resolves to a concrete model id at
+ # request time — the currently-loaded model, else the configured fallback
+ # (which LM Studio JIT-loads on first use).
+ model_override = None
+ if brain.kind == "lmstudio" and brain.model in ("auto", ""):
+ lmstudio = getattr(request.app.state, "lmstudio", None)
+ if lmstudio is not None:
+ model_override = await lmstudio.pick_model(
+ fallback=settings.lmstudio_default_model
+ )
+ else:
+ model_override = settings.lmstudio_default_model
+
api_key = (
request.headers.get("X-Provider-Key")
or request.headers.get("x-provider-key")
@@ -78,9 +100,11 @@ async def chat_completions(request: Request):
)
provider = dispatch_for_brain(
brain,
- ollama_url=request.app.state.settings.ollama_url,
- request_timeout=request.app.state.settings.request_timeout,
+ ollama_url=settings.ollama_url,
+ lmstudio_url=settings.lmstudio_url,
+ request_timeout=settings.request_timeout,
api_key=api_key or None,
+ model_override=model_override,
)
# Personality system prompt at request time.
diff --git a/services/orchestrator/orchestrator/routes/models.py b/services/orchestrator/orchestrator/routes/models.py
index 40af917..0726210 100644
--- a/services/orchestrator/orchestrator/routes/models.py
+++ b/services/orchestrator/orchestrator/routes/models.py
@@ -8,7 +8,15 @@
@router.get("/v1/models")
async def list_models(request: Request) -> JSONResponse:
- backend = request.app.state.settings.ollama_url
+ settings = request.app.state.settings
+ # Proxy whichever local backend is active. Both Ollama and LM Studio expose
+ # an OpenAI-compatible /v1/models ({data:[{id}, ...]}), which is the shape
+ # the frontend preflight + Lab pages expect.
+ backend = (
+ settings.lmstudio_url
+ if settings.llm_backend == "lmstudio"
+ else settings.ollama_url
+ ).rstrip("/")
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(f"{backend}/v1/models")
diff --git a/services/orchestrator/orchestrator/system/lmstudio.py b/services/orchestrator/orchestrator/system/lmstudio.py
new file mode 100644
index 0000000..28e40e5
--- /dev/null
+++ b/services/orchestrator/orchestrator/system/lmstudio.py
@@ -0,0 +1,122 @@
+"""LM Studio backend — model discovery + residency via the native /api/v0 API.
+
+Parallels system/residency.py (OllamaResidency) so the dashboard's /v1/state +
+/v1/swap responses are backend-agnostic. LM Studio's GET /api/v0/models returns
+rich per-model info that Ollama's /api/tags + /api/ps cannot:
+
+ {"data": [{"id", "type"("llm"|"vlm"|"embeddings"), "state"("loaded"|
+ "not-loaded"), "arch", "quantization", "max_context_length"}, ...]}
+
+Unlike Ollama, LM Studio does NOT report per-model VRAM bytes, so residency is
+binary (loaded / not). Every method is best-effort and NEVER raises — a down or
+absent LM Studio degrades to "unreachable", not a 500.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+log = logging.getLogger("orchestrator.system.lmstudio")
+
+# Model types we surface as selectable chat brains. Embeddings models can't chat.
+_CHAT_TYPES = {"llm", "vlm"}
+
+# Substrings that mark a model as emitting /reasoning output, so the
+# frontend buffers until instead of streaming tokens immediately.
+_THINKS_HINTS = ("thinking", "reasoning", "deepseek-r1", "-r1", "magistral", "qwq")
+
+
+def looks_like_thinker(model_id: str) -> bool:
+ m = model_id.lower()
+ return any(h in m for h in _THINKS_HINTS)
+
+
+class LMStudioBackend:
+ """Discovery + residency probe for a host-installed LM Studio server."""
+
+ def __init__(self, *, base_url: str, timeout: float = 2.0) -> None:
+ self._base_url = base_url.rstrip("/")
+ self._timeout = timeout
+
+ async def list_models(self) -> list[dict[str, Any]]:
+ """Chat-capable models (llm/vlm) with state. Empty list on any error."""
+ data = await self._get_models()
+ if not data:
+ return []
+ out: list[dict[str, Any]] = []
+ for m in data:
+ if m.get("type") not in _CHAT_TYPES:
+ continue
+ mid = m.get("id") or m.get("key") or ""
+ if not mid:
+ continue
+ out.append(
+ {
+ "id": mid,
+ "type": m.get("type"),
+ "state": m.get("state", "not-loaded"),
+ "arch": m.get("arch"),
+ "quant": m.get("quantization"),
+ "max_context_length": m.get("max_context_length"),
+ "loaded": m.get("state") == "loaded",
+ "thinks": looks_like_thinker(mid),
+ }
+ )
+ return out
+
+ async def query(self) -> dict[str, Any]:
+ """Residency snapshot, shaped like OllamaResidency.query(). Never raises.
+
+ Returns {"reachable": bool, "loaded": [{"model", "size_bytes",
+ "size_vram_bytes", "residency"}]}. LM Studio gives no byte sizes, so
+ those are 0 and residency is the literal "loaded" (the dashboard maps
+ it to a green chip).
+ """
+ data = await self._get_models()
+ if data is None:
+ return {"reachable": False, "loaded": []}
+ loaded: list[dict[str, Any]] = []
+ for m in data:
+ if m.get("state") != "loaded":
+ continue
+ mid = m.get("id") or m.get("key") or ""
+ if not mid:
+ continue
+ loaded.append(
+ {
+ "model": mid,
+ "size_bytes": 0,
+ "size_vram_bytes": 0,
+ "residency": "loaded",
+ }
+ )
+ return {"reachable": True, "loaded": loaded}
+
+ async def pick_model(self, *, fallback: str) -> str:
+ """Resolve the "auto" brain to a concrete model id.
+
+ Prefers a currently-loaded chat model (zero load latency); otherwise
+ returns `fallback` (which JIT-loads on first chat request).
+ """
+ for m in await self.list_models():
+ if m["loaded"]:
+ return m["id"]
+ return fallback
+
+ async def _get_models(self) -> list[dict[str, Any]] | None:
+ """GET /api/v0/models → list (possibly empty) or None when unreachable."""
+ try:
+ async with httpx.AsyncClient(timeout=self._timeout) as client:
+ resp = await client.get(f"{self._base_url}/api/v0/models")
+ if resp.status_code >= 400:
+ log.warning("lmstudio /api/v0/models HTTP %d", resp.status_code)
+ return None
+ data = resp.json()
+ except (httpx.HTTPError, ValueError) as e:
+ log.info("lmstudio /api/v0/models unreachable: %s", e)
+ return None
+ if isinstance(data, dict):
+ return data.get("data") or []
+ return data or []
diff --git a/services/orchestrator/tests/test_lmstudio.py b/services/orchestrator/tests/test_lmstudio.py
new file mode 100644
index 0000000..47ba1c3
--- /dev/null
+++ b/services/orchestrator/tests/test_lmstudio.py
@@ -0,0 +1,285 @@
+"""Tests for the LM Studio backend integration (Plan #11).
+
+Covers: config flags, the native LMStudioProvider, model discovery + residency
+(LMStudioBackend), dispatcher routing, dynamic catalog merge, and route-level
+behavior (catalog discovery + chat "auto" model resolution) when
+llm_backend == "lmstudio".
+"""
+import json
+import textwrap
+
+import httpx
+import pytest
+import respx
+from httpx import ASGITransport, AsyncClient
+
+import orchestrator.main as main_module
+from orchestrator.catalog import BrainEntry, Catalog
+from orchestrator.config import Settings
+from orchestrator.events import TokenEvent
+from orchestrator.main import create_app
+from orchestrator.providers.dispatcher import dispatch_for_brain
+from orchestrator.providers.lmstudio import LMStudioProvider
+from orchestrator.providers.ollama import OllamaProvider
+from orchestrator.system.lmstudio import LMStudioBackend, looks_like_thinker
+
+LMS = "http://lmstudio-test:1234"
+
+_MODELS_JSON = {
+ "data": [
+ {"id": "qwen/qwen3-4b-2507", "type": "llm", "state": "loaded",
+ "arch": "qwen3", "quantization": "Q4_K_M", "max_context_length": 128000},
+ {"id": "google/gemma-4-31b-it", "type": "vlm", "state": "not-loaded",
+ "max_context_length": 262144},
+ {"id": "deepseek-r1-8b-thinking", "type": "llm", "state": "not-loaded"},
+ {"id": "text-embedding-nomic", "type": "embeddings", "state": "not-loaded"},
+ ]
+}
+
+
+# ─────────────────────────── config ───────────────────────────
+def test_config_lmstudio_defaults(monkeypatch):
+ for k in ("LLM_BACKEND", "LMSTUDIO_URL", "LMSTUDIO_DEFAULT_MODEL"):
+ monkeypatch.delenv(k, raising=False)
+ s = Settings()
+ assert s.llm_backend == "ollama" # upstream default preserved
+ assert s.lmstudio_url == "http://host.docker.internal:1234"
+ assert s.lmstudio_default_model == "qwen/qwen3-4b-2507"
+
+
+def test_config_lmstudio_env_override(monkeypatch):
+ monkeypatch.setenv("LLM_BACKEND", "lmstudio")
+ monkeypatch.setenv("LMSTUDIO_URL", "http://box:4321")
+ monkeypatch.setenv("LMSTUDIO_DEFAULT_MODEL", "foo/bar")
+ s = Settings()
+ assert s.llm_backend == "lmstudio"
+ assert s.lmstudio_url == "http://box:4321"
+ assert s.lmstudio_default_model == "foo/bar"
+
+
+# ─────────────────────────── provider ───────────────────────────
+def test_lmstudio_provider_targets_native_endpoint():
+ p = LMStudioProvider(base_url=LMS, model="m")
+ assert isinstance(p, OllamaProvider) # reuses OpenAI-shaped parsing
+ assert p._CHAT_PATH == "/api/v0/chat/completions"
+
+
+@respx.mock
+async def test_lmstudio_provider_chat_hits_api_v0():
+ route = respx.post(f"{LMS}/api/v0/chat/completions").mock(
+ return_value=httpx.Response(
+ 200,
+ json={"choices": [{"message": {"role": "assistant", "content": "hi"}}]},
+ )
+ )
+ p = LMStudioProvider(base_url=LMS, model="qwen/qwen3-4b-2507")
+ events = [e async for e in p.chat([{"role": "user", "content": "x"}], stream=False)]
+ assert route.called
+ assert any(isinstance(e, TokenEvent) and e.delta == "hi" for e in events)
+
+
+# ─────────────────────── backend discovery ───────────────────────
+@respx.mock
+async def test_backend_list_models_filters_and_flags():
+ respx.get(f"{LMS}/api/v0/models").mock(
+ return_value=httpx.Response(200, json=_MODELS_JSON)
+ )
+ models = await LMStudioBackend(base_url=LMS).list_models()
+ ids = {m["id"] for m in models}
+ assert "text-embedding-nomic" not in ids # embeddings filtered out
+ assert {"qwen/qwen3-4b-2507", "google/gemma-4-31b-it"} <= ids
+ q = next(m for m in models if m["id"] == "qwen/qwen3-4b-2507")
+ assert q["loaded"] is True
+ dr = next(m for m in models if m["id"] == "deepseek-r1-8b-thinking")
+ assert dr["thinks"] is True
+
+
+@respx.mock
+async def test_backend_query_residency_shape():
+ respx.get(f"{LMS}/api/v0/models").mock(
+ return_value=httpx.Response(200, json=_MODELS_JSON)
+ )
+ snap = await LMStudioBackend(base_url=LMS).query()
+ assert snap["reachable"] is True
+ assert [m["model"] for m in snap["loaded"]] == ["qwen/qwen3-4b-2507"]
+ assert snap["loaded"][0]["residency"] == "loaded"
+
+
+@respx.mock
+async def test_backend_query_unreachable():
+ respx.get(f"{LMS}/api/v0/models").mock(side_effect=httpx.ConnectError("no"))
+ snap = await LMStudioBackend(base_url=LMS).query()
+ assert snap == {"reachable": False, "loaded": []}
+
+
+@respx.mock
+async def test_backend_pick_model_prefers_loaded():
+ respx.get(f"{LMS}/api/v0/models").mock(
+ return_value=httpx.Response(200, json=_MODELS_JSON)
+ )
+ picked = await LMStudioBackend(base_url=LMS).pick_model(fallback="fallback/x")
+ assert picked == "qwen/qwen3-4b-2507"
+
+
+@respx.mock
+async def test_backend_pick_model_falls_back_when_none_loaded():
+ j = {"data": [{"id": "a/b", "type": "llm", "state": "not-loaded"}]}
+ respx.get(f"{LMS}/api/v0/models").mock(return_value=httpx.Response(200, json=j))
+ picked = await LMStudioBackend(base_url=LMS).pick_model(fallback="fallback/x")
+ assert picked == "fallback/x"
+
+
+def test_looks_like_thinker():
+ assert looks_like_thinker("deepseek-r1-8b")
+ assert looks_like_thinker("qwen3-4b-thinking-2507")
+ assert not looks_like_thinker("qwen/qwen3-4b-2507")
+
+
+# ─────────────────────────── dispatcher ───────────────────────────
+def test_dispatch_lmstudio_kind():
+ brain = BrainEntry(id="x", label="X", kind="lmstudio", model="m")
+ p = dispatch_for_brain(
+ brain, ollama_url="http://o:11434", lmstudio_url=LMS,
+ request_timeout=10.0, api_key=None,
+ )
+ assert isinstance(p, LMStudioProvider)
+ assert p._base_url == LMS
+ assert p._model == "m"
+
+
+def test_dispatch_lmstudio_model_override():
+ brain = BrainEntry(id="x", label="X", kind="lmstudio", model="auto")
+ p = dispatch_for_brain(
+ brain, ollama_url="http://o:11434", lmstudio_url=LMS,
+ request_timeout=10.0, api_key=None, model_override="real/model",
+ )
+ assert p._model == "real/model"
+
+
+# ──────────────────────── dynamic catalog merge ────────────────────────
+def test_sync_dynamic_brains_adds_replaces_and_preserves_static():
+ cat = Catalog(brains=[BrainEntry(id="static", label="S", kind="lmstudio", model="auto")])
+ cat.sync_dynamic_brains([
+ {"id": "a/b", "loaded": True, "thinks": False},
+ {"id": "c/d-thinking", "loaded": False, "thinks": True},
+ ])
+ ids = [b.id for b in cat.brains]
+ assert "static" in ids # static (non-dynamic) preserved
+ assert {"lmstudio:a/b", "lmstudio:c/d-thinking"} <= set(ids)
+ dyn = [b for b in cat.brains if b.dynamic]
+ assert dyn[0].id == "lmstudio:a/b" # loaded sorts first
+ assert cat.brain("lmstudio:c/d-thinking").thinks is True
+
+ # re-sync REPLACES dynamic entries (no accumulation), keeps static
+ cat.sync_dynamic_brains([{"id": "e/f", "loaded": False}])
+ assert [b.id for b in cat.brains if b.dynamic] == ["lmstudio:e/f"]
+ assert [b.id for b in cat.brains if not b.dynamic] == ["static"]
+
+
+# ─────────────────────── route-level (backend=lmstudio) ───────────────────────
+@pytest.fixture
+def lmstudio_app(tmp_path):
+ settings = Settings(
+ state_path=str(tmp_path / "state.json"),
+ llm_backend="lmstudio",
+ lmstudio_url=LMS,
+ )
+ app = create_app(settings=settings)
+ app.state.state_store.set_tool("web_search", False)
+ app.state.state_store.set_tool("wiki", False)
+ return app
+
+
+@pytest.fixture
+async def lmstudio_client(lmstudio_app):
+ transport = ASGITransport(app=lmstudio_app)
+ async with AsyncClient(transport=transport, base_url="http://test") as client:
+ yield client
+
+
+def test_lmstudio_backend_promotes_default_brain(lmstudio_app):
+ # When LM Studio is active, the boot default brain is the LM Studio "auto" one.
+ assert lmstudio_app.state.catalog.default_brain().id == "lmstudio-auto"
+ assert lmstudio_app.state.state_store.get_state()["brain"] == "lmstudio-auto"
+
+
+def test_lmstudio_missing_auto_brain_resets_to_catalog_default(tmp_path, monkeypatch):
+ """Reviewer edge case: if 'lmstudio-auto' is absent from the catalog (a user
+ edited configs/catalog.yml and removed it) while LLM_BACKEND=lmstudio, boot
+ must NOT persist an invalid brain id — it falls back to the validated catalog
+ default and never writes an id the catalog doesn't know."""
+ cat = tmp_path / "catalog.yml"
+ cat.write_text(textwrap.dedent("""
+ brains:
+ - {id: qwen3-4b-instruct, label: Q, kind: ollama, model: "qwen3:4b-instruct", default: true}
+ voices:
+ - {id: bella, label: B, kokoro_voice: af_bella, default: true}
+ avatars:
+ - {id: ava, label: A, glb_path: /a.glb, default: true}
+ personalities:
+ - {id: default, label: D, system_prompt: hi, default: true}
+ """))
+ monkeypatch.setattr(main_module, "_REPO_CATALOG", cat)
+ # Pre-seed persisted state with a (stale) non-LM-Studio brain selection.
+ state = tmp_path / "state.json"
+ state.write_text(json.dumps({
+ "brain": "qwen3-4b-instruct", "voice": "bella", "avatar": "ava",
+ "personality": "default", "tools": {"web_search": False, "wiki": True},
+ }))
+ settings = Settings(state_path=str(state), llm_backend="lmstudio", lmstudio_url=LMS)
+
+ app = create_app(settings=settings) # must not raise
+
+ active = app.state.state_store.get_state()["brain"]
+ assert active != "lmstudio-auto" # the missing id is never persisted
+ app.state.catalog.brain(active) # resolvable — no CatalogError
+ assert active == "qwen3-4b-instruct" # fell back to the catalog default
+
+
+@respx.mock
+async def test_catalog_merges_lmstudio_library(lmstudio_client):
+ respx.get(f"{LMS}/api/v0/models").mock(
+ return_value=httpx.Response(200, json=_MODELS_JSON)
+ )
+ respx.get("http://host.docker.internal:11434/api/tags").mock(
+ return_value=httpx.Response(200, json={"models": []})
+ )
+ resp = await lmstudio_client.get("/v1/catalog")
+ assert resp.status_code == 200
+ brains = resp.json()["brains"]
+ ids = {b["id"] for b in brains}
+ assert "lmstudio:qwen/qwen3-4b-2507" in ids
+ assert "lmstudio:google/gemma-4-31b-it" in ids
+ assert "lmstudio:text-embedding-nomic" not in ids # embeddings excluded
+ q = next(b for b in brains if b["id"] == "lmstudio:qwen/qwen3-4b-2507")
+ assert q["available"] is True and q["loaded"] is True
+ g = next(b for b in brains if b["id"] == "lmstudio:google/gemma-4-31b-it")
+ assert g["available"] is True and g["loaded"] is False
+
+
+@respx.mock
+async def test_chat_auto_resolves_loaded_model_via_native_endpoint(lmstudio_client):
+ respx.get(f"{LMS}/api/v0/models").mock(
+ return_value=httpx.Response(200, json=_MODELS_JSON)
+ )
+ captured = {}
+
+ def _cap(request):
+ captured["url"] = str(request.url)
+ captured["body"] = json.loads(request.content)
+ return httpx.Response(
+ 200,
+ json={"choices": [{"message": {"role": "assistant", "content": "ok"}}]},
+ )
+
+ respx.post(f"{LMS}/api/v0/chat/completions").mock(side_effect=_cap)
+
+ resp = await lmstudio_client.post(
+ "/v1/chat/completions",
+ json={"messages": [{"role": "user", "content": "hi"}], "stream": False},
+ )
+ assert resp.status_code == 200
+ # active brain is lmstudio-auto → resolved to the currently-loaded model,
+ # and the request hit LM Studio's NATIVE endpoint.
+ assert captured["body"]["model"] == "qwen/qwen3-4b-2507"
+ assert "/api/v0/chat/completions" in captured["url"]
diff --git a/services/orchestrator/tests/test_routes_catalog.py b/services/orchestrator/tests/test_routes_catalog.py
index 4147185..6cd4dad 100644
--- a/services/orchestrator/tests/test_routes_catalog.py
+++ b/services/orchestrator/tests/test_routes_catalog.py
@@ -19,12 +19,12 @@ async def test_get_catalog_returns_4_sections(app_client, monkeypatch):
@respx.mock
async def test_get_catalog_marks_pulled_models_available(app_client, monkeypatch):
respx.get("http://host.docker.internal:11434/api/tags").mock(
- return_value=httpx.Response(200, json={"models": [{"name": "qwen3:4b"}]})
+ return_value=httpx.Response(200, json={"models": [{"name": "qwen3:4b-instruct"}]})
)
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
resp = await app_client.get("/v1/catalog")
body = resp.json()
- q = next(b for b in body["brains"] if b["id"] == "qwen3-4b")
+ q = next(b for b in body["brains"] if b["id"] == "qwen3-4b-instruct")
assert q["available"] is True
claude = next(b for b in body["brains"] if b["id"] == "claude-sonnet")
assert claude["available"] is False
@@ -38,9 +38,9 @@ async def test_get_catalog_unpulled_model_unavailable(app_client):
)
resp = await app_client.get("/v1/catalog")
body = resp.json()
- q = next(b for b in body["brains"] if b["id"] == "qwen3-4b")
+ q = next(b for b in body["brains"] if b["id"] == "qwen3-4b-instruct")
assert q["available"] is False
- assert "ollama pull qwen3:4b" in q["reason"]
+ assert "ollama pull qwen3:4b-instruct" in q["reason"]
@respx.mock
diff --git a/services/orchestrator/tests/test_routes_state.py b/services/orchestrator/tests/test_routes_state.py
index eb065ad..e548d0b 100644
--- a/services/orchestrator/tests/test_routes_state.py
+++ b/services/orchestrator/tests/test_routes_state.py
@@ -17,7 +17,7 @@ async def test_get_state_returns_active_and_system(app_client):
body = resp.json()
assert "active" in body
assert "system" in body
- assert body["active"]["brain"] == "qwen3-4b"
+ assert body["active"]["brain"] == "qwen3-4b-instruct"
assert body["system"]["ollama"]["reachable"] is True
assert body["system"]["ollama"]["loaded"][0]["residency"] == "gpu"
diff --git a/wiki/faqs/swap-model.md b/wiki/faqs/swap-model.md
index f7751ad..10c6724 100644
--- a/wiki/faqs/swap-model.md
+++ b/wiki/faqs/swap-model.md
@@ -1,6 +1,12 @@
# How do I swap the LLM model?
-NodeAva's [[orchestrator]] sits at port 8082 and accepts a `model` field per request, so swapping the LLM does not require restarting any service — it can be done at the request level, the deploy level, or by pointing the local llama-server at a different file.
+NodeAva's [[orchestrator]] sits at port 8082 and selects the model from server-side state, so swapping the LLM can be done live from the dashboard, at the deploy level, or by changing which backend you point at.
+
+## Easiest: pick from the dashboard (LM Studio opt-in)
+
+When you set `LLM_BACKEND=lmstudio` (the docker-compose default is `ollama`), the orchestrator auto-discovers **every model in your LM Studio library** and lists them in the dashboard's **Brain** dropdown (loaded models first, with a green "loaded" chip). Just pick one — selecting a model and chatting JIT-loads it in LM Studio. The **LM Studio (auto)** brain always follows whatever model you currently have loaded.
+
+This uses LM Studio's native API (`/api/v0`) and needs LM Studio running with "Serve on Local Network" enabled. Full guide: `docs/lmstudio-runbook.md`. TTS is unaffected — LM Studio cannot do text-to-speech, so [[kokoro-tts]] still handles the voice.
## Switch model per request
+
+
+
-
+
@@ -121,16 +131,55 @@ }]", "");
const t0 = NA.now();
@@ -171,7 +222,7 @@
What you're testing: the FastAPI orchestrator that sits between
- the browser and Ollama. It holds the personality — a system
+ the browser and the LLM backend (Ollama by default; switchable to LM Studio
+ via
-
+
diff --git a/frontend/public/lab/_shared.js b/frontend/public/lab/_shared.js
index a7c4ffd..a5fe752 100644
--- a/frontend/public/lab/_shared.js
+++ b/frontend/public/lab/_shared.js
@@ -144,6 +144,9 @@ async function preflight(elId, services) {
// Lab 1 uses this so attendees measure raw LLM TTFT without the
// orchestrator's personality prefill or agentic tool loop in the way.
ollama: () => fetch("/api/ollama/v1/models").then(r => r.ok),
+ // 'lmstudio' = bypasses the orchestrator, hits LM Studio's native API on
+ // the host directly (Plan #11). Lab 1 uses this for raw model-latency tests.
+ lmstudio: () => fetch("/api/lmstudio/api/v0/models").then(r => r.ok),
tts: () => fetch("/api/tts/v1/audio/voices").then(r => r.ok),
stt: () => fetch("/api/stt/").then(r => r.ok || r.status === 404 || r.status === 405),
orch: () => fetch("/api/orch/v1/state").then(r => r.ok),
@@ -183,6 +186,20 @@ async function listOllamaModels() {
} catch { return []; }
}
+// Fetch chat-capable models from LM Studio via its native /api/v0/models
+// (Plan #11). Returns [{name, loaded}], loaded models first; embeddings excluded.
+async function listLmStudioModels() {
+ try {
+ const r = await fetch("/api/lmstudio/api/v0/models");
+ if (!r.ok) return [];
+ const j = await r.json();
+ return (j.data || [])
+ .filter((m) => m.type === "llm" || m.type === "vlm")
+ .map((m) => ({ name: m.id, loaded: m.state === "loaded" }))
+ .sort((a, b) => (b.loaded - a.loaded) || a.name.localeCompare(b.name));
+ } catch { return []; }
+}
+
// Populate a
Console
const out = $("out"); const con = $("console"); -NA.preflight("preflight", ["ollama"]); +// Plan #11 — two-backend toggle. Both backends speak the same OpenAI-shaped wire, +// so the BACKENDS map only parameterizes the 4 things that actually differ: +// the endpoint, the model-list source, the preflight key, and the default model. +// The SSE parser, sampling controls, and chip timing below are backend-agnostic. +const BACKENDS = { + ollama: { + endpoint: "/api/ollama/v1/chat/completions", + listModels: () => NA.listOllamaModels(), + preflight: "ollama", + defaultModel: "qwen3:4b-instruct", + }, + lmstudio: { + endpoint: "/api/lmstudio/api/v0/chat/completions", + listModels: () => NA.listLmStudioModels(), + preflight: "lmstudio", + defaultModel: "qwen/qwen3-4b-2507", + }, +}; +const currentBackend = () => BACKENDS[$("backend").value]; -// Populate model dropdown from the host's Ollama -(async () => { - const models = await NA.listOllamaModels(); - if (!models.length) return; +async function refreshModels() { + const b = currentBackend(); + $("step-tag").textContent = "POST " + b.endpoint; + const models = await b.listModels(); + if (!models.length) { + NA.fillSelect($("model"), [{ + value: b.defaultModel, + label: b.defaultModel + " (backend not reachable)", + }]); + return; + } + // listLmStudioModels reports m.loaded; listOllamaModels does not — that branch is harmless. + const firstLoaded = models.find(m => m.loaded); + const defaultName = firstLoaded ? firstLoaded.name : b.defaultModel; NA.fillSelect($("model"), models.map(m => ({ - value: m.name, label: m.name, selected: m.name === "qwen3:4b-instruct", + value: m.name, + label: m.name + (m.loaded ? " (loaded)" : ""), + selected: m.name === defaultName, }))); -})(); +} + +// Initial preflight + populate. Re-run both when the Backend toggle changes. +// async/await so a rapid toggle + Run can't race ahead of the model-list fetch. +async function syncBackend() { + NA.preflight("preflight", [currentBackend().preflight]); + await refreshModels(); +} +syncBackend(); +$("backend").addEventListener("change", syncBackend); // Live value displays for sliders NA.bindSlider($("temp"), $("temp-val"), (v) => Number(v).toFixed(2)); @@ -150,13 +199,15 @@Console
$("go").onclick = async () => { const prompt = $("prompt").value.trim(); if (!prompt) return; + // Capture backend at click time so a mid-request toggle doesn't move the goalposts. + const b = currentBackend(); const streaming = $("stream").checked; out.textContent = ""; NA.clearConsole(con); NA.clearChips("chip-ttft", "chip-total", "chip-tps", "chip-tokens"); $("go").disabled = true; - NA.logLine(con, "→ POST /api/ollama/v1/chat/completions (stream=" + streaming + ")", "event"); + NA.logLine(con, "→ POST " + b.endpoint + " (stream=" + streaming + ")", "event"); NA.logLine(con, " body.messages = [{role: 'user', content:Console
if (sysPrompt) messages.push({ role: "system", content: sysPrompt }); messages.push({ role: "user", content: prompt }); - const resp = await fetch("/api/ollama/v1/chat/completions", { + const resp = await fetch(b.endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -209,9 +260,15 @@Console
const json = await resp.json(); const text = json.choices?.[0]?.message?.content || ""; out.textContent = text; - tokenCount = Math.ceil(text.length / 4); // rough estimate + tokenCount = json.usage?.completion_tokens || Math.ceil(text.length / 4); charCount = text.length; NA.logLine(con, "← non-streaming response (" + text.length + " chars)", "ok"); + // LM Studio's native API reports server-measured stats (Plan #11; Ollama does not). + if (json.stats) { + NA.logLine(con, "← LM Studio: " + (json.stats.tokens_per_second || 0).toFixed(1) + + " tok/s, TTFT " + Math.round((json.stats.time_to_first_token || 0) * 1000) + + " ms (server-measured)", "ok"); + } } const tTotal = NA.now() - t0; diff --git a/frontend/public/lab/04-orchestrator.html b/frontend/public/lab/04-orchestrator.html index 9c2461f..bd78be6 100644 --- a/frontend/public/lab/04-orchestrator.html +++ b/frontend/public/lab/04-orchestrator.html @@ -45,7 +45,8 @@Lab 4 · The Nervous System
LLM_BACKEND). It holds the personality — a system
prompt that gets prepended to every chat — and the brain selection
(which model to use). The same model under two different personalities
produces dramatically different answers; the personality is the
diff --git a/frontend/public/lab/06-pipeline.html b/frontend/public/lab/06-pipeline.html
index 2da8791..1776efc 100644
--- a/frontend/public/lab/06-pipeline.html
+++ b/frontend/public/lab/06-pipeline.html
@@ -112,7 +112,7 @@ Speak, then listen
STT (Whisper)
—
LLM (Ollama)
—
LLM (Ollama / LM Studio)
—
TTS (Kokoro)
—
Total wall-clock
—
The Brain — local LLM
Ollama on the host. Stream tokens from qwen3:4b-instruct,
- watch time-to-first-token, measure tokens-per-second. This is the
+ watch time-to-first-token, measure tokens-per-second. Toggle the lab's
+ Backend dropdown to test LM Studio (native API) instead. This is the
cognition layer; nothing else works until it does.