diff --git a/.cursor/rules/model-registry.mdc b/.cursor/rules/model-registry.mdc
index 1b9f28f88..96d3627e1 100644
--- a/.cursor/rules/model-registry.mdc
+++ b/.cursor/rules/model-registry.mdc
@@ -1,5 +1,5 @@
---
-description: Backend-owned engine model catalogue (Ollama committed list + LM Studio and llama.cpp Hugging Face catalogs)
+description: Backend-owned engine model catalogue (Ollama committed list + LM Studio, llama.cpp and Colibri Hugging Face catalogs)
alwaysApply: true
---
+
diff --git a/desktop/src/ui/components/EngineIcon.tsx b/desktop/src/ui/components/EngineIcon.tsx
index bb1618a0e..de35c37b5 100644
--- a/desktop/src/ui/components/EngineIcon.tsx
+++ b/desktop/src/ui/components/EngineIcon.tsx
@@ -6,6 +6,7 @@ import { type EngineType } from '@/shared/types/engines'
import ollamaIcon from '@/ui/assets/engine-icons/ollama.png?inline'
import lmStudioIcon from '@/ui/assets/engine-icons/lm-studio.png?inline'
import llamaCppIcon from '@/ui/assets/engine-icons/llama-cpp.svg?inline'
+import colibriIcon from '@/ui/assets/engine-icons/colibri.svg?inline'
export default function EngineIcon({ type, size = 32 }: { type: EngineType; size?: number }) {
const dimension = `${size}px`
@@ -48,5 +49,13 @@ export default function EngineIcon({ type, size = 32 }: { type: EngineType; size
)
}
+ if (type === 'colibri') {
+ return (
+
+
+
+ )
+ }
+
return null
}
diff --git a/desktop/src/ui/components/ModelHub/ModelHubContent.tsx b/desktop/src/ui/components/ModelHub/ModelHubContent.tsx
index 354709b75..c440d9a8f 100644
--- a/desktop/src/ui/components/ModelHub/ModelHubContent.tsx
+++ b/desktop/src/ui/components/ModelHub/ModelHubContent.tsx
@@ -11,6 +11,7 @@ import { ModelHubList } from './ModelHubList'
import { mergeModelHubResults, searchEngineHub } from '@/ui/utils/model-hub-search'
import { resolveStoredSort, writeStoredSort } from '@/ui/utils/model-hub-content-storage'
import { isHubEntryDownloaded } from '@/ui/utils/match-downloaded-model'
+import { EngineCapabilities } from '@/ui/constants/engine-capabilities'
import { EngineType } from '@/shared/types/engines'
import type { ModelItem } from '@/ui/types/engine-info'
import getErrorString from '@/shared/utils/get-error-string'
@@ -184,7 +185,12 @@ export const ModelHubContent = ({
async (nextQuery: string) => {
const normalized = nextQuery.trim()
setQuery(nextQuery)
- if (engine !== 'llama-cpp' || normalized.length === 0) return
+ if (
+ !engine ||
+ !EngineCapabilities[engine].engineHub?.searchable ||
+ normalized.length === 0
+ )
+ return
const generation = ++searchGenRef.current
submittedQueryRef.current = normalized
diff --git a/desktop/src/ui/components/ServiceSettings/WipeAppDataCard.tsx b/desktop/src/ui/components/ServiceSettings/WipeAppDataCard.tsx
index b29ea06bb..45e004eae 100644
--- a/desktop/src/ui/components/ServiceSettings/WipeAppDataCard.tsx
+++ b/desktop/src/ui/components/ServiceSettings/WipeAppDataCard.tsx
@@ -17,8 +17,8 @@ function wipeMessage() {
Your downloaded models (for example ~/.ollama, ~/.llamacpp
- , and ~/.lmstudio/models) are not deleted, and neither
- are engines you installed yourself.
+ , ~/colibri-models, and ~/.lmstudio/models) are{' '}
+ not deleted, and neither are engines you installed yourself.
If an engine {APP_DISPLAY_NAME} installed cannot be removed, the reset stops before
diff --git a/desktop/src/ui/constants/engine-capabilities.ts b/desktop/src/ui/constants/engine-capabilities.ts
index 2626a2e42..20ce8cf99 100644
--- a/desktop/src/ui/constants/engine-capabilities.ts
+++ b/desktop/src/ui/constants/engine-capabilities.ts
@@ -59,7 +59,34 @@ export const EngineCapabilities: Record = {
hasDeleteModel: true,
engineHub: {
label: 'llama.cpp',
- url: 'https://huggingface.co/models?library=gguf'
+ url: 'https://huggingface.co/models?library=gguf',
+ searchable: true
+ }
+ },
+ colibri: {
+ hasExpiry: false,
+ // Colibri serves one model per process: there is no model to eject
+ // short of stopping the engine. "Load" switches the served model by
+ // restarting the server onto it (nvpair-engine-manager's launch_model).
+ hasEject: false,
+ // Releases exist for Windows x64, Linux x64 and Apple silicon; other
+ // architectures report the engine as unavailable.
+ hasInstall: ['win32', 'darwin', 'linux'],
+ hasEnginePort: true,
+ hasInstallPath: false,
+ hasProxyWebUI: false,
+ hasPreferredNode: false,
+ hasCrashAlert: false,
+ // Downloads write the model store directly, so they work while the
+ // engine is stopped — the engine cannot start before a model exists.
+ hasModelSearchOnlyWhenRunning: false,
+ modelOpsWhenStopped: false,
+ // delete_model removes the model directory from ~/colibri-models.
+ hasDeleteModel: true,
+ engineHub: {
+ label: 'Colibri',
+ url: 'https://huggingface.co/models?other=colibri',
+ searchable: true
}
}
}
diff --git a/desktop/src/ui/constants/welcome.ts b/desktop/src/ui/constants/welcome.ts
index 4db425f71..855076c1a 100644
--- a/desktop/src/ui/constants/welcome.ts
+++ b/desktop/src/ui/constants/welcome.ts
@@ -14,7 +14,10 @@ export const WELCOME_STEP_SUB_HEADINGS = ['', 'You can update later by clicking
export const WELCOME_ENGINE_DEFAULT_SELECTED: Record = {
ollama: true,
'lm-studio': true,
- 'llama-cpp': true
+ 'llama-cpp': true,
+ // Colibri models are tens to hundreds of gigabytes and its launcher needs
+ // Python 3.10+, so it is offered during onboarding but not preselected.
+ colibri: false
}
export function getWelcomeEngineCandidates(os: PlatformDisplayName): EngineType[] {
diff --git a/desktop/src/ui/types/engine-manifest.ts b/desktop/src/ui/types/engine-manifest.ts
index 1cd998bfe..b4ab97523 100644
--- a/desktop/src/ui/types/engine-manifest.ts
+++ b/desktop/src/ui/types/engine-manifest.ts
@@ -13,6 +13,11 @@ export interface EngineHubConfig {
label: string
/** Public catalog URL that returns a JSON model list. */
url: string
+ /**
+ * When true, the engine's catalog answers a search of its upstream, so the
+ * hub sends Enter/Search as a query instead of only filtering the list.
+ */
+ searchable?: boolean
}
export interface EngineCaps {
diff --git a/desktop/src/ui/utils/match-downloaded-model.ts b/desktop/src/ui/utils/match-downloaded-model.ts
index cdce8576a..b01aaf103 100644
--- a/desktop/src/ui/utils/match-downloaded-model.ts
+++ b/desktop/src/ui/utils/match-downloaded-model.ts
@@ -43,7 +43,9 @@ const matchExactName: DownloadedMatcher = (hubEntry, downloaded) => downloaded.n
const MATCHERS: Partial> = {
ollama: matchOllama,
'lm-studio': matchHfPullKeyOrName,
- 'llama-cpp': matchExactName
+ 'llama-cpp': matchExactName,
+ // Both the catalog id and the downloaded model are the owner/repository id.
+ colibri: matchExactName
}
export function isHubEntryDownloaded(
diff --git a/desktop/tests/modular/delete-model-restart.test.ts b/desktop/tests/modular/delete-model-restart.test.ts
index fbdcef4fe..b02ef12fa 100644
--- a/desktop/tests/modular/delete-model-restart.test.ts
+++ b/desktop/tests/modular/delete-model-restart.test.ts
@@ -28,7 +28,8 @@ const MANIFEST_DIR = path.resolve(process.cwd(), '../services/nvpair-engine-mana
const ENGINE_TYPE_BY_MANIFEST_ID: Record = {
ollama: 'ollama',
lmstudio: 'lm-studio',
- llamacpp: 'llama-cpp'
+ llamacpp: 'llama-cpp',
+ colibri: 'colibri'
}
interface ManifestAction {
@@ -48,13 +49,25 @@ function readManifests(): Manifest[] {
}
describe('delete-model restart is scoped to LM Studio', () => {
- it('only LM Studio declares restart_after, and only on delete_model', () => {
- const declaring = readManifests().flatMap(m =>
- Object.entries(m.actions ?? {})
- .filter(([, action]) => action.restart_after === true)
- .map(([name]) => `${m.engine}.${name}`)
- )
- expect(declaring).toEqual(['lmstudio.delete_model'])
+ // Colibri's load_model also restarts — it serves one model per process, so
+ // switching models is a restart — but no delete does besides LM Studio's.
+ it('only LM Studio restarts on delete_model, and only Colibri on load_model', () => {
+ const declaring = readManifests()
+ .flatMap(m =>
+ Object.entries(m.actions ?? {})
+ .filter(([, action]) => action.restart_after === true)
+ .map(([name]) => `${m.engine}.${name}`)
+ )
+ .sort()
+ expect(declaring).toEqual(['colibri.load_model', 'lmstudio.delete_model'])
+ })
+
+ it('Colibri deletes a model directory without restarting', () => {
+ const colibri = readManifests().find(m => m.engine === 'colibri')
+ expect(colibri?.actions?.delete_model).toBeDefined()
+ expect(colibri?.actions?.delete_model?.restart_after).toBeUndefined()
+ expect(EngineCapabilities.colibri.restartsOnModelDelete).toBeFalsy()
+ expect(EngineCapabilities.colibri.hasDeleteModel).toBe(true)
})
it('Ollama deletes without a restart and therefore without a confirmation', () => {
diff --git a/desktop/tests/modular/engine-identity.test.ts b/desktop/tests/modular/engine-identity.test.ts
index aaefae02e..090e76aa8 100644
--- a/desktop/tests/modular/engine-identity.test.ts
+++ b/desktop/tests/modular/engine-identity.test.ts
@@ -16,19 +16,38 @@ import { gatewayEndpointDisplayUrl } from '@/ui/utils/gateway-inference-paths'
describe('engine identity', () => {
it('enables llama.cpp workflows on every platform', () => {
- expect(EngineTypes).toEqual(['ollama', 'lm-studio', 'llama-cpp'])
- expect(EnabledEngineTypes).toEqual(['ollama', 'lm-studio', 'llama-cpp'])
+ expect(EngineTypes).toEqual(['ollama', 'lm-studio', 'llama-cpp', 'colibri'])
+ expect(EnabledEngineTypes).toEqual(['ollama', 'lm-studio', 'llama-cpp', 'colibri'])
expect(getWelcomeEngineCandidates('Windows')).toContain('llama-cpp')
expect(getWelcomeEngineCandidates('MacOS')).toContain('llama-cpp')
expect(getWelcomeEngineCandidates('Linux')).toContain('llama-cpp')
})
- it('preselects every engine during onboarding', () => {
+ // Colibri is offered but not preselected: its models are tens to hundreds of
+ // gigabytes and its launcher needs Python.
+ it('preselects every engine but Colibri during onboarding', () => {
expect(WELCOME_ENGINE_DEFAULT_SELECTED).toEqual({
ollama: true,
'lm-studio': true,
- 'llama-cpp': true
+ 'llama-cpp': true,
+ colibri: false
})
+ expect(getWelcomeEngineCandidates('Windows')).toContain('colibri')
+ })
+
+ it('integrates Colibri as an engine with a searchable Hugging Face hub', () => {
+ expect(EngineManagerNames.colibri).toBe('colibri')
+ expect(engineTypeFromManagerName('colibri')).toBe('colibri')
+ expect(EngineDisplayNames.colibri).toBe('Colibri')
+ expect(EngineCapabilities.colibri).toMatchObject({
+ hasEject: false,
+ hasDeleteModel: true,
+ hasModelSearchOnlyWhenRunning: false,
+ engineHub: { label: 'Colibri', searchable: true }
+ })
+ expect(EngineCapabilities['llama-cpp'].engineHub?.searchable).toBe(true)
+ expect(EngineCapabilities.ollama.engineHub?.searchable).toBeFalsy()
+ expect(EngineCapabilities['lm-studio'].engineHub?.searchable).toBeFalsy()
})
it('maps the desktop id to the sole engine-manager wire id', () => {
diff --git a/desktop/tests/modular/inference-demo-lifecycle.test.ts b/desktop/tests/modular/inference-demo-lifecycle.test.ts
index 54cb20fb0..d41a4c64d 100644
--- a/desktop/tests/modular/inference-demo-lifecycle.test.ts
+++ b/desktop/tests/modular/inference-demo-lifecycle.test.ts
@@ -50,7 +50,8 @@ const POISONED_ENV: Record = {
const proxyPorts: Record = {
ollama: 11434,
'lm-studio': 1234,
- 'llama-cpp': 8080
+ 'llama-cpp': 8080,
+ colibri: 8000
}
/** Model inventory each probe returns. Mutable so a test can return none. */
diff --git a/desktop/tests/modular/match-downloaded-model.test.ts b/desktop/tests/modular/match-downloaded-model.test.ts
index 507ed1429..b26233f8b 100644
--- a/desktop/tests/modular/match-downloaded-model.test.ts
+++ b/desktop/tests/modular/match-downloaded-model.test.ts
@@ -44,4 +44,13 @@ describe('downloaded model matching', () => {
).toBe(false)
expect(isHubEntryDownloaded('llama-cpp', entry, [modelItem(id, false)])).toBe(false)
})
+
+ it('matches a Colibri repository only by its exact id', () => {
+ const id = 'Justvugg/Qwen3-Coder-30B-A3B-colibri-int4'
+ const entry = modelEntry(id)
+
+ expect(isHubEntryDownloaded('colibri', entry, [modelItem(id)])).toBe(true)
+ expect(isHubEntryDownloaded('colibri', entry, [modelItem(`${id}-other`)])).toBe(false)
+ expect(isHubEntryDownloaded('colibri', entry, [modelItem(id, false)])).toBe(false)
+ })
})
diff --git a/desktop/tests/modular/model-catalog.test.ts b/desktop/tests/modular/model-catalog.test.ts
index e65d0953f..d8900bd7b 100644
--- a/desktop/tests/modular/model-catalog.test.ts
+++ b/desktop/tests/modular/model-catalog.test.ts
@@ -127,8 +127,8 @@ describe('model catalog relay', () => {
// cost a multi-megabyte reply for nothing.
it('warms only the live-fetched catalogs', () => {
warmEngineHubs()
- expect(mocks.supervisor.callProcess).toHaveBeenCalledTimes(2)
- for (const engine of ['lmstudio', 'llamacpp']) {
+ expect(mocks.supervisor.callProcess).toHaveBeenCalledTimes(3)
+ for (const engine of ['lmstudio', 'llamacpp', 'colibri']) {
expect(mocks.supervisor.callProcess).toHaveBeenCalledWith(
'broker',
'engine:catalog',
diff --git a/desktop/tests/modular/proxy-engine-identity.test.ts b/desktop/tests/modular/proxy-engine-identity.test.ts
index 69031984f..e9a9e3fbc 100644
--- a/desktop/tests/modular/proxy-engine-identity.test.ts
+++ b/desktop/tests/modular/proxy-engine-identity.test.ts
@@ -19,11 +19,17 @@ import { engineManagerName } from '@/shared/utils/engines'
describe('proxy engine identity', () => {
it('declares the complete proxy set in stable order', () => {
- expect(PROXY_ENGINES).toEqual(['ollama', 'lm-studio', 'llama-cpp'])
- expect(PROXY_NODE_SOURCES).toEqual(['ollama-proxy', 'lmstudio-proxy', 'llamacpp-proxy'])
+ expect(PROXY_ENGINES).toEqual(['ollama', 'lm-studio', 'llama-cpp', 'colibri'])
+ expect(PROXY_NODE_SOURCES).toEqual([
+ 'ollama-proxy',
+ 'lmstudio-proxy',
+ 'llamacpp-proxy',
+ 'colibri-proxy'
+ ])
expect(isProxyEngine('ollama')).toBe(true)
expect(isProxyEngine('lm-studio')).toBe(true)
expect(isProxyEngine('llama-cpp')).toBe(true)
+ expect(isProxyEngine('colibri')).toBe(true)
})
it('round-trips engine, manager, and relay source identities', () => {
@@ -35,6 +41,7 @@ describe('proxy engine identity', () => {
expect(proxyEngineFromSource('other-proxy')).toBeNull()
expect(proxyEngineFromManagerId('other')).toBeNull()
expect(proxyEngineFromManagerId('llamacpp')).toBe('llama-cpp')
+ expect(proxyEngineFromManagerId('colibri')).toBe('colibri')
})
it('routes each proxy notification to the mapped engine', () => {
diff --git a/docs/architecture.mdx b/docs/architecture.mdx
index e91716ef9..971647186 100644
--- a/docs/architecture.mdx
+++ b/docs/architecture.mdx
@@ -449,7 +449,7 @@ sits one place lower in the order.
to an engine's own port is absent from workload events. GPU-heavy external work
can still raise pressure, but CPU-only work and queued demand remain invisible.
-**All engines are counted as one pool.** Ollama, LM Studio, and llama.cpp load is
+**All engines are counted as one pool.** Ollama, LM Studio, llama.cpp, and Colibri load is
summed, and maximum GPU pressure applies node-wide. This is conservative on a
typical single-GPU machine and can underuse a multi-GPU node where the engines
occupy different devices.
diff --git a/docs/building.mdx b/docs/building.mdx
index ee93969dc..3b816fbfa 100644
--- a/docs/building.mdx
+++ b/docs/building.mdx
@@ -255,7 +255,7 @@ application running:
On Windows use `scripts\wipe-app-data.cmd` with the same flags. Neither script
needs Node, and both exclude engine model libraries such as `~/.ollama`,
-`~/.lmstudio/models`, and `~/.llamacpp` by design, so a reset does not cost you
+`~/.lmstudio/models`, `~/.llamacpp`, and `~/colibri-models` by design, so a reset does not cost you
re-downloading models.
Start with `--dry-run`. It prints the exact paths and deletes nothing.
diff --git a/docs/engine-lifecycle.mdx b/docs/engine-lifecycle.mdx
index 63eb159e5..5ae99eb7e 100644
--- a/docs/engine-lifecycle.mdx
+++ b/docs/engine-lifecycle.mdx
@@ -1,6 +1,6 @@
---
title: Managing Engines in NVIDIA PAIR
-description: Install, start, stop, update, and remove the Ollama, LM Studio, and llama.cpp engines that NVIDIA PAIR uses to run models.
+description: Install, start, stop, update, and remove the Ollama, LM Studio, llama.cpp, and Colibri engines that NVIDIA PAIR uses to run models.
---
{/*
@@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0
*/}
An **engine** is the local inference runtime NVIDIA PAIR uses to
-run models. Today that means Ollama, LM Studio, or llama.cpp on a given machine.
+run models. Today that means Ollama, LM Studio, llama.cpp, or Colibri on a given machine.
PAIR can install and run those engines for you, or work with a supported
existing Ollama or LM Studio installation. This page explains what you can
expect when you install, start, stop, or remove an engine, and how to move one
diff --git a/docs/engine-settings.mdx b/docs/engine-settings.mdx
index 2e93c6f8b..b20b474b0 100644
--- a/docs/engine-settings.mdx
+++ b/docs/engine-settings.mdx
@@ -1,6 +1,6 @@
---
title: Engine Settings in NVIDIA PAIR
-description: Change the server port, proxy port, and launch arguments of an Ollama, LM Studio, or llama.cpp engine in NVIDIA PAIR.
+description: Change the server port, proxy port, and launch arguments of an Ollama, LM Studio, llama.cpp, or Colibri engine in NVIDIA PAIR.
---
{/*
@@ -8,7 +8,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All
SPDX-License-Identifier: Apache-2.0
*/}
-Open a device's Ollama, LM Studio, or llama.cpp row and expand **Settings**.
+Open a device's Ollama, LM Studio, llama.cpp, or Colibri row and expand **Settings**.
Edit the server port, proxy port, and engine arguments, then select the green **Apply** button.
A paired device that supports settings can be edited from another device in the
cluster.
diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx
index 721638d6f..bf40907a3 100644
--- a/docs/getting-started.mdx
+++ b/docs/getting-started.mdx
@@ -35,8 +35,9 @@ inference. Two or more on the same local network let you try pairing and routing
Nothing else has to be in place first:
-- **Engines.** PAIR installs and starts Ollama, LM Studio, or llama.cpp for you
- in step 4. Existing Ollama and LM Studio installations can also be detected
+- **Engines.** PAIR installs and starts Ollama, LM Studio, llama.cpp, or
+ Colibri for you in step 4. Colibri also needs Python 3.10 or newer on the
+ machine. Existing Ollama and LM Studio installations can also be detected
and used.
- **Models.** PAIR downloads models for you in step 4. A request needs only one
eligible node, so a single node holding the model is enough. Prepare the same
@@ -103,7 +104,8 @@ builds and for running the services directly.
On first launch, PAIR opens a setup window that can install available inference
engines on the local system. PAIR preselects Ollama, LM Studio, and llama.cpp when they are
-available for your platform.
+available for your platform. Colibri is listed but not preselected, because its
+models are much larger.
1. Review the available engines.
2. Select the engines to install, or skip installation if they are already
@@ -178,6 +180,17 @@ repositories. PAIR offers one verified `Q4_K_M` option for each result.
**Load**, **Eject**, and **Delete** use that same exact model ID. **Delete**
removes the selected entry from llama.cpp's persistent cache.
+Colibri's Add model list is every public Hugging Face repository tagged
+`colibri`, most downloaded first; pressing Enter searches within that tag. Each
+entry is a whole repository, such as
+`Justvugg/Qwen3-Coder-30B-A3B-colibri-int4`, downloaded into
+`~/colibri-models//`. These models are large — from about
+20 GB to more than 400 GB — so PAIR checks free disk space first, and an
+interrupted download resumes where it stopped. Colibri serves one model at a
+time: the first model you download is the one it starts with, and **Load**
+restarts it onto another model. There is no **Eject**. Delete removes the
+model's directory.
+
Refer to [Managing engines](engine-lifecycle.mdx) for details about:
- Installing, starting, stopping, and uninstalling engines.
@@ -265,7 +278,7 @@ curl /chat/completions \
### Ollama-Style Request (Ollama Endpoint Only)
-Sending this to the LM Studio or llama.cpp endpoint fails, because those engines
+Sending this to the LM Studio, llama.cpp, or Colibri endpoint fails, because those engines
do not implement Ollama's API. The `-N` flag tells `curl` not to buffer, so it
displays the response stream as it is generated.
@@ -416,8 +429,8 @@ official installer: `~/.lmstudio/bin/lms`, or
port. Otherwise, the CLI connects to `11434`, which is PAIR's proxy, and
`ollama list` returns the cluster's view instead of the local machine's. Use the
**Server port** value in the engine's **Settings** section to address the local
-engine directly. It defaults to `11435` for Ollama, `1235` for LM Studio, and
-`8081` for llama.cpp.
+engine directly. It defaults to `11435` for Ollama, `1235` for LM Studio,
+`8081` for llama.cpp, and `8001` for Colibri.
Use the proxy port to check what your cluster can serve. Use the engine port to
check what is installed on the local machine.
@@ -438,6 +451,14 @@ CLI, so query the router over HTTP on its engine port instead:
curl http://127.0.0.1:8081/models
```
+**Colibri** is installed the same way, under `engine-bin/colibri` (its
+launcher is `coli.cmd` on Windows and `coli` elsewhere), and keeps its models in
+`~/colibri-models`. Query the running server on its engine port:
+
+```bash
+curl http://127.0.0.1:8001/v1/models
+```
+
PAIR does not put these binaries on `PATH` yet. Until then, use the full path or
add an alias yourself.
@@ -455,8 +476,9 @@ relative to the proxy URL, before the `/v1` suffix:
| Ollama | `http://127.0.0.1:11434` | Ollama's own API (`/api/chat`, `/api/generate`, `/api/embed`, and `/api/tags`), plus the OpenAI-compatible and Anthropic APIs listed below |
| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible and Anthropic APIs listed below |
| llama.cpp | `http://127.0.0.1:8080` | The OpenAI-compatible API listed below, plus `/models` for model listings |
+| Colibri | `http://127.0.0.1:8000` | The OpenAI-compatible and Anthropic APIs listed below |
-All three endpoints expose these OpenAI-compatible paths:
+All four endpoints expose these OpenAI-compatible paths:
- `POST /v1/chat/completions` — [Conversations](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create)
- `POST /v1/completions` — [Completions](https://developers.openai.com/api/reference/resources/completions/methods/create)
@@ -464,7 +486,7 @@ All three endpoints expose these OpenAI-compatible paths:
- `POST /v1/responses` — [Responses](https://developers.openai.com/api/reference/resources/responses/methods/create)
- `GET /v1/models` — [available models](https://developers.openai.com/api/reference/resources/models/methods/list)
-Ollama and LM Studio support the following Anthropic APIs:
+Ollama, LM Studio, and Colibri support the following Anthropic APIs:
- `POST /v1/messages` — [Messages](https://platform.claude.com/docs/en/api/messages/create)
**Endpoints** includes `/v1` in the displayed and copied URL, so you can paste it
diff --git a/docs/inference-dispatcher.mdx b/docs/inference-dispatcher.mdx
index ca6634639..e89dc6259 100644
--- a/docs/inference-dispatcher.mdx
+++ b/docs/inference-dispatcher.mdx
@@ -11,7 +11,7 @@ SPDX-License-Identifier: Apache-2.0
`scripts/inference-dispatcher` is a standalone HTTP client for exercising
-Ollama, LM Studio, and llama.cpp-compatible servers. It deliberately has no
+Ollama, LM Studio, llama.cpp-compatible, and Colibri servers. It deliberately has no
knowledge of NVIDIA PAIR, its broker, Electron, JSON-RPC, discovery, or
proxy implementation.
Pointing it at a compatible proxy is indistinguishable from pointing it at a
@@ -25,6 +25,7 @@ The executable uses only the Go standard library.
./scripts/inference-dispatcher.sh --count 5 --mode parallel # five concurrent prompts
./scripts/inference-dispatcher.sh --backend lmstudio # LM Studio's OpenAI-compatible API
./scripts/inference-dispatcher.sh --backend llamacpp # llama.cpp's OpenAI-compatible API
+./scripts/inference-dispatcher.sh --backend colibri # Colibri's OpenAI-compatible API
./scripts/inference-dispatcher.sh --list-models # live inventory as JSON
./scripts/inference-dispatcher.sh --help
```
@@ -83,7 +84,7 @@ Pass the file with `--config path/to/config.json` or
`INFERENCE_DISPATCHER_*` environment equivalent.
The client always connects to `127.0.0.1`. The default ports are `11434` for
-Ollama, `1234` for LM Studio, and `8080` for llama.cpp. These are the ports the
+Ollama, `1234` for LM Studio, `8080` for llama.cpp, and `8000` for Colibri. These are the ports the
router's proxies claim, so requests enter the cluster router rather than a
single engine. Use `--port` to select another local port.
@@ -96,6 +97,8 @@ single engine. Use `--port` to select another local port.
- LM Studio inference: `POST /v1/chat/completions`
- llama.cpp inventory: `GET /v1/models`
- llama.cpp inference: `POST /v1/chat/completions`
+- Colibri inventory: `GET /v1/models`
+- Colibri inference: `POST /v1/chat/completions`
`/v1/models` is queried first because the LM Studio proxy answers it by fanning
out across the cluster, while `/api/v1/models` is forwarded to a single node.
diff --git a/docs/overview.mdx b/docs/overview.mdx
index 8d61d9928..3f8246870 100644
--- a/docs/overview.mdx
+++ b/docs/overview.mdx
@@ -42,7 +42,7 @@ These terms have specific meanings in PAIR:
- **Cluster** — the set of nodes you have paired together. A node belongs to at
most one cluster, and it must leave before it can join another.
- **Engine** — the local inference server that runs models: Ollama, LM Studio,
- or llama.cpp. PAIR can install, start, stop, and uninstall these engines. It can
+ llama.cpp, or Colibri. PAIR can install, start, stop, and uninstall these engines. It can
also adopt supported existing Ollama and LM Studio installations.
- **Model** — what you prepare on each node. Nodes do not share models, so a
node can serve a request only for a model it already holds. Preparing the same
diff --git a/docs/terminal-interface.mdx b/docs/terminal-interface.mdx
index d7ec2868c..39ecec729 100644
--- a/docs/terminal-interface.mdx
+++ b/docs/terminal-interface.mdx
@@ -348,14 +348,20 @@ press `r` to try again.
llama.cpp's catalog opens on popular GGUF models from a few well-known
publishers, but it can download any GGUF model on Hugging Face, so there `/`
-searches all of Hugging Face instead of filtering the list. Press `c` to go back
-to the starting list.
+searches all of Hugging Face instead of filtering the list. Colibri's catalog
+starts with the models of Colibri's own setup list, such as DeepSeek V4 Flash
+and Kimi K3, followed by every public Hugging Face repository tagged
+`colibri`. `/` searches within that tag and that list. Press `c` to go back to
+the starting list.
Press `n` instead if you already know exactly what you want and would rather type
it. Each engine names models its own way: an Ollama tag such as `llama3.2`, a
Hugging Face repository for LM Studio, and a repository with its quantization
-for llama.cpp, such as `ggml-org/gemma-3-1b-it-GGUF:Q4_K_M`. Either way,
-downloading needs a running engine, and progress appears on the status line.
+for llama.cpp, such as `ggml-org/gemma-3-1b-it-GGUF:Q4_K_M`, and a whole
+repository for Colibri, such as `Justvugg/Qwen3-Coder-30B-A3B-colibri-int4`.
+Downloading needs a running engine, except for Colibri, which writes its model
+store directly; progress appears on the status line. Colibri has no eject: it
+serves one model at a time, and loading another restarts it.
Restart and uninstall need PAIR to own the engine process, so they only apply
to the machine you are sitting on. They are hidden on a peer rather than
diff --git a/scripts/inference-dispatcher/client.go b/scripts/inference-dispatcher/client.go
index 545d79d5b..7042a68b8 100644
--- a/scripts/inference-dispatcher/client.go
+++ b/scripts/inference-dispatcher/client.go
@@ -117,6 +117,8 @@ func (c *backendClient) listModels(ctx context.Context) ([]RegisteredModel, erro
models, err = c.listLMStudioModels(ctx)
case "llamacpp":
models, err = c.listLlamaCPPModels(ctx)
+ case "colibri":
+ models, err = c.listColibriModels(ctx)
default:
models, err = c.listOllamaModels(ctx)
}
@@ -140,6 +142,17 @@ func (c *backendClient) listLlamaCPPModels(ctx context.Context) ([]RegisteredMod
return parseOpenAIModels(data)
}
+// listColibriModels reads Colibri's OpenAI model list. Through the PAIR proxy
+// it is the cluster's Colibri inventory; each server reports the one model it
+// was launched with.
+func (c *backendClient) listColibriModels(ctx context.Context) ([]RegisteredModel, error) {
+ data, err := c.request(ctx, http.MethodGet, "/v1/models", nil)
+ if err != nil {
+ return nil, fmt.Errorf("query Colibri models: %w", err)
+ }
+ return parseOpenAIModels(data)
+}
+
func (c *backendClient) listOllamaModels(ctx context.Context) ([]RegisteredModel, error) {
data, err := c.request(ctx, http.MethodGet, "/api/tags", nil)
if err != nil {
@@ -360,7 +373,7 @@ func (c *backendClient) inferencePath() string {
}
func (c *backendClient) usesOpenAIProtocol() bool {
- return c.cfg.Backend == "lmstudio" || c.cfg.Backend == "llamacpp"
+ return c.cfg.Backend == "lmstudio" || c.cfg.Backend == "llamacpp" || c.cfg.Backend == "colibri"
}
func (c *backendClient) infer(ctx context.Context, model, prompt string) (string, error) {
diff --git a/scripts/inference-dispatcher/config.go b/scripts/inference-dispatcher/config.go
index 892f313c7..2cd28cdc1 100644
--- a/scripts/inference-dispatcher/config.go
+++ b/scripts/inference-dispatcher/config.go
@@ -25,8 +25,11 @@ const (
defaultLMStudioPort = 1234
// llama.cpp's OpenAI-compatible proxy facade uses its stock router port.
defaultLlamaCPPPort = 8080
- defaultErrorLog = "inference_errors.txt"
- maxPromptsPerBatch = 100
+ // Colibri's proxy facade claims its stock server port; PAIR's managed
+ // Colibri runs behind it on 8001.
+ defaultColibriPort = 8000
+ defaultErrorLog = "inference_errors.txt"
+ maxPromptsPerBatch = 100
)
var defaultTopics = []string{
@@ -291,7 +294,7 @@ func parseConfig(args []string, stderr io.Writer) (Config, error) {
fs.SetOutput(stderr)
var parsedConfigPath string
fs.StringVar(&parsedConfigPath, "config", configPath, "JSON configuration file")
- fs.StringVar(&cfg.Backend, "backend", cfg.Backend, "backend: ollama, lmstudio, or llamacpp")
+ fs.StringVar(&cfg.Backend, "backend", cfg.Backend, "backend: ollama, lmstudio, llamacpp, or colibri")
fs.StringVar(&cfg.Backend, "provider", cfg.Backend, "alias for --backend")
fs.IntVar(&cfg.Port, "port", cfg.Port, "server port (backend default when omitted)")
fs.StringVar(&cfg.Model, "model", cfg.Model, "model name; omitted or auto selects an available model")
@@ -358,8 +361,8 @@ func parseConfig(args []string, stderr io.Writer) (Config, error) {
}
func validateConfig(cfg Config) error {
- if cfg.Backend != "ollama" && cfg.Backend != "lmstudio" && cfg.Backend != "llamacpp" {
- return errors.New("--backend must be ollama, lmstudio, or llamacpp")
+ if cfg.Backend != "ollama" && cfg.Backend != "lmstudio" && cfg.Backend != "llamacpp" && cfg.Backend != "colibri" {
+ return errors.New("--backend must be ollama, lmstudio, llamacpp, or colibri")
}
if cfg.Port < 0 || cfg.Port > 65535 {
return errors.New("--port must be between 1 and 65535")
@@ -421,6 +424,8 @@ func effectivePort(cfg Config) int {
return defaultLMStudioPort
case "llamacpp":
return defaultLlamaCPPPort
+ case "colibri":
+ return defaultColibriPort
default:
return defaultOllamaPort
}
diff --git a/scripts/inference-dispatcher/dispatcher_test.go b/scripts/inference-dispatcher/dispatcher_test.go
index 319b1cd88..119a4d019 100644
--- a/scripts/inference-dispatcher/dispatcher_test.go
+++ b/scripts/inference-dispatcher/dispatcher_test.go
@@ -202,6 +202,64 @@ func TestLlamaCPP_UsesOpenAIInventoryAndChat(t *testing.T) {
require.Equal(t, observedRequest{http.MethodPost, "/v1/chat/completions", "llama-demo", 1}, <-observed)
}
+func TestColibriUsesOpenAIInventoryAndChat(t *testing.T) {
+ type observedRequest struct {
+ method string
+ path string
+ model string
+ messageCount int
+ }
+ observed := make(chan observedRequest, 2)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/models":
+ observed <- observedRequest{method: r.Method, path: r.URL.Path}
+ _, _ = w.Write([]byte(`{"data":[{"id":"Justvugg/Qwen3-Coder-30B-A3B-colibri-int4"}]}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/chat/completions":
+ var request struct {
+ Model string `json:"model"`
+ Messages []struct {
+ Role string `json:"role"`
+ Content string `json:"content"`
+ } `json:"messages"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Errorf("decode chat request: %v", err)
+ }
+ observed <- observedRequest{
+ method: r.Method,
+ path: r.URL.Path,
+ model: request.Model,
+ messageCount: len(request.Messages),
+ }
+ _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"done"}}]}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ var stdout, stderr bytes.Buffer
+ exit := runAgainstServer(
+ t,
+ context.Background(),
+ []string{"--backend", "colibri", "--prompt", "test"},
+ server.URL,
+ &stdout,
+ &stderr,
+ )
+ if exit != 0 {
+ t.Fatalf("exit=%d stderr=%s", exit, stderr.String())
+ }
+ if got := <-observed; got.method != http.MethodGet || got.path != "/v1/models" {
+ t.Fatalf("inventory request = %+v, want GET /v1/models", got)
+ }
+ if got := <-observed; got.method != http.MethodPost || got.path != "/v1/chat/completions" ||
+ got.model != "Justvugg/Qwen3-Coder-30B-A3B-colibri-int4" || got.messageCount != 1 {
+ t.Fatalf("inference request = %+v, want OpenAI chat for Justvugg/Qwen3-Coder-30B-A3B-colibri-int4", got)
+ }
+}
+
// A Personal AI Router proxy answers /v1/models with the whole cluster's
// inventory and forwards /api/v1/models to one node, so the aggregated list must
// win. The native list still supplies the type and capability fields the
@@ -360,6 +418,13 @@ func TestLlamaCPP_UsesPort8080ByDefault(t *testing.T) {
require.Equal(t, 8080, effectivePort(cfg))
}
+func TestColibri_DefaultsToItsNativePort(t *testing.T) {
+ var stderr bytes.Buffer
+ cfg, err := parseConfig([]string{"--backend", "colibri"}, &stderr)
+ require.NoError(t, err, "parse config")
+ assert.Equal(t, 8000, effectivePort(cfg), "Colibri default port")
+}
+
func TestResponseText_NeverReachesStdout(t *testing.T) {
const secret = "the capital of France is Paris"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/scripts/wipe-app-data.ps1 b/scripts/wipe-app-data.ps1
index fed5459f8..1a17cd996 100644
--- a/scripts/wipe-app-data.ps1
+++ b/scripts/wipe-app-data.ps1
@@ -53,7 +53,7 @@ Usage: scripts\wipe-app-data.cmd [options]
Delete all NVIDIA PAIR-owned application data (settings, logs, cluster
identity, chat history, PAIR-managed engines under the app data root).
-Does NOT delete model libraries (%USERPROFILE%\.ollama, .llamacpp, .lmstudio\models).
+Does NOT delete model libraries (%USERPROFILE%\.ollama, .llamacpp, .lmstudio\models, colibri-models).
Does NOT uninstall the application binary.
Options:
@@ -227,7 +227,7 @@ if (-not $Confirmed) {
}
Write-Host ''
Write-Host 'WARNING: This permanently deletes all NVIDIA PAIR app data.'
- Write-Host 'Model libraries (%USERPROFILE%\.ollama, .llamacpp, .lmstudio\models) are NOT removed.'
+ Write-Host 'Model libraries (%USERPROFILE%\.ollama, .llamacpp, .lmstudio\models, colibri-models) are NOT removed.'
Write-Host ''
Write-Host 'Paths to remove:'
foreach ($t in $Targets) { Write-Host (" - {0}" -f $t.Path) }
diff --git a/scripts/wipe-app-data.sh b/scripts/wipe-app-data.sh
index 7a8cfbfc8..e8746a662 100755
--- a/scripts/wipe-app-data.sh
+++ b/scripts/wipe-app-data.sh
@@ -46,7 +46,7 @@ Usage: scripts/wipe-app-data.sh [options]
Delete all NVIDIA PAIR-owned application data (settings, logs, cluster
identity, chat history, PAIR-managed engines under the app data root).
-Does NOT delete model libraries (~/.ollama, ~/.llamacpp, ~/.lmstudio/models).
+Does NOT delete model libraries (~/.ollama, ~/.llamacpp, ~/.lmstudio/models, ~/colibri-models).
Does NOT uninstall the application binary.
Options:
@@ -216,7 +216,7 @@ if [[ "$CONFIRM" -eq 0 ]]; then
fi
echo ""
echo "WARNING: This permanently deletes all NVIDIA PAIR app data."
- echo "Model libraries (~/.ollama, ~/.llamacpp, ~/.lmstudio/models) are NOT removed."
+ echo "Model libraries (~/.ollama, ~/.llamacpp, ~/.lmstudio/models, ~/colibri-models) are NOT removed."
echo ""
echo "Paths to remove:"
for entry in "${TARGETS[@]}"; do
diff --git a/services/nvpair-engine-manager/LAUNCH_TEXT.md b/services/nvpair-engine-manager/LAUNCH_TEXT.md
index c1772df73..8e0bd4c3f 100644
--- a/services/nvpair-engine-manager/LAUNCH_TEXT.md
+++ b/services/nvpair-engine-manager/LAUNCH_TEXT.md
@@ -149,7 +149,7 @@ The existing engine-manager suite also exercises the JSON-RPC stdio path.
## Networking adapter coverage
-The bundled engines are Ollama, LM Studio, and llama.cpp. The shared parser
+The bundled engines are Ollama, LM Studio, llama.cpp, and Colibri. The shared parser
contains no engine name checks. Their mandatory networking declarations and all
platform variants are pinned by TestBundledNetworkingControls; adding an engine
requires extending that inventory and reviewing its networking alternatives.
diff --git a/services/nvpair-engine-manager/MANIFEST.md b/services/nvpair-engine-manager/MANIFEST.md
index 5c9b2d3b0..33b1446c3 100644
--- a/services/nvpair-engine-manager/MANIFEST.md
+++ b/services/nvpair-engine-manager/MANIFEST.md
@@ -122,6 +122,8 @@ and recovery. Editing `args`/`start` directly remains trusted manifest authoring
| `manifest_version` | int | yes | Must be `1`. A higher value is rejected (asks for behavior this binary lacks). Unknown optional fields within a supported version are ignored, so the schema can grow additively. |
| `platforms` | object | yes | Map of `"/"` → platform block (e.g. `"windows/amd64"`, `"darwin/arm64"`, `"linux/amd64"`). At least one entry. The runner selects the block matching the host. |
| `actions` | object | no | Map of action name → action (see below). |
+| `catalog` | array | no | Models listed by repository, for an engine with an `hf_snapshot` pull. See Listed models. |
+| `model_types` | array | no | The root `config.json` `model_type` values the engine runs, lowercase. See Listed models. |
| `detect` / `install` / `uninstall` / `models_dir` / `runtime` | — | no | Optional **shared defaults** inherited by every platform (see below). |
**Shared defaults & per-platform overrides.** The platform-level fields `detect`, `install`, `uninstall`, `models_dir`, and `runtime` may also be given once at the top level as shared defaults; each `platforms` entry is then merged onto them. Nested objects (e.g. `runtime`, `runtime.env`) merge key-by-key with the platform value winning, while arrays and scalars are replaced wholesale. So a runtime that's identical across platforms except `cli` is declared once at the top level, and each platform sets only `"runtime": { "cli": "…" }`. Omitting a key inherits the default; setting it (even to a zero value like `"port": 0`) overrides it. A manifest that fully specifies each platform with no top-level defaults behaves exactly as before.
@@ -221,6 +223,7 @@ applies to the host the test runs on.
| `env` | object | no | Extra environment (merged over the inherited env). Use `{host}`/`{port}` for the listen address, e.g. `"OLLAMA_HOST": "{host}:{port}"`. |
| `port` | int | no | Fixed port, or `0` to auto-assign a free loopback port. |
| `bind` | string | no | Listen address, substituted as `{host}`. Empty ⇒ `127.0.0.1` (the safe default); an inference engine that serves the cluster sets `"0.0.0.0"` (Ollama does), overridable per call via `engine:start {bind}`. Probes always target loopback. |
+| `model` | string | no | The persisted launch model of a single-model engine, available to `args` and `env` as `{model}`. Written to the per-user override by a `launch_model` action (and by the first `hf_snapshot` download); a bundled manifest never sets it. See Single-model engines. |
| `start` | string[][] | command mode | Ordered bring-up commands (each an argv) for `command` mode, e.g. `[["{cli}","server","start","--port","{port}"]]`. |
| `cli` | string | no | The engine's control-CLI path for this platform, referenced as `{cli}` by start/stop/actions so the manifest's **global** actions resolve to the correct per-OS binary. |
| `ready` | probe | no | Readiness probe; `start` waits for it before reporting Running. |
@@ -238,7 +241,9 @@ probes.
### Actions
Each action is a config-declared operation exposed over `engine:action`.
-Exactly one of `http`, `cmd`, or `remove_path`:
+Exactly one of `http`, `cmd`, `remove_path`, `scan_dir`, `launch_model`, or
+`hf_snapshot` (the last three serve directory model stores; see Single-model
+engines):
- **`http`** — call the engine's loopback control API. The caller's
`params` are sent as the JSON request body by default. Set
@@ -362,6 +367,97 @@ The rules the runner enforces:
Actions target the engine's control plane — **not** inference endpoints
(those stay with the proxy).
+### Single-model engines and directory stores
+
+Some engines read a model as a directory and serve one model per process
+(Colibri). Three action kinds and one placeholder cover them with no
+engine-specific code:
+
+- **`scan_dir`** — `{ "root": "{models_dir}", "marker": "config.json", "depth": 2 }`
+ lists every directory at most `depth` levels below `root` that contains
+ `marker`, as `{"models":[{"name":"owner/repo"}]}` for a
+ `{"array":"models","field":"name"}` result; a model directory is not searched
+ for further models. Hidden directories, links and downloads in progress
+ (`*.partial`) are skipped. It also defines the store's layout: a model id is
+ one to `depth` plain path segments, and a `remove_path` delete on such a store
+ removes only a directory that holds `marker`, never the store or a folder of
+ models.
+- **`launch_model`** — `{ "path": "{models_dir}/{model}", "marker": "config.json" }`
+ checks the model is downloaded and saves it as `runtime.model`. Declare
+ `restart_after` with it so a running engine restarts onto the model.
+- **`hf_snapshot`** — `{ "dest": "{models_dir}/{model}" }`, valid only as
+ `pull_model` and never with `restart_after`. Downloads every file of an open
+ Hugging Face repository (`{model}` is `owner/repo`) into `dest`, through a
+ `dest.partial` directory renamed into place when complete. It refuses
+ private, gated and marker-less repositories and unsafe file names, checks free
+ disk space, resumes `.part` files with HTTP ranges, and reports progress. It
+ needs no running engine.
+- **`{model}`** — accepted only in `runtime.args`, `runtime.env`,
+ `launch_model.path` and `hf_snapshot.dest`. At launch it is the saved
+ `runtime.model` while that model is on disk, else the first model `scan_dir`
+ finds; with none, a start is refused. It is deliberately not one of the
+ runner-owned placeholders, so `cmd` and `remove_path` actions still receive a
+ caller's `model` param. Put it in `env` rather than `args`: the launch editor
+ turns default arguments into saved literals, which would pin the model.
+
+```json
+"runtime": {
+ "args": ["serve", "--host", "{host}", "--port", "{port}"],
+ "env": { "COLI_MODEL": "{models_dir}/{model}", "COLI_MODEL_ID": "{model}" }
+},
+"actions": {
+ "list_models": { "scan_dir": { "root": "{models_dir}", "marker": "config.json", "depth": 2 }, "result": { "array": "models", "field": "name" } },
+ "pull_model": { "hf_snapshot": { "dest": "{models_dir}/{model}" } },
+ "load_model": { "launch_model": { "path": "{models_dir}/{model}", "marker": "config.json" }, "restart_after": true },
+ "delete_model": { "remove_path": { "root": "{models_dir}", "path": "{models_dir}/{model}" } }
+}
+```
+
+#### Listed models
+
+A tag search only finds what publishers tag. An engine whose maintainers name
+models by repository, such as Colibri's own `coli setup` list, which points at
+vendors' original checkpoints, declares them in a top-level `catalog`.
+`engine:catalog` shows the listed models first, with the repository's live
+downloads, likes and last change, and drops a tagged duplicate. An `hf_snapshot`
+pull of a listed repository follows the entry's rules:
+
+| Field | Required | Meaning |
+|---|---|---|
+| `repo` | yes | The `owner/name` repository, which is also the model id and its store path. |
+| `name` | yes | Display name. |
+| `parameters` | no | Display label such as `"284B"`. |
+| `size_gb` | no | Download size in gigabytes (10^9 bytes), for display. The pull measures the real size. |
+| `revision` | no | A full commit id to download instead of `main`. A branch or tag is refused because it can move. |
+| `exclude` | no | Repository paths not downloaded, as `fnmatch` patterns: `*` also matches `/`, as in the engine's own downloader. |
+| `after_download` | no | `[{ "python": "tools/x.py", "creates": "file" }]`: once the files are in, and before the model is listed, run each script from the install directory with the model directory as its argument, unless its file already exists. |
+
+An `after_download` step needs the engine installed, because the script ships
+with it. It also needs Python 3.10 or newer, found the way Colibri's launcher
+finds it: `py -3`, then `python`, then `python3` on Windows, and `python3`
+then `python` elsewhere. A failed step leaves the download as `dest.partial`,
+so the store does not list a model its engine cannot read, and the next pull
+resumes it. A model already in the store that lacks a step's file gets the step
+on its next pull, with no new download.
+
+An engine that picks its code from the root `config.json` `model_type`, as
+Colibri's family registry does, declares the values it runs in `model_types`.
+`engine:catalog` then offers a tagged repository only when the hub reports one
+of them for it, and an `hf_snapshot` pull of an unlisted repository is refused
+before any file is written when its `model_type` is missing or not listed. A
+listed repository is the engine's own choice and is not checked. Values are
+compared lowercased and trimmed, as Colibri compares them. Keep the list in
+step with the engine release the manifest pins.
+
+```json
+"catalog": [
+ { "repo": "deepseek-ai/DeepSeek-V4-Flash-0731", "name": "DeepSeek V4 Flash", "parameters": "284B", "size_gb": 166.9 },
+ { "repo": "Qwen/Qwen3.8-Flash-Next-FP8", "name": "Qwen3.8-Flash-Next", "revision": "bcd9f01ddc9cff2316eb84281bebcd5b058bddce" },
+ { "repo": "XiaomiMiMo/MiMo-V2.6-Flash-MOPD", "name": "MiMo-V2.6 Flash", "exclude": ["dflash/*", "model_mtp.safetensors"] },
+ { "repo": "moonshotai/Kimi-K3", "name": "Kimi K3", "after_download": [{ "python": "tools/k3_tokenizer.py", "creates": "tokenizer.json" }] }
+]
+```
+
## Placeholders
Resolved by the runner at execution time; any other `{token}` fails
@@ -377,6 +473,7 @@ validation at load:
| `{download_}` | Path of a verified member of `install.artifacts` | `install.run` |
| `{install_dir}` | Per-engine user-scoped install dir | `detect`, `install`, runtime |
| `{models_dir}` | The platform's `models_dir`, expanded | `detect`, runtime args/env, `uninstall.run`, actions. **Not** `install`, which resolves only `{install_dir}` and the download paths — and not `uninstall.remove`, where naming the store is rejected at load. Unset when the platform declares no store, so a template referencing it then fails to resolve rather than silently becoming a path under `/` |
+| `{model}` | The launch model of a single-model engine (`runtime.model`, else the first model in the store) | runtime `args`/`env`, `launch_model.path`, `hf_snapshot.dest`. Rejected anywhere else at load |
A `cmd` action additionally templates the action's own `params` as
placeholders (e.g. `{model}`), resolved at call time. HTTP actions send
@@ -398,7 +495,19 @@ path, an `uninstall` block sets neither `run` nor `remove`, `uninstall.remove`
is present without a `models_dir` to preserve, an `uninstall.remove` target is
the model store or inside it (whether spelled literally or as `{models_dir}`),
an action sets none or more than one of
-`http`/`cmd`/`remove_path`, a `remove_path` action omits `root` or
+`http`/`cmd`/`remove_path`/`scan_dir`/`launch_model`/`hf_snapshot`, a
+`scan_dir` omits `root` or `marker` or sets a `depth` outside 1–4, a
+`launch_model` omits `path` or `marker`, a marker is not a plain file name, an
+`hf_snapshot` is declared on any action but `pull_model` or with
+`restart_after`, a `catalog` is declared without an `hf_snapshot` pull or lists
+a repository twice, a `catalog` entry has an invalid `repo`, no `name`, a
+negative `size_gb`, a `revision` that is not a full commit id, an `exclude`
+pattern that is absolute or climbs out of the repository, or an
+`after_download` step whose `python` is not a `.py` path inside the install
+directory or whose `creates` is not a plain file name, `model_types` is declared
+without an `hf_snapshot` pull or holds a value that is not a lowercase
+`model_type` or appears twice, a `remove_path` action
+omits `root` or
`path`, `http.params_in` is not `body` or `query`, a `result` is set without
both `array` and `field` (or a
`result.match` without both `match.field` and a non-empty `match.in`), or
@@ -518,6 +627,7 @@ GPU selection, and auth.
| **Ollama** | strong (env-first) | `ollama serve` (foreground) | env: `OLLAMA_HOST`, `OLLAMA_MODELS`, `OLLAMA_KEEP_ALIVE`, `OLLAMA_NUM_PARALLEL`, `OLLAMA_MAX_LOADED_MODELS`, `OLLAMA_MAX_QUEUE`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_FLASH_ATTENTION` | HTTP `/api/tags`, `/api/pull`; CLI `ollama pull/ls/ps/stop` |
| **llama.cpp** | strong (env+flags) — **reference design** | `llama-server --host 127.0.0.1 --port {port}` (foreground) | flags + `LLAMA_ARG_*` (host/port, ctx-size, n-parallel, cont-batching, flash-attn, device, n-gpu-layers, tensor-split, main-gpu, api-key, models-dir/max) | HTTP `/v1/models`, `/models` download/delete, `/models/load`, `/models/unload`, `/health`, `/slots` |
| **LM Studio** | command / daemon | `lms daemon up` → `lms server start --port {port}` | small env (`LMS_SERVER_HOST`, `LM_API_TOKEN`); most config is flags/API/settings (`lms load --context-length/--gpu/--ttl`) | HTTP `/api/v1/models[/download\|load\|unload]`; CLI `lms ls/get/load/unload/ps` |
+| **Colibri** | single-model, directory store | `coli serve --host 127.0.0.1 --port {port}` (Python launcher, foreground) | env: `COLI_MODEL`, `COLI_MODEL_ID`, `COLI_API_KEY`, `COLI_KV_SLOTS`, `COLI_GPU(S)`, `COLI_METAL`/`COLI_VULKAN`/`COLI_CUDA`; flags `--kv-slots`, `--gpu`, `--vram`, `--ctx` | HTTP `/health`, `/v1/models`; models are Hugging Face `colibri` repos and the `catalog` of Colibri's own list (`hf_snapshot`); switch model = restart |
| **vLLM** | flags-first — **Linux/WSL only** | `vllm serve --host 127.0.0.1 --port {port}` | flags (host/port/api-key); `HF_HOME` for cache. `VLLM_PORT`/`VLLM_HOST_IP` are **not** the API bind | OpenAI `/v1/models`; one model per process (unload = restart) |
| **Jan** | hybrid (on llama.cpp router) | `jan serve --port {port}` (CLI) | forwards `LLAMA_ARG_*`; perf settings are router-preset-driven | `jan serve` auto-downloads HF repos |
| **GPT4All** | weak (GUI app) | none (Local API Server toggled in the GUI, `:4891`) | desktop settings / Python SDK — not env | OpenAI `/v1` once enabled in-app |
diff --git a/services/nvpair-engine-manager/README.md b/services/nvpair-engine-manager/README.md
index be096ed91..abbafdf8e 100644
--- a/services/nvpair-engine-manager/README.md
+++ b/services/nvpair-engine-manager/README.md
@@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0
# nvpair-engine-manager
A config-driven control plane for local inference engines. The bundled
-manifests support Ollama, LM Studio, and llama.cpp. It manages everything about
+manifests support Ollama, LM Studio, llama.cpp, and Colibri. It manages everything about
an engine **except serving inference**: detect, user-mode install,
start/stop/restart, health, and config-declared actions. Adding an engine
is a JSON manifest, not code.
@@ -39,6 +39,54 @@ supported NVIDIA/Metal devices can accelerate, while the dynamic CPU backend
remains the fallback. Hardware acceptance, not `/health` alone, is required to
claim GPU activation.
+### Colibri backend checkpoint
+
+The `colibri` manifest installs [Colibri](https://github.com/JustVugg/colibri)
+v2.1.0, a mixture-of-experts engine that streams expert weights from disk, from
+its checksum-pinned release archives: Windows x64 (the base archive with the
+CUDA archive unpacked over it), Linux x64 (Vulkan) and Apple silicon (Metal,
+`COLI_METAL=1`). Colibri's launcher is a Python script, so the install first
+checks for Python 3.10 or newer and fails with that message when it is missing.
+Other platforms have no release and report Colibri as unavailable.
+
+Colibri serves **one model per process**. The manifest runs
+`coli serve --host {host} --port {port}` on loopback port `8001` and names the
+model through `COLI_MODEL={models_dir}/{model}` and `COLI_MODEL_ID={model}`.
+`{model}` is the persisted launch model (`runtime.model`), or the first model in
+the store when none is saved or it was deleted; with no model downloaded a start
+is refused with "no model is downloaded". Readiness requires `/health` to
+answer `{"status":"ok"}`, which Colibri sends only once the model has loaded,
+so the ready budget is 600 seconds.
+
+Models are Hugging Face repositories tagged `colibri`, plus the eleven text
+models of Colibri v2.1.0's own `coli setup` list, which the manifest's
+`catalog` names by repository. Several of those are vendors' original
+checkpoints that carry no tag, among them DeepSeek V4 Flash, MiMo-V2.6 and
+Kimi K3. Models are stored as `~/colibri-models//`. Models
+that Colibri's own `coli setup` placed one level down, such as
+`~/colibri-models/qwen36-35b`, are models too:
+
+- `list_models` is a `scan_dir` of that store (directories one or two levels
+ down holding `config.json`), so the inventory is the files on disk.
+- `pull_model` is an `hf_snapshot` download of the whole repository. It refuses
+ private, gated and non-model repositories, checks free disk space, resumes
+ interrupted files with HTTP ranges, and moves the model into place only when
+ every file has arrived. It does not need the engine running. The first model
+ downloaded becomes the launch model. A listed repository follows its entry's
+ rules: Qwen3.8-Flash-Next is pinned to a commit, MiMo skips the files its
+ engine never loads, and Kimi K3 runs Colibri's `tools/k3_tokenizer.py` to
+ write the `tokenizer.json` its engine reads, which needs Colibri installed.
+- Only models Colibri v2.1.0 can run are offered or downloaded: the manifest's
+ `model_types` lists the `config.json` `model_type` values of its chat
+ engines, and a tagged repository naming another (Hy3, Laguna, MiniMax,
+ Nemotron) is neither listed nor pulled. Containers built for ports of
+ Colibri, SpeedyColibri and peng, are left out too.
+- `load_model` is a `launch_model` that saves the model as `runtime.model` and,
+ through `restart_after`, restarts a running server onto it. There is no
+ `unload_model`.
+- `delete_model` removes one model directory (one holding `config.json`),
+ never the store or an owner folder.
+
## Communication
Bidirectional newline-delimited JSON-RPC 2.0 — the same conventions as
@@ -68,7 +116,7 @@ Requests (caller → service):
| `engine:action` | `{ engine, action, params }` | the engine's raw response. `action:"pull_model"` is streamed: it emits live `engine:pull-progress` notifications and returns the pull's terminal result (see below). An action whose manifest declares `restart_after` (LM Studio's `delete_model`) restarts a running engine before replying, so the response also means the engine is back and healthy |
| `engine:logs` | `{ engine }` | `{ lines: [LogLine] }` |
| `engine:errors` | — | `{ errors: [ServiceError] }` |
-| `engine:catalog` | `{ engine, platform?, arch?, query? }` | `{ models: [{ id, name, author, url, size?, downloads, likes, updatedAt, tags, family?, parameterSize?, appleOnly? }], source, platform?, arch?, fetchedAt?, searchable?, query? }` — the models an engine can **download**, as opposed to `engine:models`, which reports what is already installed. Each engine has one curated source. Ollama's has no public API, so it is a locked list compiled into this binary from `catalog/ollama-models.json`, regenerated on demand by a developer (`npm run scrape:ollama-models` in `desktop/`) who reviews the diff; serving it needs no network. LM Studio's catalog *is* the `lmstudio-community` Hugging Face org, whose repo ids are exactly what `lms get` accepts, so it is fetched live and cached for six hours, with concurrent callers coalesced onto one request, a failed attempt — with or without a list to fall back on — followed by a backoff before the next. llama.cpp downloads any GGUF repo as `repo:quantization`; its browse list is the GGUF repos of `ggml-org`, `bartowski`, and `unsloth`, fetched and cached the same way, keeping only open, text-generating repos that ship a `Q4_K_M` file, each offered as `repo:Q4_K_M`. It is the one `searchable` source: a `query` searches all of Hugging Face under the same rules, each query cached on its own (the 20 most recently used are kept), and the reply echoes it. A query longer than 100 characters is refused rather than shortened, since a shortened one searches for something the operator did not type. Other sources ignore a query and return their whole list for the client to filter. `platform` and `arch` are the GOOS and GOARCH the models will be **installed on**, which is not always this host — a client driving a peer should name that peer. Omitting both means this host; naming a platform without an arch leaves the arch unknown rather than borrowing this host's. MLX quantizations only install on Apple Silicon, so they are marked `appleOnly` and kept only for a `darwin`/`arm64` target — an Intel Mac does not get them; the reply echoes the `platform` and `arch` it filtered for. `name` is pull-ready: it can be handed to `pull_model` verbatim. An engine with no curated source is an error, not an empty list. The Ollama reply is a single multi-megabyte frame, so every hop on its path must allow `jsonrpc.WorkerFrameBytes`. |
+| `engine:catalog` | `{ engine, platform?, arch?, query? }` | `{ models: [{ id, name, author, url, size?, downloads, likes, updatedAt, tags, family?, parameterSize?, appleOnly? }], source, platform?, arch?, fetchedAt?, searchable?, query? }` — the models an engine can **download**, as opposed to `engine:models`, which reports what is already installed. Each engine has one curated source. Ollama's has no public API, so it is a locked list compiled into this binary from `catalog/ollama-models.json`, regenerated on demand by a developer (`npm run scrape:ollama-models` in `desktop/`) who reviews the diff; serving it needs no network. LM Studio's catalog *is* the `lmstudio-community` Hugging Face org, whose repo ids are exactly what `lms get` accepts, so it is fetched live and cached for six hours, with concurrent callers coalesced onto one request, a failed attempt — with or without a list to fall back on — followed by a backoff before the next. llama.cpp downloads any GGUF repo as `repo:quantization`; its browse list is the GGUF repos of `ggml-org`, `bartowski`, and `unsloth`, fetched and cached the same way, keeping only open, text-generating repos that ship a `Q4_K_M` file, each offered as `repo:Q4_K_M`. It is the one `searchable` source: a `query` searches all of Hugging Face under the same rules, each query cached on its own (the 20 most recently used are kept), and the reply echoes it. Colibri's catalog is every open Hugging Face repository tagged `colibri` that has a root `config.json` (the speedy-colibri fork's nvfp4/mxfp4 builds are left out, as is any repository whose pipeline is not text generation), most downloaded first, fetched and cached the same way and also `searchable` within the tag; each entry is offered as its `owner/repo` id, which `hf_snapshot` downloads whole. A query longer than 100 characters is refused rather than shortened, since a shortened one searches for something the operator did not type. Other sources ignore a query and return their whole list for the client to filter. `platform` and `arch` are the GOOS and GOARCH the models will be **installed on**, which is not always this host — a client driving a peer should name that peer. Omitting both means this host; naming a platform without an arch leaves the arch unknown rather than borrowing this host's. MLX quantizations only install on Apple Silicon, so they are marked `appleOnly` and kept only for a `darwin`/`arm64` target — an Intel Mac does not get them; the reply echoes the `platform` and `arch` it filtered for. `name` is pull-ready: it can be handed to `pull_model` verbatim. An engine with no curated source is an error, not an empty list. The Ollama reply is a single multi-megabyte frame, so every hop on its path must allow `jsonrpc.WorkerFrameBytes`. |
| `engine:models` | — | `{ models: [string], modelsByEngine: { : [string] }, loadedByEngine: { : [string] } }` — the flat de-duplicated union of every running engine's models, the per-engine breakdown keyed by engine name, and the per-engine set of models currently **loaded in memory** (all normalized from each engine's `list_models` / `loaded_models` action `result` spec). `modelsByEngine` carries a key for every running engine whose inventory was successfully queried, including an empty list = "running, no models available"; a missing key means not running / not queryable / invalid response. `loadedByEngine` uses the same known-empty distinction for residency and also omits engines with no loaded endpoint. The `/v1/models` HTTP surface returns the same shape. |
| `engine:remote-get-installed` | `{ node }` | `{ engines: [EngineStatus] }` fetched from the remote node over `ec` mTLS |
| `engine:remote-install` | `{ node, engine, start? }` | `{ opId, status: EngineStatus }` after the remote install (live progress via `engine:remote-progress`) |
diff --git a/services/nvpair-engine-manager/actions.go b/services/nvpair-engine-manager/actions.go
index 023358b7c..c4136f8e4 100644
--- a/services/nvpair-engine-manager/actions.go
+++ b/services/nvpair-engine-manager/actions.go
@@ -32,6 +32,11 @@ func (e *Executor) Action(ctx context.Context, engine, action string, params jso
if !ok {
return nil, fmt.Errorf("engine %q has no action %q", engine, action)
}
+ // A snapshot download is bounded by its progress watchdog, not by the
+ // action timeout below, which a large model would outlast.
+ if act.HFSnapshot != nil {
+ return e.PullModelStream(ctx, engine, modelFromParams(params), params)
+ }
res, err := e.dispatchAction(ctx, st, engine, action, act, params)
if err != nil {
return nil, err
@@ -102,6 +107,14 @@ func (e *Executor) dispatchAction(ctx context.Context, st *engineState, engine,
if act.RemovePath != nil {
return e.runRemovePathAction(ctx, st, act, params)
}
+ // Store actions work on the files alone, so like CLI actions they are not
+ // gated on the engine running.
+ if act.ScanDir != nil {
+ return e.runScanDirAction(st, act)
+ }
+ if act.LaunchModel != nil {
+ return e.runLaunchModelAction(st, engine, act, params)
+ }
if len(act.Cmd) > 0 {
return e.runCmdAction(ctx, st, act, port, params)
}
@@ -200,6 +213,14 @@ func (e *Executor) runRemovePathAction(ctx context.Context, st *engineState, act
}
}
}
+ // In a store with a declared layout a model is one directory holding the
+ // marker, so a delete may name only such a directory — not the store, and
+ // not an owner folder holding several models.
+ marker, depth, layout := st.manifest.modelStoreLayout()
+ checkModel := layout && act.ModelResolution == ""
+ if model, named := vars["model"]; checkModel && named && !validModelID(model, depth) {
+ return nil, fmt.Errorf("delete_model: %q is not a valid model name", model)
+ }
root, err := resolvePlaceholders(act.RemovePath.Root, vars)
if err != nil {
return nil, err
@@ -235,6 +256,9 @@ func (e *Executor) runRemovePathAction(ctx context.Context, st *engineState, act
return nil, err
}
target = expandPath(target)
+ if _, named := vars["model"]; checkModel && named && !hasMarker(target, marker) {
+ return nil, fmt.Errorf("delete_model: %q is not a downloaded model", vars["model"])
+ }
if err := safeRemoveUnderRoot(root, target); err != nil {
return nil, err
}
diff --git a/services/nvpair-engine-manager/afterdownload.go b/services/nvpair-engine-manager/afterdownload.go
new file mode 100644
index 000000000..070c008f2
--- /dev/null
+++ b/services/nvpair-engine-manager/afterdownload.go
@@ -0,0 +1,92 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+// afterdownload.go runs a listed model's after_download steps: Python scripts
+// the engine ships that turn a vendor's checkpoint into a directory the engine
+// reads (Kimi K3 publishes tiktoken.model, and Colibri's engines read
+// tokenizer.json). The engine's own setup runs the same scripts, with the
+// Python its launcher finds, so PAIR finds Python the same way.
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "time"
+)
+
+// pythonCandidates are the commands tried for Python 3, in order. On Windows
+// the py launcher comes first: it is the one entry point that stays right with
+// several versions installed, and Colibri's coli.cmd asks it first too. A var
+// so tests can stand in a fake interpreter.
+var pythonCandidates = func() [][]string {
+ if runtime.GOOS == "windows" {
+ return [][]string{{"py", "-3"}, {"python"}, {"python3"}}
+ }
+ return [][]string{{"python3"}, {"python"}}
+}()
+
+// pythonVersionCheck exits zero on the Python the engines' tools require.
+const pythonVersionCheck = "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)"
+
+const (
+ // pythonProbeTimeout bounds one candidate's version check. Windows' Store
+ // alias for python opens the Store instead of answering, and must not
+ // hold the download up.
+ pythonProbeTimeout = 15 * time.Second
+ // afterDownloadTimeout bounds one step. Rewriting a tokenizer takes
+ // seconds; a step still running after this has hung.
+ afterDownloadTimeout = 30 * time.Minute
+)
+
+// findPython3 returns the first candidate that is Python 3.10 or newer.
+func findPython3(ctx context.Context) ([]string, error) {
+ for _, candidate := range pythonCandidates {
+ if _, err := exec.LookPath(candidate[0]); err != nil {
+ continue
+ }
+ probeCtx, cancel := context.WithTimeout(ctx, pythonProbeTimeout)
+ err := runManifestCommand(probeCtx, append(append([]string(nil), candidate...), "-c", pythonVersionCheck), nil)
+ cancel()
+ if err == nil {
+ return candidate, nil
+ }
+ }
+ return nil, errors.New("Python 3.10 or newer was not found on PATH")
+}
+
+// runAfterDownload runs the steps whose file is missing from dir, in order. A
+// step needs the engine installed, because the script ships with it.
+func (e *Executor) runAfterDownload(ctx context.Context, st *engineState, model string, steps []AfterDownloadStep, dir string) error {
+ for _, step := range steps {
+ if hasMarker(dir, step.Creates) {
+ continue
+ }
+ script := filepath.Join(st.installDir, filepath.FromSlash(step.Python))
+ if fi, err := os.Stat(script); st.installDir == "" || !pathWithinRoot(st.installDir, script) || err != nil || !fi.Mode().IsRegular() {
+ return fmt.Errorf("%s is finished by %s's %s, so install %s and download it again",
+ model, st.manifest.DisplayName, step.Python, st.manifest.DisplayName)
+ }
+ python, err := findPython3(ctx)
+ if err != nil {
+ return fmt.Errorf("%s is finished by a Python script, and %w", model, err)
+ }
+ slog.Info("finishing a model download", "engine", st.manifest.Engine, "model", model, "step", step.Python)
+ stepCtx, cancel := context.WithTimeout(ctx, afterDownloadTimeout)
+ err = runManifestCommand(stepCtx, append(append([]string(nil), python...), script, dir), nil)
+ cancel()
+ if err != nil {
+ return fmt.Errorf("%s could not be finished by %s: %w", model, step.Python, err)
+ }
+ if !hasMarker(dir, step.Creates) {
+ return fmt.Errorf("%s ran without writing %s for %s", step.Python, step.Creates, model)
+ }
+ }
+ return nil
+}
diff --git a/services/nvpair-engine-manager/catalog.go b/services/nvpair-engine-manager/catalog.go
index 585d84cd5..91af1ea91 100644
--- a/services/nvpair-engine-manager/catalog.go
+++ b/services/nvpair-engine-manager/catalog.go
@@ -13,6 +13,7 @@ import (
"fmt"
"io"
"log/slog"
+ "math"
"net/http"
"net/url"
"regexp"
@@ -160,10 +161,10 @@ const (
llamaCPPQuantization = "Q4_K_M"
// llamaCPPLimit is the page size for each publisher and for each search.
llamaCPPLimit = 50
- // llamaCPPSearchCacheLimit bounds how many distinct queries are kept, oldest
- // use evicted first. Each is a short list, but the set of possible queries
- // is not bounded at all.
- llamaCPPSearchCacheLimit = 20
+ // catalogSearchCacheLimit bounds how many distinct queries are kept across
+ // the searchable sources, oldest use evicted first. Each is a short list,
+ // but the set of possible queries is not bounded at all.
+ catalogSearchCacheLimit = 20
// maxCatalogQuery bounds a query passed upstream, in characters. A longer
// one is refused rather than shortened, which would search for something
// the operator did not type.
@@ -174,6 +175,20 @@ const (
// llama.cpp browse list: llama.cpp's own org and two widely used quantizers.
var llamaCPPPublishers = []string{"ggml-org", "bartowski", "unsloth"}
+// The Colibri catalogue: every open Hugging Face repository tagged colibri, the
+// tag Colibri's converted model containers are published under. A repository
+// id is the pull-ready name: hf_snapshot downloads it whole.
+const (
+ colibriTag = "colibri"
+ // colibriLimit is one page of the listing, sorted by downloads. The whole
+ // tag holds a few dozen repositories today; anything past the page is still
+ // one search or download-by-name away.
+ colibriLimit = 200
+ // colibriMarker is the file every Colibri model directory carries at its
+ // root, and so the file a repository must have to be offered.
+ colibriMarker = "config.json"
+)
+
// ollamaLibraryFile is the committed list's on-disk shape.
type ollamaLibraryFile struct {
ScrapedAt string `json:"scrapedAt"`
@@ -216,6 +231,15 @@ type hfModelRow struct {
Gated json.RawMessage `json:"gated"`
PipelineTag string `json:"pipeline_tag"`
Siblings []hfSibling `json:"siblings"`
+ // Config is the part of the root config.json the hub parses, returned
+ // when the listing asks for config=true.
+ Config *hfModelConfig `json:"config"`
+}
+
+// hfModelConfig is the field of a repository's config.json an engine that
+// picks its code by model_type is matched on.
+type hfModelConfig struct {
+ ModelType string `json:"model_type"`
}
// hfSibling is one file in a Hugging Face repo.
@@ -232,12 +256,20 @@ type catalogService struct {
lmStudio *listingCache
llamaCPP *listingCache
+ colibri *listingCache
+ // colibriListed is the current state of the models Colibri's manifest
+ // lists by repository; listed returns an engine's entries (nil lists none).
+ colibriListed *listingCache
+ listed func(engine string) []CatalogEntry
+ // modelTypes returns the config.json model_types an engine runs (nil runs
+ // every one), which a tagged repository must declare to be offered.
+ modelTypes func(engine string) []string
- // searchMu guards the llama.cpp searches, kept by lowercased query with
- // searchOrder from least to most recently used.
- searchMu sync.Mutex
- llamaSearches map[string]*listingCache
- searchOrder []string
+ // searchMu guards the searches of every searchable source, kept by engine
+ // and lowercased query, with searchOrder from least to most recently used.
+ searchMu sync.Mutex
+ searches map[string]*listingCache
+ searchOrder []string
// baseURL overrides the listing endpoint in tests. Empty means the real one;
// the caching and coalescing around the fetch is the part worth testing, and
@@ -281,10 +313,12 @@ func newCatalogService() *catalogService {
c := &catalogService{
client: newEngineHTTPClient(catalogHTTPTimeout),
ollamaCatalog: sync.OnceValues(loadOllamaCatalog),
- llamaSearches: map[string]*listingCache{},
+ searches: map[string]*listingCache{},
}
c.lmStudio = &listingCache{label: "lm studio catalog", fetch: c.fetchLmStudio}
c.llamaCPP = &listingCache{label: "llama.cpp catalog", fetch: c.fetchLlamaCPP}
+ c.colibri = &listingCache{label: "colibri catalog", fetch: c.fetchColibri}
+ c.colibriListed = &listingCache{label: "colibri listed models", fetch: c.fetchColibriListed, emptyOK: true}
return c
}
@@ -339,6 +373,30 @@ func (c *catalogService) Catalog(ctx context.Context, engine, platform, arch, qu
Searchable: true,
Query: query,
}, nil
+ case "colibri":
+ query, err := catalogQuery(query)
+ if err != nil {
+ return CatalogResult{}, err
+ }
+ models, fetchedAt, err := c.colibriCatalog(ctx, query)
+ if err != nil {
+ return CatalogResult{}, err
+ }
+ source := hfModelsAPI + "?filter=" + colibriTag
+ if query != "" {
+ source += "&search=" + url.QueryEscape(query)
+ }
+ return CatalogResult{
+ // Colibri's builds run on every platform it ships for, so nothing is
+ // filtered for the target.
+ Models: models,
+ Source: source,
+ Platform: platform,
+ Arch: arch,
+ FetchedAt: fetchedAt.UTC().Format(time.RFC3339),
+ Searchable: true,
+ Query: query,
+ }, nil
default:
return CatalogResult{}, fmt.Errorf("no model catalog for engine %q", engine)
}
@@ -396,6 +454,8 @@ func normalizeCatalogEngine(engine string) string {
return "lmstudio"
case "llama-cpp", "llama.cpp", "llamacpp":
return "llamacpp"
+ case "colibri", "colibrì":
+ return "colibri"
default:
return e
}
@@ -568,26 +628,207 @@ func (c *catalogService) llamaCPPCatalog(ctx context.Context, query string) ([]C
return c.llamaCPPSearch(query).get(ctx)
}
-// llamaCPPSearch is the cache for one query, created on first use. Queries
-// differing only in case share one.
+// llamaCPPSearch is the cache for one llama.cpp query.
func (c *catalogService) llamaCPPSearch(query string) *listingCache {
- key := strings.ToLower(query)
+ return c.searchCache("llamacpp", "llama.cpp search", query, func(ctx context.Context) ([]CatalogModel, error) {
+ rows, err := c.fetchHF(ctx, llamaCPPListing("search", query))
+ if err != nil {
+ return nil, err
+ }
+ return normalizeLlamaCPPRows(rows), nil
+ })
+}
+
+// colibriCatalog serves the browse list, or the results of a query: the models
+// Colibri's manifest lists first, then the colibri-tagged repositories. A
+// listed repository that is also tagged appears once, as listed. The listed
+// models are a garnish on the tagged list, so failing to read them leaves the
+// tagged list standing.
+func (c *catalogService) colibriCatalog(ctx context.Context, query string) ([]CatalogModel, time.Time, error) {
+ var tagged []CatalogModel
+ var fetchedAt time.Time
+ var err error
+ if query == "" {
+ tagged, fetchedAt, err = c.colibri.get(ctx)
+ } else {
+ tagged, fetchedAt, err = c.colibriSearch(query).get(ctx)
+ }
+ if err != nil {
+ return nil, time.Time{}, err
+ }
+ listed, _, err := c.colibriListed.get(ctx)
+ if err != nil {
+ slog.Warn("colibri listed models unavailable; serving the tagged ones", "err", err)
+ listed = nil
+ }
+ return mergeListedModels(listed, tagged, query), fetchedAt, nil
+}
+
+// mergeListedModels puts the listed models a query matches ahead of the tagged
+// ones, dropping a tagged copy of a listed repository. Matching mirrors what a
+// Hugging Face search matches: a substring of the repository id, here also of
+// the display name, ignoring case. A tagged hit on a listed repository brings
+// in the listed entry, since the search found it.
+func mergeListedModels(listed, tagged []CatalogModel, query string) []CatalogModel {
+ q := strings.ToLower(query)
+ byID := make(map[string]bool, len(listed))
+ for _, m := range listed {
+ byID[m.ID] = true
+ }
+ hits := make(map[string]bool, len(tagged))
+ for _, m := range tagged {
+ hits[m.ID] = true
+ }
+ out := make([]CatalogModel, 0, len(listed)+len(tagged))
+ for _, m := range listed {
+ if q == "" || hits[m.ID] || strings.Contains(strings.ToLower(m.ID), q) || strings.Contains(strings.ToLower(m.Name), q) {
+ out = append(out, m)
+ }
+ }
+ for _, m := range tagged {
+ if !byID[m.ID] {
+ out = append(out, m)
+ }
+ }
+ return out
+}
+
+// fetchColibriListed reads the current state of each model Colibri's manifest
+// lists: its downloads, likes and last change, and whether it is still open.
+// A repository that cannot be read, or has since been made private or gated, is
+// left out rather than failing the rest.
+func (c *catalogService) fetchColibriListed(ctx context.Context) ([]CatalogModel, error) {
+ var entries []CatalogEntry
+ if c.listed != nil {
+ entries = c.listed("colibri")
+ }
+ models := make([]CatalogModel, len(entries))
+ found := make([]bool, len(entries))
+ var wg sync.WaitGroup
+ for i, entry := range entries {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ row, err := c.fetchHFRepo(ctx, entry.Repo)
+ if err != nil {
+ slog.Debug("listed model unavailable", "engine", "colibri", "model", entry.Repo, "err", err)
+ return
+ }
+ models[i], found[i] = listedModel(entry, row)
+ }()
+ }
+ wg.Wait()
+ out := make([]CatalogModel, 0, len(entries))
+ for i := range entries {
+ if found[i] {
+ out = append(out, models[i])
+ }
+ }
+ return out, nil
+}
+
+// fetchHFRepo reads one repository's listing fields.
+func (c *catalogService) fetchHFRepo(ctx context.Context, repo string) (hfModelRow, error) {
+ ctx, cancel := context.WithTimeout(ctx, catalogHTTPTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint()+"/"+repo, nil)
+ if err != nil {
+ return hfModelRow{}, fmt.Errorf("build model catalog request: %w", withoutURL(err))
+ }
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("User-Agent", "PAIR/1.0")
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return hfModelRow{}, fmt.Errorf("model catalog request failed: %w", withoutURL(err))
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return hfModelRow{}, fmt.Errorf("model catalog returned %s", resp.Status)
+ }
+ var row hfModelRow
+ if err := json.NewDecoder(io.LimitReader(resp.Body, maxCatalogBody)).Decode(&row); err != nil {
+ return hfModelRow{}, fmt.Errorf("decode model catalog: %w", err)
+ }
+ return row, nil
+}
+
+// listedModel is a manifest entry as a catalog model, with the repository's
+// live fields. It is not one when the repository is private or gated, the same
+// rule the tagged list applies, or when it reports no usable date.
+func listedModel(entry CatalogEntry, row hfModelRow) (CatalogModel, bool) {
+ if row.Private || string(bytes.TrimSpace(row.Gated)) != "false" {
+ return CatalogModel{}, false
+ }
+ updated := row.LastModified
+ if updated == "" {
+ updated = row.CreatedAt
+ }
+ if _, err := time.Parse(time.RFC3339, updated); err != nil {
+ return CatalogModel{}, false
+ }
+ tags := row.Tags
+ if tags == nil {
+ tags = []string{}
+ }
+ return CatalogModel{
+ ID: entry.Repo,
+ Name: entry.Name,
+ Author: entry.Repo[:strings.Index(entry.Repo, "/")],
+ URL: "https://huggingface.co/" + entry.Repo,
+ Size: uint64(math.Round(entry.SizeGB * 1e9)),
+ Downloads: max(row.Downloads, 0),
+ Likes: max(row.Likes, 0),
+ UpdatedAt: updated,
+ Tags: tags,
+ ParameterSize: entry.Parameters,
+ }, true
+}
+
+// colibriSearch is the cache for one Colibri query.
+func (c *catalogService) colibriSearch(query string) *listingCache {
+ return c.searchCache("colibri", "colibri search", query, func(ctx context.Context) ([]CatalogModel, error) {
+ rows, err := c.fetchHF(ctx, colibriListing("search", query))
+ if err != nil {
+ return nil, err
+ }
+ return normalizeColibriRows(rows, c.servesModelType("colibri")), nil
+ })
+}
+
+// fetchColibri reads the colibri-tagged repositories, most downloaded first.
+func (c *catalogService) fetchColibri(ctx context.Context) ([]CatalogModel, error) {
+ rows, err := c.fetchHF(ctx, colibriListing("", ""))
+ if err != nil {
+ return nil, err
+ }
+ return normalizeColibriRows(rows, c.servesModelType("colibri")), nil
+}
+
+// servesModelType is the test an engine's manifest holds a tagged repository's
+// model_type to, or nil when the manifest declares no model_types.
+func (c *catalogService) servesModelType(engine string) func(string) bool {
+ if c.modelTypes == nil {
+ return nil
+ }
+ served := c.modelTypes(engine)
+ if len(served) == 0 {
+ return nil
+ }
+ m := &Manifest{ModelTypes: served}
+ return m.servesModelType
+}
+
+// searchCache is the cache for one query of one searchable source, created on
+// first use. Queries differing only in case share one. The searches of every
+// source share one bound, evicting the least recently used.
+func (c *catalogService) searchCache(engine, label, query string, fetch func(context.Context) ([]CatalogModel, error)) *listingCache {
+ key := engine + "\n" + strings.ToLower(query)
c.searchMu.Lock()
defer c.searchMu.Unlock()
- search, ok := c.llamaSearches[key]
+ search, ok := c.searches[key]
if !ok {
- search = &listingCache{
- label: "llama.cpp search",
- emptyOK: true,
- fetch: func(ctx context.Context) ([]CatalogModel, error) {
- rows, err := c.fetchHF(ctx, llamaCPPListing("search", query))
- if err != nil {
- return nil, err
- }
- return normalizeLlamaCPPRows(rows), nil
- },
- }
- c.llamaSearches[key] = search
+ search = &listingCache{label: label, emptyOK: true, fetch: fetch}
+ c.searches[key] = search
}
for i, k := range c.searchOrder {
if k == key {
@@ -596,8 +837,8 @@ func (c *catalogService) llamaCPPSearch(query string) *listingCache {
}
}
c.searchOrder = append(c.searchOrder, key)
- for len(c.searchOrder) > llamaCPPSearchCacheLimit {
- delete(c.llamaSearches, c.searchOrder[0])
+ for len(c.searchOrder) > catalogSearchCacheLimit {
+ delete(c.searches, c.searchOrder[0])
c.searchOrder = c.searchOrder[1:]
}
return search
@@ -649,6 +890,121 @@ func llamaCPPListing(field, value string) url.Values {
return q
}
+// colibriListing is the colibri-tagged listing, optionally narrowed by one
+// field. It asks for the full entry, whose file list shows whether a repository
+// is a complete model container, and for its parsed config.json, whose
+// model_type says which engine code it needs.
+func colibriListing(field, value string) url.Values {
+ q := url.Values{}
+ if field != "" {
+ q.Set(field, value)
+ }
+ q.Set("filter", colibriTag)
+ q.Set("sort", "downloads")
+ q.Set("direction", "-1")
+ q.Set("limit", fmt.Sprint(colibriLimit))
+ q.Set("full", "true")
+ q.Set("config", "true")
+ return q
+}
+
+// normalizeColibriRows converts listing entries into Colibri downloads,
+// dropping the ones Colibri cannot fetch or serve, de-duplicating, and putting
+// the most-downloaded first. serves is the engine's model_type test.
+func normalizeColibriRows(rows []hfModelRow, serves func(string) bool) []CatalogModel {
+ seen := make(map[string]int, len(rows))
+ out := make([]CatalogModel, 0, len(rows))
+ for _, r := range rows {
+ model, ok := colibriModel(r, serves)
+ if !ok {
+ continue
+ }
+ if idx, dup := seen[model.ID]; dup {
+ out[idx] = model
+ continue
+ }
+ seen[model.ID] = len(out)
+ out = append(out, model)
+ }
+ slices.SortStableFunc(out, func(a, b CatalogModel) int {
+ return cmp.Compare(b.Downloads, a.Downloads)
+ })
+ return out
+}
+
+// speedyColibriPattern matches the repositories of the "speedy colibri" fork,
+// whose nvfp4/mxfp4 containers the upstream engine PAIR installs does not read.
+var speedyColibriPattern = regexp.MustCompile(`(?i)speedy-?colibri`)
+
+// colibriForkTags mark containers built for a port of Colibri rather than for
+// Colibri itself: SpeedyColibri (Rust, nvfp4/mxfp4) and peng (zstd-packed
+// tensors). They carry the colibri tag for its expert-streaming lineage, and
+// the engine PAIR installs reads neither.
+var colibriForkTags = []string{"speedycolibri", "peng"}
+
+// colibriModel turns one listing entry into a download, or reports that it is
+// not one. The tag alone is not enough: a gated or private repository fails to
+// download without credentials, one without a root config.json is not a model
+// container (or not yet a complete one), one whose pipeline says it is not a
+// chat or completion model is nothing the OpenAI endpoint can serve, and one
+// whose config.json model_type the engine has no code for never starts.
+func colibriModel(r hfModelRow, serves func(string) bool) (CatalogModel, bool) {
+ repo := strings.TrimSpace(r.ID)
+ if repo == "" {
+ repo = strings.TrimSpace(r.ModelID)
+ }
+ if !hfRepoPattern.MatchString(repo) {
+ return CatalogModel{}, false
+ }
+ if r.Private || string(bytes.TrimSpace(r.Gated)) != "false" {
+ return CatalogModel{}, false
+ }
+ if !hasTag(r.Tags, colibriTag) || speedyColibriPattern.MatchString(repo) {
+ return CatalogModel{}, false
+ }
+ for _, fork := range colibriForkTags {
+ if hasTag(r.Tags, fork) {
+ return CatalogModel{}, false
+ }
+ }
+ if serves != nil && (r.Config == nil || !serves(r.Config.ModelType)) {
+ return CatalogModel{}, false
+ }
+ if pipeline := strings.ToLower(strings.TrimSpace(r.PipelineTag)); pipeline != "" && !generativePipelines[pipeline] {
+ return CatalogModel{}, false
+ }
+ if !hasRootSibling(r.Siblings, colibriMarker) {
+ return CatalogModel{}, false
+ }
+ updated := r.LastModified
+ if updated == "" {
+ updated = r.CreatedAt
+ }
+ if _, err := time.Parse(time.RFC3339, updated); err != nil {
+ return CatalogModel{}, false
+ }
+ return CatalogModel{
+ ID: repo,
+ Name: repo,
+ Author: repo[:strings.Index(repo, "/")],
+ URL: "https://huggingface.co/" + repo,
+ Downloads: max(r.Downloads, 0),
+ Likes: max(r.Likes, 0),
+ UpdatedAt: updated,
+ Tags: r.Tags,
+ }, true
+}
+
+// hasRootSibling reports a file directly at a repository's root.
+func hasRootSibling(files []hfSibling, name string) bool {
+ for _, f := range files {
+ if f.RFilename == name {
+ return true
+ }
+ }
+ return false
+}
+
// fetchHF reads one page of the Hugging Face model listing.
//
// A failed request is reported without its URL. net/http puts the whole URL in
@@ -744,10 +1100,6 @@ func isMLX(id string, tags []string) bool {
// The shapes a llama.cpp download depends on.
var (
- // llamaCPPRepoPattern is a repo id safe to pass to llama.cpp: owner/name,
- // each starting with a letter or digit, nothing that could read as a path
- // or a flag.
- llamaCPPRepoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// llamaCPPQuantFile matches the offered quantization in a GGUF file name.
llamaCPPQuantFile = regexp.MustCompile(`(?i)` + llamaCPPQuantization + `[.-]`)
// splitGGUFPattern matches a shard of a model split across files.
@@ -797,7 +1149,7 @@ func llamaCPPModel(r hfModelRow) (CatalogModel, bool) {
if repo == "" {
repo = strings.TrimSpace(r.ModelID)
}
- if !llamaCPPRepoPattern.MatchString(repo) {
+ if !hfRepoPattern.MatchString(repo) {
return CatalogModel{}, false
}
// Only an explicit false is open. A gated repo reports how it is gated, and
diff --git a/services/nvpair-engine-manager/catalog_test.go b/services/nvpair-engine-manager/catalog_test.go
index cb1fa8c11..1c421c270 100644
--- a/services/nvpair-engine-manager/catalog_test.go
+++ b/services/nvpair-engine-manager/catalog_test.go
@@ -4,11 +4,13 @@
package main
import (
+ "context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
+ "reflect"
"runtime"
"slices"
"strings"
@@ -213,6 +215,7 @@ func TestNormalizeCatalogEngine_ResolvesSupportedEngineNames(t *testing.T) {
test("canonical llama.cpp", "llamacpp", "llamacpp")
test("hyphenated llama.cpp", "llama-cpp", "llamacpp")
test("dotted llama.cpp", "llama.cpp", "llamacpp")
+ test("capitalized Colibri", "Colibri", "colibri")
test("other engine preserved", "vllm", "vllm")
}
@@ -553,15 +556,17 @@ func TestSearchAtTheLengthLimit_IsAnswered(t *testing.T) {
assert.Len(t, requests(), 1, "a query at the limit must make one request")
}
-// TestLlamaCPPSearchesAreBounded checks the per-query caches are evicted least
-// recently used first, since the set of possible queries is unbounded.
-func TestLlamaCPPSearch_BoundsConcurrentQueries(t *testing.T) {
+// TestCatalogSearch_BoundsQueriesAcrossSources checks the per-query caches are
+// evicted least recently used first, since the set of possible queries is
+// unbounded, and that the sources share the bound without sharing entries.
+func TestCatalogSearch_BoundsQueriesAcrossSources(t *testing.T) {
c := newCatalogService()
first := c.llamaCPPSearch("first")
- for i := range llamaCPPSearchCacheLimit {
+ require.NotSame(t, first, c.colibriSearch("first"), "one query of two sources shares a cache")
+ for i := range catalogSearchCacheLimit {
c.llamaCPPSearch(fmt.Sprint("query ", i))
}
- assert.Len(t, c.llamaSearches, llamaCPPSearchCacheLimit)
+ assert.Len(t, c.searches, catalogSearchCacheLimit)
assert.NotSame(t, first, c.llamaCPPSearch("first"), "the least recently used search was not evicted")
}
@@ -578,3 +583,259 @@ func TestQuery_IsIgnoredByASourceThatCannotSearch(t *testing.T) {
assert.Empty(t, queried.Query)
assert.Len(t, queried.Models, len(all.Models))
}
+
+// colibriRow is an open, complete Colibri model container.
+func colibriRow(id string, downloads int) hfModelRow {
+ return hfModelRow{
+ ID: id,
+ Downloads: downloads,
+ LastModified: "2026-10-02T12:00:00.000Z",
+ Tags: []string{"colibri", "moe", "int4"},
+ Gated: json.RawMessage("false"),
+ PipelineTag: "text-generation",
+ Siblings: []hfSibling{{RFilename: "config.json"}, {RFilename: "model-00000.safetensors"}},
+ }
+}
+
+// TestNormalizeColibriRowsKeepsOnlyServable checks the Colibri list offers
+// only open, complete containers the upstream engine can serve, most
+// downloaded first, each under its pull-ready repository id.
+func TestNormalizeColibriRowsKeepsOnlyServable(t *testing.T) {
+ gated := colibriRow("o/gated", 9)
+ gated.Gated = json.RawMessage(`"manual"`)
+ private := colibriRow("o/private", 9)
+ private.Private = true
+ untagged := colibriRow("o/untagged", 9)
+ untagged.Tags = []string{"moe"}
+ noConfig := colibriRow("o/no-config", 9)
+ noConfig.Siblings = []hfSibling{{RFilename: "sub/config.json"}}
+ image := colibriRow("o/image", 9)
+ image.PipelineTag = "text-to-image"
+ unknownPipeline := colibriRow("o/no-pipeline", 3)
+ unknownPipeline.PipelineTag = ""
+ speedyTag := colibriRow("o/fork", 9)
+ speedyTag.Tags = append(speedyTag.Tags, "speedycolibri")
+ badDate := colibriRow("o/bad-date", 9)
+ badDate.LastModified = "yesterday"
+
+ got := normalizeColibriRows([]hfModelRow{
+ colibriRow("o/small", 1),
+ gated, private, untagged, noConfig, image, speedyTag, badDate,
+ colibriRow("o/GLM-5.2-speedy-colibri-nvfp4", 9),
+ colibriRow("../escape", 9),
+ unknownPipeline,
+ colibriRow("o/big", 50),
+ colibriRow("o/small", 2), // a duplicate replaces the earlier entry
+ }, nil)
+ var ids []string
+ for _, m := range got {
+ ids = append(ids, m.ID)
+ if m.Name != m.ID || m.URL != "https://huggingface.co/"+m.ID || m.Author != "o" {
+ t.Errorf("%+v is not pull-ready", m)
+ }
+ }
+ if want := []string{"o/big", "o/no-pipeline", "o/small"}; !reflect.DeepEqual(ids, want) {
+ t.Fatalf("colibri catalog = %v, want %v", ids, want)
+ }
+}
+
+// TestColibriCatalogAndSearch checks the browse list asks for the colibri tag
+// in full, that a query searches within the tag, and that each query is cached
+// on its own.
+func TestColibriCatalogAndSearch(t *testing.T) {
+ var mu sync.Mutex
+ var requests []url.Values
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ mu.Lock()
+ requests = append(requests, q)
+ mu.Unlock()
+ rows := []hfModelRow{colibriRow("Justvugg/GLM-5.3-colibri-int4-g64", 10)}
+ if s := q.Get("search"); s != "" {
+ rows = []hfModelRow{colibriRow("found/"+strings.ReplaceAll(s, " ", "-"), 1)}
+ }
+ _ = json.NewEncoder(w).Encode(rows)
+ }))
+ t.Cleanup(srv.Close)
+ c := newCatalogService()
+ c.baseURL = srv.URL
+
+ res, err := c.Catalog(context.Background(), "colibri", "", "", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !res.Searchable || len(res.Models) != 1 || res.Models[0].Name != "Justvugg/GLM-5.3-colibri-int4-g64" {
+ t.Fatalf("browse = %+v", res)
+ }
+ for range 2 {
+ res, err = c.Catalog(context.Background(), "colibri", "", "", " qwen ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if res.Query != "qwen" || len(res.Models) != 1 || res.Models[0].ID != "found/qwen" {
+ t.Fatalf("search = %+v", res)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if len(requests) != 2 {
+ t.Fatalf("%d upstream requests, want one browse and one cached search", len(requests))
+ }
+ for _, q := range requests {
+ if q.Get("filter") != "colibri" || q.Get("full") != "true" || q.Get("config") != "true" || q.Get("sort") != "downloads" {
+ t.Errorf("listing asked for %v, want colibri repos in full by downloads", q)
+ }
+ }
+ if requests[1].Get("search") != "qwen" {
+ t.Errorf("search sent %v", requests[1])
+ }
+}
+
+// listedHub is a Hugging Face stand-in for the listed-model tests: the colibri
+// listing (or a search of it) at the endpoint itself, and one repository's
+// fields at endpoint//.
+func listedHub(t *testing.T, tagged []hfModelRow, repos map[string]hfModelRow) *httptest.Server {
+ t.Helper()
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if repo := strings.TrimPrefix(r.URL.Path, "/"); repo != "" {
+ row, ok := repos[repo]
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(row)
+ return
+ }
+ rows := tagged
+ if s := r.URL.Query().Get("search"); s != "" {
+ rows = nil
+ for _, row := range tagged {
+ if strings.Contains(strings.ToLower(row.ID), strings.ToLower(s)) {
+ rows = append(rows, row)
+ }
+ }
+ }
+ _ = json.NewEncoder(w).Encode(rows)
+ }))
+ t.Cleanup(srv.Close)
+ return srv
+}
+
+// TestColibriCatalogListsManifestModelsFirst checks the models Colibri's
+// manifest lists by repository lead the browse list under their manifest names
+// and sizes, with the repository's live downloads and date; that a listed
+// repository the tag also finds appears once; and that one since gated, or
+// gone, is left out without failing the rest.
+func TestColibriCatalogListsManifestModelsFirst(t *testing.T) {
+ official := colibriRow("vendor/Big-Model", 7)
+ official.Tags = []string{"text-generation"}
+ alsoTagged := colibriRow("o/tagged-official", 3)
+ gated := colibriRow("vendor/Gated", 1)
+ gated.Gated = json.RawMessage(`"manual"`)
+ srv := listedHub(t,
+ []hfModelRow{colibriRow("o/popular", 99), alsoTagged},
+ map[string]hfModelRow{"vendor/Big-Model": official, "o/tagged-official": alsoTagged, "vendor/Gated": gated})
+ c := newCatalogService()
+ c.baseURL = srv.URL
+ c.listed = func(engine string) []CatalogEntry {
+ if engine != "colibri" {
+ t.Errorf("listed asked for %q", engine)
+ }
+ return []CatalogEntry{
+ {Repo: "vendor/Big-Model", Name: "Big Model", Parameters: "284B", SizeGB: 166.9},
+ {Repo: "o/tagged-official", Name: "Tagged Official", SizeGB: 19.4},
+ {Repo: "vendor/Gated", Name: "Gated"},
+ {Repo: "vendor/Gone", Name: "Gone"},
+ }
+ }
+
+ res, err := c.Catalog(context.Background(), "colibri", "", "", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var ids []string
+ for _, m := range res.Models {
+ ids = append(ids, m.ID)
+ }
+ if strings.Join(ids, ",") != "vendor/Big-Model,o/tagged-official,o/popular" {
+ t.Fatalf("browse order = %v", ids)
+ }
+ big := res.Models[0]
+ if big.Name != "Big Model" || big.Author != "vendor" || big.Size != 166_900_000_000 || big.ParameterSize != "284B" ||
+ big.Downloads != 7 || big.UpdatedAt != official.LastModified || big.URL != "https://huggingface.co/vendor/Big-Model" {
+ t.Fatalf("listed model = %+v", big)
+ }
+ if res.Models[1].Name != "Tagged Official" {
+ t.Errorf("a tagged listed repository kept its tag-search shape: %+v", res.Models[1])
+ }
+}
+
+// TestColibriSearchFindsListedModels checks a query matches listed models by
+// repository id or display name, beside what the tag search returns.
+func TestColibriSearchFindsListedModels(t *testing.T) {
+ srv := listedHub(t,
+ []hfModelRow{colibriRow("o/kimi-colibri", 5), colibriRow("o/other", 4)},
+ map[string]hfModelRow{"moonshotai/K3": colibriRow("moonshotai/K3", 2), "vendor/DeepSeek-V4": colibriRow("vendor/DeepSeek-V4", 1)})
+ c := newCatalogService()
+ c.baseURL = srv.URL
+ c.listed = func(string) []CatalogEntry {
+ return []CatalogEntry{{Repo: "moonshotai/K3", Name: "Kimi K3"}, {Repo: "vendor/DeepSeek-V4", Name: "DeepSeek V4 Flash"}}
+ }
+
+ res, err := c.Catalog(context.Background(), "colibri", "", "", "KIMI")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var ids []string
+ for _, m := range res.Models {
+ ids = append(ids, m.ID)
+ }
+ if strings.Join(ids, ",") != "moonshotai/K3,o/kimi-colibri" {
+ t.Fatalf("search for kimi = %v, want the listed Kimi K3 (by name) then the tag hit", ids)
+ }
+}
+
+// TestColibriCatalogWithoutListedModels checks a listed model that cannot be
+// read leaves the tagged list intact.
+func TestColibriCatalogWithoutListedModels(t *testing.T) {
+ srv := listedHub(t, []hfModelRow{colibriRow("o/popular", 99)}, nil)
+ c := newCatalogService()
+ c.baseURL = srv.URL
+ c.listed = func(string) []CatalogEntry { return []CatalogEntry{{Repo: "vendor/Gone", Name: "Gone"}} }
+
+ res, err := c.Catalog(context.Background(), "colibri", "", "", "")
+ if err != nil || len(res.Models) != 1 || res.Models[0].ID != "o/popular" {
+ t.Fatalf("catalog = %+v, %v", res, err)
+ }
+}
+
+// TestNormalizeColibriRowsKeepsOnlyRunnableModels checks a tagged repository is
+// offered only when its config.json model_type is one Colibri runs, and never
+// when it is built for a port of Colibri (SpeedyColibri, peng).
+func TestNormalizeColibriRowsKeepsOnlyRunnableModels(t *testing.T) {
+ typed := func(id, modelType string) hfModelRow {
+ r := colibriRow(id, 1)
+ r.Config = &hfModelConfig{ModelType: modelType}
+ return r
+ }
+ peng := typed("o/mimo-peng", "mimo_v2")
+ peng.Tags = append(peng.Tags, "peng")
+ untyped := colibriRow("o/untyped", 1)
+ serves := (&Manifest{ModelTypes: []string{"glm_moe_dsa", "mimo_v2"}}).servesModelType
+
+ got := normalizeColibriRows([]hfModelRow{
+ typed("o/glm", "glm_moe_dsa"), typed("o/upper", "GLM_MOE_DSA"), typed("o/hy3", "hy_v3"),
+ typed("o/laguna", "laguna"), peng, untyped,
+ }, serves)
+ var ids []string
+ for _, m := range got {
+ ids = append(ids, m.ID)
+ }
+ if strings.Join(ids, ",") != "o/glm,o/upper" {
+ t.Fatalf("runnable = %v, want only the glm_moe_dsa repositories", ids)
+ }
+
+ if got := normalizeColibriRows([]hfModelRow{untyped}, nil); len(got) != 1 {
+ t.Errorf("with no model_types declared, an untyped repository was dropped: %v", got)
+ }
+}
diff --git a/services/nvpair-engine-manager/controlstream.go b/services/nvpair-engine-manager/controlstream.go
index 13bd584d6..91b4912b1 100644
--- a/services/nvpair-engine-manager/controlstream.go
+++ b/services/nvpair-engine-manager/controlstream.go
@@ -82,7 +82,7 @@ func (s *controlServer) handleInstall(w http.ResponseWriter, r *http.Request) {
return streamFrame{}, err
}
if req.Start {
- if err := s.exec.StartWith(ctx, req.Engine, startOpts{}); err != nil {
+ if err := s.exec.StartAfterInstall(ctx, req.Engine, startOpts{}); err != nil {
return streamFrame{}, err
}
}
diff --git a/services/nvpair-engine-manager/diskfree_unix.go b/services/nvpair-engine-manager/diskfree_unix.go
new file mode 100644
index 000000000..3314e0e30
--- /dev/null
+++ b/services/nvpair-engine-manager/diskfree_unix.go
@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build !windows
+
+package main
+
+import "golang.org/x/sys/unix"
+
+// diskFreeBytes reports the bytes an unprivileged user may still write on the
+// volume holding path.
+func diskFreeBytes(path string) (uint64, error) {
+ var st unix.Statfs_t
+ if err := unix.Statfs(path, &st); err != nil {
+ return 0, err
+ }
+ return uint64(st.Bavail) * uint64(st.Bsize), nil
+}
diff --git a/services/nvpair-engine-manager/diskfree_windows.go b/services/nvpair-engine-manager/diskfree_windows.go
new file mode 100644
index 000000000..579519a81
--- /dev/null
+++ b/services/nvpair-engine-manager/diskfree_windows.go
@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build windows
+
+package main
+
+import "golang.org/x/sys/windows"
+
+// diskFreeBytes reports the bytes the current user may still write on the
+// volume holding path, honouring any disk quota.
+func diskFreeBytes(path string) (uint64, error) {
+ dir, err := windows.UTF16PtrFromString(path)
+ if err != nil {
+ return 0, err
+ }
+ var available, total, free uint64
+ if err := windows.GetDiskFreeSpaceEx(dir, &available, &total, &free); err != nil {
+ return 0, err
+ }
+ return available, nil
+}
diff --git a/services/nvpair-engine-manager/executor.go b/services/nvpair-engine-manager/executor.go
index 2b5c03f85..a87197b6f 100644
--- a/services/nvpair-engine-manager/executor.go
+++ b/services/nvpair-engine-manager/executor.go
@@ -124,6 +124,13 @@ type Executor struct {
// tick. Buffered depth 1: a coalesced signal is enough.
loadedPoke chan struct{}
+ // hfBaseURL overrides the Hugging Face hub for hf_snapshot pulls in tests.
+ // Empty means the real one.
+ hfBaseURL string
+ // snapshotPulls holds the destination of every hf_snapshot pull in
+ // flight, so two pulls of one model cannot write the same files.
+ snapshotPulls sync.Map
+
reservedPort atomic.Int32
// StopAll is terminal for an Executor; the gate closes its start/snapshot race.
shuttingDown atomic.Bool
@@ -219,6 +226,26 @@ func (e *Executor) notify(method string, params any) {
}
}
+// catalogEntries returns the models an engine's manifest lists by repository,
+// for the engine catalogue.
+func (e *Executor) catalogEntries(engine string) []CatalogEntry {
+ m, ok := e.reg.Get(engine)
+ if !ok {
+ return nil
+ }
+ return m.Catalog
+}
+
+// servedModelTypes returns the config.json model_types an engine's manifest
+// says it runs, for the engine catalogue.
+func (e *Executor) servedModelTypes(engine string) []string {
+ m, ok := e.reg.Get(engine)
+ if !ok {
+ return nil
+ }
+ return m.ModelTypes
+}
+
// state returns (lazily creating) the per-engine state for a known
// engine that has a block for the host os/arch.
func (e *Executor) state(engine string) (*engineState, error) {
diff --git a/services/nvpair-engine-manager/hfsnapshot.go b/services/nvpair-engine-manager/hfsnapshot.go
new file mode 100644
index 000000000..3aa1b98b4
--- /dev/null
+++ b/services/nvpair-engine-manager/hfsnapshot.go
@@ -0,0 +1,483 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+// hfsnapshot.go is the hf_snapshot pull: it downloads every file of a public
+// Hugging Face model repository into the engine's model store, for an engine
+// that reads a model as a directory (Colibri) rather than fetching it itself.
+//
+// The download lands in ".partial" and is renamed into place only once
+// every file has arrived and matches its published size, so the store never
+// lists a half-downloaded model. Each file is written as ".part" and
+// resumed with an HTTP Range request, both after a transient error within one
+// pull and across pulls, so cancelling a 400 GB download loses nothing.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ // hfDefaultBaseURL is the Hugging Face hub, overridable per executor in
+ // tests through Executor.hfBaseURL.
+ hfDefaultBaseURL = "https://huggingface.co"
+ // hfSnapshotSpareBytes is left free on the store's volume beyond the
+ // download itself, so a pull never fills the disk to the last byte.
+ hfSnapshotSpareBytes = 1 << 30
+)
+
+var (
+ // hfSnapshotAttempts bounds how often one file is tried within a pull
+ // after a transient failure; each retry resumes where the last stopped.
+ // Vars so tests can shrink them.
+ hfSnapshotAttempts = 3
+ hfSnapshotRetryBackoff = 2 * time.Second
+ // freeDiskBytes reports the bytes available to this user on the volume
+ // holding path. A var so tests can simulate a full disk.
+ freeDiskBytes = diskFreeBytes
+)
+
+// hfRepoMeta is the subset of GET /api/models/{repo}?blobs=true used here.
+type hfRepoMeta struct {
+ Private bool `json:"private"`
+ // Gated is false for an open repo and a string ("auto", "manual") for one
+ // that needs its terms accepted first, so it is kept raw.
+ Gated json.RawMessage `json:"gated"`
+ Siblings []hfRepoFile `json:"siblings"`
+ // Config is the hub's parse of the root config.json.
+ Config *hfModelConfig `json:"config"`
+}
+
+// modelType is the repository's normalized root config.json model_type, empty
+// when the hub reports none.
+func (m hfRepoMeta) modelType() string {
+ if m.Config == nil {
+ return ""
+ }
+ return normalizeModelType(m.Config.ModelType)
+}
+
+// hfRepoFile is one file of a repository. Size is zero when unpublished.
+type hfRepoFile struct {
+ RFilename string `json:"rfilename"`
+ Size int64 `json:"size"`
+}
+
+// hfPlannedFile is one file to fetch and where it goes.
+type hfPlannedFile struct {
+ name string // repository path, "/" separated
+ size int64 // published size; zero when unknown
+ path string // destination inside the partial directory
+}
+
+// hfStatusError is an HTTP status from Hugging Face. A 5xx or 429 is worth
+// retrying; anything else is final.
+type hfStatusError struct {
+ file string
+ status int
+}
+
+func (e hfStatusError) Error() string {
+ return fmt.Sprintf("Hugging Face answered HTTP %d for %s", e.status, e.file)
+}
+
+// hfTransient reports a failure that a resumed retry can fix.
+func hfTransient(err error) bool {
+ var status hfStatusError
+ if errors.As(err, &status) {
+ return status.status >= 500 || status.status == http.StatusTooManyRequests
+ }
+ return !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded)
+}
+
+func (e *Executor) hfBase() string {
+ if e.hfBaseURL != "" {
+ return strings.TrimRight(e.hfBaseURL, "/")
+ }
+ return hfDefaultBaseURL
+}
+
+// pullHFSnapshot runs an hf_snapshot pull_model action for model, an
+// owner/name repository id, and returns {"model","files","bytes"}. A
+// repository the manifest lists is downloaded by its entry's rules.
+func (e *Executor) pullHFSnapshot(ctx context.Context, st *engineState, engine, model string, act Action) (json.RawMessage, error) {
+ if !hfRepoPattern.MatchString(model) {
+ return nil, fmt.Errorf("%q is not a Hugging Face repository id (owner/name)", model)
+ }
+ dest, err := st.resolveModelPath(act.HFSnapshot.Dest, model)
+ if err != nil {
+ return nil, err
+ }
+ entry, listed := st.manifest.catalogEntry(model)
+ marker, _, layout := st.manifest.modelStoreLayout()
+ if layout && hasMarker(dest, marker) {
+ // Complete but for a step: a copy downloaded before the step was listed,
+ // or one whose step failed after the files were in.
+ if err := e.runAfterDownload(ctx, st, model, entry.AfterDownload, dest); err != nil {
+ return nil, err
+ }
+ e.emitPullProgress(ProgressEvent{Engine: engine, Op: "pull", Stage: "success", Percent: 100, Message: model})
+ return json.Marshal(map[string]any{"model": model, "already": true})
+ }
+
+ // Two pulls of one model would write the same partial files.
+ if _, busy := e.snapshotPulls.LoadOrStore(dest, struct{}{}); busy {
+ return nil, fmt.Errorf("%s is already being downloaded", model)
+ }
+ defer e.snapshotPulls.Delete(dest)
+
+ pullCtx, watchdog := newPullProgressWatchdog(ctx, e.pullProgressTimeout)
+ defer watchdog.stop()
+ e.emitPullProgress(ProgressEvent{Engine: engine, Op: "pull", Stage: "pulling", Message: model})
+
+ meta, err := e.fetchHFRepoMeta(pullCtx, model, entry.Revision)
+ if err != nil {
+ return nil, err
+ }
+ // Checked before a byte is written: a model the engine has no code for
+ // would be hundreds of gigabytes that never start. A listed repository is
+ // the engine's own choice and is not second-guessed.
+ if !listed && !st.manifest.servesModelType(meta.modelType()) {
+ if meta.modelType() == "" {
+ return nil, fmt.Errorf("%s declares no model_type in its config.json, so %s cannot tell how to run it", model, st.manifest.DisplayName)
+ }
+ return nil, fmt.Errorf("%s is a %q model, which %s cannot run", model, meta.modelType(), st.manifest.DisplayName)
+ }
+ partial := dest + partialSuffix
+ files, total, err := planHFSnapshot(meta, partial, entry.Exclude)
+ if err != nil {
+ return nil, fmt.Errorf("%s cannot be downloaded: %w", model, err)
+ }
+ if layout && !hasRootFile(files, marker) {
+ return nil, fmt.Errorf("%s has no %s, so it is not a model %s can serve", model, marker, st.manifest.DisplayName)
+ }
+ if err := os.MkdirAll(partial, 0o755); err != nil {
+ return nil, fmt.Errorf("create the download folder: %w", err)
+ }
+ have := bytesOnDisk(files)
+ if free, err := freeDiskBytes(partial); err == nil {
+ if need := total - have + hfSnapshotSpareBytes; need > 0 && free < uint64(need) {
+ return nil, fmt.Errorf("not enough disk space for %s: it needs %s more and %s is free in %s",
+ model, formatGB(need), formatGB(int64(free)), st.modelsDir)
+ }
+ } else {
+ slog.Warn("cannot read free disk space; downloading without the check", "engine", engine, "err", err)
+ }
+
+ progress := &snapshotProgress{engine: engine, model: model, total: total, lastPct: -1, emit: e.emitPullProgress}
+ slog.Info("model download started", "engine", engine, "model", model, "files", len(files), "bytes", total)
+ for _, f := range files {
+ if err := e.downloadHFFile(pullCtx, model, entry.ref(), f, progress, watchdog); err != nil {
+ return nil, fmt.Errorf("download %s: %w", f.name, watchdog.resolveError(pullCtx, err))
+ }
+ }
+ // Before the rename, so the store never lists a model its engine cannot
+ // read yet.
+ if err := e.runAfterDownload(pullCtx, st, model, entry.AfterDownload, partial); err != nil {
+ return nil, err
+ }
+
+ // A directory at dest without the marker is an earlier, incomplete copy of
+ // this same model; the finished download replaces it.
+ if _, err := os.Lstat(dest); err == nil {
+ if err := os.RemoveAll(dest); err != nil {
+ return nil, fmt.Errorf("replace the incomplete copy of %s: %w", model, err)
+ }
+ }
+ if err := os.Rename(partial, dest); err != nil {
+ return nil, fmt.Errorf("move the downloaded model into place: %w", err)
+ }
+ slog.Info("model download finished", "engine", engine, "model", model)
+
+ // The first model an engine gets is the one it serves, so it can start.
+ st.mu.Lock()
+ current := st.plat.Runtime.Model
+ st.mu.Unlock()
+ if current == "" && runtimeReferencesModel(st.plat.Runtime) {
+ st.opMu.Lock()
+ if err := e.persistLaunchModel(engine, st, model); err != nil {
+ slog.Warn("could not save the downloaded model as the launch model", "engine", engine, "model", model, "err", err)
+ }
+ st.opMu.Unlock()
+ }
+ e.emitPullProgress(ProgressEvent{Engine: engine, Op: "pull", Stage: "success", Percent: 100, Message: model})
+ return json.Marshal(map[string]any{"model": model, "files": len(files), "bytes": total})
+}
+
+// fetchHFRepoMeta reads a repository's file list and access settings, at a
+// pinned revision when one is given, and refuses one PAIR cannot download
+// without credentials.
+func (e *Executor) fetchHFRepoMeta(ctx context.Context, repo, revision string) (hfRepoMeta, error) {
+ ctx, cancel := context.WithTimeout(ctx, catalogHTTPTimeout)
+ defer cancel()
+ endpoint := e.hfBase() + "/api/models/" + repo
+ if revision != "" {
+ endpoint += "/revision/" + url.PathEscape(revision)
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint+"?blobs=true", nil)
+ if err != nil {
+ return hfRepoMeta{}, fmt.Errorf("build the Hugging Face request: %w", withoutURL(err))
+ }
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("User-Agent", "PAIR/1.0")
+ resp, err := e.client.Do(req)
+ if err != nil {
+ return hfRepoMeta{}, fmt.Errorf("could not reach Hugging Face: %w", withoutURL(err))
+ }
+ defer resp.Body.Close()
+ switch {
+ case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound:
+ // Hugging Face answers 401 for a missing repo as well as a private one.
+ return hfRepoMeta{}, fmt.Errorf("%s was not found on Hugging Face, or it is private or gated", repo)
+ case resp.StatusCode != http.StatusOK:
+ return hfRepoMeta{}, fmt.Errorf("Hugging Face returned %s for %s", resp.Status, repo)
+ }
+ var meta hfRepoMeta
+ if err := json.NewDecoder(io.LimitReader(resp.Body, maxCatalogBody)).Decode(&meta); err != nil {
+ return hfRepoMeta{}, fmt.Errorf("read the file list of %s: %w", repo, err)
+ }
+ // Only an explicit false is open, the same rule as the catalog.
+ if meta.Private || strings.TrimSpace(string(meta.Gated)) != "false" {
+ return hfRepoMeta{}, fmt.Errorf("%s is private or gated on Hugging Face, and PAIR only downloads open models", repo)
+ }
+ return meta, nil
+}
+
+// planHFSnapshot maps a repository's files into partial, leaving out the
+// excluded ones and refusing any name that could land outside it or that the
+// filesystem cannot hold.
+func planHFSnapshot(meta hfRepoMeta, partial string, exclude []string) ([]hfPlannedFile, int64, error) {
+ var files []hfPlannedFile
+ var total int64
+ for _, sibling := range meta.Siblings {
+ name := sibling.RFilename
+ if name == "" || name == ".gitattributes" || excluded(name, exclude) {
+ continue
+ }
+ if !safeRepoFileName(name) {
+ return nil, 0, fmt.Errorf("the repository contains a file name that is not safe to write: %q", name)
+ }
+ path := filepath.Join(partial, filepath.FromSlash(name))
+ if path == filepath.Clean(partial) || !pathWithinRoot(filepath.Clean(partial), path) {
+ return nil, 0, fmt.Errorf("the repository contains a file name that is not safe to write: %q", name)
+ }
+ size := max(sibling.Size, 0)
+ files = append(files, hfPlannedFile{name: name, size: size, path: path})
+ total += size
+ }
+ if len(files) == 0 {
+ return nil, 0, errors.New("the repository has no files")
+ }
+ return files, total, nil
+}
+
+// safeRepoFileName accepts a relative "/"-separated path of plain names.
+func safeRepoFileName(name string) bool {
+ if strings.HasPrefix(name, "/") || strings.HasSuffix(name, ".part") {
+ return false
+ }
+ for _, segment := range strings.Split(name, "/") {
+ if segment == "" || segment == "." || segment == ".." || strings.TrimSpace(segment) != segment {
+ return false
+ }
+ for _, r := range segment {
+ if r < 0x20 || strings.ContainsRune(`\:*?"<>|`, r) {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+// hasRootFile reports a planned file directly at the repository root.
+func hasRootFile(files []hfPlannedFile, name string) bool {
+ for _, f := range files {
+ if f.name == name {
+ return true
+ }
+ }
+ return false
+}
+
+// bytesOnDisk counts what an earlier, interrupted pull already fetched.
+func bytesOnDisk(files []hfPlannedFile) int64 {
+ var have int64
+ for _, f := range files {
+ if fi, err := os.Stat(f.path); err == nil && (f.size == 0 || fi.Size() == f.size) {
+ have += fi.Size()
+ } else if fi, err := os.Stat(f.path + ".part"); err == nil && (f.size == 0 || fi.Size() <= f.size) {
+ have += fi.Size()
+ }
+ }
+ return have
+}
+
+// downloadHFFile fetches one file, skipping it when already complete and
+// retrying a transient failure in place, resumed from what was written.
+func (e *Executor) downloadHFFile(ctx context.Context, repo, ref string, f hfPlannedFile, progress *snapshotProgress, watchdog *pullProgressWatchdog) error {
+ if fi, err := os.Stat(f.path); err == nil && f.size > 0 && fi.Size() == f.size {
+ progress.add(f.size)
+ return nil
+ }
+ if err := os.MkdirAll(filepath.Dir(f.path), 0o755); err != nil {
+ return err
+ }
+ for attempt := 1; ; attempt++ {
+ err := e.fetchHFFileOnce(ctx, repo, ref, f, progress, watchdog)
+ if err == nil {
+ return nil
+ }
+ if attempt >= hfSnapshotAttempts || ctx.Err() != nil || !hfTransient(err) {
+ return err
+ }
+ slog.Warn("model file download failed; resuming", "model", repo, "attempt", attempt, "of", hfSnapshotAttempts, "err", err)
+ select {
+ case <-ctx.Done():
+ return err
+ case <-time.After(hfSnapshotRetryBackoff):
+ }
+ }
+}
+
+// fetchHFFileOnce makes one ranged request for the rest of a file at ref (a
+// branch or commit), appends it to the file's .part and renames that into place
+// once it is complete.
+func (e *Executor) fetchHFFileOnce(ctx context.Context, repo, ref string, f hfPlannedFile, progress *snapshotProgress, watchdog *pullProgressWatchdog) error {
+ part := f.path + ".part"
+ var offset int64
+ if fi, err := os.Stat(part); err == nil {
+ offset = fi.Size()
+ }
+ if f.size > 0 && offset > f.size {
+ // Larger than the published file: not a prefix of it, start over.
+ if err := os.Remove(part); err != nil {
+ return err
+ }
+ offset = 0
+ }
+ if f.size > 0 && offset == f.size {
+ progress.add(offset)
+ return os.Rename(part, f.path)
+ }
+
+ segments := strings.Split(f.name, "/")
+ for i, segment := range segments {
+ segments[i] = url.PathEscape(segment)
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, e.hfBase()+"/"+repo+"/resolve/"+url.PathEscape(ref)+"/"+strings.Join(segments, "/"), nil)
+ if err != nil {
+ return withoutURL(err)
+ }
+ req.Header.Set("User-Agent", "PAIR/1.0")
+ if offset > 0 {
+ req.Header.Set("Range", "bytes="+strconv.FormatInt(offset, 10)+"-")
+ }
+ resp, err := e.client.Do(req)
+ if err != nil {
+ return withoutURL(err)
+ }
+ defer resp.Body.Close()
+ flags := os.O_CREATE | os.O_WRONLY
+ switch {
+ case resp.StatusCode == http.StatusPartialContent && offset > 0 && rangeStartsAt(resp.Header.Get("Content-Range"), offset):
+ flags |= os.O_APPEND
+ case resp.StatusCode == http.StatusOK:
+ // The server sent the whole file: discard what was there.
+ offset = 0
+ flags |= os.O_TRUNC
+ case resp.StatusCode == http.StatusRequestedRangeNotSatisfiable:
+ _ = os.Remove(part)
+ return hfStatusError{file: f.name, status: http.StatusServiceUnavailable}
+ default:
+ return hfStatusError{file: f.name, status: resp.StatusCode}
+ }
+ out, err := os.OpenFile(part, flags, 0o644)
+ if err != nil {
+ return err
+ }
+ counted := &countingWriter{w: out, done: offset, onWrite: func(done int64, n int) {
+ watchdog.recordProgress(f.name, done)
+ progress.add(int64(n))
+ }}
+ progress.add(offset)
+ _, copyErr := io.Copy(counted, resp.Body)
+ closeErr := out.Close()
+ if copyErr != nil {
+ progress.add(-counted.done)
+ return copyErr
+ }
+ if closeErr != nil {
+ progress.add(-counted.done)
+ return closeErr
+ }
+ if f.size > 0 && counted.done != f.size {
+ progress.add(-counted.done)
+ return fmt.Errorf("%s arrived as %d bytes, expected %d", f.name, counted.done, f.size)
+ }
+ return os.Rename(part, f.path)
+}
+
+// rangeStartsAt reports a Content-Range that resumes at offset.
+func rangeStartsAt(contentRange string, offset int64) bool {
+ rest, ok := strings.CutPrefix(contentRange, "bytes ")
+ if !ok {
+ return false
+ }
+ start, _, ok := strings.Cut(rest, "-")
+ return ok && start == strconv.FormatInt(offset, 10)
+}
+
+// countingWriter tracks the bytes a file holds as they are written.
+type countingWriter struct {
+ w io.Writer
+ done int64
+ onWrite func(done int64, n int)
+}
+
+func (c *countingWriter) Write(p []byte) (int, error) {
+ n, err := c.w.Write(p)
+ c.done += int64(n)
+ if n > 0 {
+ c.onWrite(c.done, n)
+ }
+ return n, err
+}
+
+// snapshotProgress turns bytes into pull-progress frames, one per percent.
+type snapshotProgress struct {
+ engine, model string
+ total, done int64
+ lastPct int
+ emit func(ProgressEvent)
+}
+
+func (p *snapshotProgress) add(n int64) {
+ p.done += n
+ if p.total <= 0 {
+ return
+ }
+ // 100 is the success frame's, sent once the model is in place.
+ pct := int(min(max(p.done*100/p.total, 0), 99))
+ if pct == p.lastPct {
+ return
+ }
+ p.lastPct = pct
+ p.emit(ProgressEvent{Engine: p.engine, Op: "pull", Stage: "downloading", Percent: pct, Message: p.model})
+}
+
+// formatGB renders a byte count for an error message.
+func formatGB(n int64) string {
+ return fmt.Sprintf("%.1f GB", float64(n)/1e9)
+}
diff --git a/services/nvpair-engine-manager/hfsnapshot_test.go b/services/nvpair-engine-manager/hfsnapshot_test.go
new file mode 100644
index 000000000..d474a9924
--- /dev/null
+++ b/services/nvpair-engine-manager/hfsnapshot_test.go
@@ -0,0 +1,434 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// fakeHub is a stand-in for the Hugging Face hub: one repository's metadata
+// and its files, served with Range support.
+type fakeHub struct {
+ t *testing.T
+ repo string
+ files map[string][]byte
+ // revision, when set, is the only commit served: metadata at
+ // /api/models//revision/ and files at //resolve//.
+ revision string
+ // modelType, when set, is the config.json model_type the metadata reports.
+ modelType string
+ // meta overrides the metadata reply; empty means an open repo listing files.
+ meta string
+ status int // metadata status; zero means 200
+
+ mu sync.Mutex
+ ranges []string
+ fileHits atomic.Int32
+ failFirst map[string]int // file → how many requests answer 503 first
+ blockAfter map[string]int // file → bytes sent before the response stalls
+ stalled chan struct{}
+}
+
+func newFakeHub(t *testing.T, files map[string][]byte) *fakeHub {
+ t.Helper()
+ return &fakeHub{t: t, repo: "o/r", files: files, failFirst: map[string]int{}, blockAfter: map[string]int{}, stalled: make(chan struct{}, 1)}
+}
+
+func (h *fakeHub) metadata() string {
+ if h.meta != "" {
+ return h.meta
+ }
+ type sibling struct {
+ RFilename string `json:"rfilename"`
+ Size int `json:"size"`
+ }
+ siblings := []sibling{{RFilename: ".gitattributes", Size: 10}}
+ for name, data := range h.files {
+ siblings = append(siblings, sibling{RFilename: name, Size: len(data)})
+ }
+ meta := map[string]any{"private": false, "gated": false, "siblings": siblings}
+ if h.modelType != "" {
+ meta["config"] = map[string]string{"model_type": h.modelType}
+ }
+ out, _ := json.Marshal(meta)
+ return string(out)
+}
+
+func (h *fakeHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ metaPath, ref := "/api/models/"+h.repo, "main"
+ if h.revision != "" {
+ metaPath, ref = metaPath+"/revision/"+h.revision, h.revision
+ }
+ if r.URL.Path == metaPath {
+ if r.URL.Query().Get("blobs") != "true" {
+ h.t.Errorf("metadata requested without blobs=true: %s", r.URL)
+ }
+ if h.status != 0 {
+ w.WriteHeader(h.status)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ fmt.Fprint(w, h.metadata())
+ return
+ }
+ name, ok := strings.CutPrefix(r.URL.Path, "/"+h.repo+"/resolve/"+ref+"/")
+ data, found := h.files[name]
+ if !ok || !found {
+ http.NotFound(w, r)
+ return
+ }
+ h.fileHits.Add(1)
+ h.mu.Lock()
+ h.ranges = append(h.ranges, r.Header.Get("Range"))
+ fail := h.failFirst[name]
+ if fail > 0 {
+ h.failFirst[name] = fail - 1
+ }
+ block, blocking := h.blockAfter[name]
+ h.mu.Unlock()
+ if fail > 0 {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+ if blocking {
+ w.Header().Set("Content-Length", fmt.Sprint(len(data)))
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(data[:block])
+ w.(http.Flusher).Flush()
+ select {
+ case h.stalled <- struct{}{}:
+ default:
+ }
+ <-r.Context().Done()
+ return
+ }
+ http.ServeContent(w, r, name, time.Time{}, bytes.NewReader(data))
+}
+
+// snapshotEngine is the Colibri-shaped fake engine with an hf_snapshot pull,
+// pointed at a fake hub, recording every pull-progress frame.
+func snapshotEngine(t *testing.T, hub *fakeHub) (*Executor, string, func() []map[string]any) {
+ t.Helper()
+ e, store, _ := modelEngine(t)
+ st, _ := e.state("fake")
+ st.manifest.Actions[pullModelAction] = Action{HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}
+ srv := httptest.NewServer(hub)
+ t.Cleanup(srv.Close)
+ e.hfBaseURL = srv.URL
+ var mu sync.Mutex
+ var frames []map[string]any
+ e.emit = func(method string, params any) {
+ if method != "engine:pull-progress" {
+ return
+ }
+ mp, _ := params.(map[string]any)
+ mu.Lock()
+ frames = append(frames, mp)
+ mu.Unlock()
+ }
+ return e, store, func() []map[string]any {
+ mu.Lock()
+ defer mu.Unlock()
+ return append([]map[string]any(nil), frames...)
+ }
+}
+
+func colibriFiles() map[string][]byte {
+ return map[string][]byte{
+ "config.json": []byte(`{"model_type":"qwen3_moe"}`),
+ "model-00000.safetensors": bytes.Repeat([]byte("w"), 64*1024),
+ "tokenizer/vocab.json": []byte(`{"a":1}`),
+ }
+}
+
+func TestHFSnapshotDownloadsIntoStore(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, store, frames := snapshotEngine(t, hub)
+
+ raw, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var res struct {
+ Model string `json:"model"`
+ Files int `json:"files"`
+ Bytes int64 `json:"bytes"`
+ }
+ if err := json.Unmarshal(raw, &res); err != nil || res.Model != "o/r" || res.Files != 3 {
+ t.Fatalf("result %s (err %v)", raw, err)
+ }
+ for name, want := range hub.files {
+ got, err := os.ReadFile(filepath.Join(store, "o", "r", filepath.FromSlash(name)))
+ if err != nil || !bytes.Equal(got, want) {
+ t.Fatalf("%s: %v, content match %t", name, err, bytes.Equal(got, want))
+ }
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r", ".gitattributes")); !os.IsNotExist(err) {
+ t.Errorf(".gitattributes was downloaded: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r"+partialSuffix)); !os.IsNotExist(err) {
+ t.Errorf("the partial directory was left behind: %v", err)
+ }
+ // The first model downloaded becomes the launch model.
+ if st, _ := e.state("fake"); st.plat.Runtime.Model != "o/r" {
+ t.Errorf("launch model = %q, want the first download", st.plat.Runtime.Model)
+ }
+
+ got := frames()
+ if len(got) < 3 {
+ t.Fatalf("frames = %v", got)
+ }
+ if got[0]["stage"] != "pulling" || got[len(got)-1]["stage"] != "success" || got[len(got)-1]["percent"] != 100 {
+ t.Fatalf("frames run %v … %v, want pulling … success 100", got[0], got[len(got)-1])
+ }
+ last := -1
+ for _, f := range got[1 : len(got)-1] {
+ pct, _ := f["percent"].(int)
+ if f["stage"] != "downloading" || pct <= last || pct > 99 {
+ t.Fatalf("progress frames are not one increasing frame per percent: %v", got)
+ }
+ last = pct
+ }
+
+ // A second pull of a complete model touches nothing.
+ hits := hub.fileHits.Load()
+ raw, err = e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err != nil || !strings.Contains(string(raw), `"already":true`) || hub.fileHits.Load() != hits {
+ t.Fatalf("repeat pull = %s, %v; downloads %d → %d", raw, err, hits, hub.fileHits.Load())
+ }
+}
+
+func TestHFSnapshotResumesPartialFile(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, store, _ := snapshotEngine(t, hub)
+ weights := hub.files["model-00000.safetensors"]
+ half := len(weights) / 2
+ partial := filepath.Join(store, "o", "r"+partialSuffix)
+ if err := os.MkdirAll(partial, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(partial, "model-00000.safetensors.part"), weights[:half], 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatal(err)
+ }
+ got, err := os.ReadFile(filepath.Join(store, "o", "r", "model-00000.safetensors"))
+ if err != nil || !bytes.Equal(got, weights) {
+ t.Fatalf("resumed file differs (err %v)", err)
+ }
+ hub.mu.Lock()
+ defer hub.mu.Unlock()
+ if !slicesContains(hub.ranges, fmt.Sprintf("bytes=%d-", half)) {
+ t.Fatalf("no ranged request resumed at %d: %v", half, hub.ranges)
+ }
+}
+
+func slicesContains(list []string, want string) bool {
+ for _, s := range list {
+ if s == want {
+ return true
+ }
+ }
+ return false
+}
+
+func TestHFSnapshotRetriesTransientFailure(t *testing.T) {
+ defer func(backoff time.Duration) { hfSnapshotRetryBackoff = backoff }(hfSnapshotRetryBackoff)
+ hfSnapshotRetryBackoff = 0
+ hub := newFakeHub(t, colibriFiles())
+ hub.failFirst["model-00000.safetensors"] = 2
+ e, _, _ := snapshotEngine(t, hub)
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatalf("two 503s within the retry budget failed the pull: %v", err)
+ }
+}
+
+func TestHFSnapshotRefusals(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ model string
+ setup func(*fakeHub)
+ wanted string
+ }{
+ {"not a repo id", "../x", nil, "is not a Hugging Face repository id"},
+ {"gated", "o/r", func(h *fakeHub) {
+ h.meta = `{"private":false,"gated":"manual","siblings":[{"rfilename":"config.json","size":2}]}`
+ }, "private or gated"},
+ {"private", "o/r", func(h *fakeHub) {
+ h.meta = `{"private":true,"gated":false,"siblings":[{"rfilename":"config.json","size":2}]}`
+ }, "private or gated"},
+ {"missing or unauthorized", "o/r", func(h *fakeHub) { h.status = http.StatusUnauthorized }, "not found on Hugging Face"},
+ {"traversal", "o/r", func(h *fakeHub) {
+ h.meta = `{"private":false,"gated":false,"siblings":[{"rfilename":"config.json","size":2},{"rfilename":"../evil","size":1}]}`
+ }, "not safe to write"},
+ {"drive letter", "o/r", func(h *fakeHub) {
+ h.meta = `{"private":false,"gated":false,"siblings":[{"rfilename":"config.json","size":2},{"rfilename":"C:/evil","size":1}]}`
+ }, "not safe to write"},
+ {"not a model", "o/r", func(h *fakeHub) {
+ h.meta = `{"private":false,"gated":false,"siblings":[{"rfilename":"README.md","size":2}]}`
+ }, "has no config.json"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ if tc.setup != nil {
+ tc.setup(hub)
+ }
+ e, store, _ := snapshotEngine(t, hub)
+ _, err := e.PullModelStream(context.Background(), "fake", tc.model, nil)
+ if err == nil || !strings.Contains(err.Error(), tc.wanted) {
+ t.Fatalf("pull = %v, want an error containing %q", err, tc.wanted)
+ }
+ if hub.fileHits.Load() != 0 {
+ t.Fatal("a refused pull downloaded files")
+ }
+ if _, err := os.Stat(filepath.Join(filepath.Dir(store), "evil")); err == nil {
+ t.Fatal("a file escaped the store")
+ }
+ })
+ }
+}
+
+func TestHFSnapshotChecksDiskSpace(t *testing.T) {
+ defer func(f func(string) (uint64, error)) { freeDiskBytes = f }(freeDiskBytes)
+ freeDiskBytes = func(string) (uint64, error) { return 1 << 20, nil }
+ hub := newFakeHub(t, colibriFiles())
+ e, _, _ := snapshotEngine(t, hub)
+ _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err == nil || !strings.Contains(err.Error(), "not enough disk space") {
+ t.Fatalf("pull on a full disk = %v", err)
+ }
+ if hub.fileHits.Load() != 0 {
+ t.Fatal("a pull refused for disk space downloaded files")
+ }
+}
+
+func TestHFSnapshotCancelKeepsPartial(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ hub.blockAfter["model-00000.safetensors"] = 1000
+ e, store, _ := snapshotEngine(t, hub)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan error, 1)
+ go func() {
+ _, err := e.PullModelStream(ctx, "fake", "o/r", nil)
+ done <- err
+ }()
+ select {
+ case <-hub.stalled:
+ case <-time.After(10 * time.Second):
+ t.Fatal("the download never started")
+ }
+ // The hub has sent its bytes, but the client may not have written them
+ // yet; cancelling now would measure that race rather than the resume.
+ partPath := filepath.Join(store, "o", "r"+partialSuffix, "model-00000.safetensors.part")
+ for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(5 * time.Millisecond) {
+ if fi, err := os.Stat(partPath); err == nil && fi.Size() == 1000 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the client never wrote the bytes it was sent")
+ }
+ }
+ cancel()
+ select {
+ case err := <-done:
+ if err == nil || !errors.Is(err, context.Canceled) {
+ t.Fatalf("cancelled pull = %v, want context.Canceled", err)
+ }
+ case <-time.After(10 * time.Second):
+ t.Fatal("the pull ignored cancellation")
+ }
+ part, err := os.Stat(partPath)
+ if err != nil || part.Size() != 1000 {
+ t.Fatalf("the partial file was not kept for resume: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r")); !os.IsNotExist(err) {
+ t.Fatal("a cancelled download was moved into the store")
+ }
+ if models := scanModelDirs(store, "config.json", 2); len(models) != 0 {
+ t.Fatalf("a cancelled download is listed: %v", models)
+ }
+}
+
+// A remote initiator gets the same progress frames over the ec stream, then
+// the result.
+func TestHFSnapshotStreamsToRemoteInitiator(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, _, _ := snapshotEngine(t, hub)
+ s := &controlServer{exec: e}
+ srv := httptest.NewServer(http.HandlerFunc(s.handlePull))
+ t.Cleanup(srv.Close)
+ resp, err := srv.Client().Post(srv.URL, "application/json", strings.NewReader(`{"opId":"op","engine":"fake","model":"o/r"}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ var types []string
+ var stages []string
+ sc := bufio.NewScanner(resp.Body)
+ for sc.Scan() {
+ var frame streamFrame
+ if err := json.Unmarshal(sc.Bytes(), &frame); err != nil {
+ t.Fatal(err)
+ }
+ types = append(types, frame.Type)
+ if frame.Type == "progress" {
+ stages = append(stages, frame.Stage)
+ }
+ }
+ if len(types) == 0 || types[len(types)-1] != "result" {
+ t.Fatalf("stream frames %v, want progress then result", types)
+ }
+ if !reflect.DeepEqual(stages[:1], []string{"pulling"}) || stages[len(stages)-1] != "success" {
+ t.Fatalf("relayed stages %v", stages)
+ }
+}
+
+func TestHFSnapshotOnlyOnePullPerModel(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ hub.blockAfter["model-00000.safetensors"] = 10
+ e, _, _ := snapshotEngine(t, hub)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan struct{})
+ go func() {
+ _, _ = e.PullModelStream(ctx, "fake", "o/r", nil)
+ close(done)
+ }()
+ <-hub.stalled
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err == nil || !strings.Contains(err.Error(), "already being downloaded") {
+ t.Fatalf("concurrent pull = %v", err)
+ }
+ cancel()
+ <-done
+}
+
+func TestSafeRepoFileName(t *testing.T) {
+ for _, name := range []string{"config.json", "sub/dir/file.safetensors", "a b.txt"} {
+ if !safeRepoFileName(name) {
+ t.Errorf("safeRepoFileName(%q) = false", name)
+ }
+ }
+ for _, name := range []string{"", "/abs", "a//b", "../x", "a/../b", "./x", "C:/x", `a\b`, "x.part", " lead", "a/b?", "nul\x00"} {
+ if safeRepoFileName(name) {
+ t.Errorf("safeRepoFileName(%q) = true", name)
+ }
+ }
+}
diff --git a/services/nvpair-engine-manager/launch_controls_test.go b/services/nvpair-engine-manager/launch_controls_test.go
index 7427a3ab1..09202c4c1 100644
--- a/services/nvpair-engine-manager/launch_controls_test.go
+++ b/services/nvpair-engine-manager/launch_controls_test.go
@@ -112,7 +112,7 @@ func TestBundledLlamaCPP_DisablesCORSByDefault(t *testing.T) {
func TestBundledManifests_DeclareReviewedNetworkingControls(t *testing.T) {
reg := loadWithOverrides(t, t.TempDir())
// Adding a bundled engine requires an explicit networking review and cases.
- wantEngines := []string{"llamacpp", "lmstudio", "ollama"}
+ wantEngines := []string{"colibri", "llamacpp", "lmstudio", "ollama"}
names := reg.Names()
slices.Sort(names)
assert.Equal(t, wantEngines, names, "review networking controls for every bundled engine")
@@ -135,6 +135,14 @@ func TestBundledManifests_DeclareReviewedNetworkingControls(t *testing.T) {
require.Equal(t, []LaunchControl{{Value: "{server.host}", Flags: []string{"--host"}}, {Value: "{server.port}", Flags: []string{"--port"}}, {Value: "{cors.origins}", Flags: []string{"--cors-origins"}}}, policy.Controls, "incomplete llama.cpp controls")
valid = []string{"--host 127.0.0.1 --port 23456", "--host=127.0.0.1 --port=23456", "--port 23456 --cors-origins https://example.test"}
invalid = []string{"--host 0.0.0.0", "--host=::", "--port 0", "--port 65536", "--port", "--cors-origins=*", "--port 23456 --port 23457", "-- --host 0.0.0.0"}
+ case "colibri":
+ // Colibri's --cors-origin takes one origin per flag, which the
+ // comma-separated cors.origins field cannot express, so it has
+ // no CORS control and its defaults stay in force.
+ require.Equal(t, []string{"serve"}, policy.FixedArgs, "colibri does not start its server subcommand")
+ require.Equal(t, []LaunchControl{{Value: "{server.host}", Flags: []string{"--host"}}, {Value: "{server.port}", Flags: []string{"--port"}}}, policy.Controls, "incomplete Colibri controls")
+ valid = []string{"--host 127.0.0.1 --port 23456", "--host=127.0.0.1 --port=23456", "--port 23456 --kv-slots 2"}
+ invalid = []string{"--host 0.0.0.0", "--host=::", "--port 0", "--port 65536", "--port", "--port 23456 --port 23457", "-- --host 0.0.0.0"}
default:
require.Equal(t, []LaunchControl{{Value: "{server.host}:{server.port}", Env: []string{"OLLAMA_HOST"}}, {Value: "{cors.origins}", Env: []string{"OLLAMA_ORIGINS"}}}, policy.Controls, "incomplete Ollama controls")
valid = []string{`OLLAMA_HOST="127.0.0.1:23456"`, `OLLAMA_HOST='127.0.0.1:23456'`, "OLLAMA_HOST=127.0.0.1:23456"}
diff --git a/services/nvpair-engine-manager/launch_test.go b/services/nvpair-engine-manager/launch_test.go
index 7c1a3b561..20bc3bce0 100644
--- a/services/nvpair-engine-manager/launch_test.go
+++ b/services/nvpair-engine-manager/launch_test.go
@@ -24,8 +24,8 @@ func (command launchCommand) text() (string, error) {
func TestResolvedLaunch_MatchesBundledEngines(t *testing.T) {
reg := loadWithOverrides(t, t.TempDir())
- vars := map[string]string{"host": "127.0.0.1", "port": "12345", "cli": "/test path/lms", "install_dir": "/test path", "models_dir": "/test models"}
- for _, engine := range []string{"ollama", "lmstudio", "llamacpp"} {
+ vars := map[string]string{"host": "127.0.0.1", "port": "12345", "cli": "/test path/lms", "install_dir": "/test path", "models_dir": "/test models", "model": "o/r"}
+ for _, engine := range []string{"ollama", "lmstudio", "llamacpp", "colibri"} {
manifest, ok := reg.Get(engine)
require.True(t, ok, "missing bundled engine (%v)", engine)
// Exercise every platform even when running on Windows or macOS, so
@@ -44,6 +44,16 @@ func TestResolvedLaunch_MatchesBundledEngines(t *testing.T) {
} else if engine == "lmstudio" {
launch, err = resolveCommandLaunch(platform.Runtime.Start[0], vars)
want = []string{"/test path/lms", "server", "start", "--port", "12345", "--bind", "127.0.0.1"}
+ } else if engine == "colibri" {
+ var bin string
+ bin, err = resolvePlaceholders(platform.Runtime.Bin, vars)
+ if err == nil {
+ launch, err = resolveProcessLaunch(platform.Runtime, bin, vars)
+ }
+ want = []string{"COLI_MODEL=/test models/o/r", "COLI_MODEL_ID=o/r", bin, "serve", "--host", "127.0.0.1", "--port", "12345"}
+ if strings.HasPrefix(platformKey, "darwin/") {
+ want = append([]string{"COLI_METAL=1"}, want...)
+ }
} else {
var bin string
bin, err = resolvePlaceholders(platform.Runtime.Bin, vars)
diff --git a/services/nvpair-engine-manager/lifecycle.go b/services/nvpair-engine-manager/lifecycle.go
index ab2749ef9..1dc78563e 100644
--- a/services/nvpair-engine-manager/lifecycle.go
+++ b/services/nvpair-engine-manager/lifecycle.go
@@ -103,6 +103,30 @@ func (e *Executor) StartWith(ctx context.Context, engine string, opts startOpts)
return e.setDesiredEnabled(engine, true)
}
+// StartAfterInstall starts a freshly installed engine, unless it serves one
+// model per process and none is downloaded yet. Such an engine cannot run
+// before its first model arrives, and right after an install that is the
+// expected state rather than a failed start: it stays stopped, nothing is
+// reported, and its saved ON/OFF intent is left alone, so the first start after
+// a download is the user's. An explicit engine:start still refuses with the
+// "no model is downloaded" error.
+func (e *Executor) StartAfterInstall(ctx context.Context, engine string, opts startOpts) error {
+ st, err := e.state(engine)
+ if err != nil {
+ return err
+ }
+ st.mu.Lock()
+ rt := st.plat.Runtime
+ st.mu.Unlock()
+ if runtimeReferencesModel(rt) {
+ if _, err := st.launchModel(rt); errors.Is(err, errNoLaunchModel) {
+ slog.Info("installed; the engine starts once a model is downloaded", "engine", engine)
+ return nil
+ }
+ }
+ return e.StartWith(ctx, engine, opts)
+}
+
// doStart performs the start. Callers must hold st.opMu.
func (e *Executor) doStart(ctx context.Context, st *engineState, engine string, opts startOpts) error {
ctx, cancel := context.WithCancel(ctx)
@@ -175,11 +199,12 @@ func (e *Executor) doStart(ctx context.Context, st *engineState, engine string,
if !pathInstalled {
return fmt.Errorf("engine %q is not installed", engine)
}
- vars := st.pathVars()
- vars["host"] = effectiveBind(rt.Bind, opts.Bind)
- vars["port"] = strconv.Itoa(port)
- if rt.CLI != "" {
- vars["cli"] = expandPath(rt.CLI)
+ vars, err := st.launchVars(rt, effectiveBind(rt.Bind, opts.Bind), port)
+ if err != nil {
+ werr := fmt.Errorf("cannot start %s: %w", st.manifest.DisplayName, err)
+ e.reportStartFailedUnlessShuttingDown(ctx, engine, werr)
+ e.emitState(engine)
+ return werr
}
st.mu.Lock()
diff --git a/services/nvpair-engine-manager/manager.go b/services/nvpair-engine-manager/manager.go
index af41f9ddb..220371135 100644
--- a/services/nvpair-engine-manager/manager.go
+++ b/services/nvpair-engine-manager/manager.go
@@ -99,6 +99,8 @@ func NewManager(codec *Codec, exec *Executor, mesh *clustertrust.Mesh) *Manager
}
m.settingsRelay.send = codec.Notify
exec.settingsParent = m.settingsRelay.call
+ m.catalog.listed = exec.catalogEntries
+ m.catalog.modelTypes = exec.servedModelTypes
return m
}
@@ -323,7 +325,7 @@ func (m *Manager) runOp(ctx context.Context, msg *Message) {
switch msg.Method {
case "engine:install":
if err = m.exec.Install(ctx, p.Engine); err == nil && p.Start {
- err = start()
+ err = m.exec.StartAfterInstall(ctx, p.Engine, startOpts{Port: p.Port, Bind: p.Bind})
}
case "engine:uninstall":
err = m.exec.Uninstall(ctx, p.Engine)
diff --git a/services/nvpair-engine-manager/manifestcatalog.go b/services/nvpair-engine-manager/manifestcatalog.go
new file mode 100644
index 000000000..bdf4d62a4
--- /dev/null
+++ b/services/nvpair-engine-manager/manifestcatalog.go
@@ -0,0 +1,221 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+// manifestcatalog.go is a manifest's own model list: the models an engine's
+// maintainers publish for it by repository rather than by tag. Colibri's
+// release carries such a list (its `coli setup` catalog), and several of its
+// models are the vendors' original checkpoints, which no Hugging Face tag
+// search finds. engine:catalog shows these entries beside the tagged ones, and
+// an hf_snapshot pull of one of these repositories follows the entry's rules:
+// a pinned revision, files the engine never reads, and steps that finish the
+// model directory once the files are in.
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "path"
+ "regexp"
+ "strings"
+)
+
+// CatalogEntry is one model a manifest lists: a public Hugging Face repository
+// and how to download it.
+type CatalogEntry struct {
+ Repo string `json:"repo"`
+ Name string `json:"name"`
+ // Parameters is a display label such as "284B".
+ Parameters string `json:"parameters,omitempty"`
+ // SizeGB is the download size in gigabytes (10^9 bytes), for display. The
+ // pull measures the real size from the repository.
+ SizeGB float64 `json:"size_gb,omitempty"`
+ // Revision pins the commit downloaded. Empty is the main branch.
+ Revision string `json:"revision,omitempty"`
+ // Exclude lists repository paths that are not downloaded, as shell patterns
+ // in which "*" also matches "/": the rule of Python's fnmatch, which
+ // Colibri's own downloader applies to the same lists.
+ Exclude []string `json:"exclude,omitempty"`
+ // AfterDownload finishes the model directory before it is listed.
+ AfterDownload []AfterDownloadStep `json:"after_download,omitempty"`
+}
+
+// AfterDownloadStep runs a Python 3 script from the engine's install directory
+// with the model directory as its one argument. It is done once the file it
+// creates is in the model directory, so a step is never run twice.
+type AfterDownloadStep struct {
+ // Python is the script, relative to the install directory, "/" separated.
+ Python string `json:"python"`
+ // Creates is the file the script writes in the model directory.
+ Creates string `json:"creates"`
+}
+
+// commitPattern is a full git commit id, the only revision an entry may pin: a
+// branch or tag can move, and then the listed size and rules no longer match.
+var commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
+
+// modelTypePattern is a normalized config.json model_type.
+var modelTypePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`)
+
+// normalizeModelType lowercases and trims a model_type, the way Colibri's
+// family registry compares them.
+func normalizeModelType(modelType string) string {
+ return strings.ToLower(strings.TrimSpace(modelType))
+}
+
+// servesModelType reports a model_type the engine runs. A manifest that
+// declares no model_types runs whatever it is given, as far as PAIR can tell.
+func (m *Manifest) servesModelType(modelType string) bool {
+ if len(m.ModelTypes) == 0 {
+ return true
+ }
+ modelType = normalizeModelType(modelType)
+ for _, served := range m.ModelTypes {
+ if served == modelType {
+ return true
+ }
+ }
+ return false
+}
+
+// validateModelTypes checks the model_types list. They are what a downloaded
+// repository is held to, so they only mean something with an hf_snapshot pull.
+func (m *Manifest) validateModelTypes() error {
+ if len(m.ModelTypes) == 0 {
+ return nil
+ }
+ if act, ok := m.Actions[pullModelAction]; !ok || act.HFSnapshot == nil {
+ return errors.New("model_types requires a pull_model action with hf_snapshot")
+ }
+ seen := make(map[string]bool, len(m.ModelTypes))
+ for _, modelType := range m.ModelTypes {
+ if !modelTypePattern.MatchString(modelType) {
+ return fmt.Errorf("model_types: %q must be a lowercase config.json model_type", modelType)
+ }
+ if seen[modelType] {
+ return fmt.Errorf("model_types: %q is listed twice", modelType)
+ }
+ seen[modelType] = true
+ }
+ return nil
+}
+
+// ref is the revision to download.
+func (c CatalogEntry) ref() string {
+ if c.Revision == "" {
+ return "main"
+ }
+ return c.Revision
+}
+
+// catalogEntry returns the listed entry for a repository.
+func (m *Manifest) catalogEntry(repo string) (CatalogEntry, bool) {
+ for _, entry := range m.Catalog {
+ if entry.Repo == repo {
+ return entry, true
+ }
+ }
+ return CatalogEntry{}, false
+}
+
+// validateCatalog checks the manifest's model list. A list only means something
+// to an engine that downloads models by repository.
+func (m *Manifest) validateCatalog() error {
+ if len(m.Catalog) == 0 {
+ return nil
+ }
+ if act, ok := m.Actions[pullModelAction]; !ok || act.HFSnapshot == nil {
+ return errors.New("catalog requires a pull_model action with hf_snapshot")
+ }
+ seen := make(map[string]bool, len(m.Catalog))
+ for i, entry := range m.Catalog {
+ if !hfRepoPattern.MatchString(entry.Repo) {
+ return fmt.Errorf("catalog[%d]: repo %q is not a Hugging Face repository id (owner/name)", i, entry.Repo)
+ }
+ if seen[entry.Repo] {
+ return fmt.Errorf("catalog[%d]: %s is listed twice", i, entry.Repo)
+ }
+ seen[entry.Repo] = true
+ if strings.TrimSpace(entry.Name) == "" {
+ return fmt.Errorf("catalog %s: name is required", entry.Repo)
+ }
+ if entry.SizeGB < 0 || math.IsNaN(entry.SizeGB) || math.IsInf(entry.SizeGB, 0) {
+ return fmt.Errorf("catalog %s: size_gb must be a size in gigabytes", entry.Repo)
+ }
+ if entry.Revision != "" && !commitPattern.MatchString(entry.Revision) {
+ return fmt.Errorf("catalog %s: revision must be a full commit id", entry.Repo)
+ }
+ for _, pattern := range entry.Exclude {
+ if !validRepoPattern(pattern) {
+ return fmt.Errorf("catalog %s: exclude pattern %q must be a relative repository path", entry.Repo, pattern)
+ }
+ }
+ for _, step := range entry.AfterDownload {
+ if !safeRepoFileName(step.Python) || path.Ext(step.Python) != ".py" {
+ return fmt.Errorf("catalog %s: after_download python %q must be a .py path inside the install directory", entry.Repo, step.Python)
+ }
+ if !modelSegmentPattern.MatchString(step.Creates) {
+ return fmt.Errorf("catalog %s: after_download creates %q must be a file name", entry.Repo, step.Creates)
+ }
+ }
+ }
+ return nil
+}
+
+// validRepoPattern accepts a pattern over "/"-separated repository paths that
+// cannot climb out of the repository.
+func validRepoPattern(pattern string) bool {
+ if strings.TrimSpace(pattern) != pattern || pattern == "" || strings.HasPrefix(pattern, "/") || strings.Contains(pattern, `\`) {
+ return false
+ }
+ for _, segment := range strings.Split(pattern, "/") {
+ if segment == "" || segment == "." || segment == ".." {
+ return false
+ }
+ }
+ _, err := fnmatchRegexp(pattern)
+ return err == nil
+}
+
+// excluded reports a repository path that one of the patterns matches.
+func excluded(name string, patterns []string) bool {
+ for _, pattern := range patterns {
+ if re, err := fnmatchRegexp(pattern); err == nil && re.MatchString(name) {
+ return true
+ }
+ }
+ return false
+}
+
+// fnmatchRegexp translates a shell pattern the way Python's fnmatch does: "*"
+// is any run of characters, "/" included; "?" is one character; "[...]" is a
+// class, "[!...]" a negated one. Everything else is literal.
+func fnmatchRegexp(pattern string) (*regexp.Regexp, error) {
+ var b strings.Builder
+ b.WriteString("^")
+ for i := 0; i < len(pattern); i++ {
+ switch ch := pattern[i]; ch {
+ case '*':
+ b.WriteString(".*")
+ case '?':
+ b.WriteString(".")
+ case '[':
+ end := strings.IndexByte(pattern[i+1:], ']')
+ if end < 0 {
+ b.WriteString(`\[`)
+ continue
+ }
+ class := pattern[i+1 : i+1+end]
+ if negated, ok := strings.CutPrefix(class, "!"); ok {
+ class = "^" + negated
+ }
+ b.WriteString("[" + strings.ReplaceAll(class, `\`, `\\`) + "]")
+ i += end + 1
+ default:
+ b.WriteString(regexp.QuoteMeta(string(ch)))
+ }
+ }
+ b.WriteString("$")
+ return regexp.Compile(b.String())
+}
diff --git a/services/nvpair-engine-manager/manifestcatalog_test.go b/services/nvpair-engine-manager/manifestcatalog_test.go
new file mode 100644
index 000000000..2428e03dd
--- /dev/null
+++ b/services/nvpair-engine-manager/manifestcatalog_test.go
@@ -0,0 +1,352 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// listedEngine is snapshotEngine with o/r listed in the manifest's catalog.
+func listedEngine(t *testing.T, hub *fakeHub, entry CatalogEntry) (*Executor, *engineState, string) {
+ t.Helper()
+ e, store, _ := snapshotEngine(t, hub)
+ st, _ := e.state("fake")
+ entry.Repo, entry.Name = "o/r", "Listed"
+ st.manifest.Catalog = []CatalogEntry{entry}
+ if err := st.manifest.Validate(); err != nil {
+ t.Fatalf("listed manifest: %v", err)
+ }
+ return e, st, store
+}
+
+// fakeTool installs a script the fake interpreter follows (see fakePython in
+// testdata/fakeengine) and makes that interpreter the only Python found.
+func fakeTool(t *testing.T, st *engineState, name, script string) {
+ t.Helper()
+ path := filepath.Join(st.installDir, filepath.FromSlash(name))
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(script), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("PAIR_FAKE_PYTHON", "1")
+ saved := pythonCandidates
+ pythonCandidates = [][]string{{fakeEngineBin}}
+ t.Cleanup(func() { pythonCandidates = saved })
+}
+
+func TestValidateCatalogRejects(t *testing.T) {
+ sha := strings.Repeat("a", 40)
+ cases := map[string]CatalogEntry{
+ "not a repository id": {Repo: "just-a-name", Name: "x"},
+ "no name": {Repo: "o/r"},
+ "negative size": {Repo: "o/r", Name: "x", SizeGB: -1},
+ "branch revision": {Repo: "o/r", Name: "x", Revision: "main"},
+ "short commit": {Repo: "o/r", Name: "x", Revision: sha[:12]},
+ "exclude climbs out": {Repo: "o/r", Name: "x", Exclude: []string{"../x"}},
+ "exclude absolute": {Repo: "o/r", Name: "x", Exclude: []string{"/etc/*"}},
+ "exclude backslash": {Repo: "o/r", Name: "x", Exclude: []string{`a\b`}},
+ "step not python": {Repo: "o/r", Name: "x", AfterDownload: []AfterDownloadStep{{Python: "tools/run.sh", Creates: "t.json"}}},
+ "step climbs out": {Repo: "o/r", Name: "x", AfterDownload: []AfterDownloadStep{{Python: "../evil.py", Creates: "t.json"}}},
+ "step absolute": {Repo: "o/r", Name: "x", AfterDownload: []AfterDownloadStep{{Python: "/tmp/evil.py", Creates: "t.json"}}},
+ "creates a nested path": {Repo: "o/r", Name: "x", AfterDownload: []AfterDownloadStep{{Python: "tools/t.py", Creates: "sub/t.json"}}},
+ }
+ for name, entry := range cases {
+ m := testEngineManifest(fakeEngineBin)
+ m.Actions = map[string]Action{pullModelAction: {HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}}
+ m.Catalog = []CatalogEntry{entry}
+ if err := m.Validate(); err == nil {
+ t.Errorf("%s: accepted %+v", name, entry)
+ }
+ }
+
+ m := testEngineManifest(fakeEngineBin)
+ m.Catalog = []CatalogEntry{{Repo: "o/r", Name: "x"}}
+ if err := m.Validate(); err == nil || !strings.Contains(err.Error(), "hf_snapshot") {
+ t.Errorf("a catalog without an hf_snapshot pull was accepted: %v", err)
+ }
+
+ m = testEngineManifest(fakeEngineBin)
+ m.Actions = map[string]Action{pullModelAction: {HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}}
+ m.Catalog = []CatalogEntry{{Repo: "o/r", Name: "x"}, {Repo: "o/r", Name: "y"}}
+ if err := m.Validate(); err == nil {
+ t.Error("a repository listed twice was accepted")
+ }
+ m.Catalog = []CatalogEntry{{Repo: "o/r", Name: "x", Revision: sha, Exclude: []string{"dflash/*"},
+ AfterDownload: []AfterDownloadStep{{Python: "tools/k3_tokenizer.py", Creates: "tokenizer.json"}}}}
+ if err := m.Validate(); err != nil {
+ t.Errorf("a full entry was refused: %v", err)
+ }
+}
+
+// TestExcludedFollowsFnmatch checks exclusion matches as Python's fnmatch does,
+// so a list copied from Colibri's own catalog means the same thing here: "*"
+// crosses "/", the rest is literal.
+func TestExcludedFollowsFnmatch(t *testing.T) {
+ patterns := []string{"dflash/*", "audio_tokenizer/*", "model_mtp.safetensors", "docs/[!r]*.md", "shard-?.bin"}
+ for name, want := range map[string]bool{
+ "dflash/model.safetensors": true,
+ "dflash/deep/inside.bin": true,
+ "audio_tokenizer/config.json": true,
+ "model_mtp.safetensors": true,
+ "docs/guide.md": true,
+ "docs/readme.md": false,
+ "shard-1.bin": true,
+ "shard-10.bin": false,
+ "model-00001.safetensors": false,
+ "config.json": false,
+ "other/dflash/model.safetensors": false,
+ "model_mtpXsafetensors": false,
+ } {
+ if got := excluded(name, patterns); got != want {
+ t.Errorf("excluded(%q) = %t, want %t", name, got, want)
+ }
+ }
+}
+
+// TestBundledColibriCatalog checks the shipped list: every Colibri text model
+// is listed under its repository with a size, Qwen3.8 is pinned, MiMo skips the
+// files its engine never loads, and Kimi K3 writes the tokenizer.json its
+// engine reads.
+func TestBundledColibriCatalog(t *testing.T) {
+ m, ok := buildBundledRegistry().Get("colibri")
+ if !ok {
+ t.Fatal("no colibri manifest")
+ }
+ if len(m.Catalog) < 11 {
+ t.Fatalf("catalog has %d entries, want Colibri's eleven text models", len(m.Catalog))
+ }
+ for _, entry := range m.Catalog {
+ if entry.SizeGB <= 0 || entry.Parameters == "" {
+ t.Errorf("%s lists no size or parameter count", entry.Repo)
+ }
+ }
+ for repo, check := range map[string]func(CatalogEntry) bool{
+ "deepseek-ai/DeepSeek-V4-Flash-0731": func(e CatalogEntry) bool { return e.Revision == "" && len(e.AfterDownload) == 0 },
+ "puwaer/DeepSeek-V4-Flash-0731-reap-150b": func(e CatalogEntry) bool { return e.Revision == "" },
+ "Qwen/Qwen3.8-Flash-Next-FP8": func(e CatalogEntry) bool { return e.Revision == "bcd9f01ddc9cff2316eb84281bebcd5b058bddce" },
+ "XiaomiMiMo/MiMo-V2.6-Flash-MOPD": func(e CatalogEntry) bool { return excluded("model_mtp.safetensors", e.Exclude) },
+ "moonshotai/Kimi-K3": func(e CatalogEntry) bool {
+ return len(e.AfterDownload) == 1 && e.AfterDownload[0] == AfterDownloadStep{Python: "tools/k3_tokenizer.py", Creates: "tokenizer.json"}
+ },
+ } {
+ entry, ok := m.catalogEntry(repo)
+ if !ok || !check(entry) {
+ t.Errorf("%s: listed %t as %+v", repo, ok, entry)
+ }
+ }
+}
+
+func TestHFSnapshotListedEntryPinsRevisionAndExcludes(t *testing.T) {
+ files := colibriFiles()
+ files["dflash/draft.safetensors"] = []byte("draft")
+ files["model_mtp.safetensors"] = []byte("mtp")
+ hub := newFakeHub(t, files)
+ hub.revision = strings.Repeat("b", 40)
+ e, _, store := listedEngine(t, hub, CatalogEntry{Revision: hub.revision, Exclude: []string{"dflash/*", "model_mtp.safetensors"}})
+
+ raw, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var res struct {
+ Files int `json:"files"`
+ }
+ if err := json.Unmarshal(raw, &res); err != nil || res.Files != 3 {
+ t.Fatalf("result %s, want the three files left after the exclusions", raw)
+ }
+ for _, name := range []string{"dflash", "model_mtp.safetensors"} {
+ if _, err := os.Stat(filepath.Join(store, "o", "r", name)); !os.IsNotExist(err) {
+ t.Errorf("excluded %s was downloaded: %v", name, err)
+ }
+ }
+ got, err := os.ReadFile(filepath.Join(store, "o", "r", "config.json"))
+ if err != nil || !bytes.Equal(got, files["config.json"]) {
+ t.Fatalf("config.json from the pinned revision: %v", err)
+ }
+}
+
+func TestHFSnapshotRunsAfterDownloadBeforeListing(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, st, store := listedEngine(t, hub, CatalogEntry{AfterDownload: []AfterDownloadStep{{Python: "tools/tok.py", Creates: "tokenizer.json"}}})
+ fakeTool(t, st, "tools/tok.py", "create tokenizer.json\n")
+
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatal(err)
+ }
+ if !hasMarker(filepath.Join(store, "o", "r"), "tokenizer.json") {
+ t.Fatal("the step's file is missing from the finished model")
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r"+partialSuffix)); !os.IsNotExist(err) {
+ t.Errorf("the partial directory was left behind: %v", err)
+ }
+}
+
+// TestHFSnapshotAfterDownloadFailureIsNotListed checks a failed step leaves the
+// download as a partial, so the store does not list a model its engine cannot
+// read, and a later pull resumes rather than starting over.
+func TestHFSnapshotAfterDownloadFailureIsNotListed(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, st, store := listedEngine(t, hub, CatalogEntry{AfterDownload: []AfterDownloadStep{{Python: "tools/tok.py", Creates: "tokenizer.json"}}})
+ fakeTool(t, st, "tools/tok.py", "fail the vocabulary is unreadable\n")
+
+ _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err == nil || !strings.Contains(err.Error(), "the vocabulary is unreadable") {
+ t.Fatalf("pull = %v, want the step's own error", err)
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r")); !os.IsNotExist(err) {
+ t.Errorf("the model was moved into the store: %v", err)
+ }
+ if !hasMarker(filepath.Join(store, "o", "r"+partialSuffix), "config.json") {
+ t.Error("the downloaded files were not kept for the next pull")
+ }
+
+ fakeTool(t, st, "tools/tok.py", "create tokenizer.json\n")
+ hits := hub.fileHits.Load()
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatal(err)
+ }
+ if hub.fileHits.Load() != hits || !hasMarker(filepath.Join(store, "o", "r"), "tokenizer.json") {
+ t.Errorf("retry fetched %d files again; tokenizer present %t", hub.fileHits.Load()-hits, hasMarker(filepath.Join(store, "o", "r"), "tokenizer.json"))
+ }
+}
+
+// TestHFSnapshotFinishesAnEarlierCopy checks a model already in the store but
+// missing a step's file (downloaded before the step was listed) gets the step on
+// its next pull, without downloading anything.
+func TestHFSnapshotFinishesAnEarlierCopy(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, st, store := listedEngine(t, hub, CatalogEntry{AfterDownload: []AfterDownloadStep{{Python: "tools/tok.py", Creates: "tokenizer.json"}}})
+ dest := filepath.Join(store, "o", "r")
+ if err := os.MkdirAll(dest, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dest, "config.json"), []byte("{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ fakeTool(t, st, "tools/tok.py", "create tokenizer.json\n")
+
+ raw, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err != nil || !strings.Contains(string(raw), `"already":true`) {
+ t.Fatalf("pull = %s, %v", raw, err)
+ }
+ if hub.fileHits.Load() != 0 || !hasMarker(dest, "tokenizer.json") {
+ t.Errorf("downloads %d, tokenizer present %t", hub.fileHits.Load(), hasMarker(dest, "tokenizer.json"))
+ }
+}
+
+// TestHFSnapshotAfterDownloadNeedsTheEngine checks a step whose script is not
+// installed says what to do instead of failing on a missing file.
+func TestHFSnapshotAfterDownloadNeedsTheEngine(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ e, _, _ := listedEngine(t, hub, CatalogEntry{AfterDownload: []AfterDownloadStep{{Python: "tools/tok.py", Creates: "tokenizer.json"}}})
+
+ _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err == nil || !strings.Contains(err.Error(), "install") || !strings.Contains(err.Error(), "tools/tok.py") {
+ t.Fatalf("pull = %v, want a hint to install the engine first", err)
+ }
+}
+
+func TestValidateModelTypesRejects(t *testing.T) {
+ for name, types := range map[string][]string{
+ "uppercase": {"Qwen3_moe"},
+ "padded": {" qwen3_moe"},
+ "empty": {""},
+ "twice": {"qwen3_moe", "qwen3_moe"},
+ } {
+ m := testEngineManifest(fakeEngineBin)
+ m.Actions = map[string]Action{pullModelAction: {HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}}
+ m.ModelTypes = types
+ if err := m.Validate(); err == nil {
+ t.Errorf("%s: accepted %q", name, types)
+ }
+ }
+ m := testEngineManifest(fakeEngineBin)
+ m.ModelTypes = []string{"qwen3_moe"}
+ if err := m.Validate(); err == nil || !strings.Contains(err.Error(), "hf_snapshot") {
+ t.Errorf("model_types without an hf_snapshot pull was accepted: %v", err)
+ }
+}
+
+// TestBundledColibriRunsItsOwnCatalog checks the model_types list covers every
+// model Colibri's own setup list offers, by the model_type each repository's
+// config.json declared when the list was written, and leaves out what it has
+// no engine for.
+func TestBundledColibriRunsItsOwnCatalog(t *testing.T) {
+ m, ok := buildBundledRegistry().Get("colibri")
+ if !ok {
+ t.Fatal("no colibri manifest")
+ }
+ for repo, modelType := range map[string]string{
+ "Kreuzzelg/qwen36-35b-a3b-colibri-i4-gs64": "qwen3_5_moe_text",
+ "Justvugg/Qwen3-Coder-30B-A3B-colibri-int4": "qwen3_moe",
+ "deepseek-ai/DeepSeek-V4-Flash-0731": "deepseek_v4",
+ "Qwen/Qwen3.8-Flash-Next-FP8": "qwen4_exp",
+ "XiaomiMiMo/MiMo-V2.6-Flash-MOPD": "mimo_v2",
+ "mastouri/GLM-5.2-colibri-int4-g64-with-int8-mtp": "glm_moe_dsa",
+ "nbeerbower/Inkling-colibri-int4": "inkling_mm_model",
+ "moonshotai/Kimi-K3": "kimi_k3",
+ } {
+ if !m.servesModelType(modelType) {
+ t.Errorf("%s is a %s model, which the list does not run", repo, modelType)
+ }
+ }
+ for _, modelType := range []string{"hy_v3", "laguna", "minimax_m2", "nemotron_h", "qwenimage21pipeline", "laya_modernbert", ""} {
+ if m.servesModelType(modelType) {
+ t.Errorf("model_type %q is listed, but Colibri v2.1.0 has no chat engine for it", modelType)
+ }
+ }
+ if !m.servesModelType(" Qwen3_MoE ") {
+ t.Error("a model_type is not compared the way Colibri compares it")
+ }
+}
+
+// TestHFSnapshotRefusesAModelTheEngineCannotRun checks a tagged repository
+// whose config.json names a model_type the engine has no code for is refused
+// before anything is written, while a listed one is taken at its word.
+func TestHFSnapshotRefusesAModelTheEngineCannotRun(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ hub.modelType = "hy_v3"
+ e, store, _ := snapshotEngine(t, hub)
+ st, _ := e.state("fake")
+ st.manifest.ModelTypes = []string{"qwen3_moe"}
+
+ _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil)
+ if err == nil || !strings.Contains(err.Error(), `"hy_v3"`) {
+ t.Fatalf("pull = %v, want a refusal naming the model_type", err)
+ }
+ if hub.fileHits.Load() != 0 {
+ t.Errorf("%d files were fetched before the refusal", hub.fileHits.Load())
+ }
+ if _, err := os.Stat(filepath.Join(store, "o", "r"+partialSuffix)); !os.IsNotExist(err) {
+ t.Errorf("a refused download left a partial directory: %v", err)
+ }
+
+ hub.modelType = ""
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err == nil || !strings.Contains(err.Error(), "no model_type") {
+ t.Errorf("pull of a repository with no model_type = %v, want a refusal", err)
+ }
+
+ hub.modelType = "Qwen3_MoE"
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatalf("a served model_type was refused: %v", err)
+ }
+}
+
+func TestHFSnapshotTrustsAListedModelsType(t *testing.T) {
+ hub := newFakeHub(t, colibriFiles())
+ hub.modelType = "some_new_type"
+ e, st, _ := listedEngine(t, hub, CatalogEntry{})
+ st.manifest.ModelTypes = []string{"qwen3_moe"}
+ if _, err := e.PullModelStream(context.Background(), "fake", "o/r", nil); err != nil {
+ t.Fatalf("a listed model was refused for its model_type: %v", err)
+ }
+}
diff --git a/services/nvpair-engine-manager/manifests/colibri.json b/services/nvpair-engine-manager/manifests/colibri.json
new file mode 100644
index 000000000..20b0f39b7
--- /dev/null
+++ b/services/nvpair-engine-manager/manifests/colibri.json
@@ -0,0 +1,111 @@
+{
+ "$schema": "../manifest.schema.json",
+ "engine": "colibri",
+ "display_name": "Colibri",
+ "manifest_version": 1,
+ "install": { "mode": "user" },
+ "models_dir": "~/colibri-models",
+ "uninstall": { "remove": ["{install_dir}"] },
+ "runtime": {
+ "editable_launch": {
+ "fixed_args": ["serve"],
+ "controls": [
+ { "flags": ["--host"], "value": "{server.host}" },
+ { "flags": ["--port"], "value": "{server.port}" }
+ ]
+ },
+ "args": ["serve", "--host", "{host}", "--port", "{port}"],
+ "env": { "COLI_MODEL": "{models_dir}/{model}", "COLI_MODEL_ID": "{model}" },
+ "bind": "127.0.0.1",
+ "port": 8001,
+ "ready": { "http": "http://127.0.0.1:{port}/health", "status": 200, "timeout_s": 600, "json_match": { "field": "status", "value": "ok" } },
+ "stop": { "signal": "term", "grace_s": 15 },
+ "health": { "http": "http://127.0.0.1:{port}/health", "status": 200, "interval_s": 10 }
+ },
+ "platforms": {
+ "windows/amd64": {
+ "detect": ["{install_dir}\\coli.cmd"],
+ "install": {
+ "artifacts": [
+ { "name": "base", "url": "https://github.com/JustVugg/colibri/releases/download/v2.1.0/colibri-v2.1.0-windows-x86_64.zip", "sha256": "195ee0be129d563789f1ade2383e023bde5519230d6fb508516e63767ee85465" },
+ { "name": "cuda", "url": "https://github.com/JustVugg/colibri/releases/download/v2.1.0/colibri-v2.1.0-windows-x86_64-cuda.zip", "sha256": "69fb93c20f671533eea8d2c18c5c1f6bd1db12dd79c9e570068b9950db58f3fe" }
+ ],
+ "run": ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "& { $ErrorActionPreference = 'Continue'; $check = 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)'; $found = $false; foreach ($launcher in @('py', 'python', 'python3')) { if (Get-Command $launcher -CommandType Application -ErrorAction SilentlyContinue) { if ($launcher -eq 'py') { & $launcher -3 -c $check 2>$null | Out-Null } else { & $launcher -c $check 2>$null | Out-Null }; if ($LASTEXITCODE -eq 0) { $found = $true; break } } }; if (-not $found) { [Console]::Error.WriteLine('Colibri needs Python 3.10 or newer. Install it from https://www.python.org/downloads/ with Add python.exe to PATH ticked, then install Colibri again.'); exit 1 }; $ErrorActionPreference = 'Stop'; Expand-Archive -LiteralPath $env:NVPAIR_INSTALL_DOWNLOAD_BASE -DestinationPath $env:NVPAIR_INSTALL_DIR -Force; Expand-Archive -LiteralPath $env:NVPAIR_INSTALL_DOWNLOAD_CUDA -DestinationPath $env:NVPAIR_INSTALL_DIR -Force }"]
+ },
+ "runtime": { "bin": "{install_dir}\\coli.cmd" }
+ },
+ "linux/amd64": {
+ "detect": ["{install_dir}/coli"],
+ "install": {
+ "fetch": { "url": "https://github.com/JustVugg/colibri/releases/download/v2.1.0/colibri-v2.1.0-linux-x86_64.tar.gz", "sha256": "d54b4c57c12417fc9e784b649249ac1aa3480eec44ece2777d4cbc639a25e66a" },
+ "run": ["sh", "-c", "python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null || { echo 'Colibri needs Python 3.10 or newer as python3 on PATH; install it, then install Colibri again.' >&2; exit 1; }; tar -xzf \"$1\" -C \"$2\" && chmod +x \"$2/coli\"", "colibri-install", "{download}", "{install_dir}"]
+ },
+ "runtime": { "bin": "{install_dir}/coli" }
+ },
+ "darwin/arm64": {
+ "detect": ["{install_dir}/coli"],
+ "install": {
+ "fetch": { "url": "https://github.com/JustVugg/colibri/releases/download/v2.1.0/colibri-v2.1.0-macos-arm64.tar.gz", "sha256": "58320c9d92f84873820ced439c5dfe30afec64ebfc5ae7255ed8f34414d9e202" },
+ "run": ["sh", "-c", "python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null || { echo 'Colibri needs Python 3.10 or newer as python3 on PATH; install it, then install Colibri again.' >&2; exit 1; }; tar -xzf \"$1\" -C \"$2\" && chmod +x \"$2/coli\"", "colibri-install", "{download}", "{install_dir}"]
+ },
+ "runtime": { "bin": "{install_dir}/coli", "env": { "COLI_METAL": "1" } }
+ }
+ },
+ "actions": {
+ "list_models": {
+ "description": "List the models downloaded into the Colibri model store.",
+ "scan_dir": { "root": "{models_dir}", "marker": "config.json", "depth": 2 },
+ "result": { "array": "models", "field": "name" }
+ },
+ "loaded_models": {
+ "description": "List the model the running server serves (Colibri serves one model per process).",
+ "http": { "method": "GET", "path": "/v1/models" },
+ "result": { "array": "data", "field": "id" }
+ },
+ "pull_model": {
+ "description": "Download a Colibri model container from Hugging Face (params: {\"model\": \"\"}).",
+ "hf_snapshot": { "dest": "{models_dir}/{model}" }
+ },
+ "load_model": {
+ "description": "Serve a downloaded model: saves it as the launch model and restarts a running server onto it (params: {\"model\": \"\"}).",
+ "launch_model": { "path": "{models_dir}/{model}", "marker": "config.json" },
+ "restart_after": true
+ },
+ "delete_model": {
+ "description": "Delete a downloaded model from the store (params: {\"model\": \"\"}).",
+ "remove_path": { "root": "{models_dir}", "path": "{models_dir}/{model}" }
+ }
+ },
+ "catalog": [
+ { "repo": "Kreuzzelg/qwen36-35b-a3b-colibri-i4-gs64", "name": "Qwen3.6-35B-A3B", "parameters": "35B", "size_gb": 23.1 },
+ { "repo": "Justvugg/Qwen3-Coder-30B-A3B-colibri-int4", "name": "Qwen3-Coder-30B-A3B", "parameters": "30B", "size_gb": 19.4 },
+ { "repo": "puwaer/DeepSeek-V4-Flash-0731-reap-150b", "name": "DeepSeek V4 Flash REAP 150B", "parameters": "150B", "size_gb": 84.7 },
+ { "repo": "deepseek-ai/DeepSeek-V4-Flash-0731", "name": "DeepSeek V4 Flash", "parameters": "284B", "size_gb": 166.9 },
+ {
+ "repo": "Qwen/Qwen3.8-Flash-Next-FP8", "name": "Qwen3.8-Flash-Next", "parameters": "125B", "size_gb": 185.6,
+ "revision": "bcd9f01ddc9cff2316eb84281bebcd5b058bddce"
+ },
+ {
+ "repo": "XiaomiMiMo/MiMo-V2.6-Flash-MOPD", "name": "MiMo-V2.6 Flash", "parameters": "309B", "size_gb": 171.8,
+ "exclude": ["dflash/*", "audio_tokenizer/*", "model_mtp.safetensors"]
+ },
+ { "repo": "mastouri/GLM-5.2-colibri-int4-g64-with-int8-mtp", "name": "GLM-5.2", "parameters": "744B", "size_gb": 429.3 },
+ { "repo": "Justvugg/GLM-5.3-colibri-int4-g64", "name": "GLM-5.3", "parameters": "744B", "size_gb": 419.3 },
+ {
+ "repo": "XiaomiMiMo/MiMo-V2.6-Pro-MOPD", "name": "MiMo-V2.6 Pro", "parameters": "1.02T", "size_gb": 563.6,
+ "exclude": ["dflash/*", "audio_tokenizer/*", "model_mtp.safetensors"]
+ },
+ { "repo": "nbeerbower/Inkling-colibri-int4", "name": "Inkling", "parameters": "975B", "size_gb": 514.1 },
+ {
+ "repo": "moonshotai/Kimi-K3", "name": "Kimi K3", "parameters": "2.8T", "size_gb": 1561.0,
+ "after_download": [{ "python": "tools/k3_tokenizer.py", "creates": "tokenizer.json" }]
+ }
+ ],
+ "model_types": [
+ "glm5_next", "glm5_next_text", "glm_moe_dsa", "glm5_moe", "glm", "mistral_large4",
+ "inkling_mm_model", "inkling", "kimi_k3", "kimi_linear", "olmoe", "qwen3_5_moe",
+ "qwen3_5_moe_text", "qwen3_5", "qwen3_5_text", "qwen3_moe", "qwen3", "flint",
+ "muse_spark", "lfm3", "lfm3_vl", "qwen4_exp", "qwen4_exp_text", "deepseek_v4",
+ "deepseek_v41", "deepseek_v41_text", "mimo_v2"
+ ]
+}
diff --git a/services/nvpair-engine-manager/model_delete_test.go b/services/nvpair-engine-manager/model_delete_test.go
index f4ad92ccd..e83e51c84 100644
--- a/services/nvpair-engine-manager/model_delete_test.go
+++ b/services/nvpair-engine-manager/model_delete_test.go
@@ -184,24 +184,27 @@ func TestDeleteModelRestartFailure_FailsTheAction(t *testing.T) {
require.False(t, running, "engine reports running after a failed restart")
}
-// TestBundledManifestsRestartOnlyLMStudio guards the blast radius of
+// TestBundledManifests_RestartOnlyWhereNeeded guards the blast radius of
// restart_after. Restarting interrupts whatever the engine is serving, so it is
-// justified only for an engine that cannot see a deletion any other way. If a
-// new manifest ever needs it, that is a deliberate decision — and it also has to
-// be mirrored in the desktop's EngineCapabilities so the UI still confirms
-// first (see desktop/tests/modular/delete-model-restart.test.ts).
-func TestBundledManifests_RestartOnlyLMStudioAfterModelDeletion(t *testing.T) {
+// justified only where nothing else makes the action's effect visible: LM
+// Studio's delete_model, whose server keeps serving a deleted model until it
+// restarts, and Colibri's load_model, since Colibri serves one model per
+// process and switching models is a restart onto the new one. Any other use is
+// a deliberate decision — and a delete_model one has to be mirrored in the
+// desktop's EngineCapabilities so the UI still confirms first (see
+// desktop/tests/modular/delete-model-restart.test.ts).
+func TestBundledManifests_RestartOnlyWhereNeeded(t *testing.T) {
reg := NewRegistry()
require.NoError(t, reg.LoadFS(bundledManifests, "manifests"), "LoadFS bundled")
+ allowed := map[string]string{"lmstudio": "delete_model", "colibri": "load_model"}
for engine, m := range reg.engines {
for name, act := range m.Actions {
- if !act.RestartAfter {
- continue
+ if act.RestartAfter {
+ assert.Equal(t, allowed[engine], name, "engine %q action %q declares restart_after; only %v need one", engine, name, allowed)
}
- assert.Equal(t, "lmstudio", engine, "engine (%v, %v)", engine, name)
- assert.Equal(t, "delete_model", name, "lmstudio action")
}
}
+ require.True(t, reg.engines["colibri"].Actions["load_model"].RestartAfter, "colibri load_model lost restart_after; a running server would keep its old model")
require.True(t, reg.engines["lmstudio"].Actions["delete_model"].RestartAfter, "lmstudio delete_model lost restart_after; a deleted model would keep being served")
require.False(t, reg.engines["ollama"].Actions["delete_model"].RestartAfter, "ollama delete_model declares restart_after; Ollama reflects deletions without a bounce")
}
diff --git a/services/nvpair-engine-manager/modelactions.go b/services/nvpair-engine-manager/modelactions.go
new file mode 100644
index 000000000..9380e208e
--- /dev/null
+++ b/services/nvpair-engine-manager/modelactions.go
@@ -0,0 +1,114 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+// modelactions.go runs the two declarative actions over a directory model
+// store: scan_dir, which lists it, and launch_model, which chooses the model a
+// single-model engine serves. Neither needs the engine running.
+
+import (
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "path/filepath"
+)
+
+// modelParam reads the "model" field of an action's params.
+func modelParam(params json.RawMessage) (string, error) {
+ var p struct {
+ Model string `json:"model"`
+ }
+ if len(params) > 0 && string(params) != "null" {
+ if err := json.Unmarshal(params, &p); err != nil {
+ return "", fmt.Errorf("params must be a JSON object with a model: %w", err)
+ }
+ }
+ if p.Model == "" {
+ return "", fmt.Errorf("model is required")
+ }
+ return p.Model, nil
+}
+
+// runScanDirAction lists the store as {"models":[{"name":...}]}, the shape a
+// result spec of {"array":"models","field":"name"} extracts.
+func (e *Executor) runScanDirAction(st *engineState, act Action) (json.RawMessage, error) {
+ root, err := resolvePlaceholders(act.ScanDir.Root, st.pathVars())
+ if err != nil {
+ return nil, err
+ }
+ type row struct {
+ Name string `json:"name"`
+ }
+ rows := []row{}
+ for _, id := range scanModelDirs(expandPath(root), act.ScanDir.Marker, act.ScanDir.depthOrDefault()) {
+ rows = append(rows, row{Name: id})
+ }
+ return json.Marshal(map[string]any{"models": rows})
+}
+
+// resolveModelPath resolves an action path templated with {model} and checks
+// it names one directory strictly inside the model store.
+func (st *engineState) resolveModelPath(template, model string) (string, error) {
+ if st.modelsDir == "" {
+ return "", fmt.Errorf("%s has no model store", st.manifest.DisplayName)
+ }
+ vars := st.pathVars()
+ vars["model"] = model
+ resolved, err := resolvePlaceholders(template, vars)
+ if err != nil {
+ return "", err
+ }
+ path := filepath.Clean(expandPath(resolved))
+ store := filepath.Clean(st.modelsDir)
+ if path == store || !pathWithinRoot(store, path) {
+ return "", fmt.Errorf("model %q does not resolve inside the model store", model)
+ }
+ return path, nil
+}
+
+// runLaunchModelAction persists the model a single-model engine serves. The
+// model must already be downloaded. Pairing the action with restart_after makes
+// a running engine switch to it; a stopped one serves it on its next start.
+func (e *Executor) runLaunchModelAction(st *engineState, engine string, act Action, params json.RawMessage) (json.RawMessage, error) {
+ model, err := modelParam(params)
+ if err != nil {
+ return nil, err
+ }
+ _, depth, _ := st.manifest.modelStoreLayout()
+ if !validModelID(model, depth) {
+ return nil, fmt.Errorf("%q is not a valid model name", model)
+ }
+ path, err := st.resolveModelPath(act.LaunchModel.Path, model)
+ if err != nil {
+ return nil, err
+ }
+ if !hasMarker(path, act.LaunchModel.Marker) {
+ return nil, fmt.Errorf("model %q is not downloaded", model)
+ }
+ // The override file is shared with the port and launch settings, which
+ // write it under opMu; take it too so the writes cannot interleave.
+ st.opMu.Lock()
+ defer st.opMu.Unlock()
+ if err := e.persistLaunchModel(engine, st, model); err != nil {
+ return nil, err
+ }
+ slog.Info("launch model selected", "engine", engine, "model", model)
+ return json.Marshal(map[string]string{"model": model})
+}
+
+// persistLaunchModel writes runtime.model to the engine's override and applies
+// it to the in-memory runtime, so the next start serves model. The caller holds
+// st.opMu. The current port is passed through unchanged.
+func (e *Executor) persistLaunchModel(engine string, st *engineState, model string) error {
+ st.mu.Lock()
+ port := st.plat.Runtime.Port
+ st.mu.Unlock()
+ if err := e.persistRuntimeConfig(engine, port, nil, nil, &model); err != nil {
+ return fmt.Errorf("save the launch model: %w", err)
+ }
+ st.mu.Lock()
+ st.plat.Runtime.Model = model
+ st.mu.Unlock()
+ return nil
+}
diff --git a/services/nvpair-engine-manager/modelstore.go b/services/nvpair-engine-manager/modelstore.go
new file mode 100644
index 000000000..3bd6131ef
--- /dev/null
+++ b/services/nvpair-engine-manager/modelstore.go
@@ -0,0 +1,175 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+// modelstore.go serves engines that keep each model as a directory in their
+// store and run one model per process (Colibri). The store layout comes from
+// the manifest's scan_dir action, so the inventory, the launch and the
+// launch_model action all agree on what a model is: a directory at most a
+// fixed depth below models_dir that contains a marker file. Colibri's own
+// `coli setup` writes models one level down (~/colibri-models/qwen36-35b) and
+// PAIR's Hugging Face downloads two (~/colibri-models//), so
+// both layouts are models.
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+)
+
+// hfRepoPattern is a Hugging Face repository id that is safe to use as a path
+// and as a command-line value: owner/name, each part starting with a letter or
+// digit, so nothing can read as a parent reference, an absolute path or a flag.
+var hfRepoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
+
+// modelSegmentPattern is one directory name of a model id.
+var modelSegmentPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
+
+// partialSuffix marks a model directory still being downloaded. The download
+// renames it into place only once every file has arrived, so a listing that
+// skips it never offers a half-downloaded model.
+const partialSuffix = ".partial"
+
+// defaultModelDepth is the deepest a model sits when a manifest declares no
+// listing: the owner/repository layout of a Hugging Face snapshot.
+const defaultModelDepth = 2
+
+// errNoLaunchModel is a start refused because the store holds no model.
+var errNoLaunchModel = errors.New("no model is downloaded")
+
+// modelStoreLayout returns the file that marks a complete model directory and
+// the deepest level below the store one may sit, from the manifest's
+// list_models scan_dir action. ok is false when the manifest lists its models some other
+// way.
+func (m *Manifest) modelStoreLayout() (marker string, depth int, ok bool) {
+ if act, found := m.Actions["list_models"]; found && act.ScanDir != nil {
+ return act.ScanDir.Marker, act.ScanDir.depthOrDefault(), true
+ }
+ return "", defaultModelDepth, false
+}
+
+// validModelID reports a model id of one to depth "/"-separated segments,
+// each a plain directory name. It is what keeps a caller's model param from
+// naming anything outside the store; whether the id names a model, rather than
+// a folder of models, is the marker's to say.
+func validModelID(id string, depth int) bool {
+ parts := strings.Split(id, "/")
+ if len(parts) < 1 || len(parts) > depth {
+ return false
+ }
+ for _, part := range parts {
+ if !modelSegmentPattern.MatchString(part) || strings.HasSuffix(part, partialSuffix) {
+ return false
+ }
+ }
+ return true
+}
+
+// hasMarker reports a regular marker file directly inside dir.
+func hasMarker(dir, marker string) bool {
+ fi, err := os.Stat(filepath.Join(dir, marker))
+ return err == nil && fi.Mode().IsRegular()
+}
+
+// scanModelDirs lists the models in a store: every directory at most depth
+// levels below root that contains marker, as a "/"-joined relative id, sorted.
+// A model directory is not searched for further models. Hidden directories,
+// downloads in progress, names that could not be a model id, and links (a
+// symlink or junction is never a directory entry here) are skipped. A missing
+// or unreadable store is an empty inventory.
+func scanModelDirs(root, marker string, depth int) []string {
+ var out []string
+ var walk func(dir string, rel []string)
+ walk = func(dir string, rel []string) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return
+ }
+ for _, entry := range entries {
+ name := entry.Name()
+ if !entry.IsDir() || strings.HasSuffix(name, partialSuffix) || !modelSegmentPattern.MatchString(name) {
+ continue
+ }
+ path := filepath.Join(dir, name)
+ id := append(append([]string(nil), rel...), name)
+ if hasMarker(path, marker) {
+ out = append(out, strings.Join(id, "/"))
+ continue
+ }
+ if len(id) < depth {
+ walk(path, id)
+ }
+ }
+ }
+ if root != "" && depth > 0 {
+ walk(root, nil)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// runtimeReferencesModel reports a launch that names a model, so only an
+// engine that serves one model per process ever scans its store at start.
+func runtimeReferencesModel(rt Runtime) bool {
+ for _, arg := range rt.Args {
+ if strings.Contains(arg, "{model}") {
+ return true
+ }
+ }
+ for _, value := range rt.Env {
+ if strings.Contains(value, "{model}") {
+ return true
+ }
+ }
+ return false
+}
+
+// launchModel picks the model the next launch serves: the persisted
+// runtime.model while it is still on disk, otherwise the first model in the
+// store. A store with no model is errNoLaunchModel.
+func (st *engineState) launchModel(rt Runtime) (string, error) {
+ marker, depth, layout := st.manifest.modelStoreLayout()
+ if !layout || st.modelsDir == "" {
+ if rt.Model != "" && validModelID(rt.Model, depth) {
+ return rt.Model, nil
+ }
+ return "", errNoLaunchModel
+ }
+ if rt.Model != "" && validModelID(rt.Model, depth) &&
+ hasMarker(filepath.Join(st.modelsDir, filepath.FromSlash(rt.Model)), marker) {
+ return rt.Model, nil
+ }
+ if models := scanModelDirs(st.modelsDir, marker, depth); len(models) > 0 {
+ return models[0], nil
+ }
+ return "", errNoLaunchModel
+}
+
+// launchVars are the placeholders a launch resolves: the path placeholders,
+// the listen address, the control CLI, and {model} for a launch that names
+// one. With no model downloaded every other placeholder is still returned,
+// {model} empty, alongside an error wrapping errNoLaunchModel: a start must
+// refuse, while a settings display or preview must still render.
+func (st *engineState) launchVars(rt Runtime, host string, port int) (map[string]string, error) {
+ vars := st.pathVars()
+ vars["host"] = host
+ vars["port"] = strconv.Itoa(port)
+ if rt.CLI != "" {
+ vars["cli"] = expandPath(rt.CLI)
+ }
+ if !runtimeReferencesModel(rt) {
+ return vars, nil
+ }
+ model, err := st.launchModel(rt)
+ vars["model"] = model
+ if err != nil {
+ return vars, fmt.Errorf("%w: download a model before starting %s", err, st.manifest.DisplayName)
+ }
+ return vars, nil
+}
diff --git a/services/nvpair-engine-manager/modelstore_test.go b/services/nvpair-engine-manager/modelstore_test.go
new file mode 100644
index 000000000..c4239bcb6
--- /dev/null
+++ b/services/nvpair-engine-manager/modelstore_test.go
@@ -0,0 +1,389 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+)
+
+// writeModel creates a complete model directory (one holding the marker) at
+// id below store.
+func writeModel(t *testing.T, store, id string) {
+ t.Helper()
+ dir := filepath.Join(store, filepath.FromSlash(id))
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestScanModelDirsSkipsPartialsAndWrongDepth(t *testing.T) {
+ store := t.TempDir()
+ writeModel(t, store, "z/y")
+ writeModel(t, store, "a/b")
+ writeModel(t, store, "a/d.partial") // still downloading
+ writeModel(t, store, ".hidden/model") // not a model id
+ writeModel(t, store, "flat") // one level, as `coli setup` writes it
+ writeModel(t, store, "x/y/z") // three levels: too deep
+ if err := os.MkdirAll(filepath.Join(store, "a", "nomarker"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ // A directory named like the marker is not the marker.
+ if err := os.MkdirAll(filepath.Join(store, "a", "dirmarker", "config.json"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ got := scanModelDirs(store, "config.json", 2)
+ if want := []string{"a/b", "flat", "z/y"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("scanModelDirs = %v, want %v", got, want)
+ }
+ if got := scanModelDirs(filepath.Join(store, "missing"), "config.json", 2); len(got) != 0 {
+ t.Fatalf("a missing store listed %v", got)
+ }
+}
+
+func TestValidModelID(t *testing.T) {
+ for _, id := range []string{"owner/repo", "Justvugg/Qwen3-Coder-30B-A3B-colibri-int4", "a.b/c_d-e", "qwen36-35b"} {
+ if !validModelID(id, 2) {
+ t.Errorf("validModelID(%q) = false, want true", id)
+ }
+ }
+ for _, id := range []string{"", "a/b/c", "../x", "a/..", "-a/b", "a/-b", "/a/b", `a\b`, "a/b.partial", "a /b", "a/b?"} {
+ if validModelID(id, 2) {
+ t.Errorf("validModelID(%q) = true, want false", id)
+ }
+ }
+}
+
+// modelEngine is the fake engine set up like Colibri: a directory store, a
+// launch environment that names the model, and the store actions.
+func modelEngine(t *testing.T) (*Executor, string, string) {
+ t.Helper()
+ m := testEngineManifest(fakeEngineBin)
+ key := runtime.GOOS + "/" + runtime.GOARCH
+ p := m.Platforms[key]
+ store := t.TempDir()
+ envFile := filepath.Join(t.TempDir(), "env.json")
+ p.ModelsDir = store
+ p.Runtime.Env = map[string]string{
+ "OLLAMA_HOST": "127.0.0.1:{port}",
+ "PAIR_TEST_ENV_FILE": envFile,
+ "PAIR_TEST_LITERAL": "{models_dir}/{model}",
+ }
+ m.Platforms[key] = p
+ m.Actions = map[string]Action{
+ "list_models": {
+ ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "config.json", Depth: 2},
+ Result: &ActionResult{Array: "models", Field: "name"},
+ },
+ "load_model": {
+ LaunchModel: &ActionLaunchModel{Path: "{models_dir}/{model}", Marker: "config.json"},
+ RestartAfter: true,
+ },
+ }
+ if err := m.Validate(); err != nil {
+ t.Fatalf("model engine manifest: %v", err)
+ }
+ e := newTestExecutor(t, m)
+ e.overrideDir = t.TempDir()
+ if _, err := e.Detect("fake"); err != nil {
+ t.Fatal(err)
+ }
+ st, err := e.state("fake")
+ if err != nil {
+ t.Fatal(err)
+ }
+ st.modelsDir = store
+ t.Cleanup(func() { _ = e.Stop("fake") })
+ return e, store, envFile
+}
+
+// launchedModel reads the model path the fake engine was started with.
+func launchedModel(t *testing.T, envFile string) string {
+ t.Helper()
+ data, err := os.ReadFile(envFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var env map[string]string
+ if err := json.Unmarshal(data, &env); err != nil {
+ t.Fatal(err)
+ }
+ return env["PAIR_TEST_LITERAL"]
+}
+
+func TestStartFailsWithoutLaunchModel(t *testing.T) {
+ e, _, _ := modelEngine(t)
+ err := e.Start(context.Background(), "fake")
+ if err == nil || !errors.Is(err, errNoLaunchModel) {
+ t.Fatalf("Start with an empty store = %v, want errNoLaunchModel", err)
+ }
+ if !strings.Contains(err.Error(), "download a model before starting Fake Engine") {
+ t.Fatalf("error %q does not tell the user what to do", err)
+ }
+ if !hasErr(e.Errors(), startFailedID("fake")) {
+ t.Fatal("the refused start was not reported")
+ }
+ if st, _ := e.Status("fake"); st.Running {
+ t.Fatal("engine reported running with no model")
+ }
+}
+
+func TestStartPrefersPersistedModelThenFirstFound(t *testing.T) {
+ e, store, envFile := modelEngine(t)
+ writeModel(t, store, "a/b")
+ writeModel(t, store, "c/d")
+
+ if err := e.Start(context.Background(), "fake"); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := launchedModel(t, envFile), store+"/a/b"; got != want {
+ t.Fatalf("with nothing persisted the launch served %q, want the first model %q", got, want)
+ }
+ if err := e.Stop("fake"); err != nil {
+ t.Fatal(err)
+ }
+
+ st, _ := e.state("fake")
+ st.plat.Runtime.Model = "c/d"
+ if err := e.Start(context.Background(), "fake"); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := launchedModel(t, envFile), store+"/c/d"; got != want {
+ t.Fatalf("persisted model: launch served %q, want %q", got, want)
+ }
+ if err := e.Stop("fake"); err != nil {
+ t.Fatal(err)
+ }
+
+ // The persisted model was deleted: fall back rather than fail.
+ if err := os.RemoveAll(filepath.Join(store, "c")); err != nil {
+ t.Fatal(err)
+ }
+ if err := e.Start(context.Background(), "fake"); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := launchedModel(t, envFile), store+"/a/b"; got != want {
+ t.Fatalf("deleted persisted model: launch served %q, want %q", got, want)
+ }
+}
+
+func TestScanDirActionListsModelsWhileStopped(t *testing.T) {
+ e, store, _ := modelEngine(t)
+ writeModel(t, store, "o/r")
+ raw, err := e.Action(context.Background(), "fake", "list_models", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ st, _ := e.state("fake")
+ models, ok := extractStringsResult(raw, st.manifest.Actions["list_models"].Result)
+ if !ok || !reflect.DeepEqual(models, []string{"o/r"}) {
+ t.Fatalf("list_models = %s (%v, %v), want [o/r]", raw, models, ok)
+ }
+}
+
+func TestLaunchModelPersistsAndRestarts(t *testing.T) {
+ e, store, envFile := modelEngine(t)
+ writeModel(t, store, "a/b")
+ writeModel(t, store, "c/d")
+ if err := e.Start(context.Background(), "fake"); err != nil {
+ t.Fatal(err)
+ }
+ if got := launchedModel(t, envFile); got != store+"/a/b" {
+ t.Fatalf("initial launch served %q", got)
+ }
+
+ raw, err := e.Action(context.Background(), "fake", "load_model", json.RawMessage(`{"model":"c/d"}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(raw) != `{"model":"c/d"}` {
+ t.Fatalf("load_model result = %s", raw)
+ }
+ if st, _ := e.Status("fake"); !st.Running {
+ t.Fatal("the engine did not come back after switching models")
+ }
+ if got := launchedModel(t, envFile); got != store+"/c/d" {
+ t.Fatalf("after load_model the engine serves %q, want %q", got, store+"/c/d")
+ }
+ override, err := os.ReadFile(filepath.Join(e.overrideDir, "fake.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var saved struct {
+ Runtime struct {
+ Model string `json:"model"`
+ } `json:"runtime"`
+ }
+ if err := json.Unmarshal(override, &saved); err != nil || saved.Runtime.Model != "c/d" {
+ t.Fatalf("override %s does not persist the launch model (err %v)", override, err)
+ }
+}
+
+func TestLaunchModelStoppedEngineStaysStopped(t *testing.T) {
+ e, store, _ := modelEngine(t)
+ writeModel(t, store, "a/b")
+ if _, err := e.Action(context.Background(), "fake", "load_model", json.RawMessage(`{"model":"a/b"}`)); err != nil {
+ t.Fatal(err)
+ }
+ if st, _ := e.Status("fake"); st.Running {
+ t.Fatal("selecting a model started a stopped engine")
+ }
+ if st, _ := e.state("fake"); st.plat.Runtime.Model != "a/b" {
+ t.Fatalf("in-memory launch model = %q, want a/b", st.plat.Runtime.Model)
+ }
+}
+
+func TestLaunchModelRejectsTraversalAndMissingMarker(t *testing.T) {
+ e, store, _ := modelEngine(t)
+ writeModel(t, store, "a/b")
+ for _, tc := range []struct{ model, want string }{
+ {"", "model is required"},
+ {"../x", "not a valid model name"},
+ {"-x/y", "not a valid model name"},
+ {"a/b/c", "not a valid model name"},
+ {"a", "is not downloaded"}, // an owner folder, not a model
+ {"a/missing", "is not downloaded"},
+ } {
+ params, _ := json.Marshal(map[string]string{"model": tc.model})
+ _, err := e.Action(context.Background(), "fake", "load_model", params)
+ if err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Errorf("load_model %q = %v, want an error containing %q", tc.model, err, tc.want)
+ }
+ }
+ if entries, _ := os.ReadDir(e.overrideDir); len(entries) != 0 {
+ t.Fatal("a refused selection wrote the override")
+ }
+}
+
+// The launch settings show the model the engine will serve, and saving them
+// unchanged must not pin that model as a literal environment value — a later
+// model switch would otherwise be silently overridden.
+func TestLaunchSettingsRenderModelPlaceholder(t *testing.T) {
+ e := settingsExecutor(t, false)
+ st := settingsState(t, e)
+ store := t.TempDir()
+ st.modelsDir = store
+ st.plat.ModelsDir = store
+ st.plat.Runtime.Env["PAIR_TEST_LITERAL"] = "{models_dir}/{model}"
+ st.manifest.Actions = map[string]Action{
+ "list_models": {
+ ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "config.json", Depth: 2},
+ Result: &ActionResult{Array: "models", Field: "name"},
+ },
+ }
+
+ // No model yet: the settings still render.
+ state, err := e.LaunchSettings("fake")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !state.Editable {
+ t.Fatalf("settings with no model downloaded are not editable: %q", state.Reason)
+ }
+
+ writeModel(t, store, "o/r")
+ state, err = e.LaunchSettings("fake")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !state.Editable || !strings.Contains(state.LaunchText, "o/r") {
+ t.Fatalf("launch text %q (reason %q) does not show the launch model", state.LaunchText, state.Reason)
+ }
+ preview := previewSettings(t, e, settingsRequest(t, e))
+ if len(preview.Errors) != 0 {
+ t.Fatalf("preview errors: %v", preview.Errors)
+ }
+ if len(preview.Env) != 0 {
+ t.Fatalf("an unchanged save pinned %v as literal environment", preview.Env)
+ }
+}
+
+// A model that lands in the store while the engine runs is listed on the next
+// sweep, since the inventory is the store itself.
+func TestModelsResultListsTheStore(t *testing.T) {
+ e, store, _ := modelEngine(t)
+ writeModel(t, store, "a/b")
+ if err := e.Start(context.Background(), "fake"); err != nil {
+ t.Fatal(err)
+ }
+ writeModel(t, store, "c/d")
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ res := e.ModelsResult(context.Background())
+ if reflect.DeepEqual(res.ByEngine["fake"], []string{"a/b", "c/d"}) {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("engine:models = %+v, want both models", res)
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+}
+
+// delete_model on a layered store removes exactly one model, never the store
+// or an owner folder holding several.
+func TestDeleteModelRemovesOnlyOneModel(t *testing.T) {
+ e, store, _ := modelEngine(t)
+ st, _ := e.state("fake")
+ st.manifest.Actions["delete_model"] = Action{RemovePath: &ActionRemovePath{Root: "{models_dir}", Path: "{models_dir}/{model}"}}
+ writeModel(t, store, "o/a")
+ writeModel(t, store, "o/b")
+ for _, model := range []string{"", "o", "o/..", "../o"} {
+ params, _ := json.Marshal(map[string]string{"model": model})
+ if _, err := e.Action(context.Background(), "fake", "delete_model", params); err == nil {
+ t.Errorf("delete_model %q succeeded", model)
+ }
+ }
+ if got := scanModelDirs(store, "config.json", 2); !reflect.DeepEqual(got, []string{"o/a", "o/b"}) {
+ t.Fatalf("refused deletes changed the store: %v", got)
+ }
+ if _, err := e.Action(context.Background(), "fake", "delete_model", json.RawMessage(`{"model":"o/a"}`)); err != nil {
+ t.Fatal(err)
+ }
+ if got := scanModelDirs(store, "config.json", 2); !reflect.DeepEqual(got, []string{"o/b"}) {
+ t.Fatalf("after deleting o/a the store holds %v", got)
+ }
+}
+
+// Installing a single-model engine before any model exists is the normal first
+// step, so the install's start is skipped silently rather than reported as a
+// failure; once a model is downloaded the same call starts it.
+func TestStartAfterInstallWaitsForAModel(t *testing.T) {
+ e, store, envFile := modelEngine(t)
+ if err := e.StartAfterInstall(context.Background(), "fake", startOpts{}); err != nil {
+ t.Fatalf("StartAfterInstall with an empty store = %v, want nil", err)
+ }
+ if st, _ := e.Status("fake"); st.Running {
+ t.Fatal("the engine started with no model")
+ }
+ if len(e.Errors()) != 0 {
+ t.Fatalf("an expected post-install state was reported as errors: %+v", e.Errors())
+ }
+ if enabled, known, _ := e.desired.get("fake"); known && enabled {
+ t.Fatal("the skipped start recorded the engine as switched on")
+ }
+
+ writeModel(t, store, "o/r")
+ if err := e.StartAfterInstall(context.Background(), "fake", startOpts{}); err != nil {
+ t.Fatal(err)
+ }
+ if st, _ := e.Status("fake"); !st.Running {
+ t.Fatal("the engine did not start once a model was downloaded")
+ }
+ if got := launchedModel(t, envFile); got != store+"/o/r" {
+ t.Fatalf("launch served %q", got)
+ }
+}
diff --git a/services/nvpair-engine-manager/pull.go b/services/nvpair-engine-manager/pull.go
index 97d8fb565..ecc5332c1 100644
--- a/services/nvpair-engine-manager/pull.go
+++ b/services/nvpair-engine-manager/pull.go
@@ -122,6 +122,15 @@ func (e *Executor) PullModelStream(ctx context.Context, engine, model string, pa
params, _ = json.Marshal(map[string]string{"name": model, "model": model})
}
+ // A snapshot download writes the store itself: no engine involvement, so
+ // it neither needs the engine running nor goes through the action timeout.
+ if act.HFSnapshot != nil {
+ if model == "" {
+ model = modelFromParams(params)
+ }
+ return e.pullHFSnapshot(ctx, st, engine, model, act)
+ }
+
// CLI action (e.g. lms get): no structured line progress; emit a start
// marker and return the final result via the existing runner.
if len(act.Cmd) > 0 {
diff --git a/services/nvpair-engine-manager/registry.go b/services/nvpair-engine-manager/registry.go
index 29b9f2ecf..70a493fbc 100644
--- a/services/nvpair-engine-manager/registry.go
+++ b/services/nvpair-engine-manager/registry.go
@@ -46,6 +46,22 @@ var allowedPlaceholders = map[string]bool{
"models_dir": true,
}
+// launchPlaceholders extends allowedPlaceholders with {model}, the launch
+// model a single-model engine serves. It is accepted only where the runner
+// resolves it itself: runtime args and env, and the paths of the launch_model
+// and hf_snapshot actions. It is deliberately not in allowedPlaceholders,
+// because cmd and remove_path actions drop any caller param named after an
+// allowed placeholder — adding "model" there would strip LM Studio's
+// `lms get {model}` of its argument.
+var launchPlaceholders = func() map[string]bool {
+ out := make(map[string]bool, len(allowedPlaceholders)+1)
+ for name := range allowedPlaceholders {
+ out[name] = true
+ }
+ out["model"] = true
+ return out
+}()
+
var placeholderRe = regexp.MustCompile(`\{([a-zA-Z_][a-zA-Z0-9_]*)\}`)
var resultMatchFieldPathRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*(?:\.[a-zA-Z_][a-zA-Z0-9_-]*)*$`)
var artifactNameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,31}$`)
@@ -65,6 +81,12 @@ type Manifest struct {
ManifestVersion int `json:"manifest_version"`
Platforms map[string]Platform `json:"platforms"`
Actions map[string]Action `json:"actions,omitempty"`
+ // Catalog lists models by repository, for an engine whose models are
+ // downloaded with hf_snapshot (manifestcatalog.go).
+ Catalog []CatalogEntry `json:"catalog,omitempty"`
+ // ModelTypes are the root config.json model_type values the engine runs,
+ // for an engine that picks its code by model_type (manifestcatalog.go).
+ ModelTypes []string `json:"model_types,omitempty"`
}
// Platform is the per-`/` block. Variance lives here
@@ -176,6 +198,12 @@ type Runtime struct {
// Bind is the default listener address, used as the value of {host}.
// Empty defaults to loopback; a per-call bind can override it.
Bind string `json:"bind,omitempty"`
+ // Model is the persisted launch model of an engine that serves one model
+ // per process, available to runtime args and env as {model}. The
+ // launch_model action writes it to the per-user override; a manifest never
+ // sets it. When it is empty or no longer on disk, the first model found in
+ // the store is served instead (see engineState.launchModel).
+ Model string `json:"model,omitempty"`
// Start is the ordered list of daemon bring-up commands in command mode.
Start [][]string `json:"start,omitempty"`
// CLI is the engine's control-CLI path for this platform, referenced
@@ -282,6 +310,49 @@ type Action struct {
// only a restart makes the deletion visible to clients. A stopped engine is
// left stopped; a restart failure fails the action.
RestartAfter bool `json:"restart_after,omitempty"`
+ // ScanDir lists the models in a directory store: each directory at Depth
+ // below Root that contains Marker. It needs no running engine.
+ ScanDir *ActionScanDir `json:"scan_dir,omitempty"`
+ // LaunchModel selects the model a single-model engine serves: it checks
+ // the model is on disk and persists it as runtime.model. Pair it with
+ // restart_after so a running engine switches to the model.
+ LaunchModel *ActionLaunchModel `json:"launch_model,omitempty"`
+ // HFSnapshot downloads a whole Hugging Face model repository into Dest.
+ // Only valid as pull_model; it needs no running engine.
+ HFSnapshot *ActionHFSnapshot `json:"hf_snapshot,omitempty"`
+}
+
+// ActionScanDir is the directory-listing spec of a scan_dir action. Root is
+// templated with the path placeholders. The result is
+// {"models":[{"name":""}]}, which a result spec of
+// {"array":"models","field":"name"} extracts.
+type ActionScanDir struct {
+ Root string `json:"root"`
+ Marker string `json:"marker"`
+ // Depth is how many directory levels below Root a model sits: 2 for an
+ // owner/repository layout. Zero means 2.
+ Depth int `json:"depth,omitempty"`
+}
+
+// depthOrDefault returns the effective scan depth.
+func (s *ActionScanDir) depthOrDefault() int {
+ if s.Depth == 0 {
+ return 2
+ }
+ return s.Depth
+}
+
+// ActionLaunchModel is the spec of a launch_model action. Path is templated
+// with the path placeholders and {model}; Marker must exist inside it.
+type ActionLaunchModel struct {
+ Path string `json:"path"`
+ Marker string `json:"marker"`
+}
+
+// ActionHFSnapshot is the spec of an hf_snapshot action. Dest is templated
+// with the path placeholders and {model}, and must stay inside the store.
+type ActionHFSnapshot struct {
+ Dest string `json:"dest"`
}
// ActionResult is the list-extraction spec on an Action (see Action.Result).
@@ -728,6 +799,12 @@ func (m *Manifest) Validate() error {
}
}
}
+ if err := m.validateCatalog(); err != nil {
+ return err
+ }
+ if err := m.validateModelTypes(); err != nil {
+ return err
+ }
return m.validatePlaceholders()
}
@@ -888,17 +965,45 @@ func (a *Action) validate(name string) error {
hasCmd := len(a.Cmd) > 0
hasRemovePath := a.RemovePath != nil
kinds := 0
- if hasHTTP {
- kinds++
+ for _, present := range []bool{hasHTTP, hasCmd, hasRemovePath, a.ScanDir != nil, a.LaunchModel != nil, a.HFSnapshot != nil} {
+ if present {
+ kinds++
+ }
}
- if hasCmd {
- kinds++
+ if kinds != 1 {
+ return fmt.Errorf("action %q: exactly one of http, cmd, remove_path, scan_dir, launch_model, or hf_snapshot is required", name)
}
- if hasRemovePath {
- kinds++
+ if s := a.ScanDir; s != nil {
+ if strings.TrimSpace(s.Root) == "" || strings.TrimSpace(s.Marker) == "" {
+ return fmt.Errorf("action %q: scan_dir.root and scan_dir.marker are required", name)
+ }
+ if d := s.depthOrDefault(); d < 1 || d > 4 {
+ return fmt.Errorf("action %q: scan_dir.depth %d invalid (want 1 to 4)", name, s.Depth)
+ }
+ if !validMarkerName(s.Marker) {
+ return fmt.Errorf("action %q: scan_dir.marker %q must be a plain file name", name, s.Marker)
+ }
}
- if kinds != 1 {
- return fmt.Errorf("action %q: exactly one of http, cmd, or remove_path is required", name)
+ if l := a.LaunchModel; l != nil {
+ if strings.TrimSpace(l.Path) == "" || strings.TrimSpace(l.Marker) == "" {
+ return fmt.Errorf("action %q: launch_model.path and launch_model.marker are required", name)
+ }
+ if !validMarkerName(l.Marker) {
+ return fmt.Errorf("action %q: launch_model.marker %q must be a plain file name", name, l.Marker)
+ }
+ }
+ if h := a.HFSnapshot; h != nil {
+ if strings.TrimSpace(h.Dest) == "" {
+ return fmt.Errorf("action %q: hf_snapshot.dest is required", name)
+ }
+ if name != pullModelAction {
+ return fmt.Errorf("action %q: hf_snapshot is only valid as %s", name, pullModelAction)
+ }
+ // The streaming pull path never goes through Action, so the flag would
+ // be silently ignored. Refuse it rather than promise a restart.
+ if a.RestartAfter {
+ return fmt.Errorf("action %q: hf_snapshot does not support restart_after", name)
+ }
}
if hasRemovePath {
if strings.TrimSpace(a.RemovePath.Path) == "" || strings.TrimSpace(a.RemovePath.Root) == "" {
@@ -959,8 +1064,16 @@ func (a *Action) validate(name string) error {
return nil
}
+// validMarkerName reports a marker that names one file directly inside a
+// model directory: no separators, no parent references, no placeholders.
+func validMarkerName(name string) bool {
+ return name != "." && name != ".." && !strings.ContainsAny(name, `/\{}`) && strings.TrimSpace(name) == name
+}
+
// validatePlaceholders rejects any `{token}` outside the placeholders
// available to the platform or manifest-global action that contains it.
+// {model} is accepted only in runtime args and env and in the launch_model and
+// hf_snapshot paths, the places the runner resolves it.
func (m *Manifest) validatePlaceholders() error {
for key, platform := range m.Platforms {
allowed := make(map[string]bool, len(allowedPlaceholders))
@@ -975,8 +1088,19 @@ func (m *Manifest) validatePlaceholders() error {
if err := validatePlaceholderStrings(platform.templatedStrings(), allowed); err != nil {
return fmt.Errorf("platform %q: %w", key, err)
}
+ launch := make(map[string]bool, len(allowed)+1)
+ for name := range allowed {
+ launch[name] = true
+ }
+ launch["model"] = true
+ if err := validatePlaceholderStrings(platform.launchTemplatedStrings(), launch); err != nil {
+ return fmt.Errorf("platform %q: %w", key, err)
+ }
}
- return validatePlaceholderStrings(m.actionTemplatedStrings(), allowedPlaceholders)
+ if err := validatePlaceholderStrings(m.actionTemplatedStrings(), allowedPlaceholders); err != nil {
+ return err
+ }
+ return validatePlaceholderStrings(m.actionLaunchTemplatedStrings(), launchPlaceholders)
}
func validatePlaceholderStrings(templates []string, allowed map[string]bool) error {
@@ -1011,13 +1135,9 @@ func (p Platform) templatedStrings() []string {
out = append(out, p.Uninstall.Remove...)
}
out = append(out, p.Runtime.Bin)
- out = append(out, p.Runtime.Args...)
for _, cmd := range p.Runtime.Start {
out = append(out, cmd...)
}
- for _, v := range p.Runtime.Env {
- out = append(out, v)
- }
out = append(out, probeStrings(p.Runtime.Ready)...)
out = append(out, probeStrings(p.Runtime.Health)...)
if p.Runtime.Stop != nil {
@@ -1026,6 +1146,30 @@ func (p Platform) templatedStrings() []string {
return out
}
+// launchTemplatedStrings are the platform strings that may also use {model}:
+// the process launch's args and env.
+func (p Platform) launchTemplatedStrings() []string {
+ out := append([]string(nil), p.Runtime.Args...)
+ for _, v := range p.Runtime.Env {
+ out = append(out, v)
+ }
+ return out
+}
+
+// actionLaunchTemplatedStrings are the action paths that may use {model}.
+func (m *Manifest) actionLaunchTemplatedStrings() []string {
+ var out []string
+ for _, act := range m.Actions {
+ if act.LaunchModel != nil {
+ out = append(out, act.LaunchModel.Path)
+ }
+ if act.HFSnapshot != nil {
+ out = append(out, act.HFSnapshot.Dest)
+ }
+ }
+ return out
+}
+
func (m *Manifest) actionTemplatedStrings() []string {
var out []string
for _, act := range m.Actions {
@@ -1033,6 +1177,9 @@ func (m *Manifest) actionTemplatedStrings() []string {
out = append(out, act.RemovePath.Root)
// Path may template caller params (e.g. {model}); validated at call time.
}
+ if act.ScanDir != nil {
+ out = append(out, act.ScanDir.Root)
+ }
}
// Action http.path / cmd are resolved from runtime params (e.g.
// {model}), so they're validated at call time, not statically here.
diff --git a/services/nvpair-engine-manager/registry_test.go b/services/nvpair-engine-manager/registry_test.go
index fedd89851..514266cfe 100644
--- a/services/nvpair-engine-manager/registry_test.go
+++ b/services/nvpair-engine-manager/registry_test.go
@@ -317,7 +317,49 @@ func TestValidate_RejectsInvalidManifests(t *testing.T) {
}, "json_match.value is required"},
{"action without http or cmd", func(m *Manifest) {
m.Actions = map[string]Action{"x": {Description: "neither"}}
- }, "exactly one of http, cmd, or remove_path"},
+ }, "exactly one of http, cmd, remove_path, scan_dir, launch_model, or hf_snapshot"},
+ {"action with two kinds", func(m *Manifest) {
+ m.Actions = map[string]Action{"x": {
+ HTTP: &ActionHTTP{Method: "GET", Path: "/p"},
+ ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "config.json"},
+ }}
+ }, "exactly one of"},
+ {"scan_dir missing marker", func(m *Manifest) {
+ m.Actions["list_models"] = Action{ScanDir: &ActionScanDir{Root: "{models_dir}"}}
+ }, "scan_dir.root and scan_dir.marker are required"},
+ {"scan_dir depth out of range", func(m *Manifest) {
+ m.Actions["list_models"] = Action{ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "config.json", Depth: 9}}
+ }, "scan_dir.depth 9 invalid"},
+ {"scan_dir marker with a separator", func(m *Manifest) {
+ m.Actions["list_models"] = Action{ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "../config.json"}}
+ }, "must be a plain file name"},
+ {"scan_dir root uses model", func(m *Manifest) {
+ m.Actions["list_models"] = Action{ScanDir: &ActionScanDir{Root: "{models_dir}/{model}", Marker: "config.json"}}
+ }, "unknown placeholder {model}"},
+ {"launch_model missing marker", func(m *Manifest) {
+ m.Actions["load_model"] = Action{LaunchModel: &ActionLaunchModel{Path: "{models_dir}/{model}"}}
+ }, "launch_model.path and launch_model.marker are required"},
+ {"hf_snapshot on another action", func(m *Manifest) {
+ m.Actions["fetch"] = Action{HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}
+ }, "hf_snapshot is only valid as pull_model"},
+ {"hf_snapshot with restart_after", func(m *Manifest) {
+ m.Actions[pullModelAction] = Action{HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}, RestartAfter: true}
+ }, "hf_snapshot does not support restart_after"},
+ {"model placeholder in detect", func(m *Manifest) {
+ p := m.Platforms["linux/amd64"]
+ p.Detect = []string{"{install_dir}/{model}"}
+ m.Platforms["linux/amd64"] = p
+ }, "unknown placeholder {model}"},
+ {"model placeholder in ready probe", func(m *Manifest) {
+ p := m.Platforms["linux/amd64"]
+ p.Runtime.Ready = &Probe{HTTP: "http://127.0.0.1:{port}/{model}"}
+ m.Platforms["linux/amd64"] = p
+ }, "unknown placeholder {model}"},
+ {"model placeholder in install run", func(m *Manifest) {
+ p := m.Platforms["linux/amd64"]
+ p.Install.Run = append(p.Install.Run, "{model}")
+ m.Platforms["linux/amd64"] = p
+ }, "unknown placeholder {model}"},
{"action missing method", func(m *Manifest) {
m.Actions = map[string]Action{"x": {HTTP: &ActionHTTP{Path: "/p"}}}
}, "http.method and http.path"},
@@ -353,6 +395,24 @@ func TestValidate_RejectsInvalidManifests(t *testing.T) {
}
}
+// {model} is resolved by the runner only in the launch and in the two actions
+// that act on one model directory, so those are the only places it validates.
+func TestValidate_AcceptsModelPlaceholderInLaunchOnly(t *testing.T) {
+ m := validManifest()
+ p := m.Platforms["linux/amd64"]
+ p.ModelsDir = "~/models"
+ p.Runtime.Args = []string{"serve", "--model-id", "{model}"}
+ p.Runtime.Env = map[string]string{"MODEL_DIR": "{models_dir}/{model}"}
+ m.Platforms["linux/amd64"] = p
+ m.Actions["list_models"] = Action{
+ ScanDir: &ActionScanDir{Root: "{models_dir}", Marker: "config.json", Depth: 2},
+ Result: &ActionResult{Array: "models", Field: "name"},
+ }
+ m.Actions["load_model"] = Action{LaunchModel: &ActionLaunchModel{Path: "{models_dir}/{model}", Marker: "config.json"}, RestartAfter: true}
+ m.Actions[pullModelAction] = Action{HFSnapshot: &ActionHFSnapshot{Dest: "{models_dir}/{model}"}}
+ require.NoError(t, m.Validate())
+}
+
func TestPlatformFor_ResolvesDeclaredPlatformAndRejectsMissingPlatform(t *testing.T) {
m := validManifest()
m.Platforms["windows/amd64"] = m.Platforms["linux/amd64"]
diff --git a/services/nvpair-engine-manager/remediation_test.go b/services/nvpair-engine-manager/remediation_test.go
index 79182b621..52621b647 100644
--- a/services/nvpair-engine-manager/remediation_test.go
+++ b/services/nvpair-engine-manager/remediation_test.go
@@ -393,7 +393,7 @@ func TestWindowsLMStudioUninstall_ExpandsDeclaredPaths(t *testing.T) {
func TestBundledManifests_ValidateAndExposeRequiredActions(t *testing.T) {
reg := NewRegistry()
require.NoError(t, reg.LoadFS(bundledManifests, "manifests"), "bundled manifests invalid")
- for _, want := range []string{"ollama", "lmstudio", "llamacpp"} {
+ for _, want := range []string{"ollama", "lmstudio", "llamacpp", "colibri"} {
m, ok := reg.Get(want)
require.True(t, ok, "missing bundled engine (%v)", want)
assert.Contains(t, m.Actions, "list_models", "bundled (%v)", want)
diff --git a/services/nvpair-engine-manager/remoteclient.go b/services/nvpair-engine-manager/remoteclient.go
index 53bcc06e4..b9588e3a1 100644
--- a/services/nvpair-engine-manager/remoteclient.go
+++ b/services/nvpair-engine-manager/remoteclient.go
@@ -52,9 +52,11 @@ func waitsForEngineReadiness(path, engine string) bool {
return true
}
// controlDeletePath: LM Studio's delete_model declares restart_after, so the
- // peer replies only after the post-delete restart is ready.
+ // peer replies only after the post-delete restart is ready. Colibri's
+ // load_model does too: it switches models by restarting onto the new one,
+ // so the peer replies only once that model has loaded.
if path == controlLoadPath {
- return engine == "ollama"
+ return engine == "ollama" || engine == "colibri"
}
return path == controlStartPath || path == controlDeletePath
}
diff --git a/services/nvpair-engine-manager/remoteclient_test.go b/services/nvpair-engine-manager/remoteclient_test.go
index ce7bc9067..cefb25afd 100644
--- a/services/nvpair-engine-manager/remoteclient_test.go
+++ b/services/nvpair-engine-manager/remoteclient_test.go
@@ -166,6 +166,7 @@ func TestRemoteReadinessBudget_CoversEngineStartupAllowance(t *testing.T) {
test("delete waits", controlDeletePath, "lmstudio", true)
test("Ollama load waits", controlLoadPath, "ollama", true)
test("LM Studio load does not wait", controlLoadPath, "lmstudio", false)
+ test("Colibri load waits", controlLoadPath, "colibri", true)
test("stop does not wait", controlStopPath, "ollama", false)
test("unload does not wait", controlUnloadPath, "ollama", false)
test("listing does not wait", controlEnginesPath, "", false)
diff --git a/services/nvpair-engine-manager/removepath.go b/services/nvpair-engine-manager/removepath.go
index 9d8cf7086..4c64f31e1 100644
--- a/services/nvpair-engine-manager/removepath.go
+++ b/services/nvpair-engine-manager/removepath.go
@@ -215,6 +215,12 @@ func safeRemoveUnderRoot(root, target string) error {
if !pathWithinRoot(evalRoot, evalTarget) {
return fmt.Errorf("remove_path: %q escapes allowed root %q", evalTarget, evalRoot)
}
+ // The root bounds what may be removed; it is never itself the target. An
+ // empty model param resolves "{models_dir}/{model}" to the store, and
+ // removing it would take every model at once.
+ if filepath.Clean(evalTarget) == filepath.Clean(evalRoot) {
+ return fmt.Errorf("remove_path: refusing to remove the allowed root %q itself", evalRoot)
+ }
if _, err := os.Stat(evalTarget); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("remove_path: %q does not exist", evalTarget)
diff --git a/services/nvpair-engine-manager/removepath_test.go b/services/nvpair-engine-manager/removepath_test.go
index 4f34d068b..5a99859a2 100644
--- a/services/nvpair-engine-manager/removepath_test.go
+++ b/services/nvpair-engine-manager/removepath_test.go
@@ -38,3 +38,24 @@ func TestSafeRemoveUnderRoot_RejectsSymlinkEscape(t *testing.T) {
require.NoError(t, os.Symlink(outside, link))
require.Error(t, safeRemoveUnderRoot(root, link), "expected symlink escape to fail")
}
+
+// An empty model param resolves "{models_dir}/{model}" to the store itself;
+// removing it would delete every model at once.
+func TestSafeRemoveUnderRootRefusesTheRoot(t *testing.T) {
+ root := t.TempDir()
+ keep := filepath.Join(root, "o", "r", "config.json")
+ if err := os.MkdirAll(filepath.Dir(keep), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(keep, []byte("{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for _, target := range []string{root, root + string(os.PathSeparator), filepath.Join(root, "o", "..")} {
+ if err := safeRemoveUnderRoot(root, target); err == nil {
+ t.Fatalf("safeRemoveUnderRoot(%q) removed the root", target)
+ }
+ }
+ if _, err := os.Stat(keep); err != nil {
+ t.Fatalf("the store lost a model: %v", err)
+ }
+}
diff --git a/services/nvpair-engine-manager/setport.go b/services/nvpair-engine-manager/setport.go
index 66f0b2837..418902ca3 100644
--- a/services/nvpair-engine-manager/setport.go
+++ b/services/nvpair-engine-manager/setport.go
@@ -101,13 +101,15 @@ func (e *Executor) SetPort(ctx context.Context, engine string, port int) (Engine
// A host-platform port is updated too, when present, because platform values
// take precedence over shared runtime defaults at load time.
func (e *Executor) persistPort(engine string, port int) error {
- return e.persistRuntimeConfig(engine, port, nil, nil)
+ return e.persistRuntimeConfig(engine, port, nil, nil, nil)
}
// persistRuntimeConfig writes the port override plus, when non-nil, the launch
-// argument and environment overrides. A nil slice pointer leaves that override
-// exactly as it is on disk; an empty slice clears it to "declared, but empty".
-func (e *Executor) persistRuntimeConfig(engine string, port int, args, environment *[]string) error {
+// argument and environment overrides and the launch model. A nil slice pointer
+// leaves that override exactly as it is on disk; an empty slice clears it to
+// "declared, but empty". A nil model leaves runtime.model alone and an empty
+// one removes it.
+func (e *Executor) persistRuntimeConfig(engine string, port int, args, environment *[]string, model *string) error {
if e.overrideDir == "" {
return fmt.Errorf("no config directory available to persist the port")
}
@@ -177,6 +179,15 @@ func (e *Executor) persistRuntimeConfig(engine string, port int, args, environme
hostRuntime["port"] = port
}
}
+ if model != nil {
+ // Shared runtime only: no bundled manifest sets a per-platform model
+ // that this would have to shadow.
+ if *model == "" {
+ delete(rt, "model")
+ } else {
+ rt["model"] = *model
+ }
+ }
if args != nil {
literal := append([]string{}, (*args)...)
hostRuntime["launch_args"] = literal
diff --git a/services/nvpair-engine-manager/settings.go b/services/nvpair-engine-manager/settings.go
index 568915107..18a357e16 100644
--- a/services/nvpair-engine-manager/settings.go
+++ b/services/nvpair-engine-manager/settings.go
@@ -104,12 +104,9 @@ func resolveRuntimeCommand(rt Runtime, index int, vars map[string]string) (launc
// settings read probes ports, starts processes or writes configuration.
func launchForState(st *engineState, port int) (launchCommand, error) {
rt := st.plat.Runtime
- vars := st.pathVars()
- vars["host"] = effectiveBind(rt.Bind, "")
- vars["port"] = strconv.Itoa(port)
- if rt.CLI != "" {
- vars["cli"] = expandPath(rt.CLI)
- }
+ // A missing launch model must not hide the settings: the display resolves
+ // {model} empty, and only a start refuses.
+ vars, _ := st.launchVars(rt, effectiveBind(rt.Bind, ""), port)
st.mu.Lock()
bin := st.binPath
st.mu.Unlock()
@@ -267,9 +264,10 @@ func (e *Executor) previewLaunchLocked(st *engineState, request settings.Request
return fail("Cannot resolve the engine executable.")
}
host := effectiveBind(rt.Bind, "")
- vars := st.pathVars()
- vars["host"] = host
- vars["port"] = strconv.Itoa(request.Settings.ServerPort)
+ // The same placeholders as a launch, {model} included, so a manifest
+ // environment default that names the model is recognised as a default and
+ // not pinned as a literal user value.
+ vars, _ := st.launchVars(rt, host, request.Settings.ServerPort)
vars["bin"] = base.Bin
vars["cli"] = expandPath(rt.CLI)
if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
@@ -422,7 +420,7 @@ func (e *Executor) ConfigureLaunch(ctx context.Context, p settings.Configure, re
}
}
if changed || !reflect.DeepEqual(st.plat.Runtime.LaunchArgs, preview.Args) || !reflect.DeepEqual(st.plat.Runtime.LaunchEnv, preview.Env) {
- if err := e.persistRuntimeConfig(p.Engine, preview.Settings.ServerPort, &preview.Args, &preview.Env); err != nil {
+ if err := e.persistRuntimeConfig(p.Engine, preview.Settings.ServerPort, &preview.Args, &preview.Env, nil); err != nil {
return e.launchStateLocked(p.Engine, st), err
}
}
diff --git a/services/nvpair-engine-manager/settings_permissions_test.go b/services/nvpair-engine-manager/settings_permissions_test.go
index cb798e926..257340fc4 100644
--- a/services/nvpair-engine-manager/settings_permissions_test.go
+++ b/services/nvpair-engine-manager/settings_permissions_test.go
@@ -21,7 +21,7 @@ func TestSettingsOverride_RestrictsExistingPermissions(t *testing.T) {
path := filepath.Join(e.overrideDir, "fake.json")
require.NoError(t, os.WriteFile(path, []byte(`{"engine":"fake"}`), 0644))
environment := []string{"PAIR_TEST=private"}
- require.NoError(t, e.persistRuntimeConfig("fake", 54321, nil, &environment))
+ require.NoError(t, e.persistRuntimeConfig("fake", 54321, nil, &environment, nil))
for path, want := range map[string]os.FileMode{e.overrideDir: 0700, path: 0600} {
info, err := os.Stat(path)
require.NoError(t, err)
diff --git a/services/nvpair-engine-manager/spec.md b/services/nvpair-engine-manager/spec.md
index 02fffad7c..86418e8a9 100644
--- a/services/nvpair-engine-manager/spec.md
+++ b/services/nvpair-engine-manager/spec.md
@@ -7,7 +7,7 @@ SPDX-License-Identifier: Apache-2.0
## 1. Purpose
A declarative, config-driven control plane for **local inference engines**
-(currently Ollama, LM Studio, and llama.cpp). It owns an engine's entire
+(currently Ollama, LM Studio, llama.cpp, and Colibri). It owns an engine's entire
lifecycle *except serving inference* — locate, install, launch, stop, restart,
health, and config-declared actions — so one uniform API manages any engine
across OSes with no per-engine code. A third party drops in a JSON manifest and
@@ -35,7 +35,10 @@ extensibility story for an open-source product.
`chat`, `unload_model`, and `delete_model` (`remove_path` with
`lms-disk-path` resolution);
llama.cpp declares list, loaded-list, pull, load, unload, and native cache
- delete with exact model ids. `loaded_models` reports
+ delete with exact model ids; Colibri declares a `scan_dir` list of its
+ directory store, a loaded-list from `GET /v1/models`, an `hf_snapshot` pull, a
+ `launch_model` load that restarts the server onto the chosen model, and a
+ `remove_path` delete. `loaded_models` reports
models currently resident in memory, name-extracted via the same declarative
`result` spec with optional nested-path and row filters: Ollama uses
`GET /api/ps`, LM Studio filters nonempty `loaded_instances` from
@@ -48,6 +51,18 @@ extensibility story for an open-source product.
`/props` to report `role:"router"`; a single-model listener is incompatible
and must not be started or adopted because its `/models` contract does not
expose router residency state.
+- Single-model engines (Colibri): the `{model}` launch placeholder resolves to
+ the persisted `runtime.model` while it is on disk, else the first model in the
+ store; with no model a start is refused. Settings display and preview resolve
+ it the same way, so an unchanged save never pins the model as literal
+ environment.
+- Hugging Face snapshot downloads (`hf_snapshot`) of open repositories into a
+ model store: free-space check, per-file HTTP range resume, a `.partial`
+ directory renamed into place only when complete, progress through
+ `engine:pull-progress` and the `ec` stream, and refusal of private, gated,
+ unsafe-path and marker-less repositories. A repository the manifest's
+ `catalog` lists is downloaded at its pinned revision, without its excluded
+ files, and finished by its `after_download` Python steps before it is listed.
- Per-engine stdout/stderr log capture and structured operational error records, surfaced via the errors pipeline.
- A normalized node-level model list (`engine:models`): union of every running engine's `list_models`, name-extracted via each action's declarative `result` spec, plus the per-engine set of models loaded in memory (`loadedByEngine`, from each engine's `loaded_models` action). A successful explicit empty inventory remains an engine key with `[]`; a missing/malformed/failed inventory omits that engine key instead of being mislabeled as authoritative empty. A watcher polls the loaded set and pushes `engine:models-changed` when it changes (explicit load/unload, JIT auto-load, TTL/idle eviction).
- Expose all of the above over the `engine:*` JSON-RPC surface to whatever orchestrates the service, plus an optional plain-HTTP LAN endpoint (`--http-port`, `GET /v1/models`) that serves the model list to a peer's discovery daemon (the list moved off the size-limited mDNS TXT onto HTTP).
@@ -148,7 +163,7 @@ Requests (caller → service):
| `engine:action` | `{ engine, action, params }` | the engine's raw response |
| `engine:logs` | `{ engine }` | `{ lines: [LogLine] }` |
| `engine:errors` | — | `{ errors: [ServiceError] }` |
-| `engine:catalog` | `{ engine, platform?, arch?, query? }` | `{ models: [CatalogModel], source, platform?, arch?, fetchedAt?, searchable?, query? }` — the models an engine can **download**. One curated source per engine: Ollama's is compiled in (`catalog/ollama-models.json`, no network), LM Studio's is the `lmstudio-community` Hugging Face org fetched live and cached, and llama.cpp's is the GGUF repos of `ggml-org`, `bartowski`, and `unsloth`, fetched live and cached, offered as `repo:Q4_K_M`. Only llama.cpp's is `searchable`: a `query` searches all of Hugging Face for GGUF repos, each query cached on its own, and is echoed back; a query over 100 characters is an error rather than being shortened; other sources ignore it and return their whole list. `platform` and `arch` are the GOOS and GOARCH the models will be installed on; omitting both means this host, and naming a platform without an arch leaves the arch unknown rather than borrowing this host's. `appleOnly` rows (MLX) are kept only for a `darwin`/`arm64` target. An engine with no curated source is an error, not an empty list. The Ollama reply is a single multi-megabyte frame — every hop on its path must allow `jsonrpc.WorkerFrameBytes`. |
+| `engine:catalog` | `{ engine, platform?, arch?, query? }` | `{ models: [CatalogModel], source, platform?, arch?, fetchedAt?, searchable?, query? }` — the models an engine can **download**. One curated source per engine: Ollama's is compiled in (`catalog/ollama-models.json`, no network), LM Studio's is the `lmstudio-community` Hugging Face org fetched live and cached, and llama.cpp's is the GGUF repos of `ggml-org`, `bartowski`, and `unsloth`, fetched live and cached, offered as `repo:Q4_K_M`, and Colibri's is the models its manifest `catalog` lists by repository (Colibri v2.1.0's own `coli setup` list, vendors' original checkpoints among them), first, then every open Hugging Face repository tagged `colibri` with a root `config.json` whose `model_type` the manifest's `model_types` lists (the SpeedyColibri and peng ports' builds excluded), both fetched live and cached, offered as `owner/repo`; a listed repository appears once, and one since gated or unreadable is left out. llama.cpp's and Colibri's are `searchable`: a `query` searches Hugging Face for GGUF repos or within the `colibri` tag (and Colibri's listed models by repository or name) respectively, each query cached on its own, and is echoed back; a query over 100 characters is an error rather than being shortened; other sources ignore it and return their whole list. `platform` and `arch` are the GOOS and GOARCH the models will be installed on; omitting both means this host, and naming a platform without an arch leaves the arch unknown rather than borrowing this host's. `appleOnly` rows (MLX) are kept only for a `darwin`/`arm64` target. An engine with no curated source is an error, not an empty list. The Ollama reply is a single multi-megabyte frame — every hop on its path must allow `jsonrpc.WorkerFrameBytes`. |
| `engine:remote-get-installed` | `{ node }` | `{ engines: [EngineStatus] }` from the remote node |
| `engine:remote-install` | `{ node, engine, start? }` | `{ opId, status }` after the remote install |
| `engine:remote-pull-model` | `{ node, engine, model?, params? }` | `{ opId, result }` after the remote pull |
@@ -208,7 +223,7 @@ fails it with the peer's error message.
`~/Library/Application Support/Nvidia Corporation/Personal AI Router` on
macOS) plus bundled `manifests/*.json`. Each engine's model store is declared
as `models_dir` outside that data dir (`~/.ollama`, `~/.llamacpp`,
- `~/.lmstudio/models`) so neither an engine uninstall nor the app-level data
+ `~/.lmstudio/models`, `~/colibri-models`) so neither an engine uninstall nor the app-level data
purge removes downloaded models; clearing one is the user's own choice, made
per model through `delete_model`.
diff --git a/services/nvpair-engine-manager/testdata/fakeengine/main.go b/services/nvpair-engine-manager/testdata/fakeengine/main.go
index e58973ce8..610db1d72 100644
--- a/services/nvpair-engine-manager/testdata/fakeengine/main.go
+++ b/services/nvpair-engine-manager/testdata/fakeengine/main.go
@@ -71,6 +71,9 @@ func bstr(m map[string]any, k string) string {
}
func main() {
+ if os.Getenv("PAIR_FAKE_PYTHON") == "1" {
+ os.Exit(fakePython(os.Args[1:]))
+ }
if path := os.Getenv("PAIR_TEST_ENV_FILE"); path != "" {
values := map[string]string{}
for _, key := range []string{"OLLAMA_ORIGINS", "PAIR_TEST_LITERAL", "PAIR_TEST_EMPTY", "OLLAMA_HOST"} {
@@ -351,3 +354,36 @@ func main() {
log.Fatalf("fake-engine serve: %v", err)
}
}
+
+// fakePython stands in for Python 3 in the after-download tests (with
+// PAIR_FAKE_PYTHON=1). "-c " answers the version probe; "