Skip to content

Commit 42e4c80

Browse files
Tighten the loose edges of the model-store work
One behaviour per path, one failure mode per mistake. All from review of the preceding commits. Env values are no longer path-expanded. The expansion had been added to resolveProcessLaunch only, so a default launch and a customised one resolved the same manifest differently -- applyLiteralLaunch never had it. It was never needed: {models_dir} arrives absolute because the executor expands it once when it resolves the engine's state. Dropping it leaves the two paths identical rather than duplicating the call. An unset models_dir is no longer published as a placeholder. It resolved to "", so a template referencing it silently became a path under / rather than failing. Two tests graft a bundled platform's runtime onto the fake engine and so now need its store too; both go through one helper, since the pair only makes sense together. UninstallManaged checks for cancellation between engines. Its budget bounds the commands an uninstall runs but cannot interrupt a file walk, so the check between engines is what makes the timeout mean anything -- and the comment claiming otherwise is corrected. The budget also rises to match what one engine's worst case actually costs. The install-layout test no longer reports a broken manifest as "skipped". curl exits 22 for an HTTP error under --fail, which means the pinned URL is wrong -- a release tag that moved -- and that is the breakage the test exists to catch; only an unreachable network skips now. It also verifies each archive against the manifest's sha256, so a pin left behind by a version bump is caught here rather than by a user's failed install. Signed-off-by: Terve <ntervalon@nvidia.com>
1 parent 41df280 commit 42e4c80

6 files changed

Lines changed: 96 additions & 33 deletions

File tree

‎services/nvpair-engine-manager/executor.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -245,10 +245,14 @@ func (e *Executor) state(engine string) (*engineState, error) {
245245
// expand "~" itself — which it could not do, since only argv and a handful of
246246
// named fields pass through expandPath.
247247
func (st *engineState) pathVars() map[string]string {
248-
return map[string]string{
249-
"install_dir": st.installDir,
250-
"models_dir": st.modelsDir,
248+
vars := map[string]string{"install_dir": st.installDir}
249+
// Omitted rather than empty when the manifest declares no store, so a
250+
// template that references it fails to resolve instead of quietly becoming
251+
// a path rooted at "/".
252+
if st.modelsDir != "" {
253+
vars["models_dir"] = st.modelsDir
251254
}
255+
return vars
252256
}
253257

254258
func progress(engine, stage string, pct int) map[string]any {

‎services/nvpair-engine-manager/installlayout_test.go‎

Lines changed: 55 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
package main
55

66
import (
7+
"crypto/sha256"
8+
"encoding/hex"
9+
"errors"
10+
"io"
711
"os"
812
"os/exec"
913
"path/filepath"
@@ -44,7 +48,7 @@ func bundledManifestSet(t *testing.T) map[string]*Manifest {
4448
// published release archive.
4549
//
4650
// Whether a path matches the archive is only knowable from the archive, so the
47-
// real check is TestLiveBundledInstallLayout, which downloads each one. What is
51+
// real check is TestBundledInstallLayout, which downloads each one. What is
4852
// checkable here is that the two paths agree: detecting one file and launching
4953
// another lets an engine report installed and then fail to start.
5054
//
@@ -99,15 +103,15 @@ func TestBundledInstallLayout(t *testing.T) {
99103
}
100104
for name, manifest := range bundledManifestSet(t) {
101105
for key, platform := range manifest.Platforms {
102-
urls := archiveURLs(platform)
103-
if len(urls) == 0 {
106+
downloads := archiveDownloads(platform)
107+
if len(downloads) == 0 {
104108
continue // vendor script or detect-only engine; nothing to unpack
105109
}
106110
t.Run(name+"/"+key, func(t *testing.T) {
107111
installDir := t.TempDir()
108112
strip := strings.Contains(strings.Join(platform.Install.Run, " "), "--strip-components=1")
109-
for _, url := range urls {
110-
extractArchive(t, url, installDir, strip)
113+
for _, download := range downloads {
114+
extractArchive(t, download, installDir, strip)
111115
}
112116
for _, candidate := range platform.Detect {
113117
if !strings.HasPrefix(candidate, "{install_dir}") {
@@ -118,40 +122,49 @@ func TestBundledInstallLayout(t *testing.T) {
118122
// separator is what the extracted tree uses.
119123
relative = filepath.FromSlash(strings.ReplaceAll(relative, `\`, "/"))
120124
if _, err := os.Stat(filepath.Join(installDir, relative)); err != nil {
121-
t.Errorf("detect path %q is absent after extracting %v (strip=%v): %v",
122-
candidate, urls, strip, err)
125+
t.Errorf("detect path %q is absent after extraction (strip=%v): %v",
126+
candidate, strip, err)
123127
}
124128
}
125129
})
126130
}
127131
}
128132
}
129133

130-
// archiveURLs lists the downloads a platform's install unpacks, in the order
131-
// the manifest extracts them.
132-
func archiveURLs(platform Platform) []string {
134+
// archiveDownloads lists the downloads a platform's install unpacks, with their
135+
// pinned checksums, in the order the manifest extracts them.
136+
func archiveDownloads(platform Platform) []Fetch {
133137
if platform.Install == nil || len(platform.Install.Run) == 0 {
134138
return nil
135139
}
136-
var urls []string
140+
var downloads []Fetch
137141
if platform.Install.Fetch != nil {
138-
urls = append(urls, platform.Install.Fetch.URL)
142+
downloads = append(downloads, *platform.Install.Fetch)
139143
}
140144
for _, artifact := range platform.Install.Artifacts {
141-
urls = append(urls, artifact.URL)
145+
downloads = append(downloads, Fetch{URL: artifact.URL, SHA256: artifact.SHA256})
142146
}
143-
return urls
147+
return downloads
144148
}
145149

146-
func extractArchive(t *testing.T, url, installDir string, strip bool) {
150+
func extractArchive(t *testing.T, download Fetch, installDir string, strip bool) {
147151
t.Helper()
148-
archive := filepath.Join(t.TempDir(), filepath.Base(url))
152+
archive := filepath.Join(t.TempDir(), filepath.Base(download.URL))
149153
// curl rather than net/http: these are large, redirected downloads and the
150154
// point here is the archive's interior, not the transfer.
151-
fetch := exec.Command("curl", "-sSL", "--fail", "--max-time", "900", "-o", archive, url)
155+
fetch := exec.Command("curl", "-sSL", "--fail", "--max-time", "900", "-o", archive, download.URL)
152156
if out, err := fetch.CombinedOutput(); err != nil {
153-
t.Skipf("cannot download %s (%v): %s", url, err, strings.TrimSpace(string(out)))
157+
// With --fail, curl exits 22 for an HTTP status of 400 or above. That is
158+
// the manifest being wrong — a release tag that moved, most likely — so
159+
// it has to fail. Skipping it would report the exact breakage this test
160+
// exists to catch as "no result".
161+
var exit *exec.ExitError
162+
if errors.As(err, &exit) && exit.ExitCode() == 22 {
163+
t.Fatalf("%s is not downloadable: %s", download.URL, strings.TrimSpace(string(out)))
164+
}
165+
t.Skipf("cannot reach %s (%v): %s", download.URL, err, strings.TrimSpace(string(out)))
154166
}
167+
verifyChecksum(t, archive, download)
155168
args := []string{"-xf", archive, "-C", installDir}
156169
if strip {
157170
args = append(args, "--strip-components=1")
@@ -161,3 +174,27 @@ func extractArchive(t *testing.T, url, installDir string, strip bool) {
161174
t.Fatalf("tar %v on %s failed (%v): %s", args, runtime.GOOS, err, strings.TrimSpace(string(out)))
162175
}
163176
}
177+
178+
// verifyChecksum compares the download against the manifest's pin. The runner
179+
// does this at install time; doing it here too means a pin left behind by a
180+
// version bump is caught by the same test that checks the layout, rather than
181+
// by a user's failed install.
182+
func verifyChecksum(t *testing.T, archive string, download Fetch) {
183+
t.Helper()
184+
want := strings.TrimSpace(download.SHA256)
185+
if want == "" {
186+
return // an unpinned fetch; the runner warns rather than verifies
187+
}
188+
file, err := os.Open(archive)
189+
if err != nil {
190+
t.Fatal(err)
191+
}
192+
defer file.Close()
193+
digest := sha256.New()
194+
if _, err := io.Copy(digest, file); err != nil {
195+
t.Fatal(err)
196+
}
197+
if got := hex.EncodeToString(digest.Sum(nil)); !strings.EqualFold(got, want) {
198+
t.Errorf("%s checksum is %s, manifest pins %s", download.URL, got, want)
199+
}
200+
}

‎services/nvpair-engine-manager/launch.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,11 +117,12 @@ func resolveProcessLaunch(rt Runtime, binPath string, vars map[string]string) (l
117117
if err != nil {
118118
return launchCommand{}, err
119119
}
120-
// Env values get the same expansion as argv. A manifest env value that
121-
// names a directory ("~/.llamacpp") would otherwise reach the engine
122-
// literally, and the engine would create a directory called "~" in its
123-
// working directory instead of using the user's home.
124-
env[key] = expandPath(resolved)
120+
// Not path-expanded, here or in applyLiteralLaunch, so a default launch
121+
// and a customised one produce the same environment. A manifest env
122+
// value that names a directory therefore has to arrive already
123+
// absolute: {models_dir} does, because the executor expands it once when
124+
// it resolves the engine's state.
125+
env[key] = resolved
125126
}
126127
return applyLiteralLaunch(rt, launchCommand{Bin: binPath, Args: args, Env: env}, resolvedVars)
127128
}

‎services/nvpair-engine-manager/launch_controls_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func TestBundledLlamaCPPDisablesCORSByDefault(t *testing.T) {
115115
for platform, config := range manifest.Platforms {
116116
t.Run(platform, func(t *testing.T) {
117117
e := settingsExecutor(t, false)
118-
settingsState(t, e).plat.Runtime = config.Runtime
118+
graftPlatform(t, e, config)
119119
state, err := e.LaunchSettings("fake")
120120
if err != nil {
121121
t.Fatal(err)
@@ -145,7 +145,7 @@ func TestBundledNetworkingControls(t *testing.T) {
145145
for platform, config := range manifest.Platforms {
146146
t.Run(name+"/"+platform, func(t *testing.T) {
147147
e := settingsExecutor(t, config.Runtime.modeOrDefault() == "command")
148-
settingsState(t, e).plat.Runtime = config.Runtime
148+
graftPlatform(t, e, config)
149149
policy := config.Runtime.EditableLaunch
150150
if policy == nil {
151151
t.Fatal("missing reviewed networking controls")

‎services/nvpair-engine-manager/provenance.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ type installMarker struct {
3131
InstalledAt string `json:"installed_at"`
3232
}
3333

34-
// uninstallManagedTimeout bounds the whole --uninstall-managed pass. Removing a
35-
// model-adjacent tree walks a lot of files, and a vendor's stop command can
36-
// hang, but an uninstaller cannot wait forever for either.
37-
const uninstallManagedTimeout = 3 * time.Minute
34+
// uninstallManagedTimeout bounds the whole --uninstall-managed pass: the
35+
// commands each uninstall runs, plus the check between engines that stops the
36+
// sweep once it expires. A vendor's stop command can hang and an uninstaller
37+
// cannot wait forever, but an individual file walk is not interruptible, so the
38+
// budget has to be comfortably above one engine's worst case rather than tight.
39+
const uninstallManagedTimeout = 15 * time.Minute
3840

3941
func installMarkerPath(installDir string) string {
4042
if installDir == "" {
@@ -117,6 +119,12 @@ func (e *Executor) UninstallManaged(ctx context.Context) []ManagedUninstall {
117119
names := e.reg.Names()
118120
results := make([]ManagedUninstall, 0, len(names))
119121
for _, engine := range names {
122+
// Checked per engine, because the budget below bounds the commands an
123+
// uninstall runs but not the file walks between them.
124+
if err := ctx.Err(); err != nil {
125+
slog.Warn("stopped removing engines before finishing", "err", err, "remaining", len(names)-len(results))
126+
return results
127+
}
120128
installDir := filepath.Join(e.baseDir, engine)
121129
if !installedByPAIR(installDir) {
122130
slog.Info("leaving engine in place; PAIR did not install it", "engine", engine)

‎services/nvpair-engine-manager/settings_test.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,19 @@ import (
2121
settings "nvpair-shared/enginesettings"
2222
)
2323

24+
// graftPlatform points the fake engine at a bundled platform's runtime.
25+
//
26+
// Both fields move together: a runtime referencing {models_dir} — llama.cpp's
27+
// LLAMA_CACHE does — needs the store that goes with it, and an unset models_dir
28+
// is deliberately not published as a placeholder, so grafting the runtime alone
29+
// leaves a launch that cannot resolve.
30+
func graftPlatform(t *testing.T, e *Executor, platform Platform) {
31+
t.Helper()
32+
state := settingsState(t, e)
33+
state.plat.Runtime = platform.Runtime
34+
state.modelsDir = expandPath(platform.ModelsDir)
35+
}
36+
2437
func settingsExecutor(t *testing.T, command bool) *Executor {
2538
t.Helper()
2639
m := testEngineManifest(fakeEngineBin)
@@ -574,7 +587,7 @@ func manifestSettingsRequest(t *testing.T, platform string) (*Executor, settings
574587
t.Fatalf("platform %q missing", platform)
575588
}
576589
e := settingsExecutor(t, false)
577-
settingsState(t, e).plat.Runtime = config.Runtime
590+
graftPlatform(t, e, config)
578591
request := settingsRequest(t, e)
579592
tokens, err := parseLaunchText(request.Settings.LaunchText)
580593
if err != nil {

0 commit comments

Comments
 (0)