Skip to content

Commit 82c833b

Browse files
feat(tmachine): add the Session API and a test-runner package
Add a library API that boots an environment, runs commands in it with timeouts, copies files in and out, collects diagnostics and tears it down, so a caller can drive a machine without writing an Ansible playbook per suite. SSH and exec timeouts and QEMU cleanup on SIGTERM keep a hung guest or a cancelled CI job from leaving QEMU running. Add the `openshell-test-runner` flake package, which runs tests/runner with the toolchains and VM tools it needs. The Session API and test-runner package are adapted from #4629. Co-authored-by: Evan Lezar <elezar@nvidia.com> Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 5c533d6 commit 82c833b

13 files changed

Lines changed: 751 additions & 90 deletions

File tree

‎flake.nix‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@
127127
firmwarePkgs = testGuestPkgs;
128128
};
129129
testMachines = import ./tests/tmachine/config.nix {
130-
inherit pkgs toolchains;
130+
inherit pkgs toolchains rustToolchain;
131131
qemuPkgs = tmachineRuntimePkgs;
132132
firmwarePkgs = tmachineRuntimePkgs;
133133
};
@@ -203,6 +203,7 @@
203203
tmachine = testMachines.package;
204204
tmachine-config = testMachines.config;
205205
tmachine-unwrapped = testMachines.unwrapped;
206+
openshell-test-runner = testMachines.testRunner;
206207
};
207208

208209
devShells = {

‎tests/tmachine/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/tmachine/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ anyhow = { version = "1.0.100", default-features = false, features = ["std"] }
1515
blake3 = { version = "1.8.7", default-features = false, features = ["std"] }
1616
clap = { version = "4.6.6", default-features = false, features = ["std", "derive", "help", "usage", "color"] }
1717
directories = { version = "6.0.0", default-features = false }
18+
libc = { version = "0.2", default-features = false }
1819
serde = { version = "1.0.229", default-features = false, features = ["std", "derive"] }
1920
serde-saphyr = { version = "1.2.0", default-features = false, features = ["deserialize"] }
2021
tempfile = { version = "3.27.0", default-features = false }
21-
tokio = { version = "1.53.1", default-features = false, features = ["macros", "rt", "process", "net", "io-util"] }
22+
tokio = { version = "1.53.1", default-features = false, features = ["macros", "rt", "process", "net", "io-util", "signal", "time"] }
2223

2324
[workspace]

‎tests/tmachine/config.nix‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
{
55
pkgs,
66
toolchains,
7+
rustToolchain,
78
qemuPkgs ? pkgs,
89
firmwarePkgs ? pkgs,
910
}:
@@ -232,9 +233,42 @@ let
232233
exec ${tmachine}/bin/tmachine --config ${config} "$@"
233234
'';
234235
};
236+
237+
# Runs a test family in a tmachine environment with tests/runner
238+
# (`nix run .#openshell-test-runner -- --environment ubuntu-docker-rootful
239+
# --installer deb --family conformance`). It cross-builds guest test bundles and drives
240+
# tmachine sessions, so it needs the toolchains and the VM tools. The runner
241+
# itself is built by cargo on first use.
242+
testRunner = pkgs.writeShellApplication {
243+
name = "openshell-test-runner";
244+
runtimeInputs = [
245+
qemu
246+
pkgs.ansible
247+
pkgs.cargo-nextest
248+
pkgs.coreutils
249+
pkgs.git
250+
pkgs.gnutar
251+
pkgs.sshpass
252+
rustToolchain
253+
];
254+
runtimeEnv =
255+
pkgs.lib.foldl' (env: toolchain: env // toolchain.env) { } (builtins.attrValues toolchains)
256+
// {
257+
TMACHINE_CONFIG = "${config}";
258+
}
259+
// pkgs.lib.optionalAttrs isAarch64 {
260+
TMACHINE_FIRMWARE_CODE = firmwarePkgs.OVMF.firmware;
261+
TMACHINE_FIRMWARE_VARS = firmwarePkgs.OVMF.variables;
262+
};
263+
text = ''
264+
root=$(git rev-parse --show-toplevel)
265+
export ANSIBLE_CONFIG="$root/tests/tmachine/ansible/ansible.cfg"
266+
exec cargo run --locked --quiet --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test-runner -- "$@"
267+
'';
268+
};
235269
in
236270
{
237271
package = runner;
238272
unwrapped = tmachine;
239-
inherit config;
273+
inherit config testRunner;
240274
}

‎tests/tmachine/src/ansible.rs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44
use std::collections::BTreeMap;
55
use std::path::{Path, PathBuf};
66

7-
use anyhow::{Context, Result};
7+
use anyhow::{Context, Result, ensure};
88
use tokio::process::Command;
99

1010
pub fn requirements_path() -> PathBuf {
1111
PathBuf::from(std::env::var("ANSIBLE_CONFIG").unwrap()).with_file_name("requirements.yaml")
1212
}
1313

14-
pub async fn install_roles() {
14+
pub async fn install_roles() -> Result<()> {
1515
let status = Command::new("ansible-galaxy")
1616
.arg("role")
1717
.arg("install")
@@ -20,13 +20,18 @@ pub async fn install_roles() {
2020
.arg(requirements_path())
2121
.status()
2222
.await
23-
.unwrap();
23+
.context("run ansible-galaxy")?;
2424

25-
assert!(status.success());
25+
ensure!(status.success(), "ansible-galaxy exited with {status}");
26+
Ok(())
2627
}
2728

2829
pub async fn run(playbook: &Path, inputs: &BTreeMap<String, PathBuf>) -> Result<()> {
2930
let mut command = Command::new("ansible-playbook");
31+
// The VM's SSH forward may not be the inventory's default port.
32+
command
33+
.arg("--extra-vars")
34+
.arg(format!("ansible_port={}", crate::ssh_port()));
3035
for (name, path) in inputs {
3136
let value = match std::fs::canonicalize(path) {
3237
Ok(path) => path,
@@ -42,8 +47,19 @@ pub async fn run(playbook: &Path, inputs: &BTreeMap<String, PathBuf>) -> Result<
4247
.arg(format!("{name}={}", value.display()));
4348
}
4449

45-
let status = command.arg(playbook).status().await.unwrap();
50+
// Stop ansible-playbook if the caller is dropped, for example when the
51+
// runner is cancelled.
52+
command.kill_on_drop(true);
53+
let status = command
54+
.arg(playbook)
55+
.status()
56+
.await
57+
.context("run ansible-playbook")?;
4658

47-
assert!(status.success());
59+
ensure!(
60+
status.success(),
61+
"ansible-playbook {} exited with {status}",
62+
playbook.display()
63+
);
4864
Ok(())
4965
}

‎tests/tmachine/src/lib.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Isolated QEMU test machines provisioned with Ansible.
5+
//!
6+
//! The `tmachine` binary drives one-shot `setup`, `install`, and `test`
7+
//! commands. Callers that need to act on a running machine use [`Session`].
8+
//! A binary that starts machines should call [`cleanup_on_signals`] once, so a
9+
//! `SIGTERM` does not leave QEMU running.
10+
11+
pub mod ansible;
12+
pub mod config;
13+
mod qemu;
14+
15+
pub use qemu::{DEFAULT_SSH_PORT, Session, cleanup_on_signals, install, setup, ssh_port, test};

‎tests/tmachine/src/main.rs‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,8 @@ use std::path::PathBuf;
55

66
use anyhow::{Context, Result};
77
use clap::{Parser, Subcommand};
8-
use config::{Config, Environment, Installer, Machine};
9-
10-
mod ansible;
11-
mod config;
12-
mod qemu;
8+
use tmachine::config::{Config, Environment, Installer, Machine};
9+
use tmachine::{install, setup, test};
1310

1411
#[derive(Parser)]
1512
#[command(name = "tmachine")]
@@ -40,20 +37,22 @@ enum Command {
4037
#[tokio::main(flavor = "current_thread")]
4138
async fn main() -> Result<()> {
4239
let cli = Cli::parse();
40+
// A SIGTERM from CI cancellation must not leave QEMU holding the SSH port.
41+
tmachine::cleanup_on_signals();
4342
let config = Config::load(&cli.config)?;
4443

4544
match cli.command {
4645
Command::Setup { environment } => {
4746
let (machine, environment) = find_environment(&config, &environment)?;
48-
qemu::setup(&machine, &environment).await?;
47+
setup(&machine, &environment).await?;
4948
}
5049
Command::Install {
5150
environment,
5251
installer,
5352
} => {
5453
let (machine, environment) = find_environment(&config, &environment)?;
5554
let installer = find_installer(&config, &installer)?;
56-
qemu::install(&machine, &environment, &installer).await?;
55+
install(&machine, &environment, &installer).await?;
5756
}
5857
Command::Test {
5958
environment,
@@ -67,7 +66,7 @@ async fn main() -> Result<()> {
6766
.iter()
6867
.find(|candidate| candidate.name == testsuite)
6968
.with_context(|| format!("testsuite {testsuite:?} is not defined"))?;
70-
qemu::test(&machine, &environment, &installer, testsuite).await?;
69+
test(&machine, &environment, &installer, testsuite).await?;
7170
}
7271
}
7372

‎tests/tmachine/src/qemu/cleanup.rs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Keeping QEMU from outliving tmachine.
5+
//!
6+
//! Dropping a VM handle stops QEMU, but a destructor never runs when the
7+
//! process is killed by a signal. Two mechanisms cover that:
8+
//!
9+
//! - On Linux, QEMU asks the kernel to kill it when its parent dies
10+
//! (`PR_SET_PDEATHSIG`), whatever the reason, including `SIGKILL`.
11+
//! - [`cleanup_on_signals`] kills every tracked QEMU on `SIGTERM`, `SIGINT`
12+
//! and `SIGHUP` and exits, which also covers hosts without that kernel
13+
//! feature.
14+
//!
15+
//! A leaked QEMU holds the SSH forward port, so the next run on that host
16+
//! fails until someone kills it by hand.
17+
18+
use std::sync::Mutex;
19+
20+
use tokio::process::Command;
21+
use tokio::signal::unix::{SignalKind, signal};
22+
23+
static LIVE: Mutex<Vec<u32>> = Mutex::new(Vec::new());
24+
25+
pub(super) fn track(pid: u32) {
26+
if let Ok(mut live) = LIVE.lock() {
27+
live.push(pid);
28+
}
29+
}
30+
31+
pub(super) fn forget(pid: u32) {
32+
if let Ok(mut live) = LIVE.lock() {
33+
live.retain(|candidate| *candidate != pid);
34+
}
35+
}
36+
37+
/// Kill every QEMU this process started and has not yet reaped.
38+
fn kill_all() {
39+
let pids = LIVE.lock().map(|live| live.clone()).unwrap_or_default();
40+
for pid in pids {
41+
if let Ok(pid) = libc::pid_t::try_from(pid) {
42+
// SAFETY: `kill` has no memory-safety preconditions. A recycled
43+
// PID is not a concern: entries are removed when the VM handle
44+
// that owns the child is dropped, and the child is reaped by it.
45+
unsafe { libc::kill(pid, libc::SIGKILL) };
46+
}
47+
}
48+
}
49+
50+
/// Make a child die when this process dies, on platforms that support it.
51+
#[cfg(target_os = "linux")]
52+
pub(super) fn die_with_parent(command: &mut Command) {
53+
// SAFETY: the closure runs between fork and exec and calls only
54+
// `prctl`, which is async-signal-safe.
55+
unsafe {
56+
command.pre_exec(|| {
57+
// PR_SET_PDEATHSIG fires when the thread that spawned the child
58+
// exits, so the runtime must keep its spawning thread alive:
59+
// tmachine's binaries use a current-thread runtime.
60+
if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) == -1 {
61+
return Err(std::io::Error::last_os_error());
62+
}
63+
Ok(())
64+
});
65+
}
66+
}
67+
68+
#[cfg(not(target_os = "linux"))]
69+
pub(super) fn die_with_parent(_command: &mut Command) {}
70+
71+
/// Kill every VM started through tmachine when the process receives
72+
/// `SIGTERM`, `SIGINT` or `SIGHUP`, then exit with the conventional status.
73+
///
74+
/// Call once from a binary's `main`, inside the Tokio runtime.
75+
pub fn cleanup_on_signals() {
76+
for (kind, number) in [
77+
(SignalKind::terminate(), libc::SIGTERM),
78+
(SignalKind::interrupt(), libc::SIGINT),
79+
(SignalKind::hangup(), libc::SIGHUP),
80+
] {
81+
let Ok(mut stream) = signal(kind) else {
82+
eprintln!("tmachine: could not install the signal {number} handler; VMs may leak");
83+
continue;
84+
};
85+
tokio::spawn(async move {
86+
stream.recv().await;
87+
kill_all();
88+
std::process::exit(128 + number);
89+
});
90+
}
91+
}
92+
93+
#[cfg(test)]
94+
mod tests {
95+
use super::*;
96+
97+
#[test]
98+
fn tracks_and_forgets_pids() {
99+
// Out of range for a `pid_t`, so a concurrent `kill_all` skips them.
100+
let (kept, dropped) = (u32::MAX - 1, u32::MAX - 2);
101+
track(kept);
102+
track(dropped);
103+
forget(dropped);
104+
let live = LIVE.lock().unwrap().clone();
105+
assert!(live.contains(&kept));
106+
assert!(!live.contains(&dropped));
107+
forget(kept);
108+
}
109+
110+
#[tokio::test(flavor = "current_thread")]
111+
async fn kill_all_stops_tracked_children_and_handlers_install() {
112+
// Installing handlers must not change behavior until a signal arrives.
113+
cleanup_on_signals();
114+
let child = Command::new("sleep")
115+
.arg("60")
116+
.kill_on_drop(true)
117+
.spawn()
118+
.unwrap();
119+
let pid = child.id().unwrap();
120+
track(pid);
121+
kill_all();
122+
let mut child = child;
123+
let status = child.wait().await.unwrap();
124+
assert!(!status.success());
125+
forget(pid);
126+
}
127+
}

‎tests/tmachine/src/qemu/layer.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use blake3::Hasher;
1010
use directories::ProjectDirs;
1111

1212
use super::img::QemuImage;
13-
use super::vm::QemuVm;
13+
use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm};
1414

1515
pub(super) async fn cached_layer(
1616
base_image: &Path,
@@ -29,7 +29,7 @@ pub(super) async fn cached_layer(
2929
}
3030

3131
if use_galaxy {
32-
crate::ansible::install_roles().await;
32+
crate::ansible::install_roles().await?;
3333
}
3434

3535
let temporary_disk = disks_dir.join(format!("{hash}.tmp"));
@@ -38,10 +38,12 @@ pub(super) async fn cached_layer(
3838
}
3939

4040
let image = QemuImage::create(base_image, temporary_disk.clone()).await;
41-
let vm = QemuVm::start(&image).await;
41+
let vm = QemuVm::start(&image).await?;
4242
run_playbooks(playbooks, inputs).await?;
43-
vm.shutdown().await;
44-
vm.wait().await;
43+
// A layer is only cached after a clean power-off, so failing here is an
44+
// error rather than something to log and move past.
45+
vm.shutdown().await.context("power off the VM")?;
46+
vm.wait(LAYER_POWER_OFF_TIMEOUT).await?;
4547

4648
std::fs::rename(temporary_disk, &disk).unwrap();
4749
Ok(disk)

‎tests/tmachine/src/qemu/mod.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,18 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
mod ansible_hash;
5+
mod cleanup;
56
mod img;
67
mod install;
78
mod layer;
9+
mod session;
810
mod setup;
911
mod test;
1012
mod vm;
1113

14+
pub use cleanup::cleanup_on_signals;
1215
pub use install::install;
16+
pub use session::Session;
1317
pub use setup::setup;
1418
pub use test::test;
19+
pub use vm::{DEFAULT_SSH_PORT, ssh_port};

0 commit comments

Comments
 (0)