Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ jobs:
name: openshell-conformance-x86_64-unknown-linux-musl
path: conformance-input

- name: Run RPM gateway continuity conformance
- name: Run candidate RPM gateway conformance
shell: bash
run: |
set -euo pipefail
Expand All @@ -173,11 +173,11 @@ jobs:
--distro fedora \
--with podman-rootless \
--with selinux \
--copy "${candidate_cli_package[0]}:/var/lib/openshell-conformance/candidate/openshell.rpm" \
--copy "${candidate_gateway_package[0]}:/var/lib/openshell-conformance/candidate/openshell-gateway.rpm" \
--install "${candidate_cli_package[0]}" \
--install "${candidate_gateway_package[0]}" \
--copy conformance-input/openshell-conformance:/tmp/openshell-conformance \
--copy nix/test-guest/conformance-plans/gateway-upgrade-restart.toml:/tmp/conformance-plan.toml \
--provision openshell-rpm-latest-release \
--copy nix/test-guest/conformance-plans/gateway-restart.toml:/tmp/conformance-plan.toml \
--provision openshell-rpm \
--provision gateway-rootless-podman \
--provision openshell-rpm-gateway-upgrade \
-- /tmp/openshell-conformance run --plan /tmp/conformance-plan.toml
-- /home/openshell/.local/bin/openshell-test-guest-as-gateway-user \
/tmp/openshell-conformance run --plan /tmp/conformance-plan.toml
13 changes: 8 additions & 5 deletions architecture/build.md
Original file line number Diff line number Diff line change
Expand Up @@ -257,11 +257,14 @@ and retain that provenance with the local entry; mutable tags are used only
for explicit publication.

CLI conformance runs after target provisioning. Action-free scenarios operate
only through the configured OpenShell CLI. A versioned conformance plan may add
an ordered sequence of target-supplied host-side actions, such as a gateway
restart, while the scenario remains responsible for black-box sandbox
continuity checks. The plan exposes opaque executable paths and timeouts rather
than driver or package-manager configuration; target setup owns those details.
only through the configured OpenShell CLI. A scenario may be a named
collection of internal leaf scenarios. Collections are selected and reported
as one scenario while their runner owns cleanup for every child. A versioned
conformance plan may add an ordered sequence of target-supplied host-side
actions, such as a gateway restart, while the scenario remains responsible for
black-box sandbox continuity checks. The plan exposes opaque executable paths
and timeouts rather than driver or package-manager configuration; target setup
owns those details.

## Python Wheel Packaging

Expand Down
29 changes: 16 additions & 13 deletions crates/openshell-conformance-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,15 +103,15 @@ fn list(output: OutputFormat) -> Result<(), String> {
match output {
OutputFormat::Text => {
for candidate in scenarios() {
println!("{:<16} {}", candidate.name, candidate.description);
println!("{:<16} {}", candidate.name(), candidate.description());
}
}
OutputFormat::Json => {
let result = scenarios()
.iter()
.map(|candidate| ScenarioDescription {
name: candidate.name,
description: candidate.description,
name: candidate.name(),
description: candidate.description(),
})
.collect::<Vec<_>>();
println!(
Expand All @@ -137,7 +137,7 @@ async fn run(
let selected = select_scenarios(requested)?;
let mut results = Vec::with_capacity(selected.len());
for candidate in selected {
let plan_run = default_plan_run(candidate.name);
let plan_run = default_plan_run(candidate.name());
results.push(run_scenario(candidate, &plan_run, binary.as_ref(), None).await);
}

Expand Down Expand Up @@ -179,20 +179,20 @@ fn default_plan_run(scenario: &str) -> PlanRun {
}

async fn run_scenario(
candidate: &'static Scenario,
candidate: &'static dyn Scenario,
plan_run: &PlanRun,
binary: Option<&PathBuf>,
host_action_executor: Option<Arc<dyn HostActionExecutor>>,
) -> ScenarioResult<'static> {
let runner = binary.map_or_else(
|| OpenShellRunner::new(candidate.name),
|path| OpenShellRunner::with_binary(path.clone(), candidate.name),
|| OpenShellRunner::new(candidate.name()),
|path| OpenShellRunner::with_binary(path.clone(), candidate.name()),
);
let mut runner = match runner {
Ok(runner) => runner,
Err(error) => {
return ScenarioResult {
name: candidate.name,
name: candidate.name(),
passed: false,
diagnostic: Some(error.to_string()),
};
Expand All @@ -208,7 +208,7 @@ async fn run_scenario(
};
let outcome = runner.finish(scenario_result).await;
ScenarioResult {
name: candidate.name,
name: candidate.name(),
passed: outcome.is_ok(),
diagnostic: outcome.err(),
}
Expand Down Expand Up @@ -276,7 +276,7 @@ fn read_plan(path: &PathBuf) -> Result<ConformancePlan, String> {
ConformancePlan::parse(&contents).map_err(|error| format!("invalid conformance plan: {error}"))
}

fn select_scenarios(requested: &[String]) -> Result<Vec<&'static Scenario>, String> {
fn select_scenarios(requested: &[String]) -> Result<Vec<&'static dyn Scenario>, String> {
if requested.is_empty() {
return Ok(default_scenarios().collect());
}
Expand Down Expand Up @@ -359,19 +359,22 @@ mod tests {
#[test]
fn selects_named_scenario() {
let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke");
assert_eq!(selected[0].name, "smoke");
assert_eq!(selected[0].name(), "smoke");
}

#[test]
fn unknown_scenario_has_actionable_diagnostic() {
let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario");
let error = select_scenarios(&["missing".to_string()])
.err()
.expect("unknown scenario");
assert!(error.contains("openshell-conformance list"));
}

#[test]
fn action_scenario_requires_an_explicit_plan() {
let error = select_scenarios(&["sandbox-continuity".to_string()])
.expect_err("action scenario requires a plan");
.err()
.expect("action scenario requires a plan");

assert!(error.contains("requires an explicit --plan"));
}
Expand Down
118 changes: 84 additions & 34 deletions crates/openshell-conformance/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,41 +25,83 @@ use tokio::time::sleep;
use self::executor::{CliExecutionError, CliExecutor, ProcessCli};

pub use plan::{ConformancePlan, HostAction, PlanDiagnostics, PlanRun, WorkloadExpectation};
pub use scenarios::{SANDBOX_CONTINUITY_SCENARIO, SMOKE_SCENARIO};
pub use scenarios::{SANDBOX_CONTINUITY_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO};

/// An installed conformance scenario.
#[derive(Debug)]
pub struct Scenario {
pub name: &'static str,
pub description: &'static str,
requires_plan: bool,
run: for<'a> fn(&'a mut OpenShellRunner, &'a PlanRun) -> ScenarioFuture<'a>,
validate_plan_run: Option<PlanRunValidator>,
pub type ScenarioFuture<'a> = Pin<Box<dyn Future<Output = Result<(), String>> + Send + 'a>>;

/// A reusable `OpenShell` conformance contract.
pub trait Scenario: Send + Sync {
/// Stable command-line name for this scenario.
fn name(&self) -> &'static str;

/// Human-readable summary for scenario discovery.
fn description(&self) -> &'static str;

/// Whether this scenario may run only through an explicit target plan.
fn requires_plan(&self) -> bool {
false
}

/// Whether this scenario is selected when no scenario names are supplied.
fn runs_by_default(&self) -> bool {
!self.requires_plan()
}

/// Validates target-supplied inputs before the scenario starts.
fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> {
default_validate_plan_run(plan_run)
}

/// Execute this scenario with a suite-owned runner and target-supplied plan input.
fn run<'a>(&self, runner: &'a mut OpenShellRunner, plan_run: &'a PlanRun)
-> ScenarioFuture<'a>;
}

pub type ScenarioFuture<'a> = Pin<Box<dyn Future<Output = Result<(), String>> + Send + 'a>>;
type PlanRunValidator = fn(&PlanRun) -> Result<(), String>;
/// A scenario that executes a fixed sequence of child scenarios.
pub struct ScenarioCollection {
name: &'static str,
description: &'static str,
scenarios: &'static [&'static dyn Scenario],
}

impl Scenario {
pub async fn run(
&self,
runner: &mut OpenShellRunner,
plan_run: &PlanRun,
) -> Result<(), String> {
self.validate_plan_run(plan_run)?;
(self.run)(runner, plan_run).await
impl ScenarioCollection {
#[must_use]
pub const fn new(
name: &'static str,
description: &'static str,
scenarios: &'static [&'static dyn Scenario],
) -> Self {
Self {
name,
description,
scenarios,
}
}
}

pub fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> {
self.validate_plan_run.map_or_else(
|| default_validate_plan_run(plan_run),
|validate| validate(plan_run),
)
impl Scenario for ScenarioCollection {
fn name(&self) -> &'static str {
self.name
}

/// Whether this scenario may run only through an explicit target plan.
pub fn requires_plan(&self) -> bool {
self.requires_plan
fn description(&self) -> &'static str {
self.description
}

fn run<'a>(
&self,
runner: &'a mut OpenShellRunner,
plan_run: &'a PlanRun,
) -> ScenarioFuture<'a> {
let validation = self.validate_plan_run(plan_run);
let scenarios = self.scenarios;
Box::pin(async move {
validation?;
for scenario in scenarios {
scenario.run(runner, plan_run).await?;
}
Ok(())
})
}
}

Expand All @@ -73,23 +115,31 @@ fn default_validate_plan_run(plan_run: &PlanRun) -> Result<(), String> {
Ok(())
}

const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO, SANDBOX_CONTINUITY_SCENARIO];
const SCENARIOS: &[&dyn Scenario] = &[
SMOKE_SCENARIO,
SANDBOX_LIFECYCLE_SCENARIO,
SANDBOX_CONTINUITY_SCENARIO,
];

/// Returns every scenario compiled into this distribution.
pub fn scenarios() -> &'static [Scenario] {
pub fn scenarios() -> &'static [&'static dyn Scenario] {
SCENARIOS
}

/// Finds a scenario by its stable command-line name.
pub fn scenario(name: &str) -> Option<&'static Scenario> {
scenarios().iter().find(|candidate| candidate.name == name)
/// Finds a publicly selectable scenario by its stable command-line name.
pub fn scenario(name: &str) -> Option<&'static dyn Scenario> {
scenarios()
.iter()
.copied()
.find(|candidate| candidate.name() == name)
}

/// Returns scenarios that need no host-level disruption capability.
pub fn default_scenarios() -> impl Iterator<Item = &'static Scenario> {
pub fn default_scenarios() -> impl Iterator<Item = &'static dyn Scenario> {
scenarios()
.iter()
.filter(|scenario| !scenario.requires_plan)
.copied()
.filter(|scenario| scenario.runs_by_default())
}

const CLEANUP_TIMEOUT: Duration = Duration::from_secs(120);
Expand Down
9 changes: 6 additions & 3 deletions crates/openshell-conformance/src/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,17 @@ mod tests {
use super::*;

#[test]
fn parses_a_smoke_and_continuity_plan() {
fn parses_a_smoke_lifecycle_and_continuity_plan() {
let plan = ConformancePlan::parse(
r#"
version = 1

[[runs]]
scenario = "smoke"

[[runs]]
scenario = "sandbox-lifecycle"

[[runs]]
scenario = "sandbox-continuity"
workload_expectation = "reconciled"
Expand All @@ -145,8 +148,8 @@ mod tests {
)
.expect("valid plan");

assert_eq!(plan.runs.len(), 2);
assert_eq!(plan.runs[1].actions[0].name, "gateway-upgrade");
assert_eq!(plan.runs.len(), 3);
assert_eq!(plan.runs[2].actions[0].name, "gateway-upgrade");
}

#[test]
Expand Down
2 changes: 2 additions & 0 deletions crates/openshell-conformance/src/scenarios/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
//! Registered, portable conformance scenarios.

mod sandbox_continuity;
mod sandbox_lifecycle;
mod smoke;

pub use sandbox_continuity::SANDBOX_CONTINUITY_SCENARIO;
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
pub use smoke::SMOKE_SCENARIO;
43 changes: 31 additions & 12 deletions crates/openshell-conformance/src/scenarios/sandbox_continuity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,39 @@ struct SandboxState {
phase: String,
}

struct SandboxContinuityScenario;

/// Certify sandbox state and workspace continuity across host-side actions.
pub const SANDBOX_CONTINUITY_SCENARIO: Scenario = Scenario {
name: "sandbox-continuity",
description: "Verify sandbox state and workspace continuity across planned host actions.",
requires_plan: true,
run: run_sandbox_continuity,
validate_plan_run: Some(validate_plan_run),
};
pub static SANDBOX_CONTINUITY_SCENARIO: &dyn Scenario = &SandboxContinuityScenario;

impl Scenario for SandboxContinuityScenario {
fn name(&self) -> &'static str {
"sandbox-continuity"
}

fn description(&self) -> &'static str {
"Verify sandbox state and workspace continuity across planned host actions."
}

fn run_sandbox_continuity<'a>(
runner: &'a mut OpenShellRunner,
plan_run: &'a PlanRun,
) -> ScenarioFuture<'a> {
Box::pin(async move { run_sandbox_continuity_inner(runner, plan_run).await })
fn requires_plan(&self) -> bool {
true
}

fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> {
validate_plan_run(plan_run)
}

fn run<'a>(
&self,
runner: &'a mut OpenShellRunner,
plan_run: &'a PlanRun,
) -> ScenarioFuture<'a> {
let validation = self.validate_plan_run(plan_run);
Box::pin(async move {
validation?;
run_sandbox_continuity_inner(runner, plan_run).await
})
}
}

fn validate_plan_run(plan_run: &PlanRun) -> Result<(), String> {
Expand Down
Loading
Loading