diff --git a/.bran-export.json b/.bran-export.json index 580cc23..ccb6c48 100644 --- a/.bran-export.json +++ b/.bran-export.json @@ -1,15 +1,15 @@ { "schema_version": 1, "source_repository": "alphazede/bran-dev", - "source_commit": "0a1de91258669123a3488af9b20d47a4d16b53c2", + "source_commit": "3feeba3ab30f0504a601ffbb732f98801e25c5e8", "version": "0.1.0", "public_repository": "alphazede/bran", "files": [ { "path": ".bran/policy.yaml", "mode": "100644", - "bytes": 975, - "sha256": "41e89d763cdd6beac49d0372e1758cc86c1e7cf6e15a494ded429fbbf7b6c7ac" + "bytes": 1470, + "sha256": "1310773d9ab2c879d66823174faa2ee81fdc67596c7189ab6237e6819d75b278" }, { "path": ".branignore", @@ -17,6 +17,24 @@ "bytes": 86, "sha256": "fc3aedc03fe42a12f036c337a579efa7b47775a6f415d99e020f212cc38d4643" }, + { + "path": ".github/ISSUE_TEMPLATE/bug_report.yml", + "mode": "100644", + "bytes": 1753, + "sha256": "199b767e0392a2a9c9a1fb452aa6b7410460636b239da46e80ca905aa4e6dc5f" + }, + { + "path": ".github/ISSUE_TEMPLATE/config.yml", + "mode": "100644", + "bytes": 236, + "sha256": "639faef63a4c4deb8b3cb2ac86fdb888f12b51b06c767425c84db56bd8233558" + }, + { + "path": ".github/ISSUE_TEMPLATE/feature_request.yml", + "mode": "100644", + "bytes": 1386, + "sha256": "6fe630e14a84502ba6c59a961fb84894388369615de101aa70cf2ea3ebd3c6a0" + }, { "path": ".github/workflows/bran-fast.yml", "mode": "100644", @@ -35,6 +53,18 @@ "bytes": 100, "sha256": "e045528ae5373d1e7bbcd287702e6e7c0ccb096d2076acf9a7e6c9c708aefeab" }, + { + "path": "CODE_OF_CONDUCT.md", + "mode": "100644", + "bytes": 1346, + "sha256": "ea1c3b2cc119c76a7ce48d0ae84410eb6de22c6fbbe8a24b3aee08e86cf46ae4" + }, + { + "path": "CONTRIBUTING.md", + "mode": "100644", + "bytes": 2796, + "sha256": "3b268fa87dcfe4341f725bdb89c9d55e94139687d6e74e20f84ec71f201a7b4b" + }, { "path": "Cargo.lock", "mode": "100644", @@ -68,8 +98,8 @@ { "path": "README.md", "mode": "100644", - "bytes": 9773, - "sha256": "e1e78f08b7f5bfe79f6a922d86df0d14de8a10d6e0771b2e602afda6ce502c3d" + "bytes": 10037, + "sha256": "2e0abb9ad0a78c411b75c72d798688c24110b0c57d426af199b8e376b9e37865" }, { "path": "assets/brand/bran-repository-raven.png", @@ -134,8 +164,8 @@ { "path": "crates/bran-cli/src/main.rs", "mode": "100644", - "bytes": 322237, - "sha256": "ae35ec304b1c49cc01e3831e43f00b656af7d76fb2f4849f79a92034fa9eb924" + "bytes": 417784, + "sha256": "36b2ca5ae6b243b4d6ab0527e3e8b44100c532f14a33d7adc60a37edf7567f15" }, { "path": "crates/bran-core/Cargo.toml", @@ -386,8 +416,8 @@ { "path": "docs/integrations/agent-setup.md", "mode": "100644", - "bytes": 6176, - "sha256": "f9ecccc289fd31330021a143c2e04ffe7b40ad273d3d75a8b63052789d5b1d88" + "bytes": 6388, + "sha256": "c26af274819ea9585fd3067abbdcd8ae6c79028b223d399bb3b5cfa01ae9e47c" }, { "path": "examples/headless/README.md", @@ -674,8 +704,8 @@ { "path": "skill/use-bran/SKILL.md", "mode": "100644", - "bytes": 1214, - "sha256": "22c330b917abdf57e33b97ec44088b8278b61a7d1e0388672f358ae01b9258aa" + "bytes": 2244, + "sha256": "86006e16e96e8f5cf06b16c9bf362fa9272a228c6429a774d340918c615d6255" }, { "path": "tools/ci/build-release.sh", @@ -689,6 +719,12 @@ "bytes": 4872, "sha256": "3dbbcf9e9feeac7e84f9344620fff59810d4cd37f3de7d61456537bbdaf37163" }, + { + "path": "tools/ci/public-policy.yaml", + "mode": "100644", + "bytes": 1470, + "sha256": "1310773d9ab2c879d66823174faa2ee81fdc67596c7189ab6237e6819d75b278" + }, { "path": "tools/ci/public_boundary_check.py", "mode": "100644", @@ -698,8 +734,8 @@ { "path": "tools/ci/public_export.py", "mode": "100644", - "bytes": 20580, - "sha256": "a0fa15276e2e12a98dd3545578296ac2896602c7cc9d96bda1f77b3361f94b4a" + "bytes": 33588, + "sha256": "c262d1860bc0ff312561b13035704f97eb326acc1587739e9715b2e7104bb8fe" }, { "path": "tools/ci/release-check.sh", diff --git a/.bran/policy.yaml b/.bran/policy.yaml index 94e8276..a1ba652 100644 --- a/.bran/policy.yaml +++ b/.bran/policy.yaml @@ -10,6 +10,19 @@ frontmatter: - freshness - resource - public_boundary + - description + - status + - stale_after + - sources + - usage_window + - generated + - verified + - timestamp + - runtime + - parameters + - computation + - executor + - attester allowed: - type - title @@ -18,7 +31,23 @@ frontmatter: - freshness - resource - public_boundary - canonical_docs_frontmatter_state: legacy_baseline + - description + - status + - stale_after + - sources + - usage_window + - generated + - verified + - timestamp + - runtime + - parameters + - computation + - executor + - attester + preserved_canonical_keys: + - proposal_id + - kind + - submission_status coverage: - canonical @@ -41,6 +70,8 @@ tags: allowed: - developer - public + - internal + - bran status: allowed: @@ -51,6 +82,7 @@ status: public_boundary: values: - public + - public-compatible - private - internal path_allowlist: diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..c2376aa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,60 @@ +name: Bug report +description: Something behaves differently than it should +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Please don't paste anything private. Repository paths, source excerpts, + and query text often carry more than you'd expect. + - type: textarea + id: command + attributes: + label: What you ran + description: The exact command, verbatim. + render: shell + placeholder: bran query . "where is authentication handled" + validations: + required: true + - type: textarea + id: output + attributes: + label: What came back + description: > + The JSON BRAN printed. Include `warnings` and `failures`, since those + usually explain it. Trim it if it's long, but don't summarise it. + render: json + validations: + required: true + - type: textarea + id: expected + attributes: + label: What you expected instead + description: > + For a ranking problem, name the file you expected and where it actually + landed. "Expected src/auth.rs, got it at rank 14 behind four test + files" is something I can chase. + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: Output of `bran --version`. + placeholder: bran 0.1.0 + validations: + required: true + - type: input + id: platform + attributes: + label: Platform + placeholder: macOS 15 arm64, Ubuntu 24.04 x86_64, Windows 11 + validations: + required: true + - type: checkboxes + id: offline + attributes: + label: Mode + options: + - label: This happened with no model connected (offline) + - label: This happened with a model connected diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..42609ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Security issue + url: https://github.com/alphazede/bran/blob/main/CONTRIBUTING.md#security + about: Don't file security problems publicly. See CONTRIBUTING for how to report them. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..3a8c6da --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,38 @@ +name: Idea or feature request +description: Something BRAN should be able to do +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: What are you trying to do + description: > + Describe the problem rather than the solution. What were you doing when + you hit the wall? + validations: + required: true + - type: textarea + id: idea + attributes: + label: What you have in mind + description: If you already have a shape in mind, describe it. Optional. + - type: markdown + attributes: + value: | + Two things worth knowing before you write, because they'll shape the + answer: + + **Determinism is the point.** The same repository and question have to + return the same ranking every time. Anything that makes results depend + on timing, machine state, or hidden history changes what BRAN is. That + doesn't mean no, but it does mean we should talk about it. + + **Offline is the default.** Scanning, ranking, packets, and validation + work with no account and no network calls. + - type: checkboxes + id: constraints + attributes: + label: Does this fit + options: + - label: This works without breaking determinism, or I've explained why it's worth the trade + - label: This works offline, or only affects the optional connected mode diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3262dcb --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,42 @@ +# Code of Conduct + +## The short version + +Be decent to people. Assume they're acting in good faith. Criticise the code, +not the person who wrote it. + +## What's expected + +- Give feedback that someone can act on. "This ranking is wrong because X" + helps. "This is broken" doesn't. +- Accept that other people have different context, hardware, and constraints + than you do. +- Own your mistakes. Everyone ships bugs. This project has shipped a few + already. +- Keep discussion on the technical question at hand. + +## What isn't + +- Harassment, insults, or personal attacks. +- Sexualised language or unwanted attention of any kind. +- Publishing someone's private information without their permission. +- Sustained disruption of a discussion. + +## Where this applies + +Anywhere the project is represented: issues, discussions, pull requests, and +any public space where you're speaking for BRAN. + +## Reporting + +Email 1wgrumph@gmail.com. Reports stay private, and I'll respond as quickly as +I reasonably can. + +I'll decide what action to take, up to and including blocking someone from the +project. If your behaviour makes the project worse for other people, you'll be +asked to stop, and if it continues you'll be removed. + +## Credit + +This is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +version 2.1, rewritten to be shorter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9f13c21 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing to BRAN + +Thanks for taking a look. Bug reports are genuinely useful, and the ranking +heuristics are where I've been wrong most often, so that's a good place to +push. + +## First, a note about this repository + +This repository is a published snapshot. BRAN is developed somewhere else, and +the code here is exported from there and signed. + +**That means pull requests opened here can't be merged.** Not because they +aren't welcome, but because the next export would overwrite them. Sorry. If you +want to change something, open an issue and we'll work out the shape of it +first. If a change is worth making, I'll carry it upstream and credit you in +the commit. + +Issues, questions, and bug reports are all in the right place here. + +## Building and testing + +You need a stable Rust toolchain. Nothing else. + +```sh +cargo test # the test suite +cargo run --bin bran -- smoke # quick sanity check +./tools/ci/check.sh --fast # the gate that has to pass +``` + +`check.sh --fast` runs formatting, clippy, the tests, and the boundary checks. +If it passes, the change is in reasonable shape. Use `--full` only when you've +touched release, security, or conformance behaviour, since it's much slower. + +## Filing a good bug + +The most useful reports include the exact command you ran and what came back. +BRAN prints versioned JSON, so paste it. The templates ask for this, but the +short version is: + +- the command, verbatim +- the output, including `warnings` and `failures` +- what you expected instead +- `bran --version` + +If it's a ranking problem, say which file you expected to see and where it +actually ranked. "It returned the wrong thing" is hard to act on. "I asked X, +expected `path/to/file.rs`, and it came back at rank 14 behind four test files" +is something I can chase. + +Please don't include anything private. Repository paths, source excerpts, and +query text often carry more than you'd expect. + +## Things worth knowing + +**Determinism is the point.** The same repository and the same question have to +produce the same ranking every time. Any change that makes results depend on +timing, machine state, or hidden history is a change to what BRAN is. If you +have an idea that needs that, open an issue and let's talk it through. + +**Offline is the default.** Scanning, ranking, packets, validation, and the TUI +work with no account and no network calls. Keep it that way. + +**Nothing unavailable gets faked.** If a capability isn't there, BRAN says +`unavailable` instead of pretending. Requested and effective capability are +reported separately, on purpose. + +## Security + +Don't open a public issue for a security problem. Email 1wgrumph@gmail.com +instead and I'll deal with it. + +## Licence + +Contributions are dual-licensed under MIT or Apache-2.0, matching the project. diff --git a/README.md b/README.md index 1e6eea7..9ef1dd1 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,10 @@ bran check . okf-v0.2 All three results are reported independently and only the selected profile controls the exit code, so OKF conformance is never confused with house rules. +`okf-v0.1` remains a supported selectable compatibility profile. `okf-v0.2` is +additive and does not replace it. BRAN producer extensions (`okf_status`, +`freshness`, `public_boundary`) stay valid and are not silently renamed to +upstream `status` or `stale_after`. ## Export the knowledge graph diff --git a/crates/bran-cli/src/main.rs b/crates/bran-cli/src/main.rs index 1b5c402..b978745 100644 --- a/crates/bran-cli/src/main.rs +++ b/crates/bran-cli/src/main.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{IsTerminal, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::process::ExitCode; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Arc, Mutex}; @@ -75,9 +75,13 @@ Usage: bran [arguments] Commands: smoke query + query --record + query --add-dir + query --add-dir --record packet check [--policy-stdin] maintain ... + evidence tui agents list doctor <--onboarding|--agent> @@ -216,18 +220,53 @@ impl CliApp { Some(r) => r, None => return CliResult::usage(make_query_error("missing_root")), }; - let mut parts = vec![]; + let mut added = Vec::new(); + let mut parts = Vec::new(); + let mut accept_flags = true; + let mut expect_add_dir = false; + let mut record = false; for a in it { - match a.as_ref().to_str() { - Some(s) => parts.push(s.to_owned()), + let value = match a.as_ref().to_str() { + Some(s) => s.to_owned(), None => return CliResult::usage(make_query_error("invalid_utf8")), + }; + if expect_add_dir { + if value.is_empty() { + return CliResult::usage(make_query_error("missing_add_dir")); + } + added.push(value); + expect_add_dir = false; + continue; + } + if accept_flags && value == "--add-dir" { + expect_add_dir = true; + continue; + } + if accept_flags && value == "--record" { + if record { + return CliResult::usage(make_query_error("duplicate_record")); + } + record = true; + continue; } + accept_flags = false; + parts.push(value); + } + if expect_add_dir { + return CliResult::usage(make_query_error("missing_add_dir")); } let qtext = parts.join(" "); if qtext.trim().is_empty() { return CliResult::usage(make_query_error("missing_query")); } - match do_query(root, qtext) { + let result = if added.is_empty() { + do_query(root, qtext, record) + } else { + let mut roots = vec![root]; + roots.extend(added); + do_query_multi(roots, qtext, record) + }; + match result { Ok((status, data, warns, fails, provenance, metrics)) => { CliResult::success(make_envelope( "query", @@ -561,6 +600,34 @@ impl CliApp { _ => CliResult::usage(make_maintain_error(&sub, "unknown_subcommand")), } } + "evidence" => { + let sub = match it + .next() + .and_then(|o| o.as_ref().to_str().map(|s| s.to_owned())) + { + Some(s) => s, + None => return CliResult::usage(make_evidence_error("", "missing_subcommand")), + }; + let root = match it + .next() + .and_then(|o| o.as_ref().to_str().map(|s| s.to_owned())) + { + Some(r) => r, + None => { + return CliResult::usage(make_evidence_error(&sub, "missing_root")); + } + }; + if it.next().is_some() { + return CliResult::usage(make_evidence_error(&sub, "too_many_args")); + } + match sub.as_str() { + "summarize" => do_evidence_summarize(root), + "propose" => do_evidence_propose(root), + "replay" => do_evidence_replay(root), + "clear" => do_evidence_clear(root), + _ => CliResult::usage(make_evidence_error(&sub, "unknown_subcommand")), + } + } "tui" => { if it.next().is_some() { return CliResult::usage(UNKNOWN_COMMAND_ERROR.to_owned()); @@ -891,23 +958,32 @@ fn run_tui() -> ExitCode { } app.resolved = Some(resolved); } - TuiAction::Query { query, .. } => match do_query(".".to_owned(), query) { - Ok((status, data, warnings, failures, provenance, metrics)) => { - app.status = make_envelope( - "query", - status, - &data, - &warnings, - &failures, - &provenance, - &metrics, - ) - } - Err(error) => { - app.status = - make_envelope("query", "error", "null", &[], &[error], "{}", "{}") + TuiAction::Query { query, .. } => { + match do_query(".".to_owned(), query, false) { + Ok((status, data, warnings, failures, provenance, metrics)) => { + app.status = make_envelope( + "query", + status, + &data, + &warnings, + &failures, + &provenance, + &metrics, + ) + } + Err(error) => { + app.status = make_envelope( + "query", + "error", + "null", + &[], + &[error], + "{}", + "{}", + ) + } } - }, + } TuiAction::Apply => { let path = Path::new(".bran/settings.conf"); let resolved = bran_tui::resolve_advanced( @@ -1110,6 +1186,23 @@ fn make_maintain_error(sub: &str, detail: &str) -> String { ) } +fn make_evidence_error(sub: &str, detail: &str) -> String { + let command = if sub.is_empty() { + "evidence".to_owned() + } else { + format!("evidence.{sub}") + }; + make_envelope( + &command, + "error", + "null", + &[], + &[detail.to_owned()], + "{}", + "{}", + ) +} + fn make_agents_error(sub: &str, detail: &str) -> String { let command = if sub.is_empty() { "agents".to_owned() @@ -1413,6 +1506,7 @@ fn experimental_controls( #[derive(Clone, Debug)] struct SourceRanking { + bundle: String, id: NodeId, locator: String, rank: usize, @@ -1506,22 +1600,64 @@ fn document_body(snapshot: &ScanSnapshot, locator: &str) -> Option { Some(source[start..].to_owned()) } +/// Matched then unmatched query terms, with high-specificity entity units +/// leading each list and named whole, never as their sub-tokens. +fn query_term_coverage( + query_text: &str, + matched_terms: &BTreeSet, +) -> (Vec, Vec) { + let (terms, entities) = query_terms_and_entities(query_text); + let classify = |wanted_match: bool| { + entities + .iter() + .chain(terms.iter()) + .filter(|term| matched_terms.contains(*term) == wanted_match) + .cloned() + .collect::>() + }; + (classify(true), classify(false)) +} + +fn json_string_list(values: &[String]) -> String { + values + .iter() + .map(|value| format!("\"{}\"", json_escape(value))) + .collect::>() + .join(",") +} + +/// Semantic retrieval outcome for callers that must not treat command +/// success or a non-empty ranking as full grounding (issue #18). +/// +/// `grounded` means every extracted term matched. `miss` means there is no +/// ranked evidence. `partial_unanchored` means some terms did not match; +/// remaining rankings may still be useful. This classifies the result +/// instead of guessing whether an unmatched bare word is English. +fn query_semantic_outcome( + query_text: &str, + matched_terms: &BTreeSet, + rankings: &[SourceRanking], +) -> (&'static str, String, Vec) { + let (matched, unmatched) = query_term_coverage(query_text, matched_terms); + let outcome = if rankings.is_empty() { + "miss" + } else if unmatched.is_empty() { + "grounded" + } else { + "partial_unanchored" + }; + let coverage = format!( + "{{\"matched_terms\":[{}],\"unmatched_terms\":[{}]}}", + json_string_list(&matched), + json_string_list(&unmatched) + ); + (outcome, coverage, unmatched) +} + /// One warning naming every query term or entity unit that matched no /// document, when any. Unmatched entity units lead the list and are named /// whole, never as their sub-tokens. -fn unmatched_query_warnings(query_text: &str, matched_terms: &BTreeSet) -> Vec { - let (terms, entities) = query_terms_and_entities(query_text); - let unmatched = entities - .iter() - .filter(|term| !matched_terms.contains(*term)) - .cloned() - .chain( - terms - .iter() - .filter(|term| !matched_terms.contains(*term)) - .cloned(), - ) - .collect::>(); +fn unmatched_query_warnings(unmatched: &[String]) -> Vec { if unmatched.is_empty() { return vec![]; } @@ -1539,9 +1675,23 @@ fn source_rankings( query_text: &str, max_sources: usize, ) -> (Vec, BTreeSet) { + let (matches, matched_terms, entities) = + score_source_candidates(graph_input, snapshot, query_text); + let matches = suppress_unanchored_entity_matches(matches, &matched_terms, &entities); + ( + finalize_source_rankings(matches, max_sources, false), + matched_terms, + ) +} + +fn score_source_candidates( + graph_input: &GraphInput, + snapshot: &ScanSnapshot, + query_text: &str, +) -> (Vec, BTreeSet, BTreeSet) { let (terms, entities) = query_terms_and_entities(query_text); let mut matched_terms = BTreeSet::new(); - let mut matches = graph_input + let matches = graph_input .nodes() .iter() .filter(|node| node.role() == NodeRole::Document) @@ -1688,6 +1838,7 @@ fn source_rankings( .cloned() .unwrap_or_default(); (exact_matches + partial_matches > 0).then(|| SourceRanking { + bundle: String::new(), id: node.id().clone(), locator: node.provenance().locator().to_owned(), rank: 0, @@ -1712,37 +1863,63 @@ fn source_rankings( }) }) .collect::>(); - // A high-specificity entity unit that matched no document means the query - // names something this repository does not contain. When nothing else in - // the query matched at identity level (exact fact or path equality), the - // remaining generic body matches are not evidence for the entity: return - // no rankings so a caller cannot mistake command success for evidence - // coverage (issue #18). Exact content matches keep the rankings, with the - // unmatched unit still surfaced by name in the warnings. + (matches, matched_terms, entities) +} + +/// A high-specificity entity unit that matched no document means the query +/// names something this repository does not contain. When nothing else in +/// the query matched at identity level (exact fact or path equality), the +/// remaining generic body matches are not evidence for the entity: return +/// no rankings so a caller cannot mistake command success for evidence +/// coverage (issue #18). Exact content matches keep the rankings, with the +/// unmatched unit still surfaced by name in the warnings. +fn suppress_unanchored_entity_matches( + matches: Vec, + matched_terms: &BTreeSet, + entities: &BTreeSet, +) -> Vec { if entities .iter() .any(|entity| !matched_terms.contains(entity)) && !matches.iter().any(|ranking| ranking.exact_matches > 0) { - return (Vec::new(), matched_terms); + return Vec::new(); } + matches +} + +fn ranking_score_order(left: &SourceRanking, right: &SourceRanking) -> std::cmp::Ordering { + right + .exact_matches + .cmp(&left.exact_matches) + .then_with(|| right.partial_matches.cmp(&left.partial_matches)) + .then_with(|| right.active.cmp(&left.active)) + .then_with(|| right.canonical.cmp(&left.canonical)) + .then_with(|| right.public_safe.cmp(&left.public_safe)) + .then_with(|| right.confidence.cmp(&left.confidence)) + .then_with(|| right.freshness.cmp(&left.freshness)) +} + +fn finalize_source_rankings( + mut matches: Vec, + max_sources: usize, + cross_bundle: bool, +) -> Vec { matches.sort_by(|left, right| { - right - .exact_matches - .cmp(&left.exact_matches) - .then_with(|| right.partial_matches.cmp(&left.partial_matches)) - .then_with(|| right.active.cmp(&left.active)) - .then_with(|| right.canonical.cmp(&left.canonical)) - .then_with(|| right.public_safe.cmp(&left.public_safe)) - .then_with(|| right.confidence.cmp(&left.confidence)) - .then_with(|| right.freshness.cmp(&left.freshness)) - .then_with(|| left.id.cmp(&right.id)) + let score = ranking_score_order(left, right); + if cross_bundle { + score + .then_with(|| left.bundle.cmp(&right.bundle)) + .then_with(|| left.id.cmp(&right.id)) + } else { + score.then_with(|| left.id.cmp(&right.id)) + } }); matches.truncate(max_sources); for (index, ranking) in matches.iter_mut().enumerate() { ranking.rank = index + 1; } - (matches, matched_terms) + matches } fn query_view_spec(rankings: &[SourceRanking], max_sources: usize) -> ViewSpec { @@ -1773,6 +1950,53 @@ fn source_rankings_json(rankings: &[SourceRanking], selected_ids: &BTreeSet, +) -> String { + rankings + .iter() + .filter(|ranking| selected.contains(&(ranking.bundle.clone(), ranking.id.clone()))) + .map(|ranking| format!( + "{{\"bundle\":\"{}\",\"locator\":\"{}\",\"rank\":{},\"score\":{{\"exact\":{},\"partial\":{},\"active\":{},\"canonical\":{},\"public_safe\":{},\"confidence\":{},\"freshness\":\"{}\"}},\"match_reason\":\"{}\"}}", + json_escape(&ranking.bundle), json_escape(&ranking.locator), ranking.rank, + ranking.exact_matches, ranking.partial_matches, ranking.active, ranking.canonical, + ranking.public_safe, ranking.confidence, json_escape(&ranking.freshness), + json_escape(&ranking.match_reason) + )) + .collect::>() + .join(",") +} + +fn selected_sources_json_with_bundle( + selected: &[(String, String, &'static str)], +) -> (String, String) { + let locators = selected + .iter() + .map(|(bundle, locator, _)| { + format!( + "{{\"bundle\":\"{}\",\"locator\":\"{}\"}}", + json_escape(bundle), + json_escape(locator) + ) + }) + .collect::>() + .join(","); + let reasons = selected + .iter() + .map(|(bundle, locator, reason)| { + format!( + "{{\"bundle\":\"{}\",\"locator\":\"{}\",\"reason\":\"{}\"}}", + json_escape(bundle), + json_escape(locator), + reason + ) + }) + .collect::>() + .join(","); + (locators, reasons) +} + fn ranking_freshness(ranking: &SourceRanking) -> u64 { ranking .freshness @@ -1996,13 +2220,16 @@ fn selected_sources_json(selected: &[(String, &'static str)]) -> (String, String (locators, reasons) } -fn do_query(root: String, query_text: String) -> QueryPacketResult { +fn do_query(root: String, query_text: String, record: bool) -> QueryPacketResult { let root_path: &Path = Path::new(&root); let scanner = RepositoryScanner::new(root_path, ScanConfig::default()) .map_err(|e| format!("scan_error: {:?}", e))?; if !root_path.join(POLICY_FILENAME).is_file() { + if record { + return Err("evidence_store_unavailable".to_owned()); + } let data = format!( - "{{\"root\":\"{}\",\"query\":\"{}\",\"bran_status\":\"unavailable\",\"selected_locators\":[],\"why_selected\":[],\"source_rankings\":[],\"candidate_source_bytes\":0,\"selected_source_bytes\":0,\"context_bytes_avoided\":0,\"estimated_tokens\":0,\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\"}}", + "{{\"root\":\"{}\",\"query\":\"{}\",\"query_outcome\":\"unavailable\",\"query_coverage\":{{\"matched_terms\":[],\"unmatched_terms\":[]}},\"bran_status\":\"unavailable\",\"selected_locators\":[],\"why_selected\":[],\"source_rankings\":[],\"candidate_source_bytes\":0,\"selected_source_bytes\":0,\"context_bytes_avoided\":0,\"estimated_tokens\":0,\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\"}}", json_escape(&root), json_escape(&query_text) ); @@ -2058,14 +2285,186 @@ fn do_query(root: String, query_text: String) -> QueryPacketResult { .iter() .map(|d| format!("{:?}", d)) .collect(); - warns.extend(unmatched_query_warnings(&query_text, &matched_terms)); + let (query_outcome, query_coverage, unmatched) = + query_semantic_outcome(&query_text, &matched_terms, &rankings); + warns.extend(unmatched_query_warnings(&unmatched)); let (locs_json, why_selected_json) = selected_sources_json(&selected); let source_rankings_json = source_rankings_json(&rankings, &selected_ids); let data = format!( - "{{\"root\":\"{}\",\"query\":\"{}\",\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}],\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\"}}", + "{{\"root\":\"{}\",\"query\":\"{}\",\"query_outcome\":\"{}\",\"query_coverage\":{},\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}],\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\"}}", json_escape(&root), json_escape(&query_text), + query_outcome, + query_coverage, + locs_json, + why_selected_json, + source_rankings_json, + candidate_bytes, + selected_bytes, + context_bytes_avoided, + estimated + ); + let provenance = if locs_json.is_empty() { + "{\"sources\":[\"repository-scanner\",\"bran-core\"]}".to_owned() + } else { + format!( + "{{\"sources\":[\"repository-scanner\",\"bran-core\"],\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}]}}", + locs_json, why_selected_json, source_rankings_json + ) + }; + let metrics = format!( + "{{\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\"}}", + candidate_bytes, selected_bytes, context_bytes_avoided, estimated + ); + let evidence = query_evidence_record( + query_text, + vec![root], + query_outcome, + &matched_terms, + rankings.iter().map(evidence_ranking_from_source), + ); + if record { + persist_query_evidence(&evidence)?; + } + Ok(("ok", data, warns, vec![], provenance, metrics)) +} + +struct ScannedQueryRoot { + requested: String, + snapshot: ScanSnapshot, + graph_input: GraphInput, +} + +fn scan_policy_query_root(requested: &str) -> Result { + let root_path = Path::new(requested); + let scanner = RepositoryScanner::new(root_path, ScanConfig::default()) + .map_err(|_| format!("scan_error: {requested}"))?; + if !root_path.join(POLICY_FILENAME).is_file() { + return Err(format!("native_policy_unavailable: {requested}")); + } + let snapshot = scanner + .scan() + .map_err(|_| format!("scan_error: {requested}"))?; + let graph_input = snapshot + .graph_input() + .map_err(|_| format!("graph_input_error: {requested}"))?; + Ok(ScannedQueryRoot { + requested: requested.to_owned(), + snapshot, + graph_input, + }) +} + +fn do_query_multi(roots: Vec, query_text: String, record: bool) -> QueryPacketResult { + let mut seen = BTreeSet::new(); + for root in &roots { + if !seen.insert(root.as_str()) { + return Err(format!("duplicate_root: {root}")); + } + } + let scanned = roots + .iter() + .map(|root| scan_policy_query_root(root)) + .collect::, _>>()?; + + let mut matched_terms = BTreeSet::new(); + let mut entities = BTreeSet::new(); + let mut matches = Vec::new(); + for root in &scanned { + let (mut root_matches, root_matched, root_entities) = + score_source_candidates(&root.graph_input, &root.snapshot, &query_text); + for ranking in &mut root_matches { + ranking.bundle = root.requested.clone(); + } + matched_terms.extend(root_matched); + entities = root_entities; + matches.extend(root_matches); + } + let matches = suppress_unanchored_entity_matches(matches, &matched_terms, &entities); + let rankings = finalize_source_rankings(matches, QUERY_RESULT_LIMIT, true); + + let mut selected = BTreeMap::new(); + let mut selected_keys = BTreeSet::new(); + let mut warns = Vec::new(); + let mut candidate_bytes = 0usize; + for root in &scanned { + candidate_bytes += root.snapshot.total_bytes; + warns.extend( + root.snapshot + .diagnostics + .iter() + .map(|diagnostic| format!("{diagnostic:?}")), + ); + let root_rankings = rankings + .iter() + .filter(|ranking| ranking.bundle == root.requested) + .cloned() + .collect::>(); + if root_rankings.is_empty() { + continue; + } + let node_count = root.graph_input.nodes().len().max(1); + let edge_count = root.graph_input.edges().len().max(1); + let limits = GraphLimits::new(node_count, edge_count) + .map_err(|_| format!("limits_error: {}", root.requested))?; + let spec = query_view_spec(&root_rankings, QUERY_RESULT_LIMIT); + let graph = KnowledgeGraph::build(root.graph_input.clone(), limits) + .map_err(|_| format!("graph_error: {}", root.requested))?; + let compiled = ViewCompiler::new() + .compile(&spec, &graph) + .map_err(|_| format!("view_error: {}", root.requested))?; + let dependency_limits = DependencyClosureLimits::new(DEPENDENCY_DEPTH_LIMIT, 256) + .map_err(|_| format!("dep_limits: {}", root.requested))?; + let selected_ids = PacketAssembler::evidence_ids(&compiled, &graph, dependency_limits) + .map_err(|_| format!("packet_error: {}", root.requested))? + .into_iter() + .collect::>(); + let seed_ids = compiled + .items() + .iter() + .map(|item| item.id.clone()) + .collect::>(); + for (locator, reason) in selected_sources(&selected_ids, &seed_ids, &graph, &root.snapshot) + { + selected.insert((root.requested.clone(), locator), reason); + } + for id in selected_ids { + selected_keys.insert((root.requested.clone(), id)); + } + } + + let selected = selected + .into_iter() + .map(|((bundle, locator), reason)| (bundle, locator, reason)) + .collect::>(); + let selected_bytes = selected + .iter() + .map(|(bundle, locator, _)| { + scanned + .iter() + .find(|root| root.requested == *bundle) + .and_then(|root| root.snapshot.entries.get(locator.as_str())) + .map(|entry| entry.source.len()) + .unwrap_or(0) + }) + .sum::(); + let estimated = selected_bytes / 4 + usize::from(!selected_bytes.is_multiple_of(4)); + let context_bytes_avoided = candidate_bytes.saturating_sub(selected_bytes); + + let (query_outcome, query_coverage, unmatched) = + query_semantic_outcome(&query_text, &matched_terms, &rankings); + warns.extend(unmatched_query_warnings(&unmatched)); + + let (locs_json, why_selected_json) = selected_sources_json_with_bundle(&selected); + let source_rankings_json = source_rankings_json_with_bundle(&rankings, &selected_keys); + let data = format!( + "{{\"root\":\"{}\",\"requested_roots\":[{}],\"query\":\"{}\",\"query_outcome\":\"{}\",\"query_coverage\":{},\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}],\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\"}}", + json_escape(&roots[0]), + json_string_list(&roots), + json_escape(&query_text), + query_outcome, + query_coverage, locs_json, why_selected_json, source_rankings_json, @@ -2086,6 +2485,16 @@ fn do_query(root: String, query_text: String) -> QueryPacketResult { "{{\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\"}}", candidate_bytes, selected_bytes, context_bytes_avoided, estimated ); + let evidence = query_evidence_record( + query_text, + roots, + query_outcome, + &matched_terms, + rankings.iter().map(evidence_ranking_from_source), + ); + if record { + persist_query_evidence(&evidence)?; + } Ok(("ok", data, warns, vec![], provenance, metrics)) } @@ -2265,12 +2674,16 @@ fn do_packet(root: String, query_text: String, controls: &ExperimentalControls) .iter() .map(|d| format!("{:?}", d)) .collect(); - warns.extend(unmatched_query_warnings(&query_text, &matched_terms)); + let (query_outcome, query_coverage, unmatched) = + query_semantic_outcome(&query_text, &matched_terms, &rankings); + warns.extend(unmatched_query_warnings(&unmatched)); let data = format!( - "{{\"root\":\"{}\",\"query\":\"{}\",\"controls\":{},\"payload\":\"{}\",\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}],\"seed_ids\":[{}],\"admitted_dependency_ids\":[{}],\"selected_ids\":[{}],\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"excerpt_bytes\":{},\"raw_bytes\":{},\"encoded_packet_bytes\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\",\"runtime_token_ceiling\":{},\"truncated\":{},\"sqz\":{}}}", + "{{\"root\":\"{}\",\"query\":\"{}\",\"query_outcome\":\"{}\",\"query_coverage\":{},\"controls\":{},\"payload\":\"{}\",\"selected_locators\":[{}],\"why_selected\":[{}],\"source_rankings\":[{}],\"seed_ids\":[{}],\"admitted_dependency_ids\":[{}],\"selected_ids\":[{}],\"candidate_source_bytes\":{},\"selected_source_bytes\":{},\"context_bytes_avoided\":{},\"excerpt_bytes\":{},\"raw_bytes\":{},\"encoded_packet_bytes\":{},\"estimated_tokens\":{},\"token_estimate_method\":\"bytes-divided-by-four-ceiling\",\"actual_model_input_tokens\":\"unavailable\",\"runtime_token_ceiling\":{},\"truncated\":{},\"sqz\":{}}}", json_escape(&root), json_escape(&query_text), + query_outcome, + query_coverage, controls.controls_json(), json_escape(&pkt.payload), selected_locators_json, @@ -2906,71 +3319,1115 @@ fn do_maintain_revalidate(root: String) -> CliResult { } } -fn do_agents_list() -> CliResult { - if !agent_descriptor_configured() { - return unconfigured_agents_list(); - } - let descriptor = match ConfiguredAgentDescriptor::from_environment() { - Ok(descriptor) => descriptor, - Err(failure) => { - return CliResult::operation(make_envelope( - "agents.list", - "error", - "null", - &[], - &[failure.as_str().to_owned()], - "{}", - "{}", - )) - } - }; - let apr = match descriptor.registry() { - Ok(registry) => registry, - Err(failure) => return CliResult::operation(make_agents_error("list", failure.as_str())), - }; - let mut agent_strs = vec![]; - for p in apr.profiles() { - let name = json_escape(p.name()); - let provider = json_escape(p.provider()); - let model = json_escape(p.model()); - let account = json_escape(p.account_handle()); - let reasoning = p.default_reasoning_level().as_str(); - let tp = p.tool_policy(); - let allow = tp - .allowed() - .map(|t| format!("\"{}\"", json_escape(t))) - .collect::>() - .join(","); - let deny = tp - .denied() - .map(|t| format!("\"{}\"", json_escape(t))) - .collect::>() - .join(","); - agent_strs.push(format!( - "{{\"name\":\"{}\",\"provider\":\"{}\",\"model\":\"{}\",\"account_handle\":\"{}\",\"default_reasoning\":\"{}\",\"tool_policy\":{{\"allow\":[{}],\"deny\":[{}]}}}}", - name, provider, model, account, reasoning, allow, deny - )); - } - let data = format!("{{\"agents\":[{}]}}", agent_strs.join(",")); - CliResult::success(make_envelope( - "agents.list", - "ok", - &data, - &[], - &[], - "{}", - "{}", - )) +const EVIDENCE_STORE_VERSION: &str = "BRAN-QUERY-EVIDENCE/1"; +const EVIDENCE_STORE_RELATIVE: &str = ".bran/cache/query-evidence"; +const EVIDENCE_STORE_TMP_NAME: &str = "query-evidence.tmp"; +const MAX_EVIDENCE_RECORDS: usize = 32; +const MAX_EVIDENCE_FILE_BYTES: usize = 64 * 1024; +const MAX_EVIDENCE_REQUEST_BYTES: usize = 4096; +const MAX_EVIDENCE_ROOTS: usize = 8; +const MAX_EVIDENCE_FIELD_BYTES: usize = 4096; + +#[derive(Clone, Debug, Eq, PartialEq)] +struct QueryEvidenceRecord { + request: String, + requested_roots: Vec, + outcome: String, + matched: Vec, + unmatched: Vec, + rankings: Vec, } -fn agent_descriptor_configured() -> bool { - [ - "BRAN_AGENT_PROFILE", - "BRAN_AGENT_PROVIDER", - "BRAN_AGENT_MODEL", - "BRAN_AGENT_REASONING", - "BRAN_AGENT_ACCOUNT_REF", - ] +#[derive(Clone, Debug, Eq, PartialEq)] +struct QueryEvidenceRanking { + bundle: String, + locator: String, + rank: usize, + match_reason: String, + exact: usize, + partial: usize, + active: u8, + canonical: u8, + public_safe: u8, + confidence: u8, + freshness: String, +} + +fn evidence_ranking_from_source(ranking: &SourceRanking) -> QueryEvidenceRanking { + QueryEvidenceRanking { + bundle: ranking.bundle.clone(), + locator: ranking.locator.clone(), + rank: ranking.rank, + match_reason: ranking.match_reason.clone(), + exact: ranking.exact_matches, + partial: ranking.partial_matches, + active: ranking.active, + canonical: ranking.canonical, + public_safe: ranking.public_safe, + confidence: ranking.confidence, + freshness: ranking.freshness.clone(), + } +} + +fn query_evidence_record( + request: String, + requested_roots: Vec, + outcome: &str, + matched_terms: &BTreeSet, + rankings: I, +) -> QueryEvidenceRecord +where + I: IntoIterator, +{ + let (matched, unmatched) = query_term_coverage(&request, matched_terms); + QueryEvidenceRecord { + request, + requested_roots, + outcome: outcome.to_owned(), + matched, + unmatched, + rankings: rankings.into_iter().collect(), + } +} + +fn is_path_only_reason(reason: &str) -> bool { + matches!(reason, "exact:path" | "partial:path") +} + +fn current_knowledge_documents(snapshot: &ScanSnapshot) -> Vec { + snapshot + .entries + .keys() + .filter(|path| is_knowledge_document_path(path)) + .cloned() + .collect() +} + +fn evidence_escape(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 8); + for character in value.chars() { + match character { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + _ => out.push(character), + } + } + out +} + +fn evidence_unescape(value: &str) -> Result { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(character) = chars.next() { + if character != '\\' { + out.push(character); + continue; + } + match chars.next() { + Some('\\') => out.push('\\'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some('t') => out.push('\t'), + _ => return Err("evidence_store_corrupt"), + } + } + if out.len() > MAX_EVIDENCE_FIELD_BYTES { + return Err("evidence_store_oversized"); + } + Ok(out) +} + +fn evidence_store_path(root: &Path) -> Result { + if root + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + return Err("evidence_store_unsafe"); + } + let bran = root.join(".bran"); + match fs::symlink_metadata(&bran) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => return Err("evidence_store_unsafe"), + Err(_) => return Err("evidence_store_unavailable"), + } + let policy = root.join(POLICY_FILENAME); + match fs::symlink_metadata(&policy) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {} + Ok(_) => return Err("evidence_store_unsafe"), + Err(_) => return Err("evidence_store_unavailable"), + } + let cache = bran.join("cache"); + match fs::symlink_metadata(&cache) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => return Err("evidence_store_unsafe"), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("evidence_store_unavailable"), + } + let store = cache.join("query-evidence"); + if store + .components() + .any(|component| matches!(component, Component::ParentDir)) + || store.parent().is_none_or(|parent| { + parent.file_name() != Some(OsStr::new("cache")) + || parent + .parent() + .is_none_or(|bran_parent| bran_parent.file_name() != Some(OsStr::new(".bran"))) + }) + { + return Err("evidence_store_unsafe"); + } + Ok(store) +} + +fn inspect_evidence_file(path: &Path) -> Result, &'static str> { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(_) => Err("evidence_store_unavailable"), + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Err("evidence_store_unsafe") + } + Ok(metadata) if metadata.len() > MAX_EVIDENCE_FILE_BYTES as u64 => { + Err("evidence_store_oversized") + } + Ok(_) => Ok(Some(())), + } +} + +fn load_evidence_records(root: &str) -> Result, &'static str> { + let path = evidence_store_path(Path::new(root))?; + if inspect_evidence_file(&path)?.is_none() { + return Ok(Vec::new()); + } + let bytes = + read_regular_bounded(&path, MAX_EVIDENCE_FILE_BYTES).map_err(|error| match error { + ResultStoreError::NotFound => "evidence_store_unavailable", + ResultStoreError::Corrupt => "evidence_store_corrupt", + ResultStoreError::ItemTooLarge => "evidence_store_oversized", + _ => "evidence_store_unavailable", + })?; + parse_evidence_store(&bytes) +} + +fn parse_evidence_store(bytes: &[u8]) -> Result, &'static str> { + let text = std::str::from_utf8(bytes).map_err(|_| "evidence_store_corrupt")?; + let mut lines = text.lines(); + match lines.next() { + Some(EVIDENCE_STORE_VERSION) => {} + _ => return Err("evidence_store_corrupt"), + } + let mut records = Vec::new(); + let mut current: Option = None; + let mut seen_request = false; + let mut seen_outcome = false; + let finish = |record: QueryEvidenceRecord, + seen_request: bool, + seen_outcome: bool| + -> Result { + if !seen_request + || !seen_outcome + || record.requested_roots.is_empty() + || record.requested_roots.len() > MAX_EVIDENCE_ROOTS + || record.rankings.len() > QUERY_RESULT_LIMIT + || !matches!( + record.outcome.as_str(), + "grounded" | "miss" | "partial_unanchored" | "unavailable" + ) + { + return Err("evidence_store_corrupt"); + } + Ok(record) + }; + for line in lines { + if line == "---" { + if let Some(record) = current.take() { + records.push(finish(record, seen_request, seen_outcome)?); + if records.len() > MAX_EVIDENCE_RECORDS { + return Err("evidence_store_full"); + } + } + current = Some(QueryEvidenceRecord { + request: String::new(), + requested_roots: Vec::new(), + outcome: String::new(), + matched: Vec::new(), + unmatched: Vec::new(), + rankings: Vec::new(), + }); + seen_request = false; + seen_outcome = false; + continue; + } + let Some(record) = current.as_mut() else { + return Err("evidence_store_corrupt"); + }; + let Some((key, value)) = line.split_once('\t') else { + return Err("evidence_store_corrupt"); + }; + match key { + "request" if !seen_request => { + record.request = evidence_unescape(value)?; + if record.request.len() > MAX_EVIDENCE_REQUEST_BYTES { + return Err("evidence_store_oversized"); + } + seen_request = true; + } + "root" => { + record.requested_roots.push(evidence_unescape(value)?); + } + "outcome" if !seen_outcome => { + record.outcome = evidence_unescape(value)?; + seen_outcome = true; + } + "matched" => record.matched.push(evidence_unescape(value)?), + "unmatched" => record.unmatched.push(evidence_unescape(value)?), + "rank" => record.rankings.push(parse_evidence_rank(value)?), + _ => return Err("evidence_store_corrupt"), + } + } + if let Some(record) = current.take() { + records.push(finish(record, seen_request, seen_outcome)?); + } + if records.len() > MAX_EVIDENCE_RECORDS { + return Err("evidence_store_full"); + } + Ok(records) +} + +fn parse_evidence_rank(value: &str) -> Result { + let parts: Vec<&str> = value.split('\t').collect(); + if parts.len() != 11 { + return Err("evidence_store_corrupt"); + } + Ok(QueryEvidenceRanking { + rank: parts[0].parse().map_err(|_| "evidence_store_corrupt")?, + bundle: evidence_unescape(parts[1])?, + locator: evidence_unescape(parts[2])?, + match_reason: evidence_unescape(parts[3])?, + exact: parts[4].parse().map_err(|_| "evidence_store_corrupt")?, + partial: parts[5].parse().map_err(|_| "evidence_store_corrupt")?, + active: parts[6].parse().map_err(|_| "evidence_store_corrupt")?, + canonical: parts[7].parse().map_err(|_| "evidence_store_corrupt")?, + public_safe: parts[8].parse().map_err(|_| "evidence_store_corrupt")?, + confidence: parts[9].parse().map_err(|_| "evidence_store_corrupt")?, + freshness: evidence_unescape(parts[10])?, + }) +} + +fn encode_evidence_store(records: &[QueryEvidenceRecord]) -> Result, &'static str> { + if records.len() > MAX_EVIDENCE_RECORDS { + return Err("evidence_store_full"); + } + let mut out = String::from(EVIDENCE_STORE_VERSION); + out.push('\n'); + for record in records { + if record.request.len() > MAX_EVIDENCE_REQUEST_BYTES + || record.requested_roots.len() > MAX_EVIDENCE_ROOTS + || record.rankings.len() > QUERY_RESULT_LIMIT + { + return Err("evidence_store_oversized"); + } + out.push_str("---\nrequest\t"); + out.push_str(&evidence_escape(&record.request)); + out.push('\n'); + for root in &record.requested_roots { + out.push_str("root\t"); + out.push_str(&evidence_escape(root)); + out.push('\n'); + } + out.push_str("outcome\t"); + out.push_str(&evidence_escape(&record.outcome)); + out.push('\n'); + for term in &record.matched { + out.push_str("matched\t"); + out.push_str(&evidence_escape(term)); + out.push('\n'); + } + for term in &record.unmatched { + out.push_str("unmatched\t"); + out.push_str(&evidence_escape(term)); + out.push('\n'); + } + for ranking in &record.rankings { + out.push_str("rank\t"); + out.push_str(&ranking.rank.to_string()); + out.push('\t'); + out.push_str(&evidence_escape(&ranking.bundle)); + out.push('\t'); + out.push_str(&evidence_escape(&ranking.locator)); + out.push('\t'); + out.push_str(&evidence_escape(&ranking.match_reason)); + out.push('\t'); + out.push_str(&ranking.exact.to_string()); + out.push('\t'); + out.push_str(&ranking.partial.to_string()); + out.push('\t'); + out.push_str(&ranking.active.to_string()); + out.push('\t'); + out.push_str(&ranking.canonical.to_string()); + out.push('\t'); + out.push_str(&ranking.public_safe.to_string()); + out.push('\t'); + out.push_str(&ranking.confidence.to_string()); + out.push('\t'); + out.push_str(&evidence_escape(&ranking.freshness)); + out.push('\n'); + } + if out.len() > MAX_EVIDENCE_FILE_BYTES { + return Err("evidence_store_oversized"); + } + } + if out.len() > MAX_EVIDENCE_FILE_BYTES { + return Err("evidence_store_oversized"); + } + Ok(out.into_bytes()) +} + +fn write_evidence_records(root: &Path, records: &[QueryEvidenceRecord]) -> Result<(), String> { + let store = evidence_store_path(root).map_err(str::to_owned)?; + if inspect_evidence_file(&store).is_err() { + return Err("evidence_store_unsafe".to_owned()); + } + let cache = root.join(".bran").join("cache"); + checked_directory(&cache).map_err(|error| match error { + ResultStoreError::Corrupt => "evidence_store_unsafe".to_owned(), + _ => "evidence_store_write_failed".to_owned(), + })?; + let encoded = encode_evidence_store(records).map_err(str::to_owned)?; + let tmp = cache.join(EVIDENCE_STORE_TMP_NAME); + match fs::symlink_metadata(&tmp) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("evidence_store_unavailable".to_owned()), + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Err("evidence_store_unsafe".to_owned()); + } + Ok(_) => { + fs::remove_file(&tmp).map_err(|_| "evidence_store_write_failed".to_owned())?; + } + } + if let Err(error) = write_private_file(&tmp, &encoded) { + let _ = fs::remove_file(&tmp); + return Err(match error { + ResultStoreError::Corrupt => "evidence_store_corrupt".to_owned(), + _ => "evidence_store_write_failed".to_owned(), + }); + } + if fs::rename(&tmp, &store).is_err() { + let _ = fs::remove_file(&tmp); + return Err("evidence_store_write_failed".to_owned()); + } + Ok(()) +} + +fn persist_query_evidence(record: &QueryEvidenceRecord) -> Result<(), String> { + let primary = record + .requested_roots + .first() + .ok_or_else(|| "evidence_store_unavailable".to_owned())?; + if record.request.len() > MAX_EVIDENCE_REQUEST_BYTES + || record.requested_roots.len() > MAX_EVIDENCE_ROOTS + || record.rankings.len() > QUERY_RESULT_LIMIT + { + return Err("evidence_store_oversized".to_owned()); + } + let mut records = load_evidence_records(primary).map_err(str::to_owned)?; + records.retain(|existing| { + existing.request != record.request || existing.requested_roots != record.requested_roots + }); + if records.len() >= MAX_EVIDENCE_RECORDS { + return Err("evidence_store_full".to_owned()); + } + records.push(record.clone()); + records.sort_by(|left, right| { + left.request + .cmp(&right.request) + .then_with(|| left.requested_roots.cmp(&right.requested_roots)) + }); + write_evidence_records(Path::new(primary), &records) +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct EvidenceDocRef { + bundle: String, + locator: String, +} + +impl EvidenceDocRef { + fn new(bundle: impl Into, locator: impl Into) -> Self { + Self { + bundle: bundle.into(), + locator: locator.into(), + } + } + + fn json(&self) -> String { + format!( + "{{\"bundle\":\"{}\",\"locator\":\"{}\"}}", + json_escape(&self.bundle), + json_escape(&self.locator) + ) + } +} + +fn json_doc_refs(values: &BTreeSet) -> String { + values + .iter() + .map(EvidenceDocRef::json) + .collect::>() + .join(",") +} + +fn ranking_bundle(record: &QueryEvidenceRecord, ranking: &QueryEvidenceRanking) -> String { + if ranking.bundle.is_empty() { + record.requested_roots.first().cloned().unwrap_or_default() + } else { + ranking.bundle.clone() + } +} + +#[derive(Clone, Debug)] +struct EvidenceBundleView { + available: bool, + documents: BTreeSet, + unclassified: BTreeSet, + stale: BTreeMap>, +} + +impl EvidenceBundleView { + fn unavailable() -> Self { + Self { + available: false, + documents: BTreeSet::new(), + unclassified: BTreeSet::new(), + stale: BTreeMap::new(), + } + } +} + +fn map_primary_scan_error(error: String) -> String { + if error.starts_with("native_policy_unavailable") { + "evidence_store_unavailable".to_owned() + } else { + error + } +} + +fn inspect_evidence_bundle(root: &str, snapshot: &ScanSnapshot) -> EvidenceBundleView { + EvidenceBundleView { + available: true, + documents: current_knowledge_documents(snapshot).into_iter().collect(), + unclassified: unclassified_documents(root, snapshot).into_iter().collect(), + stale: current_stale_documents(root, snapshot), + } +} + +fn load_evidence_bundle_views( + primary: &str, + records: &[QueryEvidenceRecord], +) -> Result, String> { + let scanned = scan_policy_query_root(primary).map_err(map_primary_scan_error)?; + let mut views = BTreeMap::new(); + views.insert( + primary.to_owned(), + inspect_evidence_bundle(primary, &scanned.snapshot), + ); + let mut extra = BTreeSet::new(); + for record in records { + extra.extend(record.requested_roots.iter().cloned()); + } + extra.remove(primary); + for root in extra { + match scan_policy_query_root(&root) { + Ok(scanned) => { + views.insert( + root.clone(), + inspect_evidence_bundle(&root, &scanned.snapshot), + ); + } + Err(_) => { + views.insert(root, EvidenceBundleView::unavailable()); + } + } + } + Ok(views) +} + +fn is_stale_diagnostic_code(code: &str) -> bool { + matches!( + code, + "stale-frontmatter" | "stale-body" | "stale-after-shape" | "status-value" + ) +} + +fn current_stale_documents( + root: &str, + snapshot: &ScanSnapshot, +) -> BTreeMap> { + let mut stale: BTreeMap> = BTreeMap::new(); + let Ok(bundle) = derive_bundle_from_snapshot(snapshot) else { + return stale; + }; + let policy = RepositoryPolicy::load(Path::new(root)).ok(); + let result = ProfileValidator::validate_with_policy(&bundle, BRAN_STRICT, policy.as_ref()); + for outcome in [ + &result.bran_strict, + &result.okf_v0_2, + &result.okf_compatibility, + ] { + for diagnostic in &outcome.diagnostics { + if is_stale_diagnostic_code(&diagnostic.code) { + stale + .entry(diagnostic.path.clone()) + .or_default() + .insert(diagnostic.code.clone()); + } + } + } + for (path, doc) in bundle.docs() { + if let Some(map) = doc.frontmatter().parsed() { + if let Some(bran_core::schema::YamlValue::String(value)) = map.get("freshness") { + if value.trim().eq_ignore_ascii_case("stale") { + stale + .entry(path.clone()) + .or_default() + .insert("stale-frontmatter".to_owned()); + } + } + } + if doc.body().contains("STALE_CLAIM") { + stale + .entry(path.clone()) + .or_default() + .insert("stale-body".to_owned()); + } + } + stale +} + +fn unclassified_documents(root: &str, snapshot: &ScanSnapshot) -> Vec { + let coverage = RepositoryPolicy::load(Path::new(root)) + .ok() + .and_then(|policy| policy.document_coverage); + let mut classified = BTreeSet::new(); + let mut excluded = BTreeSet::new(); + if let Some(coverage) = coverage { + classified.extend(coverage.native_bundle); + classified.extend(coverage.canonical_documents); + classified.extend(coverage.legacy_documents); + excluded.extend(coverage.excluded_documents.into_keys()); + } + current_knowledge_documents(snapshot) + .into_iter() + .filter(|path| !excluded.contains(path) && !classified.contains(path)) + .collect() +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct StaleGap { + kind: String, + bundle: String, + locator: String, + request: Option, +} + +fn stale_gap_json(gap: &StaleGap, codes: &BTreeSet) -> String { + let request = gap.request.as_deref().map_or_else( + || "null".to_owned(), + |value| format!("\"{}\"", json_escape(value)), + ); + format!( + "{{\"kind\":\"{}\",\"bundle\":\"{}\",\"locator\":\"{}\",\"request\":{},\"codes\":[{}]}}", + json_escape(&gap.kind), + json_escape(&gap.bundle), + json_escape(&gap.locator), + request, + json_string_list(&codes.iter().cloned().collect::>()) + ) +} + +struct EvidenceGapSummary { + record_count: usize, + unanswered: BTreeSet, + retrieved: BTreeSet, + path_only: BTreeSet, + never_retrieved: BTreeSet, + unclassified: BTreeSet, + stale: BTreeMap>, +} + +fn collect_evidence_gaps( + primary: &str, + records: &[QueryEvidenceRecord], +) -> Result { + let views = load_evidence_bundle_views(primary, records)?; + let mut unanswered = BTreeSet::new(); + let mut retrieved = BTreeSet::new(); + let mut path_only = BTreeSet::new(); + let mut stale = BTreeMap::new(); + + for record in records { + unanswered.extend(record.unmatched.iter().cloned()); + for ranking in &record.rankings { + let bundle = ranking_bundle(record, ranking); + let identity = EvidenceDocRef::new(bundle.clone(), ranking.locator.clone()); + retrieved.insert(identity.clone()); + if is_path_only_reason(&ranking.match_reason) { + path_only.insert(identity); + } + let missing = match views.get(&bundle) { + Some(view) if view.available => !view.documents.contains(&ranking.locator), + _ => true, + }; + if missing { + stale.insert( + StaleGap { + kind: "missing_locator".to_owned(), + bundle, + locator: ranking.locator.clone(), + request: Some(record.request.clone()), + }, + BTreeSet::new(), + ); + } + } + for requested in &record.requested_roots { + if views.get(requested).is_none_or(|view| !view.available) { + stale.insert( + StaleGap { + kind: "unavailable_root".to_owned(), + bundle: requested.clone(), + locator: String::new(), + request: Some(record.request.clone()), + }, + BTreeSet::new(), + ); + } + } + } + + let mut never_retrieved = BTreeSet::new(); + let mut unclassified = BTreeSet::new(); + for (bundle, view) in &views { + if !view.available { + continue; + } + for locator in &view.documents { + let identity = EvidenceDocRef::new(bundle.clone(), locator.clone()); + if !retrieved.contains(&identity) { + never_retrieved.insert(identity); + } + } + for locator in &view.unclassified { + unclassified.insert(EvidenceDocRef::new(bundle.clone(), locator.clone())); + } + for (locator, codes) in &view.stale { + stale.insert( + StaleGap { + kind: "stale_metadata".to_owned(), + bundle: bundle.clone(), + locator: locator.clone(), + request: None, + }, + codes.clone(), + ); + } + } + + Ok(EvidenceGapSummary { + record_count: records.len(), + unanswered, + retrieved, + path_only, + never_retrieved, + unclassified, + stale, + }) +} + +fn evidence_summary_data(root: &str) -> Result { + let records = load_evidence_records(root).map_err(str::to_owned)?; + let summary = collect_evidence_gaps(root, &records)?; + let stale_json = summary + .stale + .iter() + .map(|(gap, codes)| stale_gap_json(gap, codes)) + .collect::>() + .join(","); + Ok(format!( + "{{\"root\":\"{}\",\"record_count\":{},\"unanswered_terms\":[{}],\"retrieved_documents\":[{}],\"path_only_retrievals\":[{}],\"never_retrieved_documents\":[{}],\"unclassified_documents\":[{}],\"stale_records\":[{}]}}", + json_escape(root), + summary.record_count, + json_string_list(&summary.unanswered.into_iter().collect::>()), + json_doc_refs(&summary.retrieved), + json_doc_refs(&summary.path_only), + json_doc_refs(&summary.never_retrieved), + json_doc_refs(&summary.unclassified), + stale_json + )) +} + +fn evidence_propose_data(root: &str) -> Result { + let records = load_evidence_records(root).map_err(str::to_owned)?; + let summary = collect_evidence_gaps(root, &records) + .map_err(|_| "evidence_store_unavailable".to_owned())?; + let mut candidates = Vec::new(); + let push_term = |candidates: &mut Vec, term: &str| { + candidates.push(format!( + "{{\"kind\":\"unanswered_term\",\"target\":null,\"term\":\"{}\",\"replacement\":null,\"authority\":null}}", + json_escape(term) + )); + }; + let push_doc = |candidates: &mut Vec, kind: &str, target: &EvidenceDocRef| { + candidates.push(format!( + "{{\"kind\":\"{}\",\"target\":{},\"term\":null,\"replacement\":null,\"authority\":null}}", + kind, + target.json() + )); + }; + for term in &summary.unanswered { + push_term(&mut candidates, term); + } + for locator in &summary.path_only { + push_doc(&mut candidates, "path_only_retrieval", locator); + } + for locator in &summary.never_retrieved { + push_doc(&mut candidates, "never_retrieved", locator); + } + for locator in &summary.unclassified { + push_doc(&mut candidates, "unclassified", locator); + } + for gap in summary.stale.keys() { + let locator = if gap.locator.is_empty() { + gap.bundle.clone() + } else { + gap.locator.clone() + }; + let target = EvidenceDocRef::new(gap.bundle.clone(), locator); + push_doc(&mut candidates, "stale_record", &target); + } + Ok(format!("{{\"candidates\":[{}]}}", candidates.join(","))) +} + +fn ranking_identity(ranking: &QueryEvidenceRanking) -> (String, String) { + (ranking.bundle.clone(), ranking.locator.clone()) +} + +fn ranking_payload(ranking: &QueryEvidenceRanking) -> String { + format!( + "{}|{}|{}|{}|{}|{}|{}|{}|{}", + ranking.rank, + ranking.match_reason, + ranking.exact, + ranking.partial, + ranking.active, + ranking.canonical, + ranking.public_safe, + ranking.confidence, + ranking.freshness + ) +} + +fn evaluate_query_record(record: &QueryEvidenceRecord) -> Result { + let roots = record.requested_roots.clone(); + if roots.is_empty() { + return Err("evidence_store_corrupt".to_owned()); + } + query_evidence_from_live( + roots, + record.request.clone(), + record.requested_roots.len() > 1, + ) +} + +fn query_evidence_from_live( + roots: Vec, + query_text: String, + multi: bool, +) -> Result { + if !multi { + let scanned = scan_policy_query_root(&roots[0])?; + let (rankings, matched_terms) = source_rankings( + &scanned.graph_input, + &scanned.snapshot, + &query_text, + QUERY_RESULT_LIMIT, + ); + let (outcome, _, _) = query_semantic_outcome(&query_text, &matched_terms, &rankings); + return Ok(query_evidence_record( + query_text, + roots, + outcome, + &matched_terms, + rankings.into_iter().map(move |ranking| { + let mut item = evidence_ranking_from_source(&ranking); + item.bundle = String::new(); + item + }), + )); + } + let scanned = roots + .iter() + .map(|root| scan_policy_query_root(root)) + .collect::, _>>()?; + let mut matched_terms = BTreeSet::new(); + let mut entities = BTreeSet::new(); + let mut matches = Vec::new(); + for root in &scanned { + let (mut root_matches, root_matched, root_entities) = + score_source_candidates(&root.graph_input, &root.snapshot, &query_text); + for ranking in &mut root_matches { + ranking.bundle = root.requested.clone(); + } + matched_terms.extend(root_matched); + entities = root_entities; + matches.extend(root_matches); + } + let matches = suppress_unanchored_entity_matches(matches, &matched_terms, &entities); + let rankings = finalize_source_rankings(matches, QUERY_RESULT_LIMIT, true); + let (outcome, _, _) = query_semantic_outcome(&query_text, &matched_terms, &rankings); + Ok(query_evidence_record( + query_text, + roots, + outcome, + &matched_terms, + rankings + .into_iter() + .map(|ranking| evidence_ranking_from_source(&ranking)), + )) +} + +fn ranking_differences( + recorded: &[QueryEvidenceRanking], + current: &[QueryEvidenceRanking], +) -> Vec { + let mut recorded_map = BTreeMap::new(); + for ranking in recorded { + recorded_map.insert(ranking_identity(ranking), ranking); + } + let mut current_map = BTreeMap::new(); + for ranking in current { + current_map.insert(ranking_identity(ranking), ranking); + } + let mut keys = BTreeSet::new(); + keys.extend(recorded_map.keys().cloned()); + keys.extend(current_map.keys().cloned()); + let mut diffs = Vec::new(); + for key in keys { + match (recorded_map.get(&key), current_map.get(&key)) { + (Some(before), Some(after)) if ranking_payload(before) == ranking_payload(after) => {} + (Some(before), Some(after)) => diffs.push(format!( + "{{\"kind\":\"changed\",\"bundle\":\"{}\",\"locator\":\"{}\",\"recorded_rank\":{},\"current_rank\":{},\"recorded_match_reason\":\"{}\",\"current_match_reason\":\"{}\"}}", + json_escape(&before.bundle), + json_escape(&before.locator), + before.rank, + after.rank, + json_escape(&before.match_reason), + json_escape(&after.match_reason) + )), + (Some(before), None) => diffs.push(format!( + "{{\"kind\":\"removed\",\"bundle\":\"{}\",\"locator\":\"{}\",\"recorded_rank\":{},\"current_rank\":null,\"recorded_match_reason\":\"{}\",\"current_match_reason\":null}}", + json_escape(&before.bundle), + json_escape(&before.locator), + before.rank, + json_escape(&before.match_reason) + )), + (None, Some(after)) => diffs.push(format!( + "{{\"kind\":\"added\",\"bundle\":\"{}\",\"locator\":\"{}\",\"recorded_rank\":null,\"current_rank\":{},\"recorded_match_reason\":null,\"current_match_reason\":\"{}\"}}", + json_escape(&after.bundle), + json_escape(&after.locator), + after.rank, + json_escape(&after.match_reason) + )), + (None, None) => {} + } + } + diffs +} + +fn do_evidence_summarize(root: String) -> CliResult { + match evidence_summary_data(&root) { + Ok(data) => CliResult::success(make_envelope( + "evidence.summarize", + "ok", + &data, + &[], + &[], + "{}", + "{}", + )), + Err(detail) => CliResult::operation(make_evidence_error("summarize", &detail)), + } +} + +fn do_evidence_propose(root: String) -> CliResult { + match evidence_propose_data(&root) { + Ok(data) => CliResult::success(make_envelope( + "evidence.propose", + "ok", + &data, + &[], + &[], + "{}", + "{}", + )), + Err(detail) => CliResult::operation(make_evidence_error("propose", &detail)), + } +} + +fn do_evidence_replay(root: String) -> CliResult { + let records = match load_evidence_records(&root) { + Ok(records) => records, + Err(detail) => return CliResult::operation(make_evidence_error("replay", detail)), + }; + if let Err(detail) = evidence_store_path(Path::new(&root)) { + return CliResult::operation(make_evidence_error("replay", detail)); + } + let mut replays = Vec::new(); + let mut difference_count = 0usize; + for record in &records { + let current = match evaluate_query_record(record) { + Ok(current) => current, + Err(_) => QueryEvidenceRecord { + request: record.request.clone(), + requested_roots: record.requested_roots.clone(), + outcome: "unavailable".to_owned(), + matched: Vec::new(), + unmatched: Vec::new(), + rankings: Vec::new(), + }, + }; + let diffs = ranking_differences(&record.rankings, ¤t.rankings); + let outcome_changed = record.outcome != current.outcome; + if outcome_changed || !diffs.is_empty() { + difference_count += 1; + } + replays.push(format!( + "{{\"request\":\"{}\",\"requested_roots\":[{}],\"recorded_outcome\":\"{}\",\"current_outcome\":\"{}\",\"outcome_changed\":{},\"ranking_differences\":[{}]}}", + json_escape(&record.request), + json_string_list(&record.requested_roots), + json_escape(&record.outcome), + json_escape(¤t.outcome), + if outcome_changed { "true" } else { "false" }, + diffs.join(",") + )); + } + let data = format!( + "{{\"root\":\"{}\",\"replays\":[{}],\"difference_count\":{}}}", + json_escape(&root), + replays.join(","), + difference_count + ); + CliResult::success(make_envelope( + "evidence.replay", + "ok", + &data, + &[], + &[], + "{}", + "{}", + )) +} + +fn do_evidence_clear(root: String) -> CliResult { + let path = match evidence_store_path(Path::new(&root)) { + Ok(path) => path, + Err(detail) => return CliResult::operation(make_evidence_error("clear", detail)), + }; + let removed = match inspect_evidence_file(&path) { + Ok(None) => false, + Err(detail) => return CliResult::operation(make_evidence_error("clear", detail)), + Ok(Some(())) => { + if fs::remove_file(&path).is_err() { + return CliResult::operation(make_evidence_error( + "clear", + "evidence_store_write_failed", + )); + } + true + } + }; + let data = format!( + "{{\"path\":\"{}\",\"removed\":{}}}", + EVIDENCE_STORE_RELATIVE, + if removed { "true" } else { "false" } + ); + CliResult::success(make_envelope( + "evidence.clear", + "ok", + &data, + &[], + &[], + "{}", + "{}", + )) +} + +fn do_agents_list() -> CliResult { + if !agent_descriptor_configured() { + return unconfigured_agents_list(); + } + let descriptor = match ConfiguredAgentDescriptor::from_environment() { + Ok(descriptor) => descriptor, + Err(failure) => { + return CliResult::operation(make_envelope( + "agents.list", + "error", + "null", + &[], + &[failure.as_str().to_owned()], + "{}", + "{}", + )) + } + }; + let apr = match descriptor.registry() { + Ok(registry) => registry, + Err(failure) => return CliResult::operation(make_agents_error("list", failure.as_str())), + }; + let mut agent_strs = vec![]; + for p in apr.profiles() { + let name = json_escape(p.name()); + let provider = json_escape(p.provider()); + let model = json_escape(p.model()); + let account = json_escape(p.account_handle()); + let reasoning = p.default_reasoning_level().as_str(); + let tp = p.tool_policy(); + let allow = tp + .allowed() + .map(|t| format!("\"{}\"", json_escape(t))) + .collect::>() + .join(","); + let deny = tp + .denied() + .map(|t| format!("\"{}\"", json_escape(t))) + .collect::>() + .join(","); + agent_strs.push(format!( + "{{\"name\":\"{}\",\"provider\":\"{}\",\"model\":\"{}\",\"account_handle\":\"{}\",\"default_reasoning\":\"{}\",\"tool_policy\":{{\"allow\":[{}],\"deny\":[{}]}}}}", + name, provider, model, account, reasoning, allow, deny + )); + } + let data = format!("{{\"agents\":[{}]}}", agent_strs.join(",")); + CliResult::success(make_envelope( + "agents.list", + "ok", + &data, + &[], + &[], + "{}", + "{}", + )) +} + +fn agent_descriptor_configured() -> bool { + [ + "BRAN_AGENT_PROFILE", + "BRAN_AGENT_PROVIDER", + "BRAN_AGENT_MODEL", + "BRAN_AGENT_REASONING", + "BRAN_AGENT_ACCOUNT_REF", + ] .iter() .filter_map(std::env::var_os) .any(|value| !value.is_empty()) @@ -7584,81 +9041,559 @@ mod tests { "bran-query-unmatched-entity-{}", std::process::id() )); - let _ = std::fs::remove_dir_all(&root); - std::fs::create_dir_all(root.join(".bran")).unwrap(); - std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); - std::fs::write( - root.join("notes.md"), - "---\ntype: concept\ntitle: Notes\n---\nGeneric notes about the nonexistent collector wrapper and runner.\n", - ) - .unwrap(); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join(".bran")).unwrap(); + std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); + std::fs::write( + root.join("notes.md"), + "---\ntype: concept\ntitle: Notes\n---\nGeneric notes about the nonexistent collector wrapper and runner.\n", + ) + .unwrap(); + + let result = CliApp::run(vec![ + "query".to_owned(), + root.to_string_lossy().into_owned(), + "zzq-entity-unit-8873".to_owned(), + ]); + + assert_eq!(result.exit_code, ExitCode::SUCCESS, "{}", result.output); + assert!(result.output.contains("\"source_rankings\":[],")); + assert!(!result.output.contains("\"locator\":\"notes.md\"")); + assert!(result + .output + .contains("unmatched_query_terms: zzq-entity-unit-8873")); + + // The same holds when generic words around the missing entity match + // bodies: their weak matches must not be presented as evidence either. + let diluted = CliApp::run(vec![ + "query".to_owned(), + root.to_string_lossy().into_owned(), + "Where are the zzq-entity-unit-8873 wrapper, runner, and tests documented?".to_owned(), + ]); + assert_eq!(diluted.exit_code, ExitCode::SUCCESS, "{}", diluted.output); + assert!(diluted.output.contains("\"source_rankings\":[],")); + assert!(!diluted.output.contains("\"locator\":\"notes.md\"")); + assert!(diluted + .output + .contains("unmatched_query_terms: zzq-entity-unit-8873")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn sub_token_does_not_count_as_entity_match() { + // Only the whole identifier unit matches: a document holding just its + // sub-tokens as separate words must not be presented as evidence for + // the entity (issue #18). + let root = std::env::temp_dir().join(format!( + "bran-query-sub-token-entity-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join(".bran")).unwrap(); + std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); + std::fs::write( + root.join("home.md"), + "---\ntype: concept\ntitle: Home\n---\nThe zzq-entity-unit-8873 wrapper and runner are implemented here.\n", + ) + .unwrap(); + std::fs::write( + root.join("noise.md"), + "---\ntype: concept\ntitle: Noise\n---\nThe entity unit wrapper notes live here.\n", + ) + .unwrap(); + + let result = CliApp::run(vec![ + "query".to_owned(), + root.to_string_lossy().into_owned(), + "zzq-entity-unit-8873".to_owned(), + ]); + + assert_eq!(result.exit_code, ExitCode::SUCCESS, "{}", result.output); + assert!(result.output.contains( + "\"source_rankings\":[{\"locator\":\"home.md\",\"rank\":1,\"score\":{\"exact\":1" + )); + assert!(result.output.contains("exact:body")); + assert!(!result.output.contains("\"locator\":\"noise.md\"")); + assert!(!result.output.contains("unmatched_query_terms")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn query_distinguishes_grounded_miss_and_partial_unanchored() { + // A bare unmatched entity in a natural-language query must never look + // fully grounded. Rankings from the generic words around it may stay + // useful, but the outcome has to be machine-readable as partial / + // unanchored rather than guessed from word length or English (issue #18). + let root = std::env::temp_dir().join(format!( + "bran-query-outcome-classification-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join(".bran")).unwrap(); + std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); + std::fs::write( + root.join("notes.md"), + "---\ntype: concept\ntitle: Notes\n---\nNotes about where the collector is configured and tested.\n", + ) + .unwrap(); + let root_arg = root.to_string_lossy().into_owned(); + + let diluted = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "where is the zephyrite collector configured and tested".to_owned(), + ]); + assert_eq!(diluted.exit_code, ExitCode::SUCCESS, "{}", diluted.output); + assert!( + diluted.output.contains("\"status\":\"ok\""), + "{}", + diluted.output + ); + assert!( + diluted + .output + .contains("\"query_outcome\":\"partial_unanchored\""), + "{}", + diluted.output + ); + assert!(!diluted.output.contains("\"query_outcome\":\"grounded\"")); + assert!( + diluted.output.contains("\"locator\":\"notes.md\""), + "{}", + diluted.output + ); + assert!( + diluted + .output + .contains("\"unmatched_terms\":[\"zephyrite\"]"), + "{}", + diluted.output + ); + assert!(diluted.output.contains("unmatched_query_terms: zephyrite")); + + let packet = CliApp::run(vec![ + "packet".to_owned(), + root_arg.clone(), + "where is the zephyrite collector configured and tested".to_owned(), + ]); + assert_eq!(packet.exit_code, ExitCode::SUCCESS, "{}", packet.output); + assert!( + packet + .output + .contains("\"query_outcome\":\"partial_unanchored\""), + "{}", + packet.output + ); + assert!( + packet.output.contains("\"locator\":\"notes.md\""), + "{}", + packet.output + ); + + let miss = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "zephyrite".to_owned(), + ]); + assert_eq!(miss.exit_code, ExitCode::SUCCESS, "{}", miss.output); + assert!( + miss.output.contains("\"query_outcome\":\"miss\""), + "{}", + miss.output + ); + assert!( + miss.output.contains("\"source_rankings\":[],"), + "{}", + miss.output + ); + assert!(!miss.output.contains("\"locator\":\"notes.md\"")); + assert!(miss.output.contains("unmatched_query_terms: zephyrite")); + + let grounded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "collector configured".to_owned(), + ]); + assert_eq!(grounded.exit_code, ExitCode::SUCCESS, "{}", grounded.output); + assert!( + grounded.output.contains("\"query_outcome\":\"grounded\""), + "{}", + grounded.output + ); + assert!( + grounded.output.contains("\"unmatched_terms\":[]"), + "{}", + grounded.output + ); + assert!(grounded.output.contains("\"locator\":\"notes.md\"")); + assert!(!grounded.output.contains("unmatched_query_terms")); + + let ordinary = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "unfortunately where is the collector configured".to_owned(), + ]); + assert_eq!(ordinary.exit_code, ExitCode::SUCCESS, "{}", ordinary.output); + assert!( + ordinary.output.contains("\"locator\":\"notes.md\""), + "{}", + ordinary.output + ); + assert!( + ordinary + .output + .contains("\"query_outcome\":\"partial_unanchored\""), + "{}", + ordinary.output + ); + assert!(ordinary + .output + .contains("unmatched_query_terms: unfortunately")); + assert!(!ordinary.output.contains("\"source_rankings\":[],")); + + let compound = CliApp::run(vec![ + "query".to_owned(), + root_arg, + "where is the zzq-entity-unit-8873 collector configured and tested".to_owned(), + ]); + assert_eq!(compound.exit_code, ExitCode::SUCCESS, "{}", compound.output); + assert!( + compound.output.contains("\"query_outcome\":\"miss\""), + "{}", + compound.output + ); + assert!( + compound.output.contains("\"source_rankings\":[],"), + "{}", + compound.output + ); + assert!(!compound.output.contains("\"locator\":\"notes.md\"")); + assert!(compound + .output + .contains("unmatched_query_terms: zzq-entity-unit-8873")); + let _ = std::fs::remove_dir_all(root); + } + + fn scratch_query_root(prefix: &str) -> std::path::PathBuf { + let root = + std::env::temp_dir().join(format!("bran-query-{}-{}", prefix, std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join(".bran")).unwrap(); + std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); + root + } + + fn write_query_doc(root: &std::path::Path, name: &str, title: &str, body: &str) { + std::fs::write( + root.join(name), + format!("---\ntype: concept\ntitle: {title}\n---\n{body}\n"), + ) + .unwrap(); + } + + #[test] + fn query_add_dir_ranks_across_two_policy_roots() { + // One query over two policy-bearing roots must return a single + // ranking that names each bundle and the ordered requested roots. + // A match in either root is returned. Repeated identical input is + // byte-identical. Ranks are unique and contiguous (issue #21). + let primary = scratch_query_root("multi-root-a"); + let added = scratch_query_root("multi-root-b"); + write_query_doc( + &primary, + "alpha.md", + "Alpha notes", + "The zzq-alpha-unit-2101 token lives only in the first bundle.", + ); + write_query_doc( + &added, + "beta.md", + "Beta notes", + "The zzq-beta-unit-2101 token lives only in the second bundle.", + ); + let primary_arg = primary.to_string_lossy().into_owned(); + let added_arg = added.to_string_lossy().into_owned(); + let args = vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + added_arg.clone(), + "zzq-alpha-unit-2101 zzq-beta-unit-2101".to_owned(), + ]; + + let first = CliApp::run(args.clone()); + let second = CliApp::run(args); + assert_eq!(first.exit_code, ExitCode::SUCCESS, "{}", first.output); + assert!(!first.is_error); + assert_eq!(first.output, second.output); + + assert!( + first.output.contains(&format!( + "\"requested_roots\":[\"{}\",\"{}\"]", + primary_arg, added_arg + )), + "{}", + first.output + ); + assert!( + first.output.contains(&format!( + "\"bundle\":\"{}\",\"locator\":\"alpha.md\",\"rank\":", + primary_arg + )), + "{}", + first.output + ); + assert!( + first.output.contains(&format!( + "\"bundle\":\"{}\",\"locator\":\"beta.md\",\"rank\":", + added_arg + )), + "{}", + first.output + ); + let data_rankings = first + .output + .split_once("\"candidate_source_bytes\"") + .map(|(prefix, _)| prefix) + .unwrap_or(&first.output); + assert!(data_rankings.contains("\"rank\":1"), "{}", first.output); + assert!(data_rankings.contains("\"rank\":2"), "{}", first.output); + assert!(!data_rankings.contains("\"rank\":3"), "{}", first.output); + assert_eq!( + data_rankings.matches("\"rank\":1").count(), + 1, + "{}", + first.output + ); + assert_eq!( + data_rankings.matches("\"rank\":2").count(), + 1, + "{}", + first.output + ); + + let only_added = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + added_arg.clone(), + "zzq-beta-unit-2101".to_owned(), + ]); + assert_eq!( + only_added.exit_code, + ExitCode::SUCCESS, + "{}", + only_added.output + ); + assert!(only_added.output.contains(&format!( + "\"bundle\":\"{}\",\"locator\":\"beta.md\"", + added_arg + ))); + assert!(!only_added.output.contains("\"locator\":\"alpha.md\"")); + + let single = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "zzq-alpha-unit-2101".to_owned(), + ]); + assert_eq!(single.exit_code, ExitCode::SUCCESS, "{}", single.output); + assert!(!single.output.contains("requested_roots")); + assert!(!single.output.contains("\"bundle\":")); + assert!(single.output.contains( + "\"source_rankings\":[{\"locator\":\"alpha.md\",\"rank\":1,\"score\":{\"exact\":1" + )); + + let _ = std::fs::remove_dir_all(primary); + let _ = std::fs::remove_dir_all(added); + } + + #[test] + fn query_add_dir_uses_one_cross_root_score_order() { + // Scores are one shared tuple over the union. An exact title match + // in the added root outranks a body-only match in the primary root. + // Equal scores break ties by bundle identity then node identity, + // not by concatenating per-root rank numbers (issue #21). + let primary = scratch_query_root("multi-score-a"); + let added = scratch_query_root("multi-score-b"); + write_query_doc( + &primary, + "notes.md", + "Primary notes", + "sharedtokenxyz appears only in this body.", + ); + write_query_doc( + &added, + "notes.md", + "sharedtokenxyz", + "title is the exact identity match.", + ); + let primary_arg = primary.to_string_lossy().into_owned(); + let added_arg = added.to_string_lossy().into_owned(); + + let result = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + added_arg.clone(), + "sharedtokenxyz".to_owned(), + ]); + assert_eq!(result.exit_code, ExitCode::SUCCESS, "{}", result.output); + let added_first = format!( + "\"bundle\":\"{}\",\"locator\":\"notes.md\",\"rank\":1", + added_arg + ); + let primary_second = format!( + "\"bundle\":\"{}\",\"locator\":\"notes.md\",\"rank\":2", + primary_arg + ); + assert!(result.output.contains(&added_first), "{}", result.output); + assert!(result.output.contains(&primary_second), "{}", result.output); + + let left = scratch_query_root("multi-tie-aaa"); + let right = scratch_query_root("multi-tie-zzz"); + write_query_doc( + &left, + "notes.md", + "Tied notes", + "tiedtokenxyz appears in this body.", + ); + write_query_doc( + &right, + "notes.md", + "Tied notes", + "tiedtokenxyz appears in this body.", + ); + let left_arg = left.to_string_lossy().into_owned(); + let right_arg = right.to_string_lossy().into_owned(); + let tied = CliApp::run(vec![ + "query".to_owned(), + left_arg.clone(), + "--add-dir".to_owned(), + right_arg.clone(), + "tiedtokenxyz".to_owned(), + ]); + assert_eq!(tied.exit_code, ExitCode::SUCCESS, "{}", tied.output); + let (first_bundle, second_bundle) = if left_arg <= right_arg { + (left_arg.as_str(), right_arg.as_str()) + } else { + (right_arg.as_str(), left_arg.as_str()) + }; + assert!(tied.output.contains(&format!( + "\"bundle\":\"{}\",\"locator\":\"notes.md\",\"rank\":1", + first_bundle + ))); + assert!(tied.output.contains(&format!( + "\"bundle\":\"{}\",\"locator\":\"notes.md\",\"rank\":2", + second_bundle + ))); + + let _ = std::fs::remove_dir_all(primary); + let _ = std::fs::remove_dir_all(added); + let _ = std::fs::remove_dir_all(left); + let _ = std::fs::remove_dir_all(right); + } + + #[test] + fn query_add_dir_refuses_invalid_or_policy_less_root() { + // One invalid or policy-less added root refuses the whole request + // and names only that root argument. Valid-root results are not + // returned (issue #21). + let primary = scratch_query_root("multi-refuse-ok"); + write_query_doc( + &primary, + "alpha.md", + "Alpha notes", + "The zzq-alpha-unit-2101 token lives here.", + ); + let policy_less = std::env::temp_dir().join(format!( + "bran-query-multi-refuse-nopolicy-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&policy_less); + std::fs::create_dir_all(&policy_less).unwrap(); + std::fs::write(policy_less.join("readme.md"), "no policy\n").unwrap(); + let missing = std::env::temp_dir().join(format!( + "bran-query-multi-refuse-missing-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&missing); - let result = CliApp::run(vec![ + let primary_arg = primary.to_string_lossy().into_owned(); + let policy_less_arg = policy_less.to_string_lossy().into_owned(); + let missing_arg = missing.to_string_lossy().into_owned(); + + let refused = CliApp::run(vec![ "query".to_owned(), - root.to_string_lossy().into_owned(), - "zzq-entity-unit-8873".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + policy_less_arg.clone(), + "zzq-alpha-unit-2101".to_owned(), ]); - - assert_eq!(result.exit_code, ExitCode::SUCCESS, "{}", result.output); - assert!(result.output.contains("\"source_rankings\":[],")); - assert!(!result.output.contains("\"locator\":\"notes.md\"")); - assert!(result + assert_eq!( + refused.exit_code, + TypedExit::Operation.code(), + "{}", + refused.output + ); + assert!(refused.is_error); + assert!(refused.output.contains("native_policy_unavailable")); + assert!(refused.output.contains(&policy_less_arg)); + assert!(!refused.output.contains("\"locator\":\"alpha.md\"")); + assert!(!refused .output - .contains("unmatched_query_terms: zzq-entity-unit-8873")); + .contains(&format!("\"bundle\":\"{primary_arg}\""))); + assert!(!refused.output.contains("source_rankings")); - // The same holds when generic words around the missing entity match - // bodies: their weak matches must not be presented as evidence either. - let diluted = CliApp::run(vec![ + let invalid = CliApp::run(vec![ "query".to_owned(), - root.to_string_lossy().into_owned(), - "Where are the zzq-entity-unit-8873 wrapper, runner, and tests documented?".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + missing_arg.clone(), + "zzq-alpha-unit-2101".to_owned(), ]); - assert_eq!(diluted.exit_code, ExitCode::SUCCESS, "{}", diluted.output); - assert!(diluted.output.contains("\"source_rankings\":[],")); - assert!(!diluted.output.contains("\"locator\":\"notes.md\"")); - assert!(diluted + assert_eq!( + invalid.exit_code, + TypedExit::Operation.code(), + "{}", + invalid.output + ); + assert!(invalid.is_error); + assert!(invalid.output.contains("scan_error")); + assert!(invalid.output.contains(&missing_arg)); + assert!(!invalid.output.contains("\"locator\":\"alpha.md\"")); + assert!(!invalid .output - .contains("unmatched_query_terms: zzq-entity-unit-8873")); - let _ = std::fs::remove_dir_all(root); - } + .contains(&format!("\"bundle\":\"{primary_arg}\""))); - #[test] - fn sub_token_does_not_count_as_entity_match() { - // Only the whole identifier unit matches: a document holding just its - // sub-tokens as separate words must not be presented as evidence for - // the entity (issue #18). - let root = std::env::temp_dir().join(format!( - "bran-query-sub-token-entity-{}", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&root); - std::fs::create_dir_all(root.join(".bran")).unwrap(); - std::fs::write(root.join(".bran/policy.yaml"), minimal_valid_policy()).unwrap(); - std::fs::write( - root.join("home.md"), - "---\ntype: concept\ntitle: Home\n---\nThe zzq-entity-unit-8873 wrapper and runner are implemented here.\n", - ) - .unwrap(); - std::fs::write( - root.join("noise.md"), - "---\ntype: concept\ntitle: Noise\n---\nThe entity unit wrapper notes live here.\n", - ) - .unwrap(); + let missing_flag = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + ]); + assert_eq!(missing_flag.exit_code, TypedExit::Usage.code()); + assert_eq!( + missing_flag.output, + super::make_query_error("missing_add_dir") + ); - let result = CliApp::run(vec![ + let late_flag = CliApp::run(vec![ "query".to_owned(), - root.to_string_lossy().into_owned(), - "zzq-entity-unit-8873".to_owned(), + primary_arg.clone(), + "zzq-alpha-unit-2101".to_owned(), + "--add-dir".to_owned(), + policy_less_arg, ]); + assert_eq!( + late_flag.exit_code, + ExitCode::SUCCESS, + "{}", + late_flag.output + ); + assert!(!late_flag.output.contains("requested_roots")); + assert!(late_flag.output.contains("\"locator\":\"alpha.md\"")); - assert_eq!(result.exit_code, ExitCode::SUCCESS, "{}", result.output); - assert!(result.output.contains( - "\"source_rankings\":[{\"locator\":\"home.md\",\"rank\":1,\"score\":{\"exact\":1" - )); - assert!(result.output.contains("exact:body")); - assert!(!result.output.contains("\"locator\":\"noise.md\"")); - assert!(!result.output.contains("unmatched_query_terms")); - let _ = std::fs::remove_dir_all(root); + let _ = std::fs::remove_dir_all(primary); + let _ = std::fs::remove_dir_all(policy_less); } #[test] @@ -8036,313 +9971,1036 @@ mod tests { } #[test] - fn body_preserved_cli_missing_or_extra_args_usage_exit_two() { + fn body_preserved_cli_missing_or_extra_args_usage_exit_two() { + let root = body_preserved_fixture(); + let missing_manifest = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + ]); + assert_eq!(missing_manifest.exit_code, TypedExit::Usage.code()); + assert!(missing_manifest.output.contains("missing_manifest")); + let extra = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + "extra".to_owned(), + ]); + assert_eq!(extra.exit_code, TypedExit::Usage.code()); + assert!(extra.output.contains("too_many_args")); + let no_args = CliApp::run(["body-preserved".to_owned()]); + assert_eq!(no_args.exit_code, TypedExit::Usage.code()); + assert!(no_args.output.contains("missing_root")); + } + + #[test] + fn body_preserved_cli_unsafe_manifest_path_safe_exit_one() { + let root = body_preserved_fixture(); + // absolute path inside JSON is rejected; the report finds it unsafe + write_bp_file(&root, "orig.md", b"body\n"); + write_bp_file(&root, "mig.md", b"body\n"); + write_bp_file( + &root, + "bp.json", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"/etc/passwd\"}]}", + ); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("\"status\":\"failed\"")); + assert!(!result.output.contains("passwd")); + } + + #[test] + fn body_preserved_cli_traversal_manifest_path_safe_exit_one() { + let root = body_preserved_fixture(); + write_bp_file(&root, "orig.md", b"body\n"); + write_bp_file(&root, "mig.md", b"body\n"); + write_bp_file( + &root, + "bp.json", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"../escape.md\"}]}", + ); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(!result.output.contains("escape")); + } + + #[test] + fn body_preserved_cli_symlink_manifest_rejected() { + let root = body_preserved_fixture(); + write_bp_file(&root, "real.json", b"{\"files\":[]}"); + #[cfg(unix)] + { + std::os::unix::fs::symlink("real.json", root.join("bp.json")).unwrap(); + } + #[cfg(not(unix))] + { + std::fs::write(root.join("bp.json"), b"{\"files\":[]}").unwrap(); + } + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("body_preserved_validation_failed")); + } + + #[test] + fn body_preserved_cli_oversized_manifest_exit_one() { + let root = body_preserved_fixture(); + let big = vec![b' '; 2 * 1024 * 1024 + 1]; + std::fs::write(root.join("big.json"), &big).unwrap(); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "big.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("body_preserved_validation_failed")); + } + + #[test] + fn body_preserved_cli_nonexistent_root_exit_three() { + let result = CliApp::run([ + "body-preserved".to_owned(), + "/nonexistent-root-zzz-bp".to_owned(), + "bp.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Operation.code()); + assert!(result.output.contains("io_unavailable")); + } + + #[test] + fn body_preserved_cli_no_secrets_or_os_strings_in_output() { + let root = body_preserved_fixture(); + write_bp_file(&root, "orig.md", b"body\n"); + write_bp_file(&root, "mig.md", b"body\n"); + write_bp_file( + &root, + "bp.json", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"missing.md\"}]}", + ); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + ]); + assert!(result.output.contains("\"status\":\"failed\"")); + // never echoes raw manifest JSON body content + assert!(!result.output.contains("missing.md")); + } + + #[test] + fn body_preserved_cli_read_only_no_writes() { + let root = body_preserved_fixture(); + write_bp_file(&root, "orig.md", b"# H\n"); + write_bp_file(&root, "mig.md", b"# H\n"); + write_bp_file( + &root, + "bp.json", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\"}]}", + ); + // snapshot before + let before: Vec<_> = std::fs::read_dir(&root) + .unwrap() + .map(|e| { + let e = e.unwrap(); + (e.file_name(), e.metadata().unwrap().modified().unwrap()) + }) + .collect(); + CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "bp.json".to_owned(), + ]); + // snapshot after + let after: Vec<_> = std::fs::read_dir(&root) + .unwrap() + .map(|e| { + let e = e.unwrap(); + (e.file_name(), e.metadata().unwrap().modified().unwrap()) + }) + .collect(); + assert_eq!(before, after); + } + + // --- 2.1-C remediation CLI tests --- + + #[test] + fn body_preserved_cli_missing_manifest_exit_one_no_echo() { + let root = body_preserved_fixture(); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "nope.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("body_preserved_validation_failed")); + assert!(!result.output.contains("nope.json")); + } + + #[test] + fn body_preserved_cli_directory_manifest_exit_one_no_echo() { let root = body_preserved_fixture(); - let missing_manifest = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - ]); - assert_eq!(missing_manifest.exit_code, TypedExit::Usage.code()); - assert!(missing_manifest.output.contains("missing_manifest")); - let extra = CliApp::run([ + std::fs::create_dir(root.join("subdir")).unwrap(); + let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), - "bp.json".to_owned(), - "extra".to_owned(), + "subdir".to_owned(), ]); - assert_eq!(extra.exit_code, TypedExit::Usage.code()); - assert!(extra.output.contains("too_many_args")); - let no_args = CliApp::run(["body-preserved".to_owned()]); - assert_eq!(no_args.exit_code, TypedExit::Usage.code()); - assert!(no_args.output.contains("missing_root")); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("body_preserved_validation_failed")); + assert!(!result.output.contains("subdir")); } #[test] - fn body_preserved_cli_unsafe_manifest_path_safe_exit_one() { + fn body_preserved_cli_absolute_manifest_path_exit_one_no_echo() { let root = body_preserved_fixture(); - // absolute path inside JSON is rejected; the report finds it unsafe - write_bp_file(&root, "orig.md", b"body\n"); - write_bp_file(&root, "mig.md", b"body\n"); - write_bp_file( - &root, - "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"/etc/passwd\"}]}", - ); let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), - "bp.json".to_owned(), + "/etc/passwd".to_owned(), ]); assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("\"status\":\"failed\"")); + assert!(result.output.contains("body_preserved_validation_failed")); assert!(!result.output.contains("passwd")); } #[test] - fn body_preserved_cli_traversal_manifest_path_safe_exit_one() { + fn body_preserved_cli_windows_style_manifest_exit_one_no_echo() { let root = body_preserved_fixture(); - write_bp_file(&root, "orig.md", b"body\n"); - write_bp_file(&root, "mig.md", b"body\n"); - write_bp_file( - &root, - "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"../escape.md\"}]}", - ); let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), - "bp.json".to_owned(), + "C:\\foo\\bar.json".to_owned(), ]); assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(!result.output.contains("escape")); + assert!(result.output.contains("body_preserved_validation_failed")); + assert!(!result.output.contains("C:")); } #[test] - fn body_preserved_cli_symlink_manifest_rejected() { + fn body_preserved_cli_unc_manifest_exit_one_no_echo() { let root = body_preserved_fixture(); - write_bp_file(&root, "real.json", b"{\"files\":[]}"); - #[cfg(unix)] - { - std::os::unix::fs::symlink("real.json", root.join("bp.json")).unwrap(); - } - #[cfg(not(unix))] - { - std::fs::write(root.join("bp.json"), b"{\"files\":[]}").unwrap(); - } let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), - "bp.json".to_owned(), + "\\\\server\\share\\file.json".to_owned(), ]); assert_eq!(result.exit_code, TypedExit::Validation.code()); assert!(result.output.contains("body_preserved_validation_failed")); + assert!(!result.output.contains("server")); } #[test] - fn body_preserved_cli_oversized_manifest_exit_one() { + fn body_preserved_cli_nul_manifest_exit_one_no_echo() { let root = body_preserved_fixture(); - let big = vec![b' '; 2 * 1024 * 1024 + 1]; - std::fs::write(root.join("big.json"), &big).unwrap(); let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), - "big.json".to_owned(), + "bad\0.json".to_owned(), ]); assert_eq!(result.exit_code, TypedExit::Validation.code()); assert!(result.output.contains("body_preserved_validation_failed")); } #[test] - fn body_preserved_cli_nonexistent_root_exit_three() { - let result = CliApp::run([ - "body-preserved".to_owned(), - "/nonexistent-root-zzz-bp".to_owned(), - "bp.json".to_owned(), - ]); - assert_eq!(result.exit_code, TypedExit::Operation.code()); - assert!(result.output.contains("io_unavailable")); + #[cfg(unix)] + fn body_preserved_cli_invalid_utf8_manifest_exits_two() { + use std::os::unix::ffi::OsStrExt; + let root = body_preserved_fixture(); + let invalid = std::ffi::OsStr::from_bytes(&[0xFF, 0xFE]); + let args = [ + std::ffi::OsString::from("body-preserved"), + root.as_os_str().to_os_string(), + invalid.to_os_string(), + ]; + let result = CliApp::run(args); + assert_eq!(result.exit_code, TypedExit::Usage.code()); } #[test] - fn body_preserved_cli_no_secrets_or_os_strings_in_output() { + fn body_preserved_cli_wrong_digest_exit_one_original_body_hash_mismatch() { let root = body_preserved_fixture(); write_bp_file(&root, "orig.md", b"body\n"); write_bp_file(&root, "mig.md", b"body\n"); write_bp_file( &root, "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"missing.md\"}]}", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\",\"body_sha256\":\"0000000000000000000000000000000000000000000000000000000000000000\"}]}", ); let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), "bp.json".to_owned(), ]); - assert!(result.output.contains("\"status\":\"failed\"")); - // never echoes raw manifest JSON body content - assert!(!result.output.contains("missing.md")); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(result.output.contains("original_body_hash_mismatch")); } #[test] - fn body_preserved_cli_read_only_no_writes() { + fn body_preserved_cli_malformed_secret_bearing_json_no_echo() { let root = body_preserved_fixture(); - write_bp_file(&root, "orig.md", b"# H\n"); - write_bp_file(&root, "mig.md", b"# H\n"); + write_bp_file(&root, "orig.md", b"body\n"); + write_bp_file(&root, "mig.md", b"body\n"); write_bp_file( &root, "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\"}]}", + b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\",\"secret\":\"abc123\",}]}", ); - // snapshot before - let before: Vec<_> = std::fs::read_dir(&root) - .unwrap() - .map(|e| { - let e = e.unwrap(); - (e.file_name(), e.metadata().unwrap().modified().unwrap()) - }) - .collect(); - CliApp::run([ + let result = CliApp::run([ "body-preserved".to_owned(), root.to_string_lossy().into_owned(), "bp.json".to_owned(), ]); - // snapshot after - let after: Vec<_> = std::fs::read_dir(&root) - .unwrap() - .map(|e| { - let e = e.unwrap(); - (e.file_name(), e.metadata().unwrap().modified().unwrap()) - }) - .collect(); - assert_eq!(before, after); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(!result.output.contains("abc123")); + assert!(!result.output.contains("secret")); + } + + #[test] + fn body_preserved_cli_exact_2mib_not_oversized() { + let root = body_preserved_fixture(); + let exact = vec![b' '; 2 * 1024 * 1024]; + std::fs::write(root.join("exact.json"), &exact).unwrap(); + let result = CliApp::run([ + "body-preserved".to_owned(), + root.to_string_lossy().into_owned(), + "exact.json".to_owned(), + ]); + assert_eq!(result.exit_code, TypedExit::Validation.code()); + assert!(!result.output.contains("oversized")); + assert!(!result.output.contains("manifest_too_large")); + } + + fn evidence_policy() -> String { + "schema_version: \"1\"\ndocument_coverage:\n roots:\n - \".\"\n canonical_documents:\n - \"notes.md\"\n" + .to_owned() + } + + fn scratch_evidence_root(prefix: &str) -> (std::path::PathBuf, String) { + let root = + std::env::temp_dir().join(format!("bran-evidence-{}-{}", prefix, std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join(".bran")).unwrap(); + std::fs::write(root.join(".bran/policy.yaml"), evidence_policy()).unwrap(); + let arg = root.to_string_lossy().into_owned(); + (root, arg) + } + + fn write_evidence_doc(root: &std::path::Path, name: &str, title: &str, body: &str) { + std::fs::write( + root.join(name), + format!("---\ntype: concept\ntitle: {title}\n---\n{body}\n"), + ) + .unwrap(); + } + + fn query_data_section(output: &str) -> &str { + let start = output.find("\"data\":").expect("data"); + let rest = &output[start..]; + let end = rest + .find(",\"warnings\":") + .or_else(|| rest.find(",\"failures\":")) + .expect("data end"); + &rest[..end] + } + + #[test] + fn query_evidence_store_is_opt_in_and_does_not_change_ranking() { + let (root, root_arg) = scratch_evidence_root("rank-stable"); + write_evidence_doc( + &root, + "notes.md", + "Notes", + "UNIQUE-BODY-SENTINEL-22 collector configured here.", + ); + write_evidence_doc(&root, "orphan.md", "Orphan", "never retrieved unique body"); + write_evidence_doc(&root, "path-zephyrite.md", "Path only", "no matching body"); + + let unrecorded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "collector configured".to_owned(), + ]); + assert_eq!( + unrecorded.exit_code, + ExitCode::SUCCESS, + "{}", + unrecorded.output + ); + assert!(unrecorded.output.contains("\"query_outcome\":\"grounded\"")); + assert!(unrecorded.output.contains("\"locator\":\"notes.md\"")); + + let recorded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + "collector configured".to_owned(), + ]); + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); + assert_eq!( + query_data_section(&unrecorded.output), + query_data_section(&recorded.output) + ); + + let store = root.join(".bran/cache/query-evidence"); + assert!(store.is_file(), "opt-in record must persist a store file"); + let stored = std::fs::read_to_string(&store).unwrap(); + assert!(stored.contains("collector configured")); + assert!(!stored.contains("UNIQUE-BODY-SENTINEL-22")); + assert_eq!( + stored.matches("collector configured").count(), + 1, + "one record per opted-in query" + ); + + let again = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + "collector configured".to_owned(), + ]); + assert_eq!(again.exit_code, ExitCode::SUCCESS, "{}", again.output); + let stored_again = std::fs::read_to_string(&store).unwrap(); + assert_eq!( + stored_again.matches("collector configured").count(), + 1, + "re-recording the same request replaces, not appends" + ); + + let with_store = CliApp::run(vec![ + "query".to_owned(), + root_arg, + "collector configured".to_owned(), + ]); + assert_eq!(unrecorded.output, with_store.output); + let _ = std::fs::remove_dir_all(root); } - // --- 2.1-C remediation CLI tests --- + #[test] + fn evidence_summarize_propose_replay_clear_and_fail_closed() { + let (root, root_arg) = scratch_evidence_root("lifecycle"); + write_evidence_doc( + &root, + "notes.md", + "Notes", + "UNIQUE-BODY-SENTINEL-22 collector configured here.", + ); + write_evidence_doc(&root, "orphan.md", "Orphan", "never retrieved unique body"); + write_evidence_doc(&root, "path-zephyrite.md", "Path only", "no matching body"); + + for request in ["collector configured", "zephyrite", "missingtokenxyz"] { + let recorded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + request.to_owned(), + ]); + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); + } + + let notes_before = std::fs::read(root.join("notes.md")).unwrap(); + let summarize = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + summarize.exit_code, + ExitCode::SUCCESS, + "{}", + summarize.output + ); + assert!(summarize + .output + .contains("\"command\":\"evidence.summarize\"")); + assert!(summarize + .output + .contains("\"unanswered_terms\":[\"missingtokenxyz\"]")); + assert!( + summarize.output.contains("\"locator\":\"notes.md\"") + || summarize.output.contains("notes.md") + ); + assert!(summarize.output.contains("path-zephyrite.md")); + assert!(summarize.output.contains("orphan.md")); + assert!(summarize.output.contains("unclassified")); + assert!( + summarize.output.contains("never_retrieved") + || summarize.output.contains("never-retrieved") + || summarize.output.contains("never_retrieved_documents") + ); + assert!( + summarize.output.contains("path_only") + || summarize.output.contains("path-only") + || summarize.output.contains("path_only_retrievals") + ); + assert!(summarize.output.contains("stale_records")); + assert!(!summarize.output.contains("UNIQUE-BODY-SENTINEL-22")); + + let propose = CliApp::run(vec![ + "evidence".to_owned(), + "propose".to_owned(), + root_arg.clone(), + ]); + assert_eq!(propose.exit_code, ExitCode::SUCCESS, "{}", propose.output); + assert!(propose.output.contains("\"command\":\"evidence.propose\"")); + assert!(propose.output.contains("candidates")); + assert!( + propose.output.contains("unanswered_term") + || propose.output.contains("missingtokenxyz") + ); + assert!(!propose.output.contains("UNIQUE-BODY-SENTINEL-22")); + assert_eq!(std::fs::read(root.join("notes.md")).unwrap(), notes_before); + assert!(!std::fs::read(root.join("orphan.md")).unwrap().is_empty()); + assert!(!std::fs::read_to_string(root.join("notes.md")) + .unwrap() + .contains("p3-replacement")); + + let replay_same = CliApp::run(vec![ + "evidence".to_owned(), + "replay".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + replay_same.exit_code, + ExitCode::SUCCESS, + "{}", + replay_same.output + ); + assert!(replay_same + .output + .contains("\"command\":\"evidence.replay\"")); + assert!( + replay_same.output.contains("\"difference_count\":0"), + "{}", + replay_same.output + ); + assert!(!replay_same.output.contains("UNIQUE-BODY-SENTINEL-22")); + + write_evidence_doc( + &root, + "notes.md", + "Notes", + "UNIQUE-BODY-SENTINEL-22 changed body.", + ); + let replay_changed = CliApp::run(vec![ + "evidence".to_owned(), + "replay".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + replay_changed.exit_code, + ExitCode::SUCCESS, + "{}", + replay_changed.output + ); + assert!( + replay_changed.output.contains("\"difference_count\":1") + || replay_changed.output.contains("\"outcome_changed\":true") + || replay_changed.output.contains("\"kind\":\"removed\""), + "{}", + replay_changed.output + ); + assert!( + replay_changed.output.contains("notes.md"), + "{}", + replay_changed.output + ); + assert!(!replay_changed.output.contains("UNIQUE-BODY-SENTINEL-22")); + + std::fs::write( + root.join("notes.md"), + "---\ntype: concept\ntitle: Notes\n---\nUNIQUE-BODY-SENTINEL-22 collector configured here.\n", + ) + .unwrap(); + std::fs::remove_file(root.join("notes.md")).unwrap(); + let summarize_stale = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + summarize_stale.exit_code, + ExitCode::SUCCESS, + "{}", + summarize_stale.output + ); + assert!( + summarize_stale.output.contains("stale"), + "{}", + summarize_stale.output + ); + assert!(!summarize_stale.output.contains("UNIQUE-BODY-SENTINEL-22")); + write_evidence_doc( + &root, + "notes.md", + "Notes", + "UNIQUE-BODY-SENTINEL-22 collector configured here.", + ); + + let other = root.join("orphan.md"); + let other_before = std::fs::read(&other).unwrap(); + let store_path = root.join(".bran/cache/query-evidence"); + assert!(store_path.is_file()); + let cleared = CliApp::run(vec![ + "evidence".to_owned(), + "clear".to_owned(), + root_arg.clone(), + ]); + assert_eq!(cleared.exit_code, ExitCode::SUCCESS, "{}", cleared.output); + assert!(cleared.output.contains("\"command\":\"evidence.clear\"")); + assert!(cleared.output.contains(".bran/cache/query-evidence")); + assert!(!store_path.exists()); + assert_eq!(std::fs::read(&other).unwrap(), other_before); + assert!(root.join("notes.md").is_file()); + assert!(root.join(".bran/policy.yaml").is_file()); + + let cleared_again = CliApp::run(vec![ + "evidence".to_owned(), + "clear".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + cleared_again.exit_code, + ExitCode::SUCCESS, + "{}", + cleared_again.output + ); + assert!(!store_path.exists()); + + let secret = "SECRET_LEAK_TOKEN_22"; + std::fs::write(&store_path, format!("not-a-valid-store\n{secret}\n")).unwrap(); + let malformed = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), + ]); + assert!(malformed.is_error, "{}", malformed.output); + assert!(!malformed.output.contains(secret)); + assert!(!malformed.output.contains("not-a-valid-store")); + + let _ = std::fs::remove_file(&store_path); + std::fs::write(root.join("outside-secret.txt"), secret).unwrap(); + #[cfg(unix)] + { + std::os::unix::fs::symlink(root.join("outside-secret.txt"), &store_path).unwrap(); + let linked = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), + ]); + assert!(linked.is_error, "{}", linked.output); + assert!(!linked.output.contains(secret)); + let _ = std::fs::remove_file(&store_path); + } - #[test] - fn body_preserved_cli_missing_manifest_exit_one_no_echo() { - let root = body_preserved_fixture(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "nope.json".to_owned(), + let oversized = vec![b'x'; 65 * 1024]; + std::fs::write(&store_path, &oversized).unwrap(); + let big = CliApp::run(vec![ + "evidence".to_owned(), + "replay".to_owned(), + root_arg.clone(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - assert!(!result.output.contains("nope.json")); - } + assert!(big.is_error, "{}", big.output); + assert!(!big.output.contains(&"x".repeat(32))); + let _ = std::fs::remove_file(&store_path); - #[test] - fn body_preserved_cli_directory_manifest_exit_one_no_echo() { - let root = body_preserved_fixture(); - std::fs::create_dir(root.join("subdir")).unwrap(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "subdir".to_owned(), + std::fs::create_dir_all(store_path.parent().unwrap()).unwrap(); + std::fs::create_dir(&store_path).unwrap(); + let blocked = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + "collector configured".to_owned(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - assert!(!result.output.contains("subdir")); + assert!(blocked.is_error, "{}", blocked.output); + assert!( + blocked.output.contains("evidence_store"), + "{}", + blocked.output + ); + let _ = std::fs::remove_dir(&store_path); + + let _ = std::fs::remove_dir_all(root); } #[test] - fn body_preserved_cli_absolute_manifest_path_exit_one_no_echo() { - let root = body_preserved_fixture(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "/etc/passwd".to_owned(), + fn evidence_multi_root_records_only_primary_and_preserves_bundles() { + let (primary, primary_arg) = scratch_evidence_root("multi-a"); + let (added, added_arg) = scratch_evidence_root("multi-b"); + write_evidence_doc( + &primary, + "alpha.md", + "Alpha notes", + "The zzq-alpha-unit-2201 token lives only in the first bundle.", + ); + write_evidence_doc( + &added, + "beta.md", + "Beta notes", + "The zzq-beta-unit-2201 token lives only in the second bundle.", + ); + + let recorded = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + added_arg.clone(), + "--record".to_owned(), + "zzq-alpha-unit-2201 zzq-beta-unit-2201".to_owned(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - assert!(!result.output.contains("passwd")); + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); + assert!(recorded.output.contains(&format!( + "\"bundle\":\"{primary_arg}\",\"locator\":\"alpha.md\"" + ))); + assert!(recorded.output.contains(&format!( + "\"bundle\":\"{added_arg}\",\"locator\":\"beta.md\"" + ))); + assert!(primary.join(".bran/cache/query-evidence").is_file()); + assert!(!added.join(".bran/cache/query-evidence").exists()); + let stored = std::fs::read_to_string(primary.join(".bran/cache/query-evidence")).unwrap(); + assert!(stored.contains(&primary_arg)); + assert!(stored.contains(&added_arg)); + assert!(stored.contains("alpha.md")); + assert!(stored.contains("beta.md")); + assert!(!stored.contains("zzq-alpha-unit-2201 token lives")); + + let replay = CliApp::run(vec![ + "evidence".to_owned(), + "replay".to_owned(), + primary_arg.clone(), + ]); + assert_eq!(replay.exit_code, ExitCode::SUCCESS, "{}", replay.output); + assert!( + replay.output.contains("\"difference_count\":0"), + "{}", + replay.output + ); + assert!(replay.output.contains(&primary_arg) || replay.output.contains("alpha.md")); + assert!(!added.join(".bran/cache/query-evidence").exists()); + + let _ = std::fs::remove_dir_all(primary); + let _ = std::fs::remove_dir_all(added); } - #[test] - fn body_preserved_cli_windows_style_manifest_exit_one_no_echo() { - let root = body_preserved_fixture(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "C:\\foo\\bar.json".to_owned(), - ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - assert!(!result.output.contains("C:")); + fn evidence_identity(bundle: &str, locator: &str) -> String { + format!("\"bundle\":\"{bundle}\",\"locator\":\"{locator}\"") } #[test] - fn body_preserved_cli_unc_manifest_exit_one_no_echo() { - let root = body_preserved_fixture(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "\\\\server\\share\\file.json".to_owned(), + fn evidence_summarize_covers_secondary_bundle_gaps_and_identities() { + let (primary, primary_arg) = scratch_evidence_root("sec-a"); + let (added, added_arg) = scratch_evidence_root("sec-b"); + write_evidence_doc( + &primary, + "notes.md", + "Primary notes", + "The zzq-sec-alpha-2201 token lives only here.", + ); + write_evidence_doc( + &primary, + "shared.md", + "Shared primary", + "The zzq-shared-both-2201 token appears in primary shared.", + ); + write_evidence_doc( + &added, + "shared.md", + "Shared secondary", + "The zzq-shared-both-2201 token appears in secondary shared.", + ); + write_evidence_doc(&added, "path-zephyrite.md", "Path only", "no matching body"); + write_evidence_doc(&added, "orphan.md", "Orphan", "never retrieved unique body"); + + let recorded = CliApp::run(vec![ + "query".to_owned(), + primary_arg.clone(), + "--add-dir".to_owned(), + added_arg.clone(), + "--record".to_owned(), + "zzq-sec-alpha-2201 zzq-shared-both-2201 zephyrite".to_owned(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - assert!(!result.output.contains("server")); - } + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); - #[test] - fn body_preserved_cli_nul_manifest_exit_one_no_echo() { - let root = body_preserved_fixture(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "bad\0.json".to_owned(), + let summarize = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + primary_arg.clone(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("body_preserved_validation_failed")); - } + assert_eq!( + summarize.exit_code, + ExitCode::SUCCESS, + "{}", + summarize.output + ); - #[test] - #[cfg(unix)] - fn body_preserved_cli_invalid_utf8_manifest_exits_two() { - use std::os::unix::ffi::OsStrExt; - let root = body_preserved_fixture(); - let invalid = std::ffi::OsStr::from_bytes(&[0xFF, 0xFE]); - let args = [ - std::ffi::OsString::from("body-preserved"), - root.as_os_str().to_os_string(), - invalid.to_os_string(), - ]; - let result = CliApp::run(args); - assert_eq!(result.exit_code, TypedExit::Usage.code()); + let primary_notes = evidence_identity(&primary_arg, "notes.md"); + let primary_shared = evidence_identity(&primary_arg, "shared.md"); + let added_shared = evidence_identity(&added_arg, "shared.md"); + let added_path = evidence_identity(&added_arg, "path-zephyrite.md"); + let added_orphan = evidence_identity(&added_arg, "orphan.md"); + assert!( + summarize.output.contains(&primary_notes), + "{}", + summarize.output + ); + assert!( + summarize.output.contains(&primary_shared), + "{}", + summarize.output + ); + assert!( + summarize.output.contains(&added_shared), + "{}", + summarize.output + ); + assert!( + summarize.output.contains(&added_path), + "{}", + summarize.output + ); + assert!( + summarize.output.contains(&added_orphan), + "{}", + summarize.output + ); + assert!( + summarize.output.contains("path_only_retrievals"), + "{}", + summarize.output + ); + assert!( + summarize.output.contains("never_retrieved_documents"), + "{}", + summarize.output + ); + + let propose = CliApp::run(vec![ + "evidence".to_owned(), + "propose".to_owned(), + primary_arg.clone(), + ]); + assert_eq!(propose.exit_code, ExitCode::SUCCESS, "{}", propose.output); + assert!( + propose.output.contains("path_only_retrieval"), + "{}", + propose.output + ); + assert!(propose.output.contains(&added_path), "{}", propose.output); + assert!( + propose.output.contains("never_retrieved"), + "{}", + propose.output + ); + assert!(propose.output.contains(&added_orphan), "{}", propose.output); + assert!(propose.output.contains("\"replacement\":null")); + assert!(propose.output.contains("\"authority\":null")); + + let _ = std::fs::remove_dir_all(&added); + let summarize_missing = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + primary_arg.clone(), + ]); + assert_eq!( + summarize_missing.exit_code, + ExitCode::SUCCESS, + "{}", + summarize_missing.output + ); + assert!( + summarize_missing.output.contains(&added_arg), + "{}", + summarize_missing.output + ); + assert!( + summarize_missing.output.contains("unavailable") + || summarize_missing.output.contains("stale"), + "{}", + summarize_missing.output + ); + let propose_missing = CliApp::run(vec![ + "evidence".to_owned(), + "propose".to_owned(), + primary_arg.clone(), + ]); + assert_eq!( + propose_missing.exit_code, + ExitCode::SUCCESS, + "{}", + propose_missing.output + ); + assert!( + propose_missing.output.contains(&added_arg), + "{}", + propose_missing.output + ); + assert!(propose_missing.output.contains("\"replacement\":null")); + assert!(propose_missing.output.contains("\"authority\":null")); + + let _ = std::fs::remove_dir_all(primary); } #[test] - fn body_preserved_cli_wrong_digest_exit_one_original_body_hash_mismatch() { - let root = body_preserved_fixture(); - write_bp_file(&root, "orig.md", b"body\n"); - write_bp_file(&root, "mig.md", b"body\n"); - write_bp_file( + fn evidence_summarize_reports_unclassified_without_coverage_roots() { + let (root, root_arg) = scratch_evidence_root("unclass"); + std::fs::write( + root.join(".bran/policy.yaml"), + "schema_version: \"1\"\ndocument_coverage:\n canonical_documents:\n - \"notes.md\"\n", + ) + .unwrap(); + write_evidence_doc( &root, - "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\",\"body_sha256\":\"0000000000000000000000000000000000000000000000000000000000000000\"}]}", + "notes.md", + "Notes", + "The zzq-unclass-2201 collector configured here.", ); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "bp.json".to_owned(), + write_evidence_doc(&root, "stray.md", "Stray", "unclassified scanned document"); + + let recorded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + "zzq-unclass-2201".to_owned(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(result.output.contains("original_body_hash_mismatch")); + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); + + let summarize = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), + ]); + assert_eq!( + summarize.exit_code, + ExitCode::SUCCESS, + "{}", + summarize.output + ); + let stray = evidence_identity(&root_arg, "stray.md"); + assert!( + summarize.output.contains("unclassified_documents"), + "{}", + summarize.output + ); + assert!(summarize.output.contains(&stray), "{}", summarize.output); + + let propose = CliApp::run(vec![ + "evidence".to_owned(), + "propose".to_owned(), + root_arg.clone(), + ]); + assert_eq!(propose.exit_code, ExitCode::SUCCESS, "{}", propose.output); + assert!( + propose.output.contains("unclassified"), + "{}", + propose.output + ); + assert!(propose.output.contains(&stray), "{}", propose.output); + assert!(propose.output.contains("\"replacement\":null")); + assert!(propose.output.contains("\"authority\":null")); + + let _ = std::fs::remove_dir_all(root); } #[test] - fn body_preserved_cli_malformed_secret_bearing_json_no_echo() { - let root = body_preserved_fixture(); - write_bp_file(&root, "orig.md", b"body\n"); - write_bp_file(&root, "mig.md", b"body\n"); - write_bp_file( + fn evidence_summarize_reports_deterministic_stale_metadata_and_validator_gaps() { + let (root, root_arg) = scratch_evidence_root("stale-meta"); + std::fs::write( + root.join(".bran/policy.yaml"), + "schema_version: \"1\"\ndocument_coverage:\n roots:\n - \".\"\n canonical_documents:\n - \"notes.md\"\n - \"stale.md\"\n - \"stale-body.md\"\n - \"bad-lifecycle.md\"\n - \"dated.md\"\n", + ) + .unwrap(); + write_evidence_doc( &root, - "bp.json", - b"{\"files\":[{\"original_path\":\"orig.md\",\"migrated_path\":\"mig.md\",\"secret\":\"abc123\",}]}", + "notes.md", + "Notes", + "The zzq-stale-2201 collector configured here.", ); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "bp.json".to_owned(), + std::fs::write( + root.join("stale.md"), + "---\ntype: concept\ntitle: Stale freshness\nfreshness: stale\n---\nCurrent but stale metadata.\n", + ) + .unwrap(); + std::fs::write( + root.join("stale-body.md"), + "---\ntype: concept\ntitle: Stale body\n---\nThis body contains STALE_CLAIM on purpose.\n", + ) + .unwrap(); + std::fs::write( + root.join("bad-lifecycle.md"), + "---\ntype: concept\ntitle: Bad lifecycle\nstatus: not-a-status\nstale_after: not-a-date\n---\nMalformed lifecycle fields.\n", + ) + .unwrap(); + std::fs::write( + root.join("dated.md"), + "---\ntype: concept\ntitle: Dated\nstale_after: 2020-01-01\n---\nValid past date is not expired without a clock.\n", + ) + .unwrap(); + + let recorded = CliApp::run(vec![ + "query".to_owned(), + root_arg.clone(), + "--record".to_owned(), + "zzq-stale-2201".to_owned(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(!result.output.contains("abc123")); - assert!(!result.output.contains("secret")); - } + assert_eq!(recorded.exit_code, ExitCode::SUCCESS, "{}", recorded.output); - #[test] - fn body_preserved_cli_exact_2mib_not_oversized() { - let root = body_preserved_fixture(); - let exact = vec![b' '; 2 * 1024 * 1024]; - std::fs::write(root.join("exact.json"), &exact).unwrap(); - let result = CliApp::run([ - "body-preserved".to_owned(), - root.to_string_lossy().into_owned(), - "exact.json".to_owned(), + let summarize = CliApp::run(vec![ + "evidence".to_owned(), + "summarize".to_owned(), + root_arg.clone(), ]); - assert_eq!(result.exit_code, TypedExit::Validation.code()); - assert!(!result.output.contains("oversized")); - assert!(!result.output.contains("manifest_too_large")); + assert_eq!( + summarize.exit_code, + ExitCode::SUCCESS, + "{}", + summarize.output + ); + + let stale_id = evidence_identity(&root_arg, "stale.md"); + let body_id = evidence_identity(&root_arg, "stale-body.md"); + let bad_id = evidence_identity(&root_arg, "bad-lifecycle.md"); + let dated_stale = format!( + "\"kind\":\"stale_record\",\"target\":{{{}}}", + evidence_identity(&root_arg, "dated.md") + ); + assert!(summarize.output.contains(&stale_id), "{}", summarize.output); + assert!(summarize.output.contains(&body_id), "{}", summarize.output); + assert!(summarize.output.contains(&bad_id), "{}", summarize.output); + assert!( + summarize.output.contains("stale-frontmatter") + || summarize.output.contains("stale-body") + || summarize.output.contains("stale-after-shape") + || summarize.output.contains("status-value"), + "{}", + summarize.output + ); + + let propose = CliApp::run(vec![ + "evidence".to_owned(), + "propose".to_owned(), + root_arg.clone(), + ]); + assert_eq!(propose.exit_code, ExitCode::SUCCESS, "{}", propose.output); + assert!(propose.output.contains(&stale_id), "{}", propose.output); + assert!( + propose.output.contains("stale_record"), + "{}", + propose.output + ); + assert!(!propose.output.contains(&dated_stale), "{}", propose.output); + assert!(propose.output.contains("\"replacement\":null")); + assert!(propose.output.contains("\"authority\":null")); + + let _ = std::fs::remove_dir_all(root); } } diff --git a/docs/integrations/agent-setup.md b/docs/integrations/agent-setup.md index f2ea515..9df7e4e 100644 --- a/docs/integrations/agent-setup.md +++ b/docs/integrations/agent-setup.md @@ -34,10 +34,14 @@ contain credential material. ```sh bran packet "" bran query "" + bran check okf-v0.1 + bran check okf-v0.2 + bran check bran-strict ``` No generated answer is expected. Preserve provenance and label byte-derived -token counts as estimates. +token counts as estimates. `okf-v0.1` remains supported; `okf-v0.2` is +additive. Only the selected profile controls the exit code. ## Journey 2: Optional connected task and removal diff --git a/skill/use-bran/SKILL.md b/skill/use-bran/SKILL.md index e419447..58d16a3 100644 --- a/skill/use-bran/SKILL.md +++ b/skill/use-bran/SKILL.md @@ -10,15 +10,29 @@ outer task; the outer agent owns all decisions and changes. 1. Default to one read-only packet: `bran packet ""`. 2. Use `bran query ""` only for a focused follow-up the - packet did not answer. + packet did not answer. After the primary root, repeat `--add-dir ` + to ask the same question across multiple policy-bearing roots. Multi-root + results list `requested_roots` in argument order and put `bundle` on every + ranked source. If any requested root lacks native policy or cannot scan, + the whole request is refused and only that root argument is named. 3. Use connected `bran -p` only when a connected profile is configured and the bounded current repository is explicitly trusted. Keep it read-only and ask only for evidence location, contents, cross-file context, and citations. -4. Read the entire result. Report failures and unavailable fields honestly, +4. Read the entire result. Treat `data.query_outcome` as the semantic result, + not command `status`. `grounded` means every extracted query term matched + repository evidence. `miss` means there is no ranked evidence; empty + selections are a miss, not a successful answer. `partial_unanchored` means + some terms did not match — rankings may still be useful but are not a + complete answer to the named request. Never treat command success or a + non-empty ranking as full grounding. Report failures and unavailable fields honestly, preserve provenance and citations, label estimates as estimates, and never invent results or compression. 5. Continue the outer task using the cited evidence; keep decisions and implementation with the outer agent. +`bran check okf-v0.1`, `okf-v0.2`, and `bran-strict` are +independent selectable profiles. `okf-v0.1` remains supported; `okf-v0.2` is +additive. Only the selected profile controls the exit code. + Run `bran -h` for command and option help. Never pass credentials on the command line. diff --git a/tools/ci/public-policy.yaml b/tools/ci/public-policy.yaml new file mode 100644 index 0000000..a1ba652 --- /dev/null +++ b/tools/ci/public-policy.yaml @@ -0,0 +1,89 @@ +schema_version: "1" + +frontmatter: + required: + - type + - title + - okf_status + - tags + optional: + - freshness + - resource + - public_boundary + - description + - status + - stale_after + - sources + - usage_window + - generated + - verified + - timestamp + - runtime + - parameters + - computation + - executor + - attester + allowed: + - type + - title + - okf_status + - tags + - freshness + - resource + - public_boundary + - description + - status + - stale_after + - sources + - usage_window + - generated + - verified + - timestamp + - runtime + - parameters + - computation + - executor + - attester + preserved_canonical_keys: + - proposal_id + - kind + - submission_status + +coverage: + - canonical + - legacy + - excluded + - unclassified + +document_coverage: + legacy_documents: + - README.md + - docs/integrations/agent-setup.md + excluded_documents: + skill/use-bran/SKILL.md: Agent skill frontmatter is parsed by the host and is not an OKF concept document. + examples/headless/README.md: Headless command reference, not an OKF concept document. + bridge_targets: + - LICENSE-APACHE + - LICENSE-MIT + +tags: + allowed: + - developer + - public + - internal + - bran + +status: + allowed: + - draft + - active + - deprecated + +public_boundary: + values: + - public + - public-compatible + - private + - internal + path_allowlist: + - . diff --git a/tools/ci/public_export.py b/tools/ci/public_export.py index 9a3d75c..871a3a7 100644 --- a/tools/ci/public_export.py +++ b/tools/ci/public_export.py @@ -17,7 +17,35 @@ CONFIG_PATH = "public-export.json" RECEIPT_PATH = ".bran-export.json" +PUBLIC_POLICY_PATH = "tools/ci/public-policy.yaml" +EXPORTED_POLICY_PATH = ".bran/policy.yaml" ALLOWED_MODES = {"100644", "100755"} +FORBIDDEN_PUBLIC_POLICY_MARKERS = ( + "AGENTS.md", + "CLAUDE.md", + "docs/bugs", + "docs/integrations/proposals", + "docs/plans", + "docs/submissions", +) +# Exact SHA-256 of the reviewed committed tools/ci/public-policy.yaml +# bytes. Any template change requires an explicit template-plus-pin review. +# Structural substring checks below are defense in depth only; they are not +# YAML parsing and must not be treated as a semantic policy validator. +EXPECTED_PUBLIC_POLICY_SHA256 = ( + "1310773d9ab2c879d66823174faa2ee81fdc67596c7189ab6237e6819d75b278" +) +CONTRACT_INTERNAL_POLICY = ( + 'schema_version: "1"\n' + "document_coverage:\n" + " canonical_documents:\n" + " - AGENTS.md\n" + " - CLAUDE.md\n" + " - docs/bugs/private.md\n" + " - docs/integrations/proposals/draft.md\n" + " - docs/plans/secret.md\n" + " - docs/submissions/secret.md\n" +) CONFIG_KEYS = { "schema_version", "source_repository", @@ -203,7 +231,7 @@ def matches_root(path: str, roots: tuple[str, ...]) -> bool: def forbid_selected_path(path: str) -> None: if PurePosixPath(path).name in {"AGENTS.md", "CLAUDE.md"}: raise ExportError(f"agent instruction file selected for public export: {path}") - if path.startswith(".bran/") and path != ".bran/policy.yaml": + if path.startswith(".bran/") and path != EXPORTED_POLICY_PATH: raise ExportError(f"local BRAN runtime path selected for public export: {path}") if matches_root(path, FORBIDDEN_PUBLIC_ROOTS): raise ExportError(f"private root selected for public export: {path}") @@ -222,11 +250,60 @@ def select_public(tree: dict[str, GitBlob], config: ExportConfig) -> dict[str, G if path == RECEIPT_PATH: raise ExportError(f"source repository may not track generated receipt {RECEIPT_PATH}") selected[path] = blob - if ".bran/policy.yaml" not in selected: - raise ExportError("public export must include .bran/policy.yaml") + if EXPORTED_POLICY_PATH not in selected: + raise ExportError(f"public export must include {EXPORTED_POLICY_PATH}") return selected +def is_sha256(value: str) -> bool: + return len(value) == 64 and all(character in "0123456789abcdef" for character in value) + + +def public_policy_blob(tree: dict[str, GitBlob]) -> GitBlob: + blob = tree.get(PUBLIC_POLICY_PATH) + if blob is None: + raise ExportError(f"source commit must contain regular {PUBLIC_POLICY_PATH}") + if blob.mode != "100644": + raise ExportError(f"{PUBLIC_POLICY_PATH} must be a regular file") + if not blob.data: + raise ExportError(f"{PUBLIC_POLICY_PATH} is empty") + if not is_sha256(EXPECTED_PUBLIC_POLICY_SHA256): + raise ExportError("public policy pin is not a lowercase 64-hex SHA-256 digest") + digest = hashlib.sha256(blob.data).hexdigest() + if digest != EXPECTED_PUBLIC_POLICY_SHA256: + raise ExportError( + f"{PUBLIC_POLICY_PATH} sha256 {digest} does not match reviewed pin " + f"{EXPECTED_PUBLIC_POLICY_SHA256}" + ) + if b"\0" in blob.data: + raise ExportError(f"{PUBLIC_POLICY_PATH} is not valid UTF-8 text") + try: + text = blob.data.decode("utf-8") + except UnicodeDecodeError as exc: + raise ExportError(f"{PUBLIC_POLICY_PATH} is not valid UTF-8") from exc + # Defense in depth only. The pin above is the fail-closed identity check. + if "schema_version" not in text: + raise ExportError(f"{PUBLIC_POLICY_PATH} is missing schema_version") + leaked = [marker for marker in FORBIDDEN_PUBLIC_POLICY_MARKERS if marker in text] + if leaked: + raise ExportError(f"{PUBLIC_POLICY_PATH} names private paths: {', '.join(leaked)}") + return blob + + +def substitute_public_policy(tree: dict[str, GitBlob], selected: dict[str, GitBlob]) -> dict[str, GitBlob]: + template = public_policy_blob(tree) + if PUBLIC_POLICY_PATH not in selected: + raise ExportError(f"{PUBLIC_POLICY_PATH} must be selected for public export") + replaced = dict(selected) + replaced[EXPORTED_POLICY_PATH] = GitBlob( + path=EXPORTED_POLICY_PATH, + mode=template.mode, + object_id=template.object_id, + data=template.data, + ) + return replaced + + CLI_MANIFEST_PATH = "crates/bran-cli/Cargo.toml" @@ -291,7 +368,7 @@ def build_export(root: Path, reference: str) -> tuple[str, ExportConfig, dict[st if config_blob is None or config_blob.mode != "100644": raise ExportError(f"source commit must contain regular {CONFIG_PATH}") config = parse_config(config_blob.data) - selected = select_public(tree, config) + selected = substitute_public_policy(tree, select_public(tree, config)) return commit, config, selected, receipt_bytes(config, commit, selected) @@ -368,19 +445,43 @@ def write_text(path: Path, value: str) -> None: path.write_text(value, encoding="utf-8") +def write_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def reviewed_public_policy_bytes() -> bytes: + """Load the workspace template and require it to match the code-owned pin. + + Used only by the standalone contract check. Export hashes the committed + blob against EXPECTED_PUBLIC_POLICY_SHA256 and never reads this file. + """ + data = (source_root() / PUBLIC_POLICY_PATH).read_bytes() + digest = hashlib.sha256(data).hexdigest() + if digest != EXPECTED_PUBLIC_POLICY_SHA256: + raise AssertionError( + f"{PUBLIC_POLICY_PATH} digest {digest} does not match reviewed pin " + f"{EXPECTED_PUBLIC_POLICY_SHA256}" + ) + return data + + def commit_all(root: Path, message: str) -> None: git_ok(root, ["add", "-A"]) git_ok(root, ["commit", "-m", message]) +CONTRACT_CHILD_ENV = "BRAN_PUBLIC_EXPORT_CONTRACT_CHILD" + + def contract_config() -> dict[str, Any]: return { "schema_version": 1, "source_repository": "example/bran-dev", "public_repository": "example/bran", "public_remote": "https://example.invalid/example/bran.git", - "allowed_files": [".bran/policy.yaml", "README.md"], - "allowed_roots": ["product"], + "allowed_files": [EXPORTED_POLICY_PATH, "README.md"], + "allowed_roots": ["product", "tools/ci"], "excluded_files": [ ".bran/settings.conf", ".closeout.json", @@ -395,10 +496,65 @@ def contract_config() -> dict[str, Any]: "docs/integrations/proposals", "docs/plans", "docs/submissions", + "tools/cutover", ], } +def prove_exported_exporter_self_contained(reviewed_policy: bytes) -> None: + """Fail if an exported snapshot cannot run this exporter's own contract. + + tools/cutover is excluded from public export. The full public gate runs + this file from the snapshot, so the pinned template must be present on + an exported root or reviewed_public_policy_bytes() cannot load it. + """ + if os.environ.get(CONTRACT_CHILD_ENV) == "1": + return + with tempfile.TemporaryDirectory(prefix="bran-export-self-") as temporary: + base = Path(temporary) + source = base / "source" + public = base / "public" + source.mkdir() + git_ok(source, ["init", "-b", "main"]) + git_ok(source, ["config", "user.name", "BRAN Export Check"]) + git_ok(source, ["config", "user.email", "bran-export@example.invalid"]) + config = contract_config() + write_text(source / "README.md", "public product\n") + write_text(source / EXPORTED_POLICY_PATH, CONTRACT_INTERNAL_POLICY) + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + write_bytes(source / "tools/ci/public_export.py", Path(__file__).read_bytes()) + write_text(source / "product/data.txt", "deterministic\n") + write_text(source / "AGENTS.md", "private instructions\n") + write_text(source / CONFIG_PATH, json.dumps(config, indent=2) + "\n") + commit_all(source, "seed self-contained export contract") + write_snapshot(source, public, "HEAD") + if not (public / "tools/ci/public_export.py").is_file(): + raise AssertionError("exported snapshot is missing tools/ci/public_export.py") + if not (public / PUBLIC_POLICY_PATH).is_file(): + raise AssertionError(f"exported snapshot is missing {PUBLIC_POLICY_PATH}") + if (public / PUBLIC_POLICY_PATH).read_bytes() != reviewed_policy: + raise AssertionError(f"exported {PUBLIC_POLICY_PATH} does not match the pinned template") + env = os.environ.copy() + env[CONTRACT_CHILD_ENV] = "1" + completed = subprocess.run( + [sys.executable, str(public / "tools/ci/public_export.py")], + cwd=str(public), + env=env, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise AssertionError( + "exported snapshot failed its own exporter contract:\n" + f"{completed.stdout}\n{completed.stderr}" + ) + if "PASS P1-PUBLIC-EXPORT" not in completed.stdout: + raise AssertionError( + "exported snapshot did not report exporter contract success:\n" + f"{completed.stdout}\n{completed.stderr}" + ) + + def run_contract_check() -> None: with tempfile.TemporaryDirectory(prefix="bran-export-") as temporary: base = Path(temporary) @@ -408,8 +564,10 @@ def run_contract_check() -> None: git_ok(source, ["init", "-b", "main"]) git_ok(source, ["config", "user.name", "BRAN Export Check"]) git_ok(source, ["config", "user.email", "bran-export@example.invalid"]) + reviewed_policy = reviewed_public_policy_bytes() write_text(source / "README.md", "public product\n") - write_text(source / ".bran/policy.yaml", 'schema_version: "1"\n') + write_text(source / EXPORTED_POLICY_PATH, CONTRACT_INTERNAL_POLICY) + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) write_text(source / "product/data.txt", "deterministic\n") write_text(source / "AGENTS.md", "private instructions\n") write_text(source / ".bran/results/run.json", "private runtime\n") @@ -423,11 +581,31 @@ def run_contract_check() -> None: expect_export_error(lambda: write_snapshot(source, nonempty, "HEAD"), "nonempty output") commit, count = write_snapshot(source, public, "HEAD") - assert count == 4 + assert count == 5 assert len(commit) == 40 assert not (public / "AGENTS.md").exists() assert not (public / ".bran/results/run.json").exists() assert not (public / "docs/integrations/proposals/draft.md").exists() + assert not (public / "tools/cutover/public-policy.yaml").exists() + exported_policy = (public / EXPORTED_POLICY_PATH).read_bytes() + exported_template = (public / PUBLIC_POLICY_PATH).read_bytes() + expected_policy = reviewed_policy + assert exported_policy == expected_policy + assert exported_template == expected_policy + for marker in FORBIDDEN_PUBLIC_POLICY_MARKERS: + assert marker.encode("utf-8") not in exported_policy + assert marker.encode("utf-8") not in exported_template + receipt = json.loads((public / RECEIPT_PATH).read_text(encoding="utf-8")) + policy_entry = next(item for item in receipt["files"] if item["path"] == EXPORTED_POLICY_PATH) + template_entry = next(item for item in receipt["files"] if item["path"] == PUBLIC_POLICY_PATH) + assert policy_entry["mode"] == template_entry["mode"] == "100644" + assert policy_entry["bytes"] == template_entry["bytes"] == len(expected_policy) + assert ( + policy_entry["sha256"] + == template_entry["sha256"] + == hashlib.sha256(expected_policy).hexdigest() + == EXPECTED_PUBLIC_POLICY_SHA256 + ) git_ok(public, ["init", "-b", "main"]) git_ok(public, ["config", "user.name", "BRAN Export Check"]) @@ -472,10 +650,107 @@ def run_contract_check() -> None: expect_export_error(lambda: build_export(source, "HEAD"), "symbolic link") link.unlink() commit_all(source, "remove unsafe link") + + git_ok(source, ["rm", PUBLIC_POLICY_PATH]) + commit_all(source, "remove public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "missing public policy template") + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + write_bytes(source / PUBLIC_POLICY_PATH, CONTRACT_INTERNAL_POLICY.encode("utf-8")) + commit_all(source, "add private paths to public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "private paths in public policy template") + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + write_bytes(source / PUBLIC_POLICY_PATH, b"") + commit_all(source, "empty public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "empty public policy template") + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + write_bytes(source / PUBLIC_POLICY_PATH, b"coverage: []\n") + commit_all(source, "malformed public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "malformed public policy template") + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + write_bytes(source / PUBLIC_POLICY_PATH, b'schema_version: "1"\n\0') + commit_all(source, "binary public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "binary public policy template") + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + (source / PUBLIC_POLICY_PATH).chmod(0o755) + commit_all(source, "unsafe public policy mode") + expect_export_error(lambda: build_export(source, "HEAD"), "unsafe public policy mode") + (source / PUBLIC_POLICY_PATH).chmod(0o644) + commit_all(source, "restore public policy mode") + + policy_mutations = ( + ( + b'schema_version: "1"\n: : :\n[[[\n', + "malformed YAML with schema_version present", + ), + ( + b'schema_version: "999"\nfrontmatter: {}\n', + "unsupported schema_version", + ), + ( + b'schema_version: "1"\npath: docs\\/plans\n', + "escaped private path", + ), + ( + b'schema_version: "1"\n- .bran/settings.conf\n', + "private .bran path", + ), + ( + reviewed_policy[:-1] + bytes([reviewed_policy[-1] ^ 0x01]), + "one-byte public policy mutation", + ), + ) + for payload, label in policy_mutations: + write_bytes(source / PUBLIC_POLICY_PATH, payload) + commit_all(source, f"mutate public policy: {label}") + expect_export_error(lambda: build_export(source, "HEAD"), label) + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + git_ok(source, ["rm", PUBLIC_POLICY_PATH]) + write_bytes(source / "tools/cutover/public-policy.yaml", reviewed_policy) + commit_all(source, "leave template only on excluded cutover path") + expect_export_error( + lambda: build_export(source, "HEAD"), + "template only at excluded cutover path", + ) + git_ok(source, ["rm", "tools/cutover/public-policy.yaml"]) + write_bytes(source / PUBLIC_POLICY_PATH, reviewed_policy) + commit_all(source, "restore public policy template") + + write_bytes(source / "tools/cutover/public-policy.yaml", CONTRACT_INTERNAL_POLICY.encode("utf-8")) + commit_all(source, "add decoy cutover policy") + _, _, selected, _ = build_export(source, "HEAD") + assert selected[PUBLIC_POLICY_PATH].data == reviewed_policy + assert selected[EXPORTED_POLICY_PATH].data == reviewed_policy + assert "tools/cutover/public-policy.yaml" not in selected + git_ok(source, ["rm", "tools/cutover/public-policy.yaml"]) + commit_all(source, "remove decoy cutover policy") + + config = contract_config() + config["allowed_roots"] = ["product"] + config["excluded_files"] = sorted({*config["excluded_files"], PUBLIC_POLICY_PATH}) + write_text(source / CONFIG_PATH, json.dumps(config, indent=2) + "\n") + commit_all(source, "exclude public policy template") + expect_export_error(lambda: build_export(source, "HEAD"), "unselected public policy template") + write_text(source / CONFIG_PATH, json.dumps(contract_config(), indent=2) + "\n") + commit_all(source, "restore export config") + write_text(source / "unknown.txt", "unclassified\n") commit_all(source, "add unclassified path") expect_export_error(lambda: build_export(source, "HEAD"), "unclassified path") + prove_exported_exporter_self_contained(reviewed_policy) + def source_root() -> Path: return Path(__file__).resolve().parents[2]