diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..72b5ef0a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# The build context of packaging/docker/Dockerfile is the repository root, because Cargo needs the manifests of +# every workspace member. Nothing here may reach the image: no history, no local data, no secrets. +.git +.github +target +**/target +/dist +/temp +/data +docs +packaging +**/*.md +.env +.env.* +**/.env +**/.env.* +*.sqlite +*.sqlite-wal +*.sqlite-shm +*~ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc9c0ee2..be5e6063 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,3 +127,26 @@ jobs: - name: Test storage, metadata and the server on every backend run: cargo test -p craft-cloud-store -p craft-cloud-meta -p craft-cloud-server --locked + + # The other jobs build with the newest stable Rust, so code or a dependency that needs a newer compiler than the + # `rust-version` in Cargo.toml would go unnoticed. Keep the toolchain here equal to that `rust-version`. + msrv: + name: minimum Rust version + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - name: Install Linux system dependencies (egui/winit) + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev + + - uses: dtolnay/rust-toolchain@1.95 + + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Check + run: cargo check --workspace --all-targets --locked diff --git a/.gitignore b/.gitignore index 6ea3d416..714e91fe 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ target/ *.sqlite-shm # a snapshot test's own render when it doesn't match *.new.png +# the Docker deployment's settings and secrets (written by packaging/docker/install.sh), and a custom TLS certificate +/packaging/docker/.env +/packaging/docker/tls/ diff --git a/AGENTS.md b/AGENTS.md index a0d3a8e6..e40b3efb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ The design lives in `docs/design/sync.md`; the live protocols (the server's live ```text crates/ - api craft-cloud-api what apps need of the protocol, with no postcard: IDs, names, limits, the agent's local API (`local.rs`, `/v1/blobs` too), error codes and the account's devices (`api.rs`) + api craft-cloud-api what apps need of the protocol, with no postcard: IDs, names, limits, the agent's local API (`local.rs`, `/v1/blobs` too), error codes, the account's devices and the admin API's bodies (`api.rs`) proto craft-cloud-proto wire types: hashes, manifests, commit ops, API bodies, join codes and join strings (`join.rs`), the live protocol (`live.rs`); re-exports `craft-cloud-api` under the same paths chunker craft-cloud-chunker FastCDC `fastcdc-v1` + BLAKE3, frozen by golden vectors store craft-cloud-store blob store: disk, S3, memory; checks every blob in and out @@ -34,14 +34,15 @@ crates/ sync craft-cloud-sync the sync engine over `Fs`/`Transport` traits; `MemFs`; scenario tests and the simulation kit craft-cloud-kit the app kit (`docs/kit.md`): an app's model live through the agent; the `LiveDocument` trait and `Session` (`session.rs`: open, the `local_change` hook, `poll`, undo through Loro's undo manager, reconnect and the 64 MiB queue, presence, saves), the change writer that cuts commits under 512 KiB (`writer.rs`), the mappings (`tree.rs`, `text.rs`, `table.rs`, `timeline.rs`, `opaque.rs`, `list.rs`), values, chunk references and blob stores (`value.rs`) egui craft-cloud-egui Craft Cloud's piece of an app's window: the account line, the shared recents, a live document's badge, and `CloudWatch` that keeps them current - app craft-cloud-app what apps link: a blocking client of the agent's local API (status, sign-in, joining, computers, pause, autostart, recents, live documents at the level of bytes: `live.rs`) + app craft-cloud-app what apps link: a blocking client of the agent's local API (status, sign-in, joining, computers, pause, autostart, recents, live documents at the level of bytes: `live.rs`, the admin calls: `admin.rs`, team spaces: `spaces.rs`) apps/ - craft-cloud-server HTTP API (`/api/v1`), the live service (`/api/v1/live`, `live.rs`: logs, documents by import, snapshots and compaction, presence), join codes (`join.rs`: pairings, join, the `/join/{code}` page), auth, admin CLI (`init`, `serve`, `user` with `invite`, `device`, `reset-epoch`, `live`) - craft-cloud-agent the sync agent: real disk `Fs`, HTTP `Transport` over a `Connection`, saved account and state, root marker, CLI (`login`, `join`, `run`, `sync`, `status`, `logout`, `recents`, `doc` (`doc.rs`), `add-computer`, `devices`, `pause`, `resume`, `quit`, `autostart`), the `run` loop, the local API (`local_api.rs`), the live hub (`live/`: a document's Loro copy, recents document, replica (recents and documents of files), store (with the documents' journals), link, hub, the local API's docs socket), start at login (`autostart/`: Windows, macOS, Linux), the background build (`src/bin/bg.rs`, logging to `runlog.rs`) + craft-cloud-server HTTP API (`/api/v1`), the live service (`/api/v1/live`, `live.rs`: logs, documents by import, snapshots and compaction, presence), join codes (`join.rs`: pairings, join, the `/join/{code}` page), the space layer (`space.rs`: the `craft-space` header, `require_space`, a role per route, `read_only`), the team-space routes (`spaces.rs`: `/spaces…`, `/admin/spaces…`, invitations, members), auth, admin pages (`web/`: server-rendered HTML with browser sessions, off unless `[web_admin]` is on), admin CLI (`init`, `serve`, `user` with `invite`, `device`, `reset-epoch`, `live`, `web-sessions`) + craft-cloud-agent the sync agent: real disk `Fs`, HTTP `Transport` over a `Connection`, saved account and state, root marker, CLI (`login`, `join`, `run`, `sync`, `status`, `logout`, `recents`, `doc` (`doc.rs`), `add-computer`, `devices`, `pause`, `resume`, `quit`, `autostart`), the `run` loop, local API (`local_api.rs`, including server admin calls in `admin.rs`), team spaces (`spaces.rs`: discovery by `GET /spaces`, a folder and an engine per synced space, losing access), live hub (`live/`: a document's Loro copy, recents document, replica (recents and documents of files), store (with the documents' journals), link, hub, the local API's docs socket), start at login (`autostart/`: Windows, macOS, Linux), background build (`src/bin/bg.rs`, logging to `runlog.rs`) + tests/team_spaces.rs several spaces against a real server: invitations, the team folder both ways, viewer, removal, rename, folder clashes, the echo, an old server tests/network/ the simulation over a simulated network (turmoil): `main.rs` histories and scenarios, `harness.rs` server and agent hosts, `live.rs` live documents over WebSockets on it tests/recents_sim.rs the recents convergence simulation: replicas against the server's live sessions tests/documents_sim.rs the documents convergence simulation (shares `tests/docs_common/` with `documents.rs` and `live_speed.rs`) - craft-cloud-desktop the Craft Cloud app (`craft-cloud`, eframe/egui): the pure model (`model.rs`), the agent behind a trait and calls on threads (`agent.rs`), starting the agent (`start_agent.rs`), one app per user (`single_instance.rs`), the window (`ui/`, `app.rs`), the tray (`tray/`: tray-icon in the window's process on Windows and macOS; on Linux a StatusNotifierItem over zbus in a process of its own, `craft-cloud --tray`, `tray/process.rs`, since Wayland can't hide a window), the app's own start at login (`autostart.rs`), its icons (`assets/`, drawn by `cargo xtask icons`) + craft-cloud-desktop the Craft Cloud app (`craft-cloud`, eframe/egui): the pure model (`model.rs`, the admin pages' part of it in `admin.rs`), the agent behind a trait and calls on threads (`agent.rs`), starting the agent (`start_agent.rs`), one app per user (`single_instance.rs`), the window (`ui/`, `app.rs`), the tray (`tray/`: tray-icon in the window's process on Windows and macOS; on Linux a StatusNotifierItem over zbus in a process of its own, `craft-cloud --tray`, `tray/process.rs`, since Wayland can't hide a window), the app's own start at login (`autostart.rs`), its icons (`assets/`, drawn by `cargo xtask icons`) tests/snapshots/ every screen as kittest draws it (`UPDATE_SNAPSHOTS=1` redraws them) craft-cloud-setup the Windows setup (`craft-cloud-setup.exe`): install, upgrade, move from a hand-made install, `--uninstall`; the exes built in by `cargo xtask package` craft-cloud-kit-demo the app kit's demo (never packaged or installed): a story and a layer tree, live through the kit; its model and adapter (`model.rs`), its window (`ui.rs`); tests drive two windows on one fake hub, and time the kit on a big document diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index e0b2760c..a0c69d3c 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -14,3 +14,4 @@ next to them. | `apps/craft-cloud-desktop/assets/tray-*.png` | Craft Cloud's tray icon in each state (synced, syncing, offline, needs attention, paused): a cloud with a badge | Craft Cloud contributors | Drawn by `cargo xtask icons` (`xtask/src/icons.rs`) for this repository | MIT OR Apache-2.0 | | `apps/craft-cloud-desktop/tests/autostart/*` | Expected start-at-login entries of the app per OS (golden files) | Craft Cloud contributors | Written for this repository | MIT OR Apache-2.0 | | `apps/craft-cloud-desktop/tests/snapshots/*.png` | The Craft Cloud app's screens as its snapshot tests expect them (the text in egui's default fonts, see `epaint_default_fonts`) | Craft Cloud contributors | Rendered by the tests (`UPDATE_SNAPSHOTS=1`) | MIT OR Apache-2.0 | +| `apps/craft-cloud-server/src/web/style.css` | The admin pages' stylesheet: system fonts, no images, no external files | Craft Cloud contributors | Written for this repository | MIT OR Apache-2.0 | diff --git a/Cargo.lock b/Cargo.lock index 7b4cbe7c..b38279b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,6 +121,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -910,6 +945,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.7" @@ -1028,6 +1073,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1260,6 +1311,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio", + "tracing", "uuid", ] @@ -1283,6 +1335,7 @@ dependencies = [ name = "craft-cloud-server" version = "0.1.0" dependencies = [ + "aes-gcm", "argon2", "axum", "base64 0.23.1", @@ -1291,13 +1344,17 @@ dependencies = [ "clap", "craft-cloud-meta", "craft-cloud-proto", + "craft-cloud-server", "craft-cloud-store", "futures-util", "getrandom 0.4.3", + "hmac", "reqwest", "serde", "serde_json", + "sha1 0.11.0", "sqlx", + "subtle", "tempfile", "thiserror 2.0.21", "tokio", @@ -1306,6 +1363,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zeroize", ] [[package]] @@ -1326,6 +1384,7 @@ dependencies = [ "blake3", "bytes", "craft-cloud-proto", + "futures-util", "object_store", "tempfile", "thiserror 2.0.21", @@ -1429,6 +1488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1441,6 +1501,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.3" @@ -1562,6 +1631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -2286,6 +2356,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gl_generator" version = "0.14.0" @@ -2894,6 +2974,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -4107,6 +4196,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -4395,6 +4490,18 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -6230,6 +6337,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 3c65eb18..254366e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = ["crates/*", "apps/*", "xtask"] version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" -rust-version = "1.94" +rust-version = "1.95" repository = "https://github.com/itrycodinganditsbad/craft-cloud" [workspace.lints.rust] @@ -28,6 +28,13 @@ craft-cloud-agent = { path = "apps/craft-cloud-agent" } craft-cloud-app = { path = "crates/app" } craft-cloud-kit = { path = "crates/kit" } craft-cloud-egui = { path = "crates/egui" } +# Two-factor sign-in: AES-256-GCM for the stored TOTP secrets (RustCrypto, pure Rust), and HMAC-SHA-1 for the codes +# themselves (authenticator apps speak SHA-1). `subtle` compares codes in constant time, `zeroize` wipes secrets. +aes-gcm = "0.10.3" +hmac = "0.13" +sha1 = "0.11" +subtle = "2.6" +zeroize = "1.9" argon2 = "0.6" async-trait = "0.1.89" # `ws`: the server's live service and the agent's local API are WebSockets (tokio-tungstenite, MIT). diff --git a/README.md b/README.md index 48d61966..3f0ecc60 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ that is signed in, and enter its code on the new one. ## Documentation - [Installing the Craft Cloud app](docs/install.md) on Windows, macOS and Linux -- [Running the server](docs/server.md): reverse proxy, Postgres and S3, accounts, invites, backups +- [Running the server](docs/server.md): reverse proxy, Postgres and S3, accounts, invites, backups, Docker - [The agent on the command line](docs/agent.md): `login`, `run`, `sync`, `join`, start at login - [Development](docs/development.md): how it fits together, building, testing, the sync engine's rules - [The app kit](docs/kit.md): how a Craft app makes its documents live diff --git a/apps/craft-cloud-agent/src/account.rs b/apps/craft-cloud-agent/src/account.rs index 4a04ac33..13bf137d 100644 --- a/apps/craft-cloud-agent/src/account.rs +++ b/apps/craft-cloud-agent/src/account.rs @@ -28,6 +28,8 @@ use crate::http::{DeviceToken, ServerUrl}; pub const ACCOUNT_FILE: &str = "account.json"; pub const STATE_FILE: &str = "state.json"; +/// The data folder's folder of the team spaces' own state, one folder per space. +pub const SPACES_DIR: &str = "spaces"; /// What the `run` loop is doing ([`LoopStatus`]), next to `state.json`. pub const STATUS_FILE: &str = "status.json"; /// An approval handed to the running loop ([`Approval`]). @@ -86,6 +88,37 @@ pub struct Account { pub space: SpaceId, pub root: NodeId, pub token: DeviceToken, + /// The team spaces this computer knows (appended in a later format of the same file: an agent from before team + /// spaces ignores it, and a file without it has none). Left out of the file while there is none, so an installation + /// that never saw a team space keeps its `account.json` byte for byte. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub spaces: Vec, +} + +/// What this computer remembers of one team space (`account.json`, `spaces`): where it syncs, whether it does, and what +/// the server last called it. Its state lives in `spaces//state.json` of the data folder. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SpaceSetting { + pub space: SpaceId, + /// The space's folder (absolute); it keeps its name when the space is renamed on the server. + pub folder: PathBuf, + /// Whether the space syncs on this computer. A team space starts with syncing off. + pub sync: bool, + /// The name the server last gave the space, to say which one lost access when the server no longer lists it. + #[serde(default)] + pub name: Option, + /// The space's root, as the server listed it: what the folder's marker names. + #[serde(default)] + pub root: Option, + /// Which saved state of the space may be used: a state records the generation it was made under and is used only under + /// the same one. It is raised (and saved here) before a state is retired or a folder is set up afresh, so a state that + /// survives its retirement (a file that couldn't be removed, a crash in between) is never trusted again. + #[serde(default, skip_serializing_if = "is_zero")] + pub generation: u64, +} + +fn is_zero(n: &u64) -> bool { + *n == 0 } /// How a pass ended. @@ -111,6 +144,11 @@ pub struct PassSummary { pub created_dirs: usize, pub conflict_copies: Vec, pub refused: usize, + /// Whether the pass ran in read-only mode: the space is read-only for this account (a viewer), nothing was sent, + /// and what is only here is listed in `skipped` with the reason. + pub read_only: bool, + /// Files and folders deleted here that the server still has, put back (read-only mode only). + pub restored: Vec, /// Entries that didn't sync, with the reason. pub skipped: Vec<(String, String)>, /// Deletions on the server waiting for approval. @@ -142,6 +180,8 @@ impl PassSummary { created_dirs: report.created_dirs.len(), conflict_copies: paths(&report.conflict_copies), refused: report.refused.len(), + read_only: report.read_only, + restored: paths(&report.restored), skipped: report.skipped.clone(), pending_deletes: paths(&report.pending_deletes), pending_local_deletes: paths(&report.pending_local_deletes), @@ -272,6 +312,28 @@ pub struct SavedState { /// `run` loop only while no pass runs (a pass writes `false`: passes run only when not paused). #[serde(default)] pub paused: bool, + /// A team space's state only: the canonical folder it was made for. A state is used only for the folder it names + /// (see [`DataDir::load_state_for`]); the personal space's state has none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub folder: Option, + /// A team space's state only: the ID of the folder it was made for (see [`crate::disk::FolderIdentity`]). A folder + /// replaced at the same path (restored from a backup, made again) has another ID, and a state made for the first is not + /// used for it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub folder_id: Option, + /// A team space's state only: the nonce in the folder's `.craft-cloud/nonce` when this state was saved (see + /// [`crate::root::read_nonce`]). A state is used only for a folder whose nonce is this one (both absent counts: a folder + /// and a state from before nonces). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub folder_nonce: Option, + /// A team space's state only: the server's stamp of the account's membership when the state was made + /// (`SpaceListItem::joined_ms`). A state made under another stamp (the member was removed and invited again meanwhile) + /// is not used. `None`: the server gave none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub joined_ms: Option, + /// A team space's state only: the generation of the space's setting it was made under ([`SpaceSetting::generation`]). + #[serde(default, skip_serializing_if = "is_zero")] + pub generation: u64, } /// `status.json`: what the `run` loop is doing, written atomically on every change and as a heartbeat, for `status` @@ -383,6 +445,11 @@ impl DataDir { &self.0 } + /// The place of a team space's own `state.json` and `status.json`: `spaces//` (not created). + pub fn space_dir(&self, space: SpaceId) -> DataDir { + DataDir(self.0.join(SPACES_DIR).join(space.to_string())) + } + fn file(&self, name: &str) -> PathBuf { self.0.join(name) } @@ -433,6 +500,56 @@ impl DataDir { Ok(load(&self.file(STATE_FILE), STATE_SCHEMA)?.unwrap_or(SavedState { schema: STATE_SCHEMA, ..SavedState::default() })) } + /// A team space's saved state for `folder`: the state only if it was made for this space and this folder (compared + /// as canonical paths), else a fresh one. A state remembers what the folder it syncs holds; used for another folder it + /// would find that folder's files missing and delete them on the server. `joined_ms` is the server's current stamp of + /// the account's membership: a state made under a different one is set aside too, since the access was interrupted + /// and the folder and the server changed meanwhile without this state seeing it. A server that gives no stamp (an older + /// one) is not compared; a state without one, while the server gives one, is not trusted. A state made under another + /// `generation` (the setting's, see [`SpaceSetting::generation`]) is not used either. The `bool` says the saved + /// state was set aside because it belongs elsewhere. + pub fn load_state_for(&self, space: SpaceId, folder: &Path, joined_ms: Option, generation: u64) -> Result<(SavedState, bool), AccountError> { + self.load_state_with(space, crate::disk::folder_identity(folder).as_ref(), joined_ms, generation) + } + + /// As [`DataDir::load_state_for`], for a folder looked at once already (`identity`): a session takes that one look for + /// both the state it chooses and the folder it later compares with. The state must also have been made for the folder's + /// ID, when the folder has one. + pub fn load_state_with( + &self, + space: SpaceId, + identity: Option<&crate::disk::FolderIdentity>, + joined_ms: Option, + generation: u64, + ) -> Result<(SavedState, bool), AccountError> { + let saved = self.load_state()?; + let here = identity.map(|i| i.path.clone()); + let same_folder_id = match identity.and_then(|i| i.id) { + Some(now) => saved.folder_id == Some(now), + None => true, + }; + // The same path and ID are not enough on a file system that reuses IDs: the folder's nonce must be the one the state + // recorded. A folder without a nonce suits a state without one only. + let same_nonce = identity.and_then(|i| crate::root::read_nonce(&i.path)) == saved.folder_nonce; + let same_membership = match (joined_ms, saved.joined_ms) { + (Some(now), Some(then)) => now == then, + // The server gives a stamp and the state has none: it was made before the membership was stamped. + (Some(_), None) => false, + (None, _) => true, + }; + let ours = saved.state.space.is_none_or(|s| s == space) + && saved.folder.is_some() + && saved.folder == here + && same_folder_id + && same_nonce + && same_membership + && saved.generation == generation; + if ours || (saved.state.space.is_none() && saved.last_pass.is_none() && saved.folder.is_none()) { + return Ok((saved, false)); + } + Ok((SavedState { schema: STATE_SCHEMA, ..SavedState::default() }, true)) + } + pub fn save_state(&self, saved: &SavedState) -> Result<(), AccountError> { self.create()?; save(&self.file(STATE_FILE), saved) @@ -442,6 +559,24 @@ impl DataDir { remove(&self.file(STATE_FILE)) } + /// Removes the saved state of every team space (`spaces//state.json`), as signing out does for the personal one. + /// Their folders, markers and files are not touched. + pub fn remove_space_states(&self) -> Result<(), AccountError> { + let dir = self.0.join(SPACES_DIR); + let entries = match fs::read_dir(&dir) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(io_at(&dir)(e)), + }; + for entry in entries { + let entry = entry.map_err(io_at(&dir))?; + if entry.path().is_dir() { + DataDir(entry.path()).remove_state()?; + } + } + Ok(()) + } + /// Removes the temporary files a crash left while `account.json`, `state.json`, `status.json`, `approval.json` or /// `shown.json` was written (`.state.json..tmp`), and nothing else. Call it while holding the data /// folder's lock. Only those at least [`TEMP_MIN_AGE`] old go: `status` and `sync --approve-deletes` write files @@ -733,6 +868,7 @@ mod tests { space: SpaceId::generate(), root: NodeId::generate(), token: DeviceToken::new("tok".into()), + spaces: Vec::new(), } } @@ -758,6 +894,33 @@ mod tests { assert_eq!(fs::read_dir(data.path()).unwrap().count(), 0); } + /// An `account.json` from before team spaces has no `spaces` and is written back without one; team spaces round-trip, + /// and each has its own place for state. + #[test] + fn team_spaces_are_appended_to_the_account_and_left_out_while_there_are_none() { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let mut acc = account(dir.path()); + data.save_account(&acc).unwrap(); + let written = fs::read_to_string(dir.path().join("data/account.json")).unwrap(); + assert!(!written.contains("spaces"), "{written}"); + assert_eq!(data.load_account().unwrap().unwrap().spaces, Vec::new()); + let setting = SpaceSetting { + space: SpaceId::generate(), + folder: dir.path().join("Craft Cloud Shared").join("Studio"), + sync: false, + name: Some("Studio".into()), + root: Some(NodeId::generate()), + generation: 0, + }; + acc.spaces.push(setting.clone()); + data.save_account(&acc).unwrap(); + assert_eq!(data.load_account().unwrap().unwrap().spaces, vec![setting.clone()]); + let sub = data.space_dir(setting.space); + assert_eq!(sub.path(), dir.path().join("data").join("spaces").join(setting.space.to_string())); + assert_eq!(sub.load_state().unwrap().state.space, None, "no state yet"); + } + /// Only the data folder is owner-only: missing folders above it are made as any program makes them. #[cfg(unix)] #[test] @@ -897,6 +1060,109 @@ mod tests { assert_eq!(data.load_state().unwrap(), saved); } + /// A team space's state is used for the space and the folder it was made for, and for nothing else: another folder, another + /// space, or a state that records no folder (one from before it did) starts afresh. A missing state is a fresh one. + #[test] + fn a_team_state_is_used_only_for_its_space_and_folder() { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (a, b) = (dir.path().join("A"), dir.path().join("B")); + fs::create_dir_all(&a).unwrap(); + fs::create_dir_all(&b).unwrap(); + let space = SpaceId::generate(); + assert_eq!(data.load_state_for(space, &a, None, 0).unwrap(), (SavedState { schema: STATE_SCHEMA, ..SavedState::default() }, false)); + let id = crate::disk::folder_identity(&a).and_then(|i| i.id).expect("the folder has an ID"); + let mut saved = SavedState { schema: STATE_SCHEMA, folder: fs::canonicalize(&a).ok(), folder_id: Some(id), ..SavedState::default() }; + saved.state.space = Some(space); + saved.state.seq = 9; + data.save_state(&saved).unwrap(); + assert_eq!(data.load_state_for(space, &a, None, 0).unwrap(), (saved.clone(), false), "the folder it was made for"); + let (other, set_aside) = data.load_state_for(space, &b, None, 0).unwrap(); + assert!(set_aside && other.state.seq == 0 && other.state.space.is_none(), "another folder starts afresh"); + let (other, set_aside) = data.load_state_for(SpaceId::generate(), &a, None, 0).unwrap(); + assert!(set_aside && other.state.space.is_none(), "another space starts afresh"); + let (gone, set_aside) = data.load_state_for(space, &dir.path().join("missing"), None, 0).unwrap(); + assert!(set_aside && gone.state.seq == 0, "a folder that can't be found is not the one"); + // A state from before the folder was recorded is not trusted either. + saved.folder = None; + data.save_state(&saved).unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1); + // The server's stamp of the membership: a state made under another one is not used; without a stamp on either side + // (an older server, a state from before stamps) nothing is compared. + saved.folder = fs::canonicalize(&a).ok(); + saved.folder_id = Some(id); + saved.joined_ms = Some(100); + data.save_state(&saved).unwrap(); + assert!(!data.load_state_for(space, &a, Some(100), 0).unwrap().1, "the same membership"); + assert!(data.load_state_for(space, &a, Some(200), 0).unwrap().1, "removed and invited again meanwhile"); + assert!(!data.load_state_for(space, &a, None, 0).unwrap().1, "a server without stamps"); + saved.joined_ms = None; + data.save_state(&saved).unwrap(); + assert!(data.load_state_for(space, &a, Some(200), 0).unwrap().1, "a state without a stamp is not trusted when the server gives one"); + assert!(!data.load_state_for(space, &a, None, 0).unwrap().1, "and is used with a server that gives none"); + // A state is made for one folder ID: the same path replaced by another folder (a restored backup) is not the folder, and a + // state without an ID is not trusted when the folder has one. + saved.folder_id = None; + data.save_state(&saved).unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "a state without a folder ID is not trusted"); + saved.folder_id = Some(id.wrapping_add(1)); + data.save_state(&saved).unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "another folder ID at the same path: not the folder"); + saved.folder_id = Some(id); + data.save_state(&saved).unwrap(); + assert!(!data.load_state_for(space, &a, None, 0).unwrap().1, "the folder it was made for"); + // The folder's nonce: the state is used only for a folder that carries the nonce it recorded. A folder without one suits a + // state without one only, and the other way round. + saved.folder_id = Some(id); + saved.folder_nonce = None; + data.save_state(&saved).unwrap(); + assert!(!data.load_state_for(space, &a, None, 0).unwrap().1, "neither has a nonce"); + fs::create_dir_all(a.join(".craft-cloud")).unwrap(); + fs::write(a.join(".craft-cloud/nonce"), "00ff").unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "the folder has a nonce and the state none"); + saved.folder_nonce = Some("00ff".into()); + data.save_state(&saved).unwrap(); + assert!(!data.load_state_for(space, &a, None, 0).unwrap().1, "the same nonce"); + saved.folder_nonce = Some("11aa".into()); + data.save_state(&saved).unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "an older folder (another nonce) at the same path and ID"); + fs::remove_file(a.join(".craft-cloud/nonce")).unwrap(); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "the state has a nonce and the folder none"); + saved.folder_nonce = None; + data.save_state(&saved).unwrap(); + // A state made under another generation of the space's setting is not used; the generation is written only when raised. + saved.joined_ms = None; + saved.generation = 3; + data.save_state(&saved).unwrap(); + assert!(!data.load_state_for(space, &a, None, 3).unwrap().1, "the same generation"); + assert!(data.load_state_for(space, &a, None, 4).unwrap().1, "the setting was raised since: the state is not trusted"); + assert!(data.load_state_for(space, &a, None, 0).unwrap().1, "and not under an older generation either"); + // The folder is written only when there is one: the personal space's state is as it was. + let personal = SavedState { schema: STATE_SCHEMA, ..SavedState::default() }; + assert!(!serde_json::to_string(&personal).unwrap().contains("folder")); + } + + /// Signing out removes every team space's state and leaves the rest of their folders alone. + #[test] + fn the_states_of_all_team_spaces_can_be_removed() { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path()); + data.remove_space_states().unwrap(); + let (one, two) = (SpaceId::generate(), SpaceId::generate()); + for space in [one, two] { + data.space_dir(space).save_state(&SavedState { schema: STATE_SCHEMA, ..SavedState::default() }).unwrap(); + } + data.space_dir(one).give_approval("set", &[], 1).unwrap(); + data.save_state(&SavedState { schema: STATE_SCHEMA, ..SavedState::default() }).unwrap(); + data.remove_space_states().unwrap(); + for space in [one, two] { + assert!(!data.space_dir(space).path().join(STATE_FILE).exists()); + } + assert!(data.space_dir(one).path().join(APPROVAL_FILE).is_file(), "nothing else goes"); + assert!(data.path().join(STATE_FILE).is_file(), "nor the personal state"); + data.remove_space_states().unwrap(); + } + /// Held deletes an older agent saved are still read, as the link's path, and keep holding: fix round 3 wrote the /// bare path, fix rounds 4 and 5 a map with the link's folder ID and name as well (and `fresh_dirs`, no longer /// kept). The current record (every candidate link) and the links the last finished push saw round-trip. diff --git a/apps/craft-cloud-agent/src/admin.rs b/apps/craft-cloud-agent/src/admin.rs new file mode 100644 index 00000000..f18ae7ae --- /dev/null +++ b/apps/craft-cloud-agent/src/admin.rs @@ -0,0 +1,379 @@ +//! The local API's admin calls (`/v1/admin/…`): the server's admin API (`/api/v1/admin/…`, `craft-cloud-server`'s +//! `admin.rs`) for the Craft Cloud app's admin pages, passed on with the signed-in device's token. +//! +//! - **Nothing is decided here.** The agent checks no role and keeps no confirmation: the acting admin's password +//! travels in the body of each change, from the app through this agent to the server, and is dropped with the +//! request. Bodies are read as the shared types (so the `Debug` output of every one hides the password), at most +//! [`MAX_ADMIN_JSON_BYTES`] long, and written to the server again; a field this agent doesn't know is dropped. +//! Nothing of a body is logged. +//! - **The server's answer is the answer**: the same body and, for a refusal, the same HTTP status, error code and +//! message, and `Retry-After` for a `429`. A `403` (`wrong_password`, `forbidden`, `account_disabled`) stays a `403`, +//! a `409` (`last_admin`) a `409`. The one exception is the server's `401`: for a signed-in route it means this +//! device was revoked, and is told as every other call tells it (`409 needs_you`), not as `401`, which on the local +//! API means a wrong token. +//! - The same `Host`, `Origin` and token checks as every local call (`local_api::guard`) run first. + +use axum::body::Body; +use axum::extract::rejection::{JsonRejection, PathRejection, QueryRejection}; +use axum::extract::{DefaultBodyLimit, Path, Query, State}; +use axum::http::StatusCode; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{ + AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditPage, codes as api_codes, valid_audit_action, +}; +use craft_cloud_proto::local::{MAX_ADMIN_JSON_BYTES, paths}; +use reqwest::Method; +use serde::Deserialize; +use uuid::Uuid; + +use crate::cli::AgentError; +use crate::http::{CallError, MAX_DEVICES}; +use crate::local_api::{Api, LocalError, path_id, server_api}; + +/// Most accounts one page of the user list may hold (the server's own limit; a longer page is refused). +const MAX_PAGE_USERS: usize = 200; + +impl From for LocalError { + /// A refusal keeps the server's status, code and message and its `Retry-After` (never a 401: that is a revoked device, + /// told as every signed-in call tells it); anything else is what the sync paths make of it. + fn from(e: CallError) -> Self { + AgentError::from(e).into() + } +} + +type Answer = Result; + +/// The admin calls, to be merged into the local API's router (behind its `guard`). +pub(crate) fn routes() -> Router { + let small = || DefaultBodyLimit::max(MAX_ADMIN_JSON_BYTES); + let user = format!("{}/{{id}}", paths::ADMIN_USERS); + Router::new() + .route(paths::ACCOUNT, get(account)) + .route(paths::ACCOUNT_ACTIVITY, get(activity)) + .route(paths::ADMIN_AUDIT, get(audit)) + .route(paths::ADMIN_AUDIT_EXPORT, get(audit_export)) + .route(paths::ADMIN_SERVER, get(server)) + .route(paths::ADMIN_STORAGE, get(storage)) + .route(paths::ADMIN_USERS, get(list_users).post(create_user).layer(small())) + .route(&user, axum::routing::patch(update_user).delete(remove_user).layer(small())) + .route(&format!("{user}/usage"), get(user_usage)) + .route(&format!("{user}/disable"), post(disable_user).layer(small())) + .route(&format!("{user}/enable"), post(enable_user).layer(small())) + .route(&format!("{user}/invite"), post(issue_invite).delete(revoke_invite).layer(small())) + .route(&format!("{user}/password-reset"), post(issue_reset).layer(small())) + .route(&format!("{user}/reset-two-factor"), post(reset_two_factor).layer(small())) + .route(&format!("{user}/devices"), get(list_devices)) + .route(&format!("{user}/devices/{{device_id}}"), axum::routing::delete(revoke_device).layer(small())) +} + +async fn server(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/server", &[]).await?)) +} + +async fn storage(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/storage", &[]).await?)) +} + +async fn user_usage(State(state): State, id: Result, PathRejection>) -> Answer> { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get(&format!("admin/users/{id}/usage"), &[]).await?)) +} + +#[derive(Debug, Deserialize)] +struct ListQuery { + cursor: Option, + limit: Option, + query: Option, +} + +async fn list_users(State(state): State, q: Result, QueryRejection>) -> Answer> { + let Query(q) = q.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let (_, transport) = server_api(&state)?; + let limit = q.limit.map(|l| l.to_string()); + let mut pairs: Vec<(&str, &str)> = Vec::new(); + if let Some(cursor) = q.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(("cursor", cursor)); + } + if let Some(limit) = limit.as_deref() { + pairs.push(("limit", limit)); + } + if let Some(query) = q.query.as_deref().filter(|s| !s.is_empty()) { + pairs.push(("query", query)); + } + let list: AdminUserList = transport.admin_get("admin/users", &pairs).await?; + if list.users.len() > MAX_PAGE_USERS { + return Err(AgentError::from(craft_cloud_sync::TransportError::Protocol(format!( + "the server listed {} accounts on a page, more than the {MAX_PAGE_USERS} this agent reads", + list.users.len() + ))) + .into()); + } + Ok(Json(list)) +} + +async fn create_user(State(state): State, body: Result, JsonRejection>) -> Answer> { + let Json(body) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, "admin/users", &body).await?)) +} + +async fn update_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let id = path_id(id)?; + let Json(body) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::PATCH, &format!("admin/users/{id}"), &body).await?)) +} + +/// A change with only the confirmation as its body, answered without one: the server's own status (`204`, or `202`). +async fn confirmed(state: &Api, method: Method, path: String, body: Result, JsonRejection>) -> Answer { + let Json(body) = body?; + let (_, transport) = server_api(state)?; + let status = transport.admin_empty(method, &path, &body).await?; + Ok(StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::NO_CONTENT)) +} + +async fn disable_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/disable", path_id(id)?), body).await +} + +async fn enable_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/enable", path_id(id)?), body).await +} + +async fn remove_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::DELETE, format!("admin/users/{}", path_id(id)?), body).await +} + +async fn revoke_invite( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::DELETE, format!("admin/users/{}/invite", path_id(id)?), body).await +} + +async fn revoke_device( + State(state): State, + ids: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + let (user, device) = path_id(ids)?; + confirmed(&state, Method::DELETE, format!("admin/users/{user}/devices/{device}"), body).await +} + +/// A new invite: shown to the admin once, by the app; the agent keeps nothing of it and says nothing of it in its log. +async fn issue_invite( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let Json(body) = body?; + let path = format!("admin/users/{}/invite", path_id(id)?); + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, &path, &body).await?)) +} + +/// A password reset code, with the confirmation and whether redeeming it also turns the account's two-factor sign-in off +/// ([`AdminPasswordReset::reset_two_factor`]): shown once, kept and logged by nobody. +async fn issue_reset( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let Json(body) = body?; + let path = format!("admin/users/{}/password-reset", path_id(id)?); + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, &path, &body).await?)) +} + +async fn reset_two_factor( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/reset-two-factor", path_id(id)?), body).await +} + +async fn list_devices(State(state): State, id: Result, PathRejection>) -> Answer> { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + let list: AdminDevices = transport.admin_get(&format!("admin/users/{id}/devices"), &[]).await?; + if list.devices.len() > MAX_DEVICES { + return Err(AgentError::from(craft_cloud_sync::TransportError::Protocol(format!( + "the server listed {} devices, more than the {MAX_DEVICES} this agent reads", + list.devices.len() + ))) + .into()); + } + Ok(Json(list)) +} + +// ---- the account and the audit log ---- + +/// The most events a page of the account's own activity holds, and of the admin's audit list (the server's limits). +const ACCOUNT_MAX_PAGE: u32 = 100; +const ADMIN_MAX_PAGE: u32 = 200; +/// The longest cursor passed on, in bytes (the server's are a few dozen). +const MAX_CURSOR_BYTES: usize = 256; + +/// `GET /v1/account`: the signed-in account as the server says it (its role, and whether it has two-factor sign-in). +async fn account(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.account_info().await?)) +} + +/// The query of the audit routes. Unknown parameters are ignored. +#[derive(Debug, Deserialize)] +struct AuditParams { + cursor: Option, + before_ms: Option, + limit: Option, + user: Option, + action: Option, +} + +fn invalid(message: &str) -> LocalError { + LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, message) +} + +/// The query as pairs for the server, after checking what is checked here (the server checks it all again): a cursor is passed +/// on as it came, but bounded and made of the characters of the server's own; `user` is an account ID, `action` one of the +/// closed list or a prefix of it, `before_ms` a time, `limit` 1 to `max_limit`. `admin`: whether `user` and `action` are read +/// (an account's own activity has neither). +fn audit_pairs(params: &AuditParams, max_limit: u32, admin: bool, list: bool) -> Result, LocalError> { + let mut pairs = Vec::new(); + if let Some(cursor) = params.cursor.as_deref().filter(|c| !c.is_empty()) { + if !list || cursor.len() > MAX_CURSOR_BYTES || !cursor.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) { + return Err(invalid("the cursor is not one this server gave")); + } + pairs.push(("cursor", cursor.to_owned())); + } + if let Some(before) = params.before_ms { + if before < 0 { + return Err(invalid("before_ms is a time in milliseconds")); + } + pairs.push(("before_ms", before.to_string())); + } + if let Some(limit) = params.limit { + if !list || !(1..=max_limit).contains(&limit) { + return Err(invalid(&format!("limit is 1 to {max_limit}"))); + } + pairs.push(("limit", limit.to_string())); + } + if admin { + if let Some(user) = params.user.as_deref().filter(|u| !u.is_empty()) { + let id = Uuid::parse_str(user).map_err(|_| invalid("user is an account ID"))?; + pairs.push(("user", id.to_string())); + } + if let Some(action) = params.action.as_deref().filter(|a| !a.is_empty()) { + if !valid_audit_action(action) { + return Err(invalid("action is an event name like auth.login, or a prefix like auth.")); + } + pairs.push(("action", action.to_owned())); + } + } + Ok(pairs) +} + +fn as_refs<'a>(pairs: &'a [(&'static str, String)]) -> Vec<(&'static str, &'a str)> { + pairs.iter().map(|(k, v)| (*k, v.as_str())).collect() +} + +/// `GET /v1/account/activity?cursor&before_ms&limit`: the account's own security events (any account; `user` and `action` are +/// not read). +async fn activity(State(state): State, params: Result, QueryRejection>) -> Answer> { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ACCOUNT_MAX_PAGE, false, true)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("account/activity", &as_refs(&pairs)).await?)) +} + +/// `GET /v1/admin/audit?cursor&before_ms&limit&user&action`: the whole log, filtered; the server refuses a member. +async fn audit(State(state): State, params: Result, QueryRejection>) -> Answer> { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ADMIN_MAX_PAGE, true, true)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/audit", &as_refs(&pairs)).await?)) +} + +/// `GET /v1/admin/audit/export?before_ms&user&action`: the server's NDJSON export passed through as it comes, chunk by chunk: +/// the agent never holds more than the chunk in hand, and a connection that breaks in the middle ends this one without its +/// last chunk, so the client sees a broken stream. The last line (`complete` or `truncated`) is the server's. +async fn audit_export(State(state): State, params: Result, QueryRejection>) -> Answer { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ADMIN_MAX_PAGE, true, false)?; + let (_, transport) = server_api(&state)?; + let answer = transport.admin_export_stream(&as_refs(&pairs)).await?; + let stream = futures_util::stream::unfold(Some(answer.body), |body| async move { + let mut body = body?; + match body.next().await { + Ok(Some(bytes)) => Some((Ok::<_, std::io::Error>(bytes), Some(body))), + Ok(None) => None, + // The error ends the stream with it: the connection breaks, and nothing is made up for the missing end. + Err(e) => Some((Err(std::io::Error::other(e.to_string())), None)), + } + }); + let mut response = Response::new(Body::from_stream(stream)); + let headers = response.headers_mut(); + headers.insert(axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static("application/x-ndjson")); + headers.insert(axum::http::header::CACHE_CONTROL, axum::http::HeaderValue::from_static("no-store")); + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::http::Refusal; + use axum::response::IntoResponse; + use craft_cloud_proto::local::codes; + use craft_cloud_sync::TransportError; + + fn refused(status: u16, code: &str, retry: Option) -> LocalError { + LocalError::from(CallError::Refused(Refusal { status, error: code.into(), message: "m".into(), two_factor: None, retry_after_secs: retry })) + } + + #[test] + fn a_refusal_keeps_its_status_code_and_retry_after() { + let wrong = refused(403, "wrong_password", None); + assert_eq!((wrong.status, wrong.code.as_str()), (StatusCode::FORBIDDEN, "wrong_password")); + let invalid = refused(403, "two_factor_invalid", None); + assert_eq!((invalid.status, invalid.code.as_str()), (StatusCode::FORBIDDEN, "two_factor_invalid")); + let last = refused(409, "last_admin", None); + assert_eq!((last.status, last.code.as_str()), (StatusCode::CONFLICT, "last_admin")); + let locked = refused(429, "two_factor_locked", Some(30)).into_response(); + assert_eq!(locked.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(locked.headers().get(axum::http::header::RETRY_AFTER).and_then(|v| v.to_str().ok()), Some("30")); + } + + /// The server's `401` (this device was revoked) is never passed on as a `401`: on the local API that says the + /// token is wrong. + #[test] + fn a_revoked_device_is_not_a_wrong_token() { + let failure = LocalError::from(CallError::Transport(TransportError::Unauthorized)); + assert_ne!(failure.status, StatusCode::UNAUTHORIZED); + assert_eq!(failure.code, codes::NEEDS_YOU); + } +} diff --git a/apps/craft-cloud-agent/src/cli.rs b/apps/craft-cloud-agent/src/cli.rs index 194d4713..72bce54e 100644 --- a/apps/craft-cloud-agent/src/cli.rs +++ b/apps/craft-cloud-agent/src/cli.rs @@ -33,11 +33,11 @@ use jiff::tz::TimeZone; use crate::account::{ACCOUNT_SCHEMA, Account, AccountError, DataDir, LastPass, LoopState, Outcome, PassSummary, STATE_SCHEMA, SavedState}; use crate::disk::{AGENT_DIR, DiskFs, create_private_dir}; -use crate::http::{self, DeviceToken, HttpTransport, JoinError, ReachError, ServerUrl}; +use crate::http::{self, CallError, DeviceToken, Factor, HttpTransport, JoinError, LoginStep, ReachError, Refusal, ServerUrl}; use crate::lock::{self, DATA_LOCK, FOLDER_LOCK, Lock, LockError}; use crate::root::{self, RootError, RootMarker}; use crate::settle::Stability; -use craft_cloud_proto::api::LoginResponse; +use craft_cloud_proto::api::{LoginResponse, TwoFactorChallenge}; use craft_cloud_proto::join::JoinString; #[derive(Debug, Parser)] @@ -73,6 +73,10 @@ pub enum Command { /// still wait). With `=` (the set the list was printed with): only if that set is the one waiting. #[arg(long, value_name = "SET", num_args = 0..=1, require_equals = true, value_parser = set_name)] approve_deletes: Option>, + /// With `--approve-deletes`: approve the waiting deletions of this team space (its ID or its name) instead of the + /// personal space's. The running agent carries them out. + #[arg(long, value_name = "SPACE", requires = "approve_deletes")] + space: Option, }, /// Keep the folder in sync until stopped (Ctrl-C): sync whenever something changes here or on the server, and /// at least every 10 minutes. Ctrl-C lets a running pass finish; a second Ctrl-C stops at once. Exits 0 at once, @@ -248,7 +252,7 @@ impl AgentError { | AgentError::Lock(LockError::Busy(_) | LockError::NotAFile(_)) | AgentError::Revoked | AgentError::OtherAccountState(_) - | AgentError::Server(TransportError::Protocol(_) | TransportError::QuotaExceeded { .. }) + | AgentError::Server(TransportError::Protocol(_) | TransportError::QuotaExceeded { .. } | TransportError::ReadOnly) | AgentError::Sync(SyncError::Corrupt(_)) | AgentError::Account(AccountError::Newer { .. } | AccountError::Damaged { .. }) | AgentError::NeedsYou(_) => Exit::NeedsYou, @@ -336,6 +340,24 @@ pub enum AgentError { .0.display() )] FolderHasFiles(PathBuf), + /// `login`: the account has two-factor sign-in. The challenge goes back with a code from its authenticator app (or a + /// recovery code) to finish the sign-in. For `join` the challenge is empty: the same join goes again with the code. + #[error("this account has two-factor sign-in: a code from its authenticator app (or a recovery code) is needed")] + TwoFactorRequired(TwoFactorChallenge), + /// The code or recovery code was wrong (the server's message). Never a revoked device. + #[error("{0}")] + TwoFactorInvalid(String), + /// Too many wrong codes: the second factor is locked for a while (`retry_after_secs`, the server's `Retry-After`). + #[error("{message}")] + TwoFactorLocked { message: String, retry_after_secs: Option }, + /// The second step of a sign-in found its challenge gone (it works five minutes and takes five tries, and a password + /// change ends it): the sign-in starts again. + #[error("this sign-in step has run out (it works for five minutes and takes five tries): sign in again")] + ChallengeGone, + /// The server refused a call of the account (a wrong password, a conflict, a lock, a limit): its status, code and + /// message as it said them, and its `Retry-After`. Never a revoked device (that is a 401, [`AgentError::Revoked`]). + #[error("{message}")] + Refused { status: u16, code: String, message: String, retry_after_secs: Option }, /// Something found as `run` started that only a person can put right (the message says what); the agent waits, /// serving the local API, until it is. #[error("{0}")] @@ -352,6 +374,61 @@ impl From for AgentError { } } +impl From for AgentError { + /// For a signed-in call of the account: the server's refusal with its status, code and message; anything else as sync's + /// transport errors are (a 401 is a revoked device, offline is offline). + fn from(e: CallError) -> Self { + match e { + CallError::Refused(Refusal { status, error, message, retry_after_secs, .. }) => { + AgentError::Refused { status, code: error, message, retry_after_secs } + } + CallError::Transport(t) => t.into(), + } + } +} + +/// What a refusal said, as the transport error the sign-in used to see for it (a refusal with a code of its own is handled +/// before this), so that messages and exits of the first step don't change. +fn legacy(refusal: &Refusal) -> TransportError { + if refusal.status == 429 { + TransportError::Unavailable(format!("the server is busy: {}", refusal.message)) + } else { + TransportError::Protocol(format!("the server or a proxy refused the request (HTTP {}): {}", refusal.status, refusal.message)) + } +} + +/// A wait in words, for a lock: "about 14 minutes". +fn wait_in_words(secs: u64) -> String { + match secs { + 0..=90 => format!("{secs} seconds"), + 91..=5_400 => format!("about {} minutes", secs.div_ceil(60)), + _ => format!("about {} hours", secs.div_ceil(3_600)), + } +} + +/// The errors of a sign-in's second step and of a join with a code, from what the server said. +fn factor_error(e: CallError) -> AgentError { + match e { + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_INVALID => AgentError::TwoFactorInvalid(r.message), + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_LOCKED => locked(r.message, r.retry_after_secs), + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_REQUIRED => { + AgentError::TwoFactorRequired(r.two_factor.unwrap_or(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })) + } + // The challenge doesn't work any more: not a revoked device (nothing is signed in yet). + CallError::Transport(TransportError::Unauthorized) => AgentError::ChallengeGone, + CallError::Refused(r) => legacy(&r).into(), + CallError::Transport(t) => t.into(), + } +} + +fn locked(message: String, retry_after_secs: Option) -> AgentError { + let message = match retry_after_secs { + Some(secs) => format!("{message} (try again in {})", wait_in_words(secs)), + None => message, + }; + AgentError::TwoFactorLocked { message, retry_after_secs } +} + impl From for AgentError { fn from(e: SyncError) -> Self { match e { @@ -384,19 +461,45 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re return Err(already_signed_in(&account)); } let password = read_password(input)?; - match login(&data, &args, &password, out).await { + let mut through_agent = false; + let first = match login(&data, &args, &password, out).await { // `run` holds the data folder: it signs in, through its local API. - Err(AgentError::Lock(LockError::Busy(path))) => login_through_agent(&data, &args, password, out).await.map_err(|e| match e { - AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), - other => other, - })?, - other => { - other?; + Err(AgentError::Lock(LockError::Busy(path))) => { + through_agent = true; + login_through_agent(&data, &args, password, out).await.map_err(|e| match e { + AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), + other => other, + }) } + other => other.map(|_| ()), + }; + match first { + // An account with two-factor sign-in: the code comes from standard input, like the password (never from the + // command line), and a wrong one may be typed again twice. + Err(AgentError::TwoFactorRequired(challenge)) => { + let mut tries = 0; + loop { + let factor = read_factor(input, out)?; + let done = if through_agent { + second_step_through_agent(&data, &args, &challenge, &factor, out).await + } else { + login_second_factor(&data, &args, &challenge, &factor, out).await.map(|_| ()) + }; + match done { + Err(AgentError::TwoFactorInvalid(message)) if tries < MAX_CODE_TRIES - 1 => { + tries += 1; + say!(out, "{message}")?; + } + other => break other?, + } + } + } + other => other?, } Ok(Exit::Done) } - Command::Sync { approve_deletes } => Ok(sync(&data, Approve::from_flag(approve_deletes), out).await?.0), + Command::Sync { approve_deletes, space: Some(space) } => approve_team_deletes(&data, &space, &Approve::from_flag(approve_deletes), out), + Command::Sync { approve_deletes, space: None } => Ok(sync(&data, Approve::from_flag(approve_deletes), out).await?.0), Command::Run { .. } => { let (stop, stops) = tokio::sync::mpsc::unbounded_channel(); let signals = tokio::spawn(forward_stop_signals(stop)); @@ -440,11 +543,13 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re return Err(already_signed_in(&account)); } let mut password: Option = None; + let mut factor: Option = None; + let mut wrong_codes = 0; loop { - let joined = match join(&data, &args, password.as_deref(), out).await { + let joined = match join_with(&data, &args, password.as_deref(), factor.as_ref(), out).await { // `run` holds the data folder: it joins, through its local API. Err(AgentError::Lock(LockError::Busy(path))) => { - join_through_agent(&data, &args, password.clone(), out).await.map_err(|e| match e { + join_through_agent(&data, &args, password.clone(), factor.clone(), out).await.map_err(|e| match e { AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), other => other, }) @@ -453,6 +558,13 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re }; match joined { Err(AgentError::PasswordRequired) if password.is_none() => password = Some(new_password(input, out)?), + // The code was not used up: the same join goes again with the second factor. + Err(AgentError::TwoFactorRequired(_)) if factor.is_none() => factor = Some(read_factor(input, out)?), + Err(AgentError::TwoFactorInvalid(message)) if wrong_codes < MAX_CODE_TRIES - 1 => { + wrong_codes += 1; + say!(out, "{message}")?; + factor = Some(read_factor(input, out)?); + } other => break other?, } } @@ -572,8 +684,29 @@ fn new_password(input: &mut impl BufRead, out: &mut impl Write) -> Result Result { + say!(out, "This account has two-factor sign-in. Type the code from your authenticator app (or a recovery code):")?; + out.flush().map_err(io_at(Path::new("standard output")))?; + let mut line = String::new(); + input.take(200).read_line(&mut line).map_err(|e| AgentError::Usage(format!("can't read the code from standard input: {e}")))?; + if line.trim().is_empty() { + return Err(AgentError::Usage("no code on standard input".into())); + } + Ok(Factor::typed(&line)) +} + /// `join` while `run` runs: the running agent joins. -async fn join_through_agent(data: &DataDir, args: &JoinArgs, password: Option, out: &mut impl Write) -> Result<(), AgentError> { +async fn join_through_agent( + data: &DataDir, + args: &JoinArgs, + password: Option, + factor: Option, + out: &mut impl Write, +) -> Result<(), AgentError> { let folder = args.folder.as_ref().map(|f| std::path::absolute(f).map_err(io_at(f))).transpose()?; let request = craft_cloud_app::JoinRequest { join: args.join.clone(), @@ -581,6 +714,8 @@ async fn join_through_agent(data: &DataDir, args: &JoinArgs, password: Option(f: impl FnOnce() -> Result AgentError { use craft_cloud_proto::local::codes; + // A refusal that carries the challenge or the wait of a lock. + if let AgentError::Local(inner @ craft_cloud_app::Error::RefusedWith { .. }) = &e { + let message = inner.to_string(); + return match inner.code() { + Some(codes::TWO_FACTOR_REQUIRED) => { + AgentError::TwoFactorRequired(inner.challenge().cloned().unwrap_or(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })) + } + Some(codes::TWO_FACTOR_LOCKED) => locked(message, inner.retry_after_secs()), + _ => AgentError::Usage(message), + }; + } match e { - AgentError::Local(craft_cloud_app::Error::Refused { code, message }) => match code.as_str() { + AgentError::Local(craft_cloud_app::Error::Refused { code, message, .. }) => match code.as_str() { + codes::TWO_FACTOR_INVALID => AgentError::TwoFactorInvalid(message), + codes::TWO_FACTOR_EXPIRED => AgentError::ChallengeGone, + codes::TWO_FACTOR_REQUIRED => AgentError::TwoFactorRequired(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 }), codes::WRONG_PASSWORD => AgentError::WrongPassword, codes::NOT_SIGNED_IN => AgentError::NotSignedIn, codes::OFFLINE => AgentError::Offline(message), @@ -749,6 +898,36 @@ async fn login_through_agent(data: &DataDir, args: &LoginArgs, password: String, ) } +/// The second step of `login` while `run` runs: the running agent finishes the sign-in. +async fn second_step_through_agent( + data: &DataDir, + args: &LoginArgs, + challenge: &TwoFactorChallenge, + factor: &Factor, + out: &mut impl Write, +) -> Result<(), AgentError> { + let folder = args.folder.as_ref().map(|f| std::path::absolute(f).map_err(io_at(f))).transpose()?; + let request = craft_cloud_app::SecondFactorLoginRequest { + server: args.url.clone(), + user: args.user.clone(), + folder: folder.map(|f| f.display().to_string()), + device_name: args.device_name.clone(), + adopt: args.adopt, + challenge: challenge.challenge.clone(), + totp_code: factor.totp_code.clone(), + recovery_code: factor.recovery_code.clone(), + }; + let data_path = data.path().to_owned(); + let status = blocking(move || craft_cloud_app::Client::connect_in(&data_path)?.login_second_factor(&request)).await.map_err(local_error)?; + say!( + out, + "signed in as {} on {} (through the running agent, which syncs {} now)", + status.account.unwrap_or_default(), + status.server.unwrap_or_default(), + status.folder.unwrap_or_default() + ) +} + /// `recents [add ] [--watch]`: a client of the running agent like an app. async fn recents(data: &DataDir, args: RecentsArgs, out: &mut impl Write) -> Result { let data_path = data.path().to_owned(); @@ -986,7 +1165,7 @@ fn hold_folder(folder: &Path) -> Result { } /// Whether `folder` holds anything besides the agent's own folder. -fn has_files(folder: &Path) -> Result { +pub(crate) fn has_files(folder: &Path) -> Result { for entry in std::fs::read_dir(folder).map_err(io_at(folder))? { let entry = entry.map_err(io_at(folder))?; if entry.file_name() != AGENT_DIR { @@ -1023,7 +1202,7 @@ fn resolved(path: &Path) -> PathBuf { /// Whether one of the two paths lies inside the other (or they are the same), as the OS resolves them. Names are /// compared ignoring case on Windows and macOS, whose filesystems usually do. -fn nested(a: &Path, b: &Path) -> bool { +pub(crate) fn nested(a: &Path, b: &Path) -> bool { let (a, b) = (folded(&resolved(a)), folded(&resolved(b))); a.starts_with(&b) || b.starts_with(&a) } @@ -1060,7 +1239,7 @@ fn check_private_place(data: &DataDir, local_app_data: Option<&Path>) -> Result< /// Refuses a synced folder and a data folder that lie inside one another: the data folder holds the device token, /// which would then be synced (or the synced folder would sync the agent's own state). -fn check_apart(data: &DataDir, folder: &Path) -> Result<(), AgentError> { +pub(crate) fn check_apart(data: &DataDir, folder: &Path) -> Result<(), AgentError> { if nested(data.path(), folder) { return Err(AgentError::DataInFolder { data: resolved(data.path()), folder: resolved(folder) }); } @@ -1095,20 +1274,65 @@ pub(crate) async fn login_locked(data: &DataDir, args: &LoginArgs, password: &st let server = ServerUrl::parse(&args.url).map_err(|e| AgentError::Usage(e.to_string()))?; let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; ask_server(&server, |_, message| AgentError::Offline(message)).await?; - let signed_in = match http::login(&server, &args.user, password, &prepared.device_name).await { - Err(TransportError::Unauthorized) => return Err(AgentError::WrongPassword), - other => other?, + let signed_in = match http::login_step(&server, &args.user, password, &prepared.device_name).await { + Ok(LoginStep::Device(signed_in)) => signed_in, + // An account with two-factor sign-in: no device yet. The caller asks for the code and finishes with + // [`login_second_factor_locked`], which needs nothing of this call but its arguments. + Ok(LoginStep::SecondFactor(challenge)) => return Err(AgentError::TwoFactorRequired(challenge)), + Err(CallError::Transport(TransportError::Unauthorized)) => return Err(AgentError::WrongPassword), + Err(CallError::Refused(refusal)) => return Err(legacy(&refusal).into()), + Err(CallError::Transport(e)) => return Err(e.into()), }; set_up(data, &server, signed_in, prepared).await } +/// The second step of a sign-in: the folder's checks again (the agent keeps nothing between the steps), then the challenge and +/// the second factor make the device, and the folder is set up as after [`login`]. Prints nothing. +pub(crate) async fn login_second_factor_locked(data: &DataDir, args: &LoginArgs, challenge: &str, factor: &Factor) -> Result { + if let Some(account) = data.load_account()? { + return Err(already_signed_in(&account)); + } + let server = ServerUrl::parse(&args.url).map_err(|e| AgentError::Usage(e.to_string()))?; + let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; + ask_server(&server, |_, message| AgentError::Offline(message)).await?; + let signed_in = http::second_factor(&server, challenge, factor).await.map_err(factor_error)?; + set_up(data, &server, signed_in, prepared).await +} + +/// [`login_second_factor_locked`] for the command line: holds the data folder and says it signed in. +pub async fn login_second_factor( + data: &DataDir, + args: &LoginArgs, + challenge: &TwoFactorChallenge, + factor: &Factor, + out: &mut impl Write, +) -> Result { + let _data_lock = hold_data(data)?; + let account = login_second_factor_locked(data, args, &challenge.challenge, factor).await?; + say!(out, "signed in as {} on {} (device \"{}\")", account.username, account.server, account.device_name)?; + say!(out, "syncing {}; run `craft-cloud-agent sync` to sync it", account.folder.display())?; + Ok(account) +} + /// Joins with a join string (an invite or an add-a-computer code), creates a device and sets up the folder, as /// [`login`] does. Everything that can be checked here is checked before the code is spent (the folder, the device /// name, what the server speaks); a failure after it (the folder is set up for another space) removes the new device /// again, and says the code is used up. `password`: the new account's, for an invite that needs one. pub async fn join(data: &DataDir, args: &JoinArgs, password: Option<&str>, out: &mut impl Write) -> Result { + join_with(data, args, password, None, out).await +} + +/// [`join`] with the second factor of an account that has two-factor sign-in (without one, such an account answers +/// [`AgentError::TwoFactorRequired`] and the code is not used up). +pub async fn join_with( + data: &DataDir, + args: &JoinArgs, + password: Option<&str>, + factor: Option<&Factor>, + out: &mut impl Write, +) -> Result { let _data_lock = hold_data(data)?; - let account = join_locked(data, args, password).await?; + let account = join_locked(data, args, password, factor).await?; say!(out, "joined as {} on {} (device \"{}\")", account.username, account.server, account.device_name)?; say!(out, "syncing {}; run `craft-cloud-agent run` to keep it in sync", account.folder.display())?; Ok(account) @@ -1116,7 +1340,7 @@ pub async fn join(data: &DataDir, args: &JoinArgs, password: Option<&str>, out: /// [`join`] for a caller that holds the data folder's lock already: the `run` loop, joining for the local API. /// Prints nothing. -pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Option<&str>) -> Result { +pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Option<&str>, factor: Option<&Factor>) -> Result { if let Some(account) = data.load_account()? { return Err(already_signed_in(&account)); } @@ -1125,8 +1349,13 @@ pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Optio let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; let unreachable = |server: &ServerUrl, message| AgentError::Unreachable { server: server.to_string(), message }; ask_server(&server, unreachable).await?; - let signed_in = match http::join(&server, join.code.as_str(), &prepared.device_name, password).await { + let signed_in = match http::join_with_factor(&server, join.code.as_str(), &prepared.device_name, password, factor).await { Ok(signed_in) => signed_in, + Err(JoinError::TwoFactorRequired) => { + return Err(AgentError::TwoFactorRequired(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })); + } + Err(JoinError::TwoFactorInvalid(message)) => return Err(AgentError::TwoFactorInvalid(message)), + Err(JoinError::TwoFactorLocked { message, retry_after_secs }) => return Err(locked(message, retry_after_secs)), Err(JoinError::InvalidCode) => return Err(AgentError::InvalidCode), Err(JoinError::PasswordRequired) => return Err(AgentError::PasswordRequired), Err(JoinError::WeakPassword(message)) => return Err(AgentError::WeakPassword(message)), @@ -1206,6 +1435,7 @@ async fn set_up(data: &DataDir, server: &ServerUrl, signed_in: LoginResponse, pr space: marker.space, root: marker.root, token, + spaces: Vec::new(), }; data.remove_state()?; data.save_account(&account)?; @@ -1339,6 +1569,12 @@ fn approve_for_loop(data: &DataDir, approve: &Approve, out: &mut impl Write) -> if !data.load_status()?.is_some_and(|s| s.running(now_ms())) { return Ok(None); } + approve_waiting(data, approve, out).map(|(exit, summary, _)| Some((exit, summary))) +} + +/// The approval itself, for the state in `data` (the data folder, or a team space's folder in it): the rules of +/// [`approve_for_loop`] without asking whether a loop runs. The `bool` is whether an approval was handed over. +pub(crate) fn approve_waiting(data: &DataDir, approve: &Approve, out: &mut impl Write) -> Result<(Exit, PassSummary, bool), AgentError> { let waiting = data.load_state()?.last_pass.and_then(|l| l.summary).map(|s| s.pending_only()).unwrap_or_default(); // The set of deletes held behind a link goes only when named. if let Approve::Set(named) = approve @@ -1353,7 +1589,7 @@ fn approve_for_loop(data: &DataDir, approve: &Approve, out: &mut impl Write) -> for (path, links) in &waiting.held_deletes { say!(out, " delete on the server: {} (held while any of {} is a link)", path, links.join(", "))?; } - return Ok(Some((Exit::Done, waiting))); + return Ok((Exit::Done, waiting, true)); } // The bare flag never releases a hold: say how to, rather than "nothing was waiting". let held = waiting.held_set().filter(|_| *approve == Approve::Listed); @@ -1364,22 +1600,22 @@ fn approve_for_loop(data: &DataDir, approve: &Approve, out: &mut impl Write) -> if let Approve::Set(named) = approve && waiting.approvable() { - return wrong_set(data, named, &waiting, out).map(Some); + return wrong_set(data, named, &waiting, out).map(|(exit, summary)| (exit, summary, false)); } if held.is_none() { say!(out, "nothing was waiting for approval")?; } - return Ok(Some((Exit::Done, waiting))); + return Ok((Exit::Done, waiting, false)); }; if let Approve::Set(named) = approve { if *named != set { - return wrong_set(data, named, &waiting, out).map(Some); + return wrong_set(data, named, &waiting, out).map(|(exit, summary)| (exit, summary, false)); } } else if data.load_shown()?.is_none_or(|shown| shown.set != set) { print_waiting(out, &waiting)?; data.save_shown(&set, now_ms())?; say!(out, "nothing approved: these deletions changed since `status` listed them (or it didn't); check them, then run this again")?; - return Ok(Some((Exit::NeedsApproval, waiting))); + return Ok((Exit::NeedsApproval, waiting, false)); } data.give_approval(&set, &waiting.pending_nodes, now_ms())?; say!(out, "the running agent will carry out these deletions (set {set}):")?; @@ -1389,19 +1625,82 @@ fn approve_for_loop(data: &DataDir, approve: &Approve, out: &mut impl Write) -> for path in &waiting.pending_local_deletes { say!(out, " delete here: {path}")?; } - Ok(Some((Exit::Done, waiting))) + Ok((Exit::Done, waiting, true)) +} + +/// `sync --approve-deletes --space `: the waiting deletions of a team space go to the running agent, by the +/// rules of [`approve_for_loop`] (the space's own `state.json`, `shown.json` and `approval.json` in `spaces//`). A team +/// space syncs only while `run` runs, so without a running agent there is nobody to hand the approval to. +fn approve_team_deletes(data: &DataDir, space: &str, approve: &Approve, out: &mut impl Write) -> Result { + let account = data.load_account()?.ok_or(AgentError::NotSignedIn)?; + let setting = find_team_space(&account.spaces, space)?; + if !data.load_status()?.is_some_and(|s| s.running(now_ms())) { + return Err(AgentError::Usage( + "team spaces sync only while `craft-cloud-agent run` is running; there is no running agent to approve for".into(), + )); + } + approve_team(data, setting, approve, out).map(|(exit, _, _)| exit) +} + +/// The team space the user means by its ID or its name (as the server last called it, ignoring case). +fn find_team_space<'a>(spaces: &'a [crate::account::SpaceSetting], text: &str) -> Result<&'a crate::account::SpaceSetting, AgentError> { + let text = text.trim(); + if let Ok(id) = text.parse::() { + return spaces.iter().find(|s| s.space == id).ok_or_else(|| AgentError::Usage(format!("no team space {id} on this computer"))); + } + let mut named = spaces.iter().filter(|s| s.name.as_deref().is_some_and(|n| n.to_lowercase() == text.to_lowercase())); + match (named.next(), named.next()) { + (Some(only), None) => Ok(only), + (Some(_), Some(_)) => Err(AgentError::Usage("several team spaces have that name; give the space's ID".into())), + (None, _) => Err(AgentError::Usage("no team space of that name or ID on this computer".into())), + } +} + +/// The approval of a team space's waiting deletions (see [`approve_waiting`]), for the state saved for its folder: a state +/// made for another folder has nothing waiting. The `bool` is whether an approval was handed over. +pub(crate) fn approve_team( + data: &DataDir, + setting: &crate::account::SpaceSetting, + approve: &Approve, + out: &mut impl Write, +) -> Result<(Exit, PassSummary, bool), AgentError> { + let state = data.space_dir(setting.space); + let (saved, set_aside) = state.load_state_for(setting.space, &setting.folder, None, setting.generation)?; + if set_aside || saved.state.space.is_none() { + say!(out, "nothing was waiting for approval")?; + return Ok((Exit::Done, PassSummary::default(), false)); + } + approve_waiting(&state, approve, out) } /// A signed-in folder ready for passes: its account, the engine on the real disk and the HTTP API, and the folder's /// lock. The caller holds the data folder's lock for as long as the session lives. One-off `sync` opens one for one /// pass; the `run` loop keeps one for many (so the stability window's looks carry over between passes). pub struct Session { + /// Where `state.json` is: the data folder, or a team space's own folder in it (`spaces//`). data: DataDir, + /// The agent's data folder (the synced folder must lie apart from it). + root: DataDir, + /// The space is a team space: the transport is bound to it, and the server must speak team spaces. + bound: bool, account: Account, engine: Engine, /// The last pass's outcome, also written with every checkpoint (so it is shared with the checkpoint closure). last_pass: std::sync::Arc>>, _folder_lock: Lock, + /// A team space's folder as `state.json` records it (canonical); `None` for the personal space. + state_folder: Option, + /// The ID of that folder (see [`SavedState::folder_id`]). + state_folder_id: Option, + /// The folder the session was opened on (a team space's), to notice a link that leads elsewhere later. + opened_on: Option, + /// Set when the folder was found to lead elsewhere during a pass: the engine's state took in part of that other folder. + poisoned: std::sync::Arc, + /// The server's stamp of the account's membership of a team space that `state.json` records (see + /// [`SavedState::joined_ms`]); `None` for the personal space. + joined_ms: Option, + /// The generation of the space's setting that `state.json` is made under (see [`crate::account::SpaceSetting::generation`]). + generation: u64, /// The stability window's looks, shared with the `run` loop (`None`: a one-off `sync` trusts every file). stability: Option, /// When the earliest look at a file still changing is due, after the last pass (if it finished). @@ -1420,10 +1719,55 @@ impl Session { data.clean_tmp()?; } let account = data.load_account()?.ok_or(AgentError::NotSignedIn)?; - check_apart(data, &account.folder)?; + Self::open_in(data, data, account, false, (None, 0), stability) + } + + /// Opens a team space's folder for passes: as [`Session::open`], with the space's own state in `spaces//`, its + /// root marker, and a transport bound to the space (every request names it, every answer must echo it). `account` + /// is the signed-in account; the space's folder and root are `setting`'s. `joined_ms` is the server's current stamp of + /// the account's membership, which a saved state must have been made under. + pub fn open_team( + data: &DataDir, + account: &Account, + setting: &crate::account::SpaceSetting, + joined_ms: Option, + stability: Option, + ) -> Result { + let root = setting + .root + .ok_or_else(|| AgentError::NeedsYou("the server hasn't said where this team space starts yet; it is asked again soon".into()))?; + let state = data.space_dir(setting.space); + state.create()?; + state.clean_tmp()?; + let team = Account { folder: setting.folder.clone(), space: setting.space, root, spaces: Vec::new(), ..account.clone() }; + Self::open_in(data, &state, team, true, (joined_ms, setting.generation), stability) + } + + fn open_in( + root_data: &DataDir, + data: &DataDir, + account: Account, + bound: bool, + (joined_ms, generation): (Option, u64), + stability: Option, + ) -> Result { + check_apart(root_data, &account.folder)?; root::check(&account.folder, &RootMarker { space: account.space, root: account.root })?; let folder_lock = hold_folder(&account.folder)?; - let mut saved = data.load_state()?; + // A team space's state is used only for the folder it was made for; otherwise the folder starts afresh (a first sync: + // what is there and what the server has are merged, and nothing is deleted on either side). + // One look at the folder decides both which state is used and what the session later compares the folder with. + let opened_on = bound.then(|| crate::disk::folder_identity(&account.folder)).flatten(); + let (mut saved, set_aside) = + if bound { data.load_state_with(account.space, opened_on.as_ref(), joined_ms, generation)? } else { (data.load_state()?, false) }; + if set_aside { + crate::runlog::note(format_args!( + "the saved state of team space {} was made for another folder, membership or generation: starting afresh, nothing is deleted", + account.space + )); + } + let state_folder = opened_on.as_ref().map(|i| i.path.clone()); + let state_folder_id = opened_on.as_ref().and_then(|i| i.id); if saved.state.space.is_some_and(|s| s != account.space) { return Err(AgentError::OtherAccountState(data.path().join(crate::account::STATE_FILE))); } @@ -1436,26 +1780,98 @@ impl Session { } fs.clean_tmp().map_err(io_at(&fs.tmp_dir()))?; let transport = HttpTransport::new(account.server.clone(), &account.token)?; + let transport = if bound { transport.bound_to(account.space)? } else { transport }; let config = Config::new(account.device_name.clone(), account.device); let mut engine = Engine::new(fs, transport, std::mem::take(&mut saved.state), config, conflict_stamp)?; // What Craft apps saved from live documents (their local API's `saved`): see `live::saves`. - engine.set_save_records(std::sync::Arc::new(crate::live::saves::SaveBook::new(data.path(), account.space))); + engine.set_save_records(std::sync::Arc::new(crate::live::saves::SaveBook::new(root_data.path(), account.space))); // `state.json` is also written at the pass's checkpoints, so a crash or a kill loses at most what the pass did // since the last one (the last pass's outcome stays as it was until this one ends). let last_pass = std::sync::Arc::new(std::sync::Mutex::new(saved.last_pass)); - let (checkpoint_data, checkpoint_last) = (data.clone(), last_pass.clone()); + let (checkpoint_data, checkpoint_last, checkpoint_folder) = (data.clone(), last_pass.clone(), state_folder.clone()); + // A team folder that was pointed at another folder during a pass is noticed before the engine saves its state and before + // every commit: the pass stops without sending or saving anything of what it saw there. What the pass has taken in + // since was about the other folder, so the session is finished with even if the link is pointed back (see + // `Session::folder_moved`). + let poisoned = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let still_there: std::sync::Arc std::io::Result<()> + Send + Sync> = { + let (folder, identity, poisoned) = (account.folder.clone(), opened_on.clone(), poisoned.clone()); + std::sync::Arc::new(move || { + if let Some(then) = &identity + && let Some(now) = crate::disk::folder_identity(&folder) + && *then != now + { + poisoned.store(true, std::sync::atomic::Ordering::SeqCst); + return Err(std::io::Error::other("the folder leads somewhere else than when the session opened")); + } + Ok(()) + }) + }; + let guard_check = still_there.clone(); + engine.on_guard(move || guard_check()); + // The folder's nonce is rewritten at every checkpoint and at the end of a pass, and recorded in the state saved with it. + let checkpoint_folder_path = account.folder.clone(); engine.on_checkpoint(move |state| { + still_there()?; + let folder_nonce = if bound { Some(root::rotate_nonce(&checkpoint_folder_path)?) } else { None }; let last_pass = checkpoint_last.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); - let saved = SavedState { schema: STATE_SCHEMA, state: state.clone(), last_pass, paused: false }; + let saved = SavedState { + schema: STATE_SCHEMA, + state: state.clone(), + last_pass, + paused: false, + folder: checkpoint_folder.clone(), + folder_id: state_folder_id, + folder_nonce, + joined_ms, + generation, + }; checkpoint_data.save_state(&saved).map_err(|e| std::io::Error::other(e.to_string())) }); - Ok(Self { data: data.clone(), account, engine, last_pass, _folder_lock: folder_lock, stability, recheck_in: None }) + Ok(Self { + data: data.clone(), + root: root_data.clone(), + bound, + account, + engine, + last_pass, + _folder_lock: folder_lock, + opened_on, + poisoned, + state_folder, + state_folder_id, + joined_ms, + generation, + stability, + recheck_in: None, + }) + } + + /// The server's stamp of the account's membership that this session's state was opened under (team spaces only). + pub fn joined_ms(&self) -> Option { + self.joined_ms + } + + /// A team space's folder now leads to another folder than the one the session was opened on (a link was pointed + /// elsewhere, or the folder was replaced): what the session remembers is not about this folder. A folder that can't be + /// looked at is not "moved" (the pass says what is wrong with it). + pub fn folder_moved(&self) -> bool { + if self.poisoned.load(std::sync::atomic::Ordering::SeqCst) { + return true; + } + let now = self.bound.then(|| crate::disk::folder_identity(&self.account.folder)).flatten(); + matches!((&self.opened_on, now), (Some(then), Some(now)) if *then != now) } pub fn account(&self) -> &Account { &self.account } + /// Tells the engine the account is a viewer here (or not): see `Engine::set_read_only`. + pub fn set_read_only(&mut self, read_only: bool) { + self.engine.set_read_only(read_only); + } + /// The engine's state (its `seq` is where the change feed was read to). pub fn state(&self) -> &craft_cloud_sync::State { self.engine.state() @@ -1479,7 +1895,7 @@ impl Session { /// saved, whether the pass finished or not. The folder and its marker are checked first: the folder may have /// gone since the session opened (a drive unmounted while the loop runs). pub async fn pass(&mut self, approve: Vec) -> Result { - check_apart(&self.data, &self.account.folder)?; + check_apart(&self.root, &self.account.folder)?; root::check(&self.account.folder, &RootMarker { space: self.account.space, root: self.account.root })?; if !approve.is_empty() { self.engine.approve_deletes_of(approve); @@ -1488,7 +1904,15 @@ impl Session { // The server is asked what it speaks before every pass (it may have been updated since the last one); a // mismatch fails the pass like any other server answer, so it is saved and shown by `status`. let compatible = match http::info(&self.account.server).await { - Ok(info) => http::check_compatible(&info), + Ok(info) => http::check_compatible(&info).and_then(|()| { + // A team space is only ever asked of a server that does team spaces: no `craft-space` header goes to an + // older one. + if self.bound && info.protocol_max < crate::spaces::TEAM_PROTOCOL { + Err(TransportError::Protocol(crate::spaces::NEEDS_UPDATE.into())) + } else { + Ok(()) + } + }), Err(e) => Err(e), }; let started = self.stability.as_ref().map(Stability::now); @@ -1521,9 +1945,31 @@ impl Session { (Outcome::Failed(failure_message(e)), previous.map(|s| s.pending_only()).filter(|s| s.needs_approval() || !s.held_deletes.is_empty())) } }; + // A folder that leads elsewhere than at the start (also one that was pointed back since): what the engine holds may + // be about the other folder, so nothing is saved. The pass thread drops the session and removes the state. + if self.folder_moved() { + self.poisoned.store(true, std::sync::atomic::Ordering::SeqCst); + result?; + return Err(AgentError::Usage("the folder of this space leads somewhere else now; the next pass starts afresh there".into())); + } let last = LastPass { finished_ms: now_ms(), outcome, summary: summary.clone() }; *self.last_pass.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(last.clone()); - let saved = SavedState { schema: STATE_SCHEMA, state: self.engine.state().clone(), last_pass: Some(last), paused: false }; + let folder_nonce = if self.bound { + Some(root::rotate_nonce(&self.account.folder).map_err(|source| AgentError::Io { path: self.account.folder.clone(), source })?) + } else { + None + }; + let saved = SavedState { + schema: STATE_SCHEMA, + state: self.engine.state().clone(), + last_pass: Some(last), + paused: false, + folder: self.state_folder.clone(), + folder_id: self.state_folder_id, + folder_nonce, + joined_ms: self.joined_ms, + generation: self.generation, + }; self.data.save_state(&saved)?; result?; Ok(summary.unwrap_or_default()) @@ -1568,6 +2014,15 @@ pub(crate) fn print_summary(out: &mut impl Write, s: &PassSummary) -> Result<(), } else { say!(out, "synced: {}", done.join(", "))?; } + if s.read_only { + say!(out, "this space is read-only for you: nothing is uploaded, and what you change here stays on this computer")?; + } + if !s.restored.is_empty() { + say!(out, "deleted here, downloaded again (the space is read-only for you):")?; + for path in &s.restored { + say!(out, " {path}")?; + } + } if !s.conflict_copies.is_empty() { say!(out, "conflict copies (both versions kept):")?; for path in &s.conflict_copies { @@ -1761,6 +2216,8 @@ pub(crate) async fn revoke_device(account: &Account) -> Result<(), AgentError> { pub(crate) fn forget_account(data: &DataDir) -> Result<(), AgentError> { data.remove_account()?; data.remove_state()?; + // The team spaces' states too: one kept would be reused if the same folder is chosen after a later sign-in. + data.remove_space_states()?; Ok(()) } @@ -1823,24 +2280,48 @@ mod tests { assert_eq!(serde_json::from_value::(serde_json::to_value(&both).unwrap()).unwrap(), both, "round trip"); } + #[test] + fn a_team_space_is_found_by_its_id_or_its_name() { + use crate::account::SpaceSetting; + let setting = |name: &str| SpaceSetting { + space: craft_cloud_proto::SpaceId::generate(), + folder: PathBuf::from("/x"), + sync: true, + name: Some(name.into()), + root: None, + generation: 0, + }; + let (studio, plans, twin) = (setting("Studio"), setting("Plans"), setting("plans")); + let spaces = vec![studio.clone(), plans.clone()]; + assert_eq!(find_team_space(&spaces, &studio.space.to_string()).unwrap().space, studio.space); + assert_eq!(find_team_space(&spaces, " STUDIO ").unwrap().space, studio.space, "a name, whatever the case"); + assert!(matches!(find_team_space(&spaces, "Nothing"), Err(AgentError::Usage(_)))); + assert!(matches!(find_team_space(&spaces, &craft_cloud_proto::SpaceId::generate().to_string()), Err(AgentError::Usage(_)))); + let spaces = vec![studio, plans, twin]; + assert!(matches!(find_team_space(&spaces, "Plans"), Err(AgentError::Usage(m)) if m.contains("ID")), "two of that name: the ID is needed"); + } + #[test] fn the_cli_parses() { let cli = Cli::try_parse_from(["craft-cloud-agent", "login", "https://c.example", "--user", "ana", "--adopt", "--data-dir", "d"]).unwrap(); assert_eq!(cli.data_dir.as_deref(), Some(Path::new("d"))); assert!(matches!(cli.command, Command::Login(LoginArgs { adopt: true, ref user, .. }) if user == "ana")); let cli = Cli::try_parse_from(["craft-cloud-agent", "sync", "--approve-deletes"]).unwrap(); - let Command::Sync { approve_deletes } = cli.command else { panic!("{:?}", cli.command) }; + let Command::Sync { approve_deletes, .. } = cli.command else { panic!("{:?}", cli.command) }; assert_eq!(Approve::from_flag(approve_deletes), Approve::Listed); let cli = Cli::try_parse_from(["craft-cloud-agent", "sync", "--approve-deletes=3F9A0b1c2d3e"]).unwrap(); - let Command::Sync { approve_deletes } = cli.command else { panic!("{:?}", cli.command) }; + let Command::Sync { approve_deletes, .. } = cli.command else { panic!("{:?}", cli.command) }; assert_eq!(Approve::from_flag(approve_deletes), Approve::Set("3f9a0b1c2d3e".into())); let cli = Cli::try_parse_from(["craft-cloud-agent", "sync"]).unwrap(); - let Command::Sync { approve_deletes } = cli.command else { panic!("{:?}", cli.command) }; + let Command::Sync { approve_deletes, .. } = cli.command else { panic!("{:?}", cli.command) }; assert_eq!(Approve::from_flag(approve_deletes), Approve::Nothing); for empty in ["--approve-deletes=", "--approve-deletes= "] { let e = Cli::try_parse_from(["craft-cloud-agent", "sync", empty]).unwrap_err(); assert_eq!(e.kind(), clap::error::ErrorKind::ValueValidation, "{empty:?} is a usage error: {e}"); } + let cli = Cli::try_parse_from(["craft-cloud-agent", "sync", "--approve-deletes", "--space", "Studio"]).unwrap(); + assert!(matches!(cli.command, Command::Sync { space: Some(ref space), .. } if space == "Studio")); + assert!(Cli::try_parse_from(["craft-cloud-agent", "sync", "--space", "Studio"]).is_err(), "--space needs --approve-deletes"); assert!(Cli::try_parse_from(["craft-cloud-agent", "login", "https://c.example"]).is_err(), "--user is required"); } @@ -1920,6 +2401,9 @@ mod tests { (SyncError::Transport(TransportError::Protocol("a refusal naming no operation".into())), Exit::NeedsYou), // Only with no room left at all: a file that doesn't fit while others still may is skipped instead. (SyncError::Transport(TransportError::QuotaExceeded { used: 10, limit: 10 }), Exit::NeedsYou), + // The engine keeps a read-only space read-only by itself; one that gets this far (another caller of the + // transport) is a state the user is told about, not retried. + (SyncError::Transport(TransportError::ReadOnly), Exit::NeedsYou), (SyncError::Io { path: "an interrupted download".into(), source: std::io::Error::other("x") }, Exit::Failed), (SyncError::Corrupt("chunk x of a.txt is damaged".into()), Exit::NeedsYou), (SyncError::Checkpoint(std::io::Error::other("disk full")), Exit::Failed), @@ -1935,7 +2419,8 @@ mod tests { | TransportError::Unauthorized | TransportError::Unavailable(_) | TransportError::Protocol(_) - | TransportError::QuotaExceeded { .. }, + | TransportError::QuotaExceeded { .. } + | TransportError::ReadOnly, ) | SyncError::Io { .. } | SyncError::Corrupt(_) diff --git a/apps/craft-cloud-agent/src/daemon.rs b/apps/craft-cloud-agent/src/daemon.rs index 0841eea7..5152ba4a 100644 --- a/apps/craft-cloud-agent/src/daemon.rs +++ b/apps/craft-cloud-agent/src/daemon.rs @@ -47,6 +47,7 @@ use crate::local_api::{Command, Control, LocalApi}; use crate::lock::Lock; use crate::schedule::{Ms, Next, PassEnd, Phase, Recheck, Scheduler, Timing}; use crate::settle::Stability; +use crate::spaces::{PassGate, Spaces, SpacesConfig}; use crate::watch::{FolderWatcher, WatchEvent}; /// The loop's settings. [`RunOptions::default`] is what `craft-cloud-agent run` uses; tests make the times short. @@ -127,7 +128,7 @@ struct PassDone { /// How a pass that ended with `error` is scheduled: a pause for what needs the user (rechecked by itself where that /// is safe and may help; see [`Recheck`]), a retry with backoff for the rest. A finished pass that left files still /// changing runs again when the disk says the earliest look at them is due (`recheck_in`). -fn pass_end(result: &Result, recheck_in: Option) -> PassEnd { +pub(crate) fn pass_end(result: &Result, recheck_in: Option) -> PassEnd { match result { // Deletes held behind a link can be approved too, so the loop looks for an approval while any is held (more // slowly: nothing else waits for it). @@ -140,13 +141,17 @@ fn pass_end(result: &Result, recheck_in: Option Recheck { match e { AgentError::Root(_) | AgentError::DataInFolder { .. } | AgentError::Lock(_) => Recheck::Soon, - AgentError::Server(craft_cloud_sync::TransportError::Protocol(_) | craft_cloud_sync::TransportError::QuotaExceeded { .. }) => Recheck::Slowly, + AgentError::Server( + craft_cloud_sync::TransportError::Protocol(_) + | craft_cloud_sync::TransportError::QuotaExceeded { .. } + | craft_cloud_sync::TransportError::ReadOnly, + ) => Recheck::Slowly, _ => Recheck::Never, } } @@ -154,7 +159,13 @@ fn recheck_for(e: &AgentError) -> Recheck { /// The pass thread: opens the folder when a pass is asked for (again after a stop that needs the user, so a folder /// that came back is opened afresh), runs the pass on its own current-thread runtime and reports it. Ends when the /// loop drops its end of `requests`, releasing the folder's lock. -fn pass_thread(data: DataDir, stability: Stability, requests: std::sync::mpsc::Receiver>, events: mpsc::UnboundedSender) { +fn pass_thread( + data: DataDir, + stability: Stability, + gate: PassGate, + requests: std::sync::mpsc::Receiver>, + events: mpsc::UnboundedSender, +) { let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { Ok(runtime) => Some(runtime), Err(e) => { @@ -170,6 +181,8 @@ fn pass_thread(data: DataDir, stability: Stability, requests: std::sync::mpsc::R let Some(runtime) = runtime else { return }; let mut session: Option = None; while let Ok(approve) = requests.recv() { + // The team spaces' passes and this one take turns. + let _turn = gate.hold(); let result = runtime.block_on(async { let open = match session.take() { Some(open) => open, @@ -776,9 +789,11 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou // The stability window's looks, shared: the disk asks, the loop leaves the watcher's events for waiting files alone. let stability = Stability::new(options.timing.stability); let (thread_data, thread_stability) = (data.clone(), stability.clone()); + let gate = PassGate::default(); + let thread_gate = gate.clone(); let pass = std::thread::Builder::new() .name("craft-cloud-pass".into()) - .spawn(move || pass_thread(thread_data, thread_stability, requests_rx, pass_events)) + .spawn(move || pass_thread(thread_data, thread_stability, thread_gate, requests_rx, pass_events)) .map_err(|source| AgentError::Io { path: data.path().to_owned(), source })?; // Where the saved state read the change feed to, once a first sync is done. let since = data.load_state().ok().and_then(|saved| saved.state.space.map(|_| saved.state.seq)); @@ -798,6 +813,16 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou tokio::spawn(long_poll(transport.clone(), since_rx.clone(), tell, options, random)) }; let mut poll = (!user_paused).then(&start_poll); + // The account's team spaces (a server without them has none): looked for now and then, and synced when turned on. + let spaces = Spaces::start(SpacesConfig { + data: data.clone(), + account: account.clone(), + options, + control: control.clone(), + gate, + paused: user_paused, + }); + control.set_spaces_handle(Some(spaces.handle())); let mut scheduler = Scheduler::new(options.timing, status.now(), random); let mut watching = Watching::default(); @@ -988,13 +1013,14 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou break End::Exit(Exit::Failed); } say(out, "stopping after the running pass (press Ctrl-C again to stop at once)"); + spaces.handle().quiesce(); scheduler.stop(); watching.watcher = None; } None => stop_open = false, }, command = commands.recv(), if commands_open => match command { - Some(Command::Login(_, reply) | Command::Join(_, reply)) => { + Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::LoginSecondFactor(_, reply)) => { let _ = reply.send(Err(crate::cli::already_signed_in(&account))); } // Signed out on the server first (an unreachable server changes nothing, and syncing goes on); then @@ -1015,6 +1041,7 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou poll.abort(); } watching.watcher = None; + spaces.handle().set_paused(true); if let Some(hub) = &hub { // Bounded: a hub that doesn't answer never holds up the loop (its stop, quit, beats). let _ = tokio::time::timeout(HUB_ANSWER, hub.handle().set_paused(true)).await; @@ -1039,6 +1066,7 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou remember_paused(data, false, out); } poll = Some(start_poll()); + spaces.handle().set_paused(false); if let Some(hub) = &hub { let _ = tokio::time::timeout(HUB_ANSWER, hub.handle().set_paused(false)).await; } @@ -1051,6 +1079,7 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou } Some(Command::Quit(reply)) => { say(out, "quitting after the running pass"); + spaces.handle().quiesce(); scheduler.stop(); watching.watcher = None; let _ = reply.send(Ok(())); @@ -1071,6 +1100,12 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou } drop(watching); drop(requests); + // The team spaces stop with the account: a running pass finishes, then their folders are let go of (after a second + // stop at once: nothing queued starts, and the pass thread is not waited for, as the personal one is not). Only the + // second stop asks for that; a loop that ends because its pass thread is gone also exits with `Failed`. + control.set_spaces_handle(None); + spaces.stop(stops > 1).await; + control.set_spaces(Vec::new(), Vec::new()); // Recents stop with the account: their followers are told, and what they were given is on disk. control.set_hub(None); drop(hub); @@ -1178,7 +1213,7 @@ async fn wait_needing_you( let _ = reply.send(Err(e)); } }, - Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::Pause(reply) | Command::Resume(reply)) => { + Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::LoginSecondFactor(_, reply) | Command::Pause(reply) | Command::Resume(reply)) => { let _ = reply.send(Err(AgentError::NeedsYou(reason.to_owned()))); } Some(Command::Quit(reply)) => { @@ -1227,6 +1262,20 @@ async fn wait_for_sign_in( return SignedOut::SignedIn; } } + Some(Command::LoginSecondFactor(request, reply)) => { + let args = LoginArgs { + url: request.server.clone(), + user: request.user.clone(), + folder: request.folder.as_ref().map(std::path::PathBuf::from), + device_name: request.device_name.clone(), + adopt: request.adopt, + }; + let factor = crate::http::Factor { totp_code: request.totp_code.clone(), recovery_code: request.recovery_code.clone() }; + let result = crate::cli::login_second_factor_locked(data, &args, &request.challenge, &factor).await; + if signed_in(result, "signed in", control, reply, out) { + return SignedOut::SignedIn; + } + } Some(Command::Join(request, reply)) => { let args = JoinArgs { join: request.join.clone(), @@ -1236,7 +1285,9 @@ async fn wait_for_sign_in( // The caller asks for it with `POST /v1/autostart` once joined. autostart: false, }; - let result = crate::cli::join_locked(data, &args, request.password.as_deref()).await; + let factor = (request.totp_code.is_some() || request.recovery_code.is_some()) + .then(|| crate::http::Factor { totp_code: request.totp_code.clone(), recovery_code: request.recovery_code.clone() }); + let result = crate::cli::join_locked(data, &args, request.password.as_deref(), factor.as_ref()).await; if signed_in(result, "joined", control, reply, out) { return SignedOut::SignedIn; } @@ -1284,7 +1335,7 @@ fn signed_in( /// Whether a watcher event runs a pass: it does unless every path in it is a file waiting to settle (an event without /// paths may be anything). Each waiting file's event is recorded, so its cheap look follows once it goes quiet (see /// [`crate::settle::Settling::event`]). -fn counts_as_change(roots: &[std::path::PathBuf], stability: &Stability, paths: &[std::path::PathBuf], closed: bool) -> bool { +pub(crate) fn counts_as_change(roots: &[std::path::PathBuf], stability: &Stability, paths: &[std::path::PathBuf], closed: bool) -> bool { let mut change = paths.is_empty(); for p in paths { let waiting = crate::watch::relative(roots, p).is_some_and(|r| stability.event(&r, closed)); @@ -1295,7 +1346,7 @@ fn counts_as_change(roots: &[std::path::PathBuf], stability: &Stability, paths: /// The cheap looks that are due (one stat each, see [`crate::settle::Settling::peeked`]): whether one found a file /// settled (or gone), so a pass should run. -fn take_due_peeks(folder: &std::path::Path, stability: &Stability) -> bool { +pub(crate) fn take_due_peeks(folder: &std::path::Path, stability: &Stability) -> bool { let wall = now_ms(); let mut settled = false; for path in stability.take_peeks() { @@ -1325,6 +1376,7 @@ fn summary_to_show(summary: &PassSummary, last_held: &mut Option) -> (Op fn summary_worth_showing(s: &PassSummary) -> bool { s.downloaded + s.uploaded + s.removed_locally + s.removed_remotely + s.moved_locally + s.moved_remotely + s.created_dirs + s.refused > 0 || !s.conflict_copies.is_empty() + || !s.restored.is_empty() || !s.skipped.is_empty() || !s.held_deletes.is_empty() || s.needs_approval() diff --git a/apps/craft-cloud-agent/src/disk.rs b/apps/craft-cloud-agent/src/disk.rs index 0abf4f72..03902b5c 100644 --- a/apps/craft-cloud-agent/src/disk.rs +++ b/apps/craft-cloud-agent/src/disk.rs @@ -298,6 +298,21 @@ pub fn fold_file_id(id: FileId) -> u64 { u64::from_le_bytes(first) } +/// Which folder a path leads to: its canonical path and its ID (a link at `dir` itself is followed, as the synced root may be +/// one). A session compares it with what it opened with, so a link that was pointed at another folder meanwhile is noticed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FolderIdentity { + pub path: PathBuf, + pub id: Option, +} + +/// The identity of the folder `dir` leads to, or `None` if it can't be looked at (it is gone, a drive is unmounted). +pub fn folder_identity(dir: &Path) -> Option { + let path = fs::canonicalize(dir).ok()?; + let md = fs::metadata(&path).ok().filter(|md| md.is_dir())?; + Some(FolderIdentity { id: local_id(&md, &path), path }) +} + /// The ID of the file or folder at `path`, whose metadata (not following links) is `md`. #[cfg(unix)] fn local_id(md: &Metadata, _path: &Path) -> Option { diff --git a/apps/craft-cloud-agent/src/http.rs b/apps/craft-cloud-agent/src/http.rs index 16efff9f..7b6c1969 100644 --- a/apps/craft-cloud-agent/src/http.rs +++ b/apps/craft-cloud-agent/src/http.rs @@ -17,7 +17,7 @@ //! ~35 KB/s). A stalled connection fails after a minute either way. //! - **Answers map onto [`TransportError`]** by the server's error codes first ([`codes`]), then by status: //! a refused batch with its conflicts, missing chunks, `resync_required` and `quota_exceeded` (with how much the -//! account stores and may store) by code; 401 is `Unauthorized`, +//! account stores and may store) and `read_only` (the account's role in the space doesn't allow the write: a viewer) by code; 401 is `Unauthorized`, //! 404 `NotFound`; 408, 429, 5xx, connection failures and timeouts are `Unavailable` (try again later); //! anything else, and any answer that doesn't parse, is `Protocol` (another 4xx says "the server or a proxy //! refused the request (HTTP )": a reverse proxy in front of the server may be the one refusing). @@ -35,14 +35,16 @@ use std::time::Duration; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use craft_cloud_proto::api::{ - CommitRequest, CommitResponse, DeviceInfo, ErrorBody, InfoResponse, JoinRequest, LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, - MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, PairingCreated, PairingStatus, SpaceResponse, codes, + AccountInfo, ChangePasswordRequest, CommitRequest, CommitResponse, DeviceInfo, ErrorBody, HistoryResponse, InfoResponse, JoinRequest, + LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, PairingCreated, PairingStatus, + RecoveryCodes, RenameDeviceRequest, RestoreRequest, RestoreResponse, RevokedOthers, SecondFactorRequest, SpaceList, SpaceListItem, SpaceResponse, + TwoFactorChallenge, TwoFactorConfirmRequest, TwoFactorProofRequest, TwoFactorSetupRequest, TwoFactorSetupResponse, TwoFactorStatus, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::local::MAX_MACHINE_BYTES; -use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, ManifestId, NodeId, Op, PageCursor, SaveRecord, VersionId}; +use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, ManifestId, NodeId, Op, PageCursor, SaveRecord, SpaceId, VersionId}; use craft_cloud_sync::{Transport, TransportError}; -use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, HeaderValue, RETRY_AFTER, USER_AGENT}; +use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, RETRY_AFTER, USER_AGENT}; use reqwest::{Client, Method, StatusCode, Url}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -59,6 +61,8 @@ pub const MAX_DEVICES: usize = 1000; const MAX_CODE_BYTES: usize = 64; /// Longest server message shown to the user, in characters. const MAX_MESSAGE_CHARS: usize = 300; +/// The whole of an audit export, from connecting to the last byte (it is read as it comes, with the usual idle limit). +const EXPORT_TIMEOUT: Duration = Duration::from_secs(60 * 60); const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// No data at all for this long fails the request. const READ_TIMEOUT: Duration = Duration::from_secs(60); @@ -438,6 +442,7 @@ fn error_for(status: StatusCode, body: Option) -> TransportError { return TransportError::MissingChunks(missing.clone()); } (Some(codes::RESYNC_REQUIRED), _) => return TransportError::ResyncRequired, + (Some(codes::READ_ONLY), _) => return TransportError::ReadOnly, (Some(codes::QUOTA_EXCEEDED), Some(ErrorBody { quota: Some(quota), .. })) => { return TransportError::QuotaExceeded { used: quota.used, limit: quota.limit }; } @@ -492,6 +497,25 @@ pub fn check_compatible(info: &InfoResponse) -> Result<(), TransportError> { Ok(()) } +/// The most bytes the server takes of a device's `platform` or `app_version`. +const MAX_DEVICE_DETAIL_BYTES: usize = 32; + +/// What this agent says about itself when it signs in or joins: `agent`, the operating system and the crate's +/// version, for the account's device list (display only). The server takes at most 32 bytes without control +/// characters per field, so a longer value is cut at a character boundary rather than refused. +fn device_details() -> (Option, Option, Option) { + fn limited(text: &str) -> String { + text.chars() + .filter(|c| !c.is_control()) + .scan(0usize, |len, c| { + *len += c.len_utf8(); + (*len <= MAX_DEVICE_DETAIL_BYTES).then_some(c) + }) + .collect() + } + (Some("agent".to_owned()), Some(limited(std::env::consts::OS)), Some(limited(env!("CARGO_PKG_VERSION")))) +} + /// Signs in and creates a device. `Unauthorized` means a wrong username or password. pub async fn login(server: &ServerUrl, username: &str, password: &str, device_name: &str) -> Result { login_over(&Reqwest::new(server)?, server, username, password, device_name).await @@ -505,11 +529,178 @@ pub async fn login_over( password: &str, device_name: &str, ) -> Result { - let body = LoginRequest { username: username.to_owned(), password: password.to_owned(), device_name: device_name.to_owned() }; + let (client_kind, platform, app_version) = device_details(); + let body = LoginRequest { + username: username.to_owned(), + password: password.to_owned(), + device_name: device_name.to_owned(), + client_kind, + platform, + app_version, + }; let req = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body)?; json(connection.send(req).await?, MAX_SMALL_JSON_BYTES).await } +/// A server's refusal of a call that the agent keeps whole (its status, code and message, the challenge of +/// `two_factor_required`, the `Retry-After` of a lock), unlike [`TransportError`], which sync relies on and which has no +/// room for them. Only the calls that need it return this ([`CallError`]); the sync paths, [`error_for`] and +/// `TransportError` are as they were. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Refusal { + pub status: u16, + pub error: String, + pub message: String, + pub two_factor: Option, + pub retry_after_secs: Option, +} + +/// Why a call that returns a [`Refusal`] failed: the server refused it (a wrong password or code, a conflict, a lock: never a +/// reason to think this device was revoked), or it failed as any call does (offline, a 401 on a signed-in call, a 404). +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum CallError { + #[error("{}", .0.message)] + Refused(Refusal), + #[error(transparent)] + Transport(TransportError), +} + +impl From for CallError { + fn from(e: TransportError) -> Self { + CallError::Transport(e) + } +} + +/// The server's `Retry-After` in seconds, cut to a sane size. +fn retry_after_of(headers: &HeaderMap) -> Option { + headers.get(RETRY_AFTER).and_then(|v| v.to_str().ok()).and_then(|s| s.trim().parse::().ok()).map(|secs| secs.min(24 * 60 * 60)) +} + +/// A failed answer as a [`CallError`]: a 400, 403, 409, 422 or 429 with an error body, a 401 that is `two_factor_required` or +/// `two_factor_invalid` (the not-signed-in routes' answers; on a signed-in route a 401 is only ever a revoked device) and a +/// 503 `two_factor_unavailable` are a [`Refusal`]; everything else is what [`error_for`] makes of it. +async fn failure(resp: Answer) -> CallError { + let status = resp.status; + let retry = retry_after_of(&resp.headers); + let body = body_capped(resp, MAX_SMALL_JSON_BYTES).await.ok().and_then(|b| serde_json::from_slice::(&b).ok()); + if let Some(b) = &body { + let refused = match status.as_u16() { + 400 | 403 | 409 | 422 | 429 => true, + 401 => matches!(b.error.as_str(), codes::TWO_FACTOR_REQUIRED | codes::TWO_FACTOR_INVALID), + 503 => b.error == codes::TWO_FACTOR_UNAVAILABLE, + _ => false, + }; + if refused { + return CallError::Refused(Refusal { + status: status.as_u16(), + error: b.error.clone(), + message: clean(&b.message), + two_factor: b.two_factor.clone(), + retry_after_secs: retry, + }); + } + } + CallError::Transport(error_for(status, body)) +} + +/// The second factor a person gave: a code from the authenticator app, or a recovery code. Its `Debug` output hides both. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Factor { + pub totp_code: Option, + pub recovery_code: Option, +} + +impl Factor { + /// Reads what a person typed: six digits (spaces allowed) are a code, anything else a recovery code. + pub fn typed(text: &str) -> Self { + let trimmed = text.trim(); + let digits: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect(); + if digits.len() == 6 && digits.bytes().all(|b| b.is_ascii_digit()) { + Self { totp_code: Some(digits), recovery_code: None } + } else { + Self { totp_code: None, recovery_code: Some(trimmed.to_owned()) } + } + } +} + +impl std::fmt::Debug for Factor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Factor") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// The first step of a sign-in: a device, or the challenge of an account that has two-factor sign-in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LoginStep { + Device(LoginResponse), + SecondFactor(TwoFactorChallenge), +} + +/// Signs in as [`login`] does, but tells the account's second factor from a wrong password: `Unauthorized` (a wrong username +/// or password) is [`CallError::Transport`], a challenge is [`LoginStep::SecondFactor`]. +pub async fn login_step(server: &ServerUrl, username: &str, password: &str, device_name: &str) -> Result { + login_step_over(&Reqwest::new(server)?, server, username, password, device_name).await +} + +/// [`login_step`] over another connection (a simulated network, in tests). +pub async fn login_step_over( + connection: &C, + server: &ServerUrl, + username: &str, + password: &str, + device_name: &str, +) -> Result { + let (client_kind, platform, app_version) = device_details(); + let body = LoginRequest { + username: username.to_owned(), + password: password.to_owned(), + device_name: device_name.to_owned(), + client_kind, + platform, + app_version, + }; + let req = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body)?; + let resp = connection.send(req).await?; + if resp.status.is_success() { + return Ok(LoginStep::Device(json(resp, MAX_SMALL_JSON_BYTES).await?)); + } + match failure(resp).await { + CallError::Refused(Refusal { error, two_factor, .. }) if error == codes::TWO_FACTOR_REQUIRED => match two_factor { + Some(challenge) => Ok(LoginStep::SecondFactor(challenge)), + None => Err(CallError::Transport(TransportError::Protocol("the server asked for a second factor without a challenge".into()))), + }, + other => Err(other), + } +} + +/// The second step of a sign-in: the `challenge` and the second factor make the device. A challenge that doesn't work any +/// more (unknown, expired, used, from before a password change) is `Transport(Unauthorized)` here, which is no revoked device +/// (the caller says the sign-in has to start again); a wrong code is `Refused` with `two_factor_invalid`, a lock with +/// `two_factor_locked` and its wait. +pub async fn second_factor(server: &ServerUrl, challenge: &str, factor: &Factor) -> Result { + second_factor_over(&Reqwest::new(server)?, server, challenge, factor).await +} + +/// [`second_factor`] over another connection (a simulated network, in tests). +pub async fn second_factor_over( + connection: &C, + server: &ServerUrl, + challenge: &str, + factor: &Factor, +) -> Result { + let body = + SecondFactorRequest { challenge: challenge.to_owned(), totp_code: factor.totp_code.clone(), recovery_code: factor.recovery_code.clone() }; + let req = request(Method::POST, server.endpoint("auth/login/second-factor"), None, 0).json(&body)?; + let resp = connection.send(req).await?; + if resp.status.is_success() { + return Ok(json(resp, MAX_SMALL_JSON_BYTES).await?); + } + Err(failure(resp).await) +} + /// Why a join ([`join`]) was refused. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum JoinError { @@ -525,13 +716,34 @@ pub enum JoinError { /// Too many failed joins from this address; the server's message. #[error("{0}")] RateLimited(String), + /// The account has two-factor sign-in: join again with a code from its authenticator app or a recovery code. The code + /// was not used up. + #[error("the account has two-factor sign-in: a code from its authenticator app is needed")] + TwoFactorRequired, + /// The code or recovery code was wrong (the join code still works); the server's message. + #[error("{0}")] + TwoFactorInvalid(String), + /// Too many wrong codes: the second factor is locked for `retry_after_secs`. + #[error("{message}")] + TwoFactorLocked { message: String, retry_after_secs: Option }, #[error(transparent)] Transport(TransportError), } /// Redeems a join code (normalised) for a new device named `device_name`; `password` for an invite to a new account. pub async fn join(server: &ServerUrl, code: &str, device_name: &str, password: Option<&str>) -> Result { - join_over(&Reqwest::new(server).map_err(JoinError::Transport)?, server, code, device_name, password).await + join_with_factor(server, code, device_name, password, None).await +} + +/// [`join`] with the second factor of an account that has two-factor sign-in. +pub async fn join_with_factor( + server: &ServerUrl, + code: &str, + device_name: &str, + password: Option<&str>, + factor: Option<&Factor>, +) -> Result { + join_over_with(&Reqwest::new(server).map_err(JoinError::Transport)?, server, code, device_name, password, factor).await } /// [`join`] over another connection (a simulated network, in tests). @@ -542,15 +754,43 @@ pub async fn join_over( device_name: &str, password: Option<&str>, ) -> Result { - let body = JoinRequest { code: code.to_owned(), device_name: device_name.to_owned(), password: password.map(str::to_owned) }; + join_over_with(connection, server, code, device_name, password, None).await +} + +/// [`join_over`] with a second factor. +pub async fn join_over_with( + connection: &C, + server: &ServerUrl, + code: &str, + device_name: &str, + password: Option<&str>, + factor: Option<&Factor>, +) -> Result { + let (client_kind, platform, app_version) = device_details(); + let body = JoinRequest { + code: code.to_owned(), + device_name: device_name.to_owned(), + password: password.map(str::to_owned), + totp_code: factor.and_then(|f| f.totp_code.clone()), + recovery_code: factor.and_then(|f| f.recovery_code.clone()), + client_kind, + platform, + app_version, + }; let req = request(Method::POST, server.endpoint("join"), None, 0).json(&body).map_err(JoinError::Transport)?; let resp = connection.send(req).await.map_err(JoinError::Transport)?; if resp.status.is_success() { return json(resp, MAX_SMALL_JSON_BYTES).await.map_err(JoinError::Transport); } let status = resp.status; + let retry = retry_after_of(&resp.headers); let body = body_capped(resp, MAX_SMALL_JSON_BYTES).await.ok().and_then(|b| serde_json::from_slice::(&b).ok()); Err(match body.as_ref().map(|b| b.error.as_str()) { + Some(codes::TWO_FACTOR_REQUIRED) => JoinError::TwoFactorRequired, + Some(codes::TWO_FACTOR_INVALID) => JoinError::TwoFactorInvalid(body.as_ref().map(|b| clean(&b.message)).unwrap_or_default()), + Some(codes::TWO_FACTOR_LOCKED) => { + JoinError::TwoFactorLocked { message: body.as_ref().map(|b| clean(&b.message)).unwrap_or_default(), retry_after_secs: retry } + } Some(codes::INVALID_CODE) => JoinError::InvalidCode, Some(codes::PASSWORD_REQUIRED) => JoinError::PasswordRequired, Some(codes::WEAK_PASSWORD) => JoinError::WeakPassword(body.as_ref().map(|b| clean(&b.message)).unwrap_or_default()), @@ -580,6 +820,51 @@ pub struct HttpTransport { connection: C, server: ServerUrl, auth: HeaderValue, + /// The team space every request works on (`craft-space`); `None`: the account's personal space, as before. Every + /// answer that the server's space layer gives must echo it ([`TOO_OLD_FOR_TEAM_SPACES`]). + space: Option, +} + +/// What the agent says when a server answers a request for a team space without echoing the space, or with another one: +/// it is a server from before team spaces (it ignored the `craft-space` header and answered about the personal space). +/// The space stops syncing with this message, so two trees are never mixed. +pub const TOO_OLD_FOR_TEAM_SPACES: &str = "the server is too old for team spaces; ask the administrator to update it"; + +/// Whether an answer to a request for team space `asked` carries the echo the server's space layer adds to every answer +/// it produces (`craft-space` with the space it worked on). Checked on the answers that the layer or the handler behind it +/// gives (success, `403`, `404`, `409`, `410`); a `400`, `401` or `429` can come from before the layer (or from a proxy in +/// front of the server) and carries none. +fn echoed(asked: &HeaderValue, answer: &Answer) -> Result<(), TransportError> { + let layered = answer.status.is_success() || matches!(answer.status.as_u16(), 403 | 404 | 409 | 410); + if !layered { + return Ok(()); + } + match answer.headers.get(HeaderName::from_static("craft-space")) { + Some(echo) if echo.as_bytes().eq_ignore_ascii_case(asked.as_bytes()) => Ok(()), + _ => Err(TransportError::Protocol(TOO_OLD_FOR_TEAM_SPACES.into())), + } +} + +/// A history or restore refusal, retaining the server's HTTP status and stable error code. +#[derive(Debug)] +pub enum HistoryCallError { + Transport(TransportError), + Refused { status: StatusCode, body: Box }, +} + +async fn history_answer(answer: Answer) -> Result { + if !answer.status.is_success() { + let status = answer.status; + let body = body_capped(answer, MAX_SMALL_JSON_BYTES) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_else(|| ErrorBody { error: codes::INTERNAL.into(), message: status.to_string(), ..ErrorBody::default() }); + return Err(HistoryCallError::Refused { status, body: Box::new(body) }); + } + let bytes = body_capped(answer, MAX_SMALL_JSON_BYTES).await.map_err(HistoryCallError::Transport)?; + serde_json::from_slice(&bytes) + .map_err(|e| HistoryCallError::Transport(TransportError::Protocol(format!("the server's answer doesn't parse: {e}")))) } impl HttpTransport { @@ -591,19 +876,101 @@ impl HttpTransport { impl HttpTransport { /// The API over another connection (a simulated network, in tests). `server` names the endpoints. pub fn over(connection: C, server: ServerUrl, token: &DeviceToken) -> Result { - Ok(Self { connection, server, auth: token.bearer()? }) + Ok(Self { connection, server, auth: token.bearer()?, space: None }) + } + + /// The same API for one space: every request names it in the `craft-space` header. Without it, the account's personal + /// space, which is also what a server from before team spaces answers about. + pub fn bound_to(mut self, space: SpaceId) -> Result { + let value = HeaderValue::from_str(&space.to_string()).map_err(|_| TransportError::Protocol("a space ID that can't be sent".into()))?; + self.space = Some(value); + Ok(self) } pub fn server(&self) -> &ServerUrl { &self.server } + /// The server's newest-first history for one file node. + pub async fn file_history(&self, node: NodeId, cursor: Option<&str>, limit: Option) -> Result { + let mut url = self.server.endpoint(&format!("nodes/{node}/history")); + { + let mut query = url.query_pairs_mut(); + if let Some(cursor) = cursor { + query.append_pair("cursor", cursor); + } + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + } + let req = request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Makes a prior version the file's content as a new version. + pub async fn restore_file(&self, node: NodeId, version: VersionId, expected_version: VersionId) -> Result { + let req = self + .request(Method::POST, &format!("nodes/{node}/restore"), MAX_SMALL_JSON_BYTES) + .json(&RestoreRequest { version, expected_version }) + .map_err(HistoryCallError::Transport)?; + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// The server's newest-first trash page. + pub async fn trash(&self, cursor: Option<&str>, limit: Option) -> Result { + let mut url = self.server.endpoint("trash"); + { + let mut query = url.query_pairs_mut(); + if let Some(cursor) = cursor { + query.append_pair("cursor", cursor); + } + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + } + let req = request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Restores a deleted node from the server's trash. + pub async fn restore_from_trash(&self, node: NodeId) -> Result { + let req = self.request(Method::POST, &format!("trash/{node}/restore"), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Marks a deleted node out of the server's trash. This does not free stored data yet. + pub async fn remove_from_trash(&self, node: NodeId) -> Result<(), HistoryCallError> { + let req = self.request(Method::DELETE, &format!("trash/{node}"), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + if answer.status.is_success() { + return Ok(()); + } + let status = answer.status; + let body = body_capped(answer, MAX_SMALL_JSON_BYTES) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_else(|| ErrorBody { error: codes::INTERNAL.into(), message: status.to_string(), ..ErrorBody::default() }); + Err(HistoryCallError::Refused { status, body: Box::new(body) }) + } + fn request(&self, method: Method, path: &str, max_body: usize) -> Request { request(method, self.server.endpoint(path), Some(&self.auth), max_body) } - async fn send(&self, req: Request) -> Result { - self.connection.send(req).await + async fn send(&self, mut req: Request) -> Result { + if let Some(space) = &self.space { + req.headers.insert(HeaderName::from_static("craft-space"), space.clone()); + } + let answer = self.connection.send(req).await?; + if let Some(space) = &self.space { + echoed(space, &answer)?; + } + Ok(answer) } fn page_request(&self, path: &str, since: Option, after: Option) -> Request { @@ -660,11 +1027,88 @@ impl HttpTransport { } } + /// A signed-in call whose refusals the agent keeps whole. + async fn account_call(&self, req: Request) -> Result { + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(json(resp, MAX_SMALL_JSON_BYTES).await?); + } + Err(failure(resp).await) + } + + /// [`HttpTransport::account_call`] for an answer without a body. + async fn account_unit(&self, req: Request) -> Result<(), CallError> { + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(()); + } + Err(failure(resp).await) + } + + /// Whether the account has two-factor sign-in on (`GET /account/two-factor`). + pub async fn two_factor_status(&self) -> Result { + self.account_call(self.request(Method::GET, "account/two-factor", 0)).await + } + + /// Starts a set-up with the account's password: the new secret, shown once. + pub async fn two_factor_setup(&self, password: &str) -> Result { + let req = self.request(Method::POST, "account/two-factor/setup", 0).json(&TwoFactorSetupRequest { password: password.to_owned() })?; + self.account_call(req).await + } + + /// Turns it on with the first code: the recovery codes, shown once. + pub async fn two_factor_confirm(&self, totp_code: &str) -> Result { + let req = self.request(Method::POST, "account/two-factor/confirm", 0).json(&TwoFactorConfirmRequest { totp_code: totp_code.to_owned() })?; + self.account_call(req).await + } + + /// Turns it off: the password and a code or recovery code. + pub async fn two_factor_disable(&self, body: &TwoFactorProofRequest) -> Result<(), CallError> { + self.account_unit(self.request(Method::POST, "account/two-factor/disable", 0).json(body)?).await + } + + /// New recovery codes: the password and a code from the authenticator app. + pub async fn two_factor_recovery_codes(&self, body: &TwoFactorProofRequest) -> Result { + self.account_call(self.request(Method::POST, "account/two-factor/recovery-codes", 0).json(body)?).await + } + + /// Changes the account's password. + pub async fn change_password(&self, body: &ChangePasswordRequest) -> Result<(), CallError> { + self.account_unit(self.request(Method::POST, "account/password", 0).json(body)?).await + } + + /// Signs every other device of the account out. + pub async fn revoke_other_devices(&self) -> Result { + self.account_call(self.request(Method::POST, "devices/revoke-others", 0)).await + } + + /// Renames one of the account's devices. + pub async fn rename_device(&self, device: DeviceId, name: &str) -> Result { + let req = self.request(Method::PATCH, &format!("devices/{device}"), 0).json(&RenameDeviceRequest { name: name.to_owned() })?; + self.account_call(req).await + } + /// Revokes a device of this account (this one, to sign out). pub async fn revoke(&self, device: DeviceId) -> Result<(), TransportError> { self.no_content(self.request(Method::DELETE, &format!("devices/{device}"), 0)).await } + /// `GET /spaces`: the personal space and the team spaces of the account, invitations among them. A server from before + /// team spaces answers `404` here (a refusal with that status, or `NotFound`). + pub async fn spaces(&self) -> Result { + self.account_call(self.request(Method::GET, "spaces", MAX_SMALL_JSON_BYTES)).await + } + + /// `POST /spaces/{id}/accept`: the space as `GET /spaces` lists it. + pub async fn accept_space(&self, space: SpaceId) -> Result { + self.account_call(self.request(Method::POST, &format!("spaces/{space}/accept"), MAX_SMALL_JSON_BYTES)).await + } + + /// `POST /spaces/{id}/decline`. + pub async fn decline_space(&self, space: SpaceId) -> Result<(), CallError> { + self.account_unit(self.request(Method::POST, &format!("spaces/{space}/decline"), MAX_SMALL_JSON_BYTES)).await + } + /// The account's devices, this one marked `current`. pub async fn devices(&self) -> Result, TransportError> { let list: Vec = json(self.send(self.request(Method::GET, "devices", MAX_SMALL_JSON_BYTES)).await?, MAX_SMALL_JSON_BYTES).await?; @@ -674,6 +1118,11 @@ impl HttpTransport { Ok(list) } + /// The account's stored bytes and labeled estimates, fetched without caching. + pub async fn usage(&self) -> Result { + json(self.send(self.request(Method::GET, "account/usage", MAX_SMALL_JSON_BYTES)).await?, MAX_SMALL_JSON_BYTES).await + } + /// A new add-a-computer code from this device (replacing its open one). pub async fn create_pairing(&self) -> Result { let created: PairingCreated = json(self.send(self.request(Method::POST, "pairings", 0)).await?, MAX_SMALL_JSON_BYTES).await?; @@ -696,6 +1145,51 @@ impl HttpTransport { pub async fn delete_pairing(&self, id: Uuid) -> Result<(), TransportError> { self.no_content(self.request(Method::DELETE, &format!("pairings/{id}"), 0)).await } + + /// The signed-in account (`GET /account`): its role and whether it has two-factor sign-in. + pub async fn account_info(&self) -> Result { + self.account_call(self.request(Method::GET, "account", MAX_SMALL_JSON_BYTES)).await + } + + /// The audit export (`GET /admin/audit/export?`) as the server sends it: the good answer, whose body is read as it + /// comes ([`AnswerBody`]), never held whole; a refusal is kept whole. The whole-request time is long (a big export over a + /// slow link), the idle time is the usual minute. + pub async fn admin_export_stream(&self, query: &[(&str, &str)]) -> Result { + let mut url = self.server.endpoint("admin/audit/export"); + if !query.is_empty() { + url.query_pairs_mut().extend_pairs(query); + } + let mut req = request(Method::GET, url, Some(&self.auth), 0); + req.timeout = EXPORT_TIMEOUT; + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(resp); + } + Err(failure(resp).await) + } + + /// `GET /api/v1/?` of the admin API (or the account's activity), read as `T`; a refusal is kept whole. + pub async fn admin_get(&self, path: &str, query: &[(&str, &str)]) -> Result { + let mut url = self.server.endpoint(path); + if !query.is_empty() { + url.query_pairs_mut().extend_pairs(query); + } + self.account_call(request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES)).await + } + + /// `method /api/v1/` of the admin API with `body` (the confirmation among it), answered with a body read as `T`. + pub async fn admin_json(&self, method: Method, path: &str, body: &B) -> Result { + self.account_call(self.request(method, path, MAX_SMALL_JSON_BYTES).json(body)?).await + } + + /// [`HttpTransport::admin_json`] for a call answered without a body (`204`, or `202`): the status it succeeded with. + pub async fn admin_empty(&self, method: Method, path: &str, body: &B) -> Result { + let resp = self.send(self.request(method, path, MAX_SMALL_JSON_BYTES).json(body)?).await?; + if resp.status.is_success() { + return Ok(resp.status); + } + Err(failure(resp).await) + } } #[async_trait] @@ -763,6 +1257,26 @@ mod tests { use super::*; use craft_cloud_proto::{Conflict, ConflictKind}; + #[test] + fn a_second_factor_is_read_as_typed_and_hidden_in_debug() { + assert_eq!(Factor::typed(" 123 456 "), Factor { totp_code: Some("123456".into()), recovery_code: None }); + assert_eq!(Factor::typed("abcd-efgh-jkmn-pqrs"), Factor { totp_code: None, recovery_code: Some("abcd-efgh-jkmn-pqrs".into()) }); + assert_eq!(Factor::typed("12345").recovery_code.as_deref(), Some("12345"), "five digits are not a code"); + let shown = format!("{:?}", Factor::typed("123456")); + assert!(!shown.contains("123456"), "{shown}"); + let shown = format!( + "{:?}", + Refusal { + status: 401, + error: "x".into(), + message: "m".into(), + two_factor: Some(TwoFactorChallenge { challenge: "chal-secret".into(), expires_ms: 1 }), + retry_after_secs: None + } + ); + assert!(!shown.contains("chal-secret"), "{shown}"); + } + #[test] fn https_is_required_except_on_this_machine() { assert!(ServerUrl::parse("https://cloud.example.com").is_ok()); @@ -811,7 +1325,12 @@ mod tests { #[test] fn requests_keep_bodies_and_tokens_out_of_debug_output() { let server = ServerUrl::parse("http://127.0.0.1:1").unwrap(); - let body = LoginRequest { username: "ana".into(), password: "correct horse battery".into(), device_name: "laptop".into() }; + let body = LoginRequest { + username: "ana".into(), + password: "correct horse battery".into(), + device_name: "laptop".into(), + ..LoginRequest::default() + }; let login = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body).unwrap(); let shown = format!("{login:?}"); assert!(!shown.contains("correct horse") && !shown.contains("ana"), "{shown}"); @@ -823,6 +1342,22 @@ mod tests { assert!(!shown.contains("secret-token") && shown.contains("authorization"), "{shown}"); } + /// An admin change carries the admin's password in its body, to the server only: no `Debug` output of the request + /// or of the bodies shows it. + #[test] + fn admin_confirmations_stay_out_of_debug_output() { + use craft_cloud_proto::api::{AdminConfirm, AdminCreateUser, AdminUpdateUser}; + let server = ServerUrl::parse("http://127.0.0.1:1").unwrap(); + let confirm = AdminConfirm { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), ..AdminConfirm::default() }; + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + let update = AdminUpdateUser { password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + let sent = request(Method::POST, server.endpoint("admin/users/x/disable"), None, 0).json(&confirm).unwrap(); + assert!(sent.body.as_deref().is_some_and(|b| String::from_utf8_lossy(b).contains("admin-secret-pw")), "the body carries it"); + let shown = format!("{sent:?} {confirm:?} {create:?} {update:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + } + fn body(code: &str) -> ErrorBody { ErrorBody { error: code.into(), message: "m".into(), ..ErrorBody::default() } } @@ -835,6 +1370,9 @@ mod tests { let missing = ErrorBody { missing: Some(vec![ChunkId::from_bytes([1; 32])]), ..body(codes::MISSING_CHUNKS) }; assert_eq!(error_for(StatusCode::CONFLICT, Some(missing)), TransportError::MissingChunks(vec![ChunkId::from_bytes([1; 32])])); assert_eq!(error_for(StatusCode::GONE, Some(body(codes::RESYNC_REQUIRED))), TransportError::ResyncRequired); + // A viewer's write: told apart from every other refusal, so the engine keeps the space read-only. + assert_eq!(error_for(StatusCode::FORBIDDEN, Some(body(codes::READ_ONLY))), TransportError::ReadOnly); + assert!(matches!(error_for(StatusCode::FORBIDDEN, Some(body(codes::FORBIDDEN))), TransportError::Protocol(_))); let full = ErrorBody { quota: Some(craft_cloud_proto::api::QuotaUse { used: 9, limit: 10 }), ..body(codes::QUOTA_EXCEEDED) }; assert_eq!(error_for(StatusCode::INSUFFICIENT_STORAGE, Some(full)), TransportError::QuotaExceeded { used: 9, limit: 10 }); // Without the numbers it is only a server error: tried again later. @@ -881,8 +1419,8 @@ mod tests { max_missing_ids: MAX_MISSING_IDS as u32, }; assert!(check_compatible(&info).is_ok()); - info.protocol_min = 2; - info.protocol_max = 3; + info.protocol_min = PROTOCOL_MAX + 1; + info.protocol_max = PROTOCOL_MAX + 2; assert!(check_compatible(&info).is_err()); info.protocol_min = 1; info.chunker_params = vec!["other".into()]; diff --git a/apps/craft-cloud-agent/src/lib.rs b/apps/craft-cloud-agent/src/lib.rs index ace6aa3e..2d2de8d7 100644 --- a/apps/craft-cloud-agent/src/lib.rs +++ b/apps/craft-cloud-agent/src/lib.rs @@ -11,6 +11,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod account; +pub mod admin; pub mod autostart; pub mod cli; pub mod daemon; @@ -24,4 +25,5 @@ pub mod root; pub mod runlog; pub mod schedule; pub mod settle; +pub mod spaces; pub mod watch; diff --git a/apps/craft-cloud-agent/src/live/copy.rs b/apps/craft-cloud-agent/src/live/copy.rs index cd8cc722..205558f8 100644 --- a/apps/craft-cloud-agent/src/live/copy.rs +++ b/apps/craft-cloud-agent/src/live/copy.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use craft_cloud_proto::live::MAX_UPDATE_BYTES; -use loro::{ExportMode, IdSpan, LoroDoc, VersionVector}; +use loro::{ExportMode, IdSpan, LoroDoc}; /// Why an update or a stored document was refused. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -271,17 +271,39 @@ impl LiveCopy { } /// A snapshot of the document as it was at `version` (its history up to there, and the state then; nothing - /// later). `Invalid` if this copy doesn't hold every operation `version` names. + /// later). `Invalid` if this copy doesn't hold every operation `version` names, if those operations need others + /// past `version`, or if the document is shallow (Loro can't take a historical snapshot of one). + /// + /// Built by replaying the history up to `version` into a fresh document and taking its snapshot: Loro's own + /// `SnapshotAt` checks the document out at `version`, which on a long text history can need many GiB. pub fn snapshot_at(&self, version: &Version) -> Result, UpdateError> { - self.guard(|doc| { + let updates = self.guard(|doc| { let have = doc.oplog_vv(); if version.iter().any(|(peer, end)| *end > 0 && have.get(peer).is_none_or(|h| h < end)) { return Err(UpdateError::Invalid("the copy doesn't hold that version")); } - let vv: VersionVector = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| (*peer, *end)).collect(); - let frontiers = doc.vv_to_frontiers(&vv); - doc.export(ExportMode::SnapshotAt { version: std::borrow::Cow::Owned(frontiers) }).map_err(|e| UpdateError::Malformed(e.to_string())) - }) + if doc.is_shallow() { + return Err(UpdateError::Invalid("a shallow document has no historical snapshot")); + } + let spans: Vec<_> = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| IdSpan::new(*peer, 0, *end)).collect(); + doc.export(ExportMode::updates_in_range(spans)).map_err(|e| UpdateError::Malformed(e.to_string())) + })?; + let copy = Self::new(self.peer()); + let result = (|| { + copy.import_any_size(&updates)?; + let want: Version = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| (*peer, *end)).collect(); + if copy.version()? != want { + return Err(UpdateError::Invalid("the version is not causally complete")); + } + copy.snapshot() + })(); + if copy.is_poisoned() { + // Dropping a poisoned Loro document panics again; this one is only a scratch copy, so it is let go of. + // A panic in it says nothing about this document, so it isn't reported as one. + std::mem::forget(copy); + return Err(UpdateError::Malformed("the historical snapshot couldn't be built".into())); + } + result } /// The changes (Loro commits: peer, first and end counter) of this document that `has` doesn't wholly hold, in @@ -645,4 +667,46 @@ mod tests { assert!(log.at(3).is_some()); assert_eq!(version_of(&snapshot).unwrap().get(&1), Some(&8)); } + + #[test] + fn historical_snapshot_preserves_large_text_and_accepts_later_updates() { + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + let copy = LiveCopy::new(9); + let text = "a".repeat(200_000); + let first = text_update(&source, 0, &text); + copy.import(&first).unwrap(); + let version = copy.version().unwrap(); + let later = text_update(&source, text.len(), "later"); + copy.import(&later).unwrap(); + let snapshot = copy.snapshot_at(&version).unwrap(); + let fresh = LiveCopy::load(&snapshot, 5).unwrap(); + assert_eq!(fresh.doc.get_text("text").to_string(), text); + fresh.import(&later).unwrap(); + assert_eq!(fresh.doc.get_text("text").to_string(), format!("{text}later")); + } + + /// A version whose operations depend on others past it (peer 2 typed after more of peer 1 than the version + /// holds) has no snapshot of its own: refused, never silently widened. + #[test] + fn a_historical_snapshot_of_a_version_that_needs_later_operations_is_refused() { + let (x, y) = (LoroDoc::new(), LoroDoc::new()); + x.set_peer_id(1).unwrap(); + y.set_peer_id(2).unwrap(); + let first = text_update(&x, 0, "ab"); + let second = text_update(&x, 2, "cd"); + y.import(&first).unwrap(); + y.import(&second).unwrap(); + let from_y = text_update(&y, 4, "e"); + let copy = LiveCopy::new(9); + for update in [&first, &second, &from_y] { + copy.import(update).unwrap(); + } + let partial: Version = [(1, 2), (2, 1)].into_iter().collect(); + assert!(matches!(copy.snapshot_at(&partial), Err(UpdateError::Invalid(_)))); + assert!(!copy.is_poisoned()); + let whole = copy.version().unwrap(); + let snapshot = copy.snapshot_at(&whole).unwrap(); + assert_eq!(LiveCopy::load(&snapshot, 5).unwrap().doc.get_text("text").to_string(), "abcde"); + } } diff --git a/apps/craft-cloud-agent/src/local_api.rs b/apps/craft-cloud-agent/src/local_api.rs index 3e2f8bb5..dc2e86bb 100644 --- a/apps/craft-cloud-agent/src/local_api.rs +++ b/apps/craft-cloud-agent/src/local_api.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; use axum::extract::rejection::{JsonRejection, PathRejection}; -use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::extract::{DefaultBodyLimit, Path, Query, Request, State}; use axum::http::StatusCode; use axum::http::header::{AUTHORIZATION, HOST, ORIGIN}; use axum::middleware::{self, Next}; @@ -25,11 +25,16 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; use craft_cloud_proto::DeviceId; -use craft_cloud_proto::api::{DeviceInfo, ErrorBody, PairingStatus, codes as api_codes}; +use craft_cloud_proto::api::{ + AccountUsage, DeviceInfo, HistoryResponse, PairingStatus, RecoveryCodes, RestoreResponse, RevokedOthers, TwoFactorChallenge, + TwoFactorSetupResponse, TwoFactorStatus, codes as api_codes, +}; use craft_cloud_proto::join::{Code, JoinString, MAX_JOIN_BYTES, PAIRING_CHARS}; use craft_cloud_proto::local::{ - AutostartRequest, JoinRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_SCHEMA, LocalApiFile, LoginRequest, LogoutRequest, MAX_LOCAL_JSON_BYTES, - PairingInfo, RunState, Status, codes, paths, + ApproveDeletesAnswer, ApproveDeletesRequest, AutostartRequest, ChangePasswordRequest, JoinRequest, LOCAL_API_FILE, LOCAL_API_MAX, + LOCAL_API_SCHEMA, LocalApiFile, LoginRequest, LogoutRequest, MAX_LOCAL_JSON_BYTES, PairingInfo, PasswordRequest, RenameDeviceRequest, RunState, + SecondFactorLoginRequest, SpaceInfo, SpaceList, SpaceRun, SpaceStatus, SpaceSync, SpaceSyncRequest, Status, TotpRequest, TwoFactorProofRequest, + codes, paths, }; use tokio::sync::{mpsc, oneshot}; @@ -41,11 +46,14 @@ use crate::autostart::{self, Roots}; use crate::cli::{AgentError, Exit}; use crate::http::HttpTransport; use crate::live::app_blobs::{self, BlobError}; +use crate::spaces::SpacesHandle; /// What the local API asks the `run` loop to do, with where to send the outcome. pub enum Command { Login(LoginRequest, oneshot::Sender>), Join(JoinRequest, oneshot::Sender>), + /// The second step of a sign-in that answered `two_factor_required`. + LoginSecondFactor(SecondFactorLoginRequest, oneshot::Sender>), Logout(LogoutRequest, oneshot::Sender>), /// Hold syncing and the live connection until [`Command::Resume`], also after a restart. Pause(oneshot::Sender>), @@ -70,6 +78,11 @@ struct Shared { last_sync_ms: Option, /// The last pass found this device refused by the server. removed: bool, + /// The team spaces, as the spaces task last saw them (`GET /v1/spaces`, after the personal space), and the ones that sync + /// or lost access (`GET /v1/status`). + spaces: Vec, + space_status: Vec, + spaces_handle: Option, } /// The loop's side of the local API: the status it reports, and the commands it receives. Cheap to clone. @@ -85,8 +98,18 @@ pub struct Control { impl Control { /// A control for a loop that receives its commands from `commands`' other end. pub fn new(commands: mpsc::Sender) -> Self { - let shared = - Shared { account: None, run_state: RunState::SignedOut, online: false, hub: None, autostart: None, last_sync_ms: None, removed: false }; + let shared = Shared { + account: None, + run_state: RunState::SignedOut, + online: false, + hub: None, + autostart: None, + last_sync_ms: None, + removed: false, + spaces: Vec::new(), + space_status: Vec::new(), + spaces_handle: None, + }; let autostart_refusal = "this agent doesn't handle starting at login here".to_owned(); Self { shared: Arc::new(Mutex::new(shared)), commands, autostart: None, autostart_refusal } } @@ -131,10 +154,29 @@ impl Control { s.online = false; s.last_sync_ms = None; s.removed = false; + s.spaces.clear(); + s.space_status.clear(); } }); } + /// What the spaces task sees: the team spaces (for `GET /v1/spaces`) and the status of those that sync. + pub fn set_spaces(&self, spaces: Vec, status: Vec) { + self.with(|s| { + s.spaces = spaces; + s.space_status = status; + }); + } + + /// The spaces task's handle, while signed in. + pub fn set_spaces_handle(&self, handle: Option) { + self.with(|s| s.spaces_handle = handle); + } + + pub fn spaces_handle(&self) -> Option { + self.with(|s| s.spaces_handle.clone()) + } + /// Whether the server refused this device at the last pass (it was removed from the account). pub fn set_removed(&self, removed: bool) { self.with(|s| s.removed = removed); @@ -187,27 +229,50 @@ impl Control { autostart: s.autostart, last_sync_ms: s.last_sync_ms, removed: s.account.is_some() && s.removed, + spaces: if s.account.is_some() { s.space_status.clone() } else { Vec::new() }, }) } } -/// An error answer: the HTTP status and the [`ErrorBody`]. +/// An error answer: the HTTP status, the typed local API code and message, and optional details such as a two-factor +/// challenge, a lock wait, or a trash-restore name conflict. #[derive(Debug)] pub struct LocalError { pub status: StatusCode, - pub code: &'static str, + pub code: String, pub message: String, + pub two_factor: Option, + pub retry_after_secs: Option, + pub name_taken: Option, } impl LocalError { - pub fn new(status: StatusCode, code: &'static str, message: impl Into) -> Self { - Self { status, code, message: message.into() } + pub fn new(status: StatusCode, code: impl Into, message: impl Into) -> Self { + Self { status, code: code.into(), message: message.into(), two_factor: None, retry_after_secs: None, name_taken: None } + } + + fn with_code(status: StatusCode, code: String, message: impl Into) -> Self { + Self { status, code, message: message.into(), two_factor: None, retry_after_secs: None, name_taken: None } } } impl IntoResponse for LocalError { fn into_response(self) -> Response { - (self.status, Json(ErrorBody { error: self.code.to_owned(), message: self.message, ..ErrorBody::default() })).into_response() + let extras = craft_cloud_proto::local::ErrorExtras { two_factor: self.two_factor, retry_after_secs: self.retry_after_secs }; + let mut body = serde_json::json!({ "error": self.code, "message": self.message }); + if let (Some(object), Ok(serde_json::Value::Object(more))) = (body.as_object_mut(), serde_json::to_value(&extras)) { + object.extend(more); + } + if let Some(name_taken) = self.name_taken + && let Some(object) = body.as_object_mut() + { + object.insert("name_taken".to_owned(), serde_json::Value::Bool(name_taken)); + } + let mut response = (self.status, Json(body)).into_response(); + if let Some(secs) = self.retry_after_secs { + response.headers_mut().insert(axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from(secs)); + } + response } } @@ -217,7 +282,33 @@ impl From for LocalError { // A setup that failed after the code was spent answers as what failed, with the whole message. if let AgentError::JoinSetup(inner) = e { let inner = LocalError::from(*inner); - return Self::new(inner.status, inner.code, message); + return Self::with_code(inner.status, inner.code, message); + } + // The second factor: a challenge, a wrong code, a lock and a server's refusal keep what they carry. None of them is a + // revoked device (that is `Revoked`, from a 401 on a signed-in call). + match &e { + AgentError::TwoFactorRequired(challenge) => { + let mut error = Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, message); + // `join` has no challenge to hand on: the same join goes again with the code. + error.two_factor = Some(challenge.clone()); + return error; + } + AgentError::TwoFactorInvalid(_) => return Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_INVALID, message), + AgentError::ChallengeGone => return Self::new(StatusCode::CONFLICT, codes::TWO_FACTOR_EXPIRED, message), + AgentError::TwoFactorLocked { retry_after_secs, .. } => { + let mut error = Self::new(StatusCode::TOO_MANY_REQUESTS, codes::TWO_FACTOR_LOCKED, message); + error.retry_after_secs = *retry_after_secs; + return error; + } + AgentError::Refused { status, code, retry_after_secs, .. } => { + // The server's own status and code (never a 401: that is `Revoked`). + let status = + StatusCode::from_u16(*status).ok().filter(|s| (s.is_client_error() || s.is_server_error()) && *s != StatusCode::UNAUTHORIZED); + let mut error = Self::new(status.unwrap_or(StatusCode::BAD_REQUEST), code.clone(), message); + error.retry_after_secs = *retry_after_secs; + return error; + } + _ => {} } let (status, code) = match &e { AgentError::WrongPassword => (StatusCode::UNAUTHORIZED, codes::WRONG_PASSWORD), @@ -233,6 +324,7 @@ impl From for LocalError { AgentError::NotSignedIn => (StatusCode::CONFLICT, codes::NOT_SIGNED_IN), AgentError::Offline(_) | AgentError::LogoutOffline(_) => (StatusCode::SERVICE_UNAVAILABLE, codes::OFFLINE), AgentError::Usage(m) if m.starts_with("already signed in") => (StatusCode::CONFLICT, codes::ALREADY_SIGNED_IN), + AgentError::Usage(m) if m == crate::spaces::CHANGING => (StatusCode::CONFLICT, codes::CONFLICT), AgentError::Usage(_) => (StatusCode::BAD_REQUEST, api_codes::INVALID), other if other.exit() == Exit::NeedsYou => (StatusCode::CONFLICT, codes::NEEDS_YOU), _ => (StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL), @@ -337,24 +429,44 @@ impl Drop for LocalApi { } } -type Api = Arc; +pub(crate) type Api = Arc; fn router(state: Api) -> Router { Router::new() .route(paths::STATUS, get(status)) .route(paths::LOGIN, post(login)) + .route(paths::LOGIN_SECOND_FACTOR, post(login_second_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route(paths::LOGOUT, post(logout)) + .route(paths::TWO_FACTOR, get(two_factor_status)) + .route(paths::TWO_FACTOR_SETUP, post(two_factor_setup).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_CONFIRM, post(two_factor_confirm).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_DISABLE, post(two_factor_disable).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_RECOVERY_CODES, post(two_factor_recovery_codes).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::ACCOUNT_PASSWORD, post(change_password).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::DEVICES_REVOKE_OTHERS, post(revoke_other_devices)) .route(paths::RECENTS, get(crate::live::recents_socket)) .route(paths::DOCS, get(crate::live::docs::docs_socket)) .route(paths::JOIN, post(join)) .route(paths::PAIRINGS, post(create_pairing)) .route(&format!("{}/{{id}}", paths::PAIRINGS), get(pairing_status).delete(delete_pairing)) .route(paths::DEVICES, get(devices)) - .route(&format!("{}/{{id}}", paths::DEVICES), delete(remove_device)) + .route(paths::USAGE, get(usage)) + .route(&format!("{}/{{id}}", paths::DEVICES), delete(remove_device).patch(rename_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::SPACES, get(spaces)) + .route(&format!("{}/{{id}}/accept", paths::SPACES), post(accept_space)) + .route(&format!("{}/{{id}}/decline", paths::SPACES), post(decline_space)) + .route(&format!("{}/{{id}}/sync", paths::SPACES), post(sync_space).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(&format!("{}/{{id}}/approve-deletes", paths::SPACES), post(approve_space_deletes).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route(paths::AUTOSTART, post(set_autostart)) .route(paths::PAUSE, post(pause)) .route(paths::RESUME, post(resume)) .route(paths::QUIT, post(quit)) + .merge(crate::admin::routes()) + .route(paths::FILE_HISTORY, get(file_history)) + .route(paths::FILE_RESTORE, post(file_restore)) + .route(paths::TRASH, get(trash)) + .route(paths::TRASH_RESTORE, post(trash_restore)) + .route(paths::TRASH_REMOVE, post(trash_remove)) .route(paths::BLOBS, axum::routing::put(put_blob).layer(DefaultBodyLimit::max(MAX_BLOB_BYTES))) .route(&format!("{}/{{hash}}", paths::BLOBS), get(get_blob)) .fallback(|| async { LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such call") }) @@ -363,6 +475,139 @@ fn router(state: Api) -> Router { .with_state(state) } +#[derive(serde::Deserialize)] +struct HistoryQuery { + path: String, + cursor: Option, + limit: Option, +} + +fn file_node(state: &ApiState, path: &str) -> Result { + if path.is_empty() || path.starts_with('/') || path.contains('\\') || path.contains(':') || path.as_bytes().contains(&0) { + return Err(LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "path must be relative to the sync root")); + } + let names = path + .split('/') + .map(craft_cloud_proto::Name::new) + .collect::, _>>() + .map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "path contains an invalid name"))?; + let saved = + DataDir::new(&state.data).load_state().map_err(|e| LocalError::new(StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL, e.to_string()))?; + let rel = craft_cloud_sync::RelPath::from_names(names); + let node = + saved.state.base_at(&rel).ok_or_else(|| LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no synced file at that path"))?; + if !matches!(saved.state.base.get(&node).map(|entry| &entry.kind), Some(craft_cloud_sync::BaseKind::File { .. })) { + return Err(LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "history is only available for files")); + } + Ok(node) +} + +fn history_error(error: crate::http::HistoryCallError) -> LocalError { + match error { + crate::http::HistoryCallError::Refused { status: StatusCode::UNAUTHORIZED, body } => { + LocalError::new(StatusCode::CONFLICT, codes::NEEDS_YOU, body.message) + } + crate::http::HistoryCallError::Refused { status, body } => { + let name_taken = body + .conflicts + .as_ref() + .map(|conflicts| conflicts.iter().any(|conflict| matches!(conflict.kind, craft_cloud_proto::ConflictKind::NameTaken { .. }))); + LocalError { status, code: body.error, message: body.message, name_taken, two_factor: None, retry_after_secs: None } + } + crate::http::HistoryCallError::Transport(craft_cloud_sync::TransportError::NotFound) => { + LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such file or version") + } + crate::http::HistoryCallError::Transport(craft_cloud_sync::TransportError::Unauthorized) => { + LocalError::new(StatusCode::CONFLICT, codes::NEEDS_YOU, "sign in again to continue") + } + crate::http::HistoryCallError::Transport(error) => AgentError::from(error).into(), + } +} + +async fn file_history( + State(state): State, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result, LocalError> { + let Query(query) = query.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let node = file_node(&state, &query.path)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.file_history(node, query.cursor.as_deref(), query.limit).await.map_err(history_error)?)) +} + +async fn file_restore( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let node = file_node(&state, &request.path)?; + let (_, transport) = server_api(&state)?; + let answer = transport.restore_file(node, request.version, request.expected_version).await.map_err(history_error)?; + state.control.request_pass(); + Ok(Json(answer)) +} + +#[derive(serde::Deserialize)] +struct TrashQuery { + cursor: Option, + limit: Option, +} + +fn trash_path_hint(saved: Option<&crate::account::SavedState>, parent: craft_cloud_proto::NodeId, name: &str) -> Option { + let saved = saved?; + let parent_path = saved.state.base_path(parent)?; + if parent_path.is_root() { Some(name.to_owned()) } else { Some(format!("{parent_path}/{name}")) } +} + +async fn trash( + State(state): State, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result, LocalError> { + let Query(query) = query.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let (_, transport) = server_api(&state)?; + let response = transport.trash(query.cursor.as_deref(), query.limit).await.map_err(history_error)?; + let saved = DataDir::new(&state.data).load_state().ok(); + Ok(Json(craft_cloud_proto::local::TrashResponse { + items: response + .items + .into_iter() + .map(|item| craft_cloud_proto::local::TrashItem { + path: trash_path_hint(saved.as_ref(), item.parent, &item.name), + id: item.id, + parent: item.parent, + name: item.name, + is_dir: item.is_dir, + size: item.size, + deleted_ms: item.deleted_ms, + purge_after_ms: item.purge_after_ms, + }) + .collect(), + next: response.next, + })) +} + +async fn trash_restore( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let id = request.id.parse().map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "id must be a node ID"))?; + let (_, transport) = server_api(&state)?; + let answer = transport.restore_from_trash(id).await.map_err(history_error)?; + state.control.request_pass(); + Ok(Json(answer)) +} + +async fn trash_remove( + State(state): State, + body: Result, JsonRejection>, +) -> Result { + let Json(request) = body?; + let id = request.id.parse().map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "id must be a node ID"))?; + let (_, transport) = server_api(&state)?; + transport.remove_from_trash(id).await.map_err(history_error)?; + Ok(StatusCode::NO_CONTENT) +} + /// Equal strings, compared in time that doesn't depend on where they differ. fn same_secret(a: &str, b: &str) -> bool { a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 @@ -408,6 +653,101 @@ async fn login(State(state): State, body: Result, JsonRe ask(&state, |reply| Command::Login(request, reply)).await } +/// Most bytes of the small bodies (a password, a code, a name): far less than a join string's megabyte. +const TINY_JSON_BYTES: usize = 4 * 1024; + +async fn login_second_factor( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + ask(&state, |reply| Command::LoginSecondFactor(request, reply)).await +} + +/// A call of the signed-in account to the server, its refusals kept whole: never mistaken for a revoked device. +fn account_call_error(e: crate::http::CallError) -> LocalError { + AgentError::from(e).into() +} + +async fn two_factor_status(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_status().await.map_err(account_call_error)?)) +} + +async fn two_factor_setup( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_setup(&request.password).await.map_err(account_call_error)?)) +} + +async fn two_factor_confirm(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_confirm(&request.totp_code).await.map_err(account_call_error)?)) +} + +async fn two_factor_disable(State(state): State, body: Result, JsonRejection>) -> Result { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + transport + .two_factor_disable(&craft_cloud_proto::api::TwoFactorProofRequest { + password: request.password, + totp_code: request.totp_code, + recovery_code: request.recovery_code, + }) + .await + .map_err(account_call_error)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn two_factor_recovery_codes( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let sent = craft_cloud_proto::api::TwoFactorProofRequest { + password: request.password, + totp_code: request.totp_code, + recovery_code: request.recovery_code, + }; + Ok(Json(transport.two_factor_recovery_codes(&sent).await.map_err(account_call_error)?)) +} + +async fn change_password(State(state): State, body: Result, JsonRejection>) -> Result { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let sent = craft_cloud_proto::api::ChangePasswordRequest { current_password: request.current_password, new_password: request.new_password }; + transport.change_password(&sent).await.map_err(account_call_error)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn revoke_other_devices(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.revoke_other_devices().await.map_err(account_call_error)?)) +} + +async fn rename_device( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let id = path_id(id)?; + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let renamed = transport.rename_device(id, &request.name).await.map_err(|e| match e { + // A device that isn't there (or isn't ours, or is signed out): the server's `not_found`. + crate::http::CallError::Transport(craft_cloud_sync::TransportError::NotFound) => { + LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such computer in this cloud") + } + other => account_call_error(other), + })?; + Ok(Json(renamed)) +} + async fn logout(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { let Json(request) = body?; ask(&state, |reply| Command::Logout(request, reply)).await @@ -475,14 +815,14 @@ async fn join(State(state): State, body: Result, JsonReje } /// The signed-in account's server API, for the calls the local API makes itself (they touch nothing here). -fn server_api(state: &ApiState) -> Result<(Account, HttpTransport), LocalError> { +pub(crate) fn server_api(state: &ApiState) -> Result<(Account, HttpTransport), LocalError> { let account = state.control.account().ok_or(AgentError::NotSignedIn)?; let transport = HttpTransport::new(account.server.clone(), &account.token).map_err(AgentError::from)?; Ok((account, transport)) } /// A path's ID, or `400 invalid`. -fn path_id(id: Result, PathRejection>) -> Result { +pub(crate) fn path_id(id: Result, PathRejection>) -> Result { id.map(|Path(id)| id).map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text())) } @@ -526,6 +866,12 @@ async fn devices(State(state): State) -> Result>, Loca Ok(Json(transport.devices().await.map_err(AgentError::from)?)) } +/// `GET /v1/usage`: asks the signed-in account's server each time; nothing is cached. +async fn usage(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.usage().await.map_err(AgentError::from)?)) +} + /// Removes (revokes) another computer of the account; answered with the list as it is then. This computer can't be /// removed this way: that is signing out. async fn remove_device(State(state): State, id: Result, PathRejection>) -> Result>, LocalError> { @@ -541,6 +887,110 @@ async fn remove_device(State(state): State, id: Result, Path Ok(Json(transport.devices().await.map_err(AgentError::from)?)) } +/// The personal space as `GET /v1/spaces` lists it: always syncing (to the account's folder), the account its owner. +fn personal_space(control: &Control, account: &Account) -> SpaceInfo { + let status = control.status(); + SpaceInfo { + space: account.space, + kind: "personal".into(), + name: None, + role: craft_cloud_proto::api::SpaceRole::Owner, + state: "active".into(), + archived: false, + folder: Some(account.folder.display().to_string()), + sync: SpaceSync::On, + run_state: Some(match status.run_state { + RunState::Retrying { message } => SpaceRun::Retrying { message }, + RunState::Paused { reason } => SpaceRun::Paused { reason }, + RunState::PausedByUser => SpaceRun::PausedByUser, + RunState::Syncing => SpaceRun::Syncing, + RunState::SignedOut | RunState::Idle | RunState::Stopping => SpaceRun::Idle, + }), + last_sync_ms: status.last_sync_ms, + read_only: false, + } +} + +/// `GET /v1/spaces`: the personal space, then the team spaces (invitations among them) as the agent last saw the server's list. +async fn spaces(State(state): State) -> Result, LocalError> { + let account = state.control.account().ok_or(AgentError::NotSignedIn)?; + let mut list = vec![personal_space(&state.control, &account)]; + list.extend(state.control.with(|s| s.spaces.clone())); + Ok(Json(SpaceList { spaces: list })) +} + +/// `POST /v1/spaces/{id}/accept`: passed on to the server; the agent looks at its list again before it answers. +async fn accept_space( + State(state): State, + id: Result, PathRejection>, +) -> Result, LocalError> { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + let accepted = transport.accept_space(id).await?; + if let Some(spaces) = state.control.spaces_handle() { + spaces.refresh().await; + } + Ok(Json(accepted)) +} + +/// `POST /v1/spaces/{id}/decline`: passed on to the server. +async fn decline_space(State(state): State, id: Result, PathRejection>) -> Result { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + transport.decline_space(id).await?; + if let Some(spaces) = state.control.spaces_handle() { + spaces.refresh().await; + } + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /v1/spaces/{id}/sync`: turns the sync of a team space on this computer on or off. +async fn sync_space( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let id = path_id(id)?; + let Json(request) = body?; + let spaces = state.control.spaces_handle().ok_or(AgentError::NotSignedIn)?; + Ok(Json(spaces.sync(id, request).await?)) +} + +/// `POST /v1/spaces/{id}/approve-deletes`: hands the approval of a synced team space's waiting deletions to this agent +/// (`spaces//approval.json`), by the rules of `sync --approve-deletes --space`: with a `set`, that set if it is the +/// one waiting; without, the set the previous call listed. +async fn approve_space_deletes( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let id = path_id(id)?; + let Json(request) = body?; + let approve = match request.set.as_deref().map(str::trim) { + None => crate::cli::Approve::Listed, + Some("") => { + return Err(LocalError::new( + StatusCode::BAD_REQUEST, + api_codes::INVALID, + "the set is empty; leave it out to approve the set last listed", + )); + } + Some(set) => crate::cli::Approve::Set(set.to_ascii_lowercase()), + }; + let data = DataDir::new(&state.data); + let answer = tokio::task::spawn_blocking(move || { + let account = data.load_account()?.ok_or(AgentError::NotSignedIn)?; + let setting = account.spaces.iter().find(|s| s.space == id && s.sync); + let Some(setting) = setting else { return Ok(None) }; + let mut out = Vec::new(); + let (_, _, approved) = crate::cli::approve_team(&data, setting, &approve, &mut out)?; + Ok::<_, AgentError>(Some(ApproveDeletesAnswer { approved, message: String::from_utf8_lossy(&out).trim().to_owned() })) + }) + .await + .map_err(|e| LocalError::new(StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL, e.to_string()))??; + answer.map(Json).ok_or_else(|| LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such team space syncing on this computer")) +} + /// Registers this agent to start at login, or removes the entry; the status after says which. async fn set_autostart(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { let Json(AutostartRequest { on }) = body?; @@ -596,6 +1046,7 @@ mod tests { assert_eq!(code(AgentError::LogoutOffline("x".into())), codes::OFFLINE); assert_eq!(code(AgentError::Usage("already signed in as ana".into())), codes::ALREADY_SIGNED_IN); assert_eq!(code(AgentError::Usage("not a URL".into())), api_codes::INVALID); + assert_eq!(code(AgentError::Usage(crate::spaces::CHANGING.into())), codes::CONFLICT); assert_eq!(code(AgentError::Revoked), codes::NEEDS_YOU); assert_eq!(code(AgentError::InvalidCode), codes::INVALID_CODE); assert_eq!(code(AgentError::PasswordRequired), codes::PASSWORD_REQUIRED); diff --git a/apps/craft-cloud-agent/src/root.rs b/apps/craft-cloud-agent/src/root.rs index 95f6b457..01ca378b 100644 --- a/apps/craft-cloud-agent/src/root.rs +++ b/apps/craft-cloud-agent/src/root.rs @@ -95,6 +95,31 @@ pub fn write(folder: &Path, marker: &RootMarker) -> Result<(), RootError> { write_atomic(&path, &json, false).map_err(|source| RootError::Io { path, source }) } +pub const NONCE_FILE: &str = "nonce"; +/// A nonce is 32 hex digits; anything much bigger is not ours. +const MAX_NONCE_BYTES: u64 = 128; + +/// The nonce `/.craft-cloud/nonce`, if the folder has one. The agent rewrites it with a new random value at every +/// checkpoint and records the same value in the space's `state.json`, so a folder restored from an older copy (which holds an +/// older nonce, or none) is not the folder the state was made for, even if its path and ID are the same. +pub fn read_nonce(folder: &Path) -> Option { + let path = folder.join(AGENT_DIR).join(NONCE_FILE); + let md = fs::symlink_metadata(&path).ok().filter(|md| md.file_type().is_file() && md.len() <= MAX_NONCE_BYTES)?; + let text = fs::read_to_string(&path).ok().filter(|_| md.len() > 0)?; + let text = text.trim(); + (!text.is_empty() && text.bytes().all(|b| b.is_ascii_hexdigit())).then(|| text.to_owned()) +} + +/// Writes a new random nonce into `/.craft-cloud/nonce` and returns it. +pub fn rotate_nonce(folder: &Path) -> io::Result { + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).map_err(|e| io::Error::other(e.to_string()))?; + let nonce: String = bytes.iter().map(|b| format!("{b:02x}")).collect(); + let path = folder.join(AGENT_DIR).join(NONCE_FILE); + write_atomic(&path, nonce.as_bytes(), false)?; + Ok(nonce) +} + /// Checks that `folder` is there and its marker names `expected`. pub fn check(folder: &Path, expected: &RootMarker) -> Result<(), RootError> { if !folder.is_dir() { diff --git a/apps/craft-cloud-agent/src/spaces.rs b/apps/craft-cloud-agent/src/spaces.rs new file mode 100644 index 00000000..1f5c4da2 --- /dev/null +++ b/apps/craft-cloud-agent/src/spaces.rs @@ -0,0 +1,1652 @@ +//! Team spaces on this computer (`craft-cloud-agent run`): finding out which spaces the account is in, one engine per +//! synced space, and what the local API says about them. +//! +//! - **Discovery**: `GET /spaces` is asked at the start, then every two long-poll waits, and at once when a change may +//! have happened (an invitation accepted or declined here, a space that answered `404`). A server that speaks +//! protocol 1, or has no `/spaces`, has no team spaces: the personal space syncs as before, the agent asks about the +//! server's version only (ten times less often), and a team space it remembers shows as unavailable. Nothing sends the +//! `craft-space` header to such a server. A failed discovery changes nothing; when someone waits for the list (an invitation +//! accepted or declined here, the sync turned on) it is tried again after a few seconds, for up to [`REFRESH_BOUND`], before +//! they are released with the old list. +//! - **Folders**: a team space syncs to `Craft Cloud Shared//` beside the personal folder. The name goes through the +//! node name rules (anything a file name can't hold becomes `_`), a name that is taken (by another space or by a folder +//! already there) gets ` (2)`, ` (3)`. The folder is chosen when the space is first seen and kept when the space is +//! renamed on the server: renaming a folder under running apps would break their open paths. +//! - **Off by default**: an accepted team space does not sync until the person turns it on ([`SpacesHandle::sync`]): +//! an administrator may add someone to a space of terabytes. Turning it on sets the folder up (a new or empty folder, +//! or one adopted with consent, with the space's root marker), and the sync begins. +//! - **One engine per space**: each synced space has its own session ([`crate::cli::Session::open_team`]): its own state in +//! `spaces//state.json`, root, marker, guards, scheduler, folder watcher and long-poll, all through a transport +//! bound to the space that refuses an answer without the space's echo. A problem in one space never pauses another. +//! - **A state belongs to its folder**: `state.json` records the space and the canonical folder it was made for, and is used +//! for that pair only. For any other folder the space starts afresh, as a first sync: what is there and what the server +//! has are merged, and nothing is deleted on either side. Changing a space's folder, or turning it off, first closes its +//! session and waits until the pass thread has let go of it, so no pass for the old folder can write a state after the +//! new folder is set up. The state also records the server's stamp of the account's membership (`joined_ms` in +//! `GET /spaces`, new when someone is removed and invited again): a state made under another stamp is not used either, +//! which covers an agent that was not running while access was lost and regained. A server without the stamp is not +//! compared. Turning a space off forgets its state: turned on again, in the same folder or another, it is a first sync. +//! Waiting for a session to close never holds up the task: other spaces, the local API and stopping go on, and the caller +//! is answered when the session is closed (or after [`CLOSE_WAIT`]). +//! - **A first sync merges** (a fresh state, whatever the reason): nothing is deleted on either side, because nothing is known +//! to have been deleted. A file that was deleted on the server meanwhile and is still in the folder is uploaded again. A +//! file that exists on both sides with different content is not overwritten: the local one is moved aside as a conflict +//! copy and the server's version takes its path. +//! - **A state is trusted only when nothing doubtful happened to it**: the setting's `generation` (saved in `account.json`) is +//! raised before a state is retired or a new folder's marker is written, and a state made under another generation is not +//! used; a state that can't be removed fails the turn-off; and a session whose folder now leads somewhere else (a link +//! pointed at another folder) is dropped with its state before the next pass. +//! A raised generation that can't be saved (account.json can't be written) is kept in memory and saved again at every look at +//! the list; that space is not started meanwhile (its state shows why), so a session that begins cannot use a state that was +//! meant to be dropped. Other spaces are not held back. +//! - **One pass at a time**: passes of the spaces run one after another on a thread of their own, and take turns with the +//! personal space's passes ([`PassGate`]). A pass that is queued when its space is closed (turned off, moved to another +//! folder, lost) or the agent stops is not run; the running pass finishes. A second change of a space that is still +//! being closed is refused ([`CHANGING`], `409` in the local API). +//! - **Losing access**: a space that is no longer listed (or answers `404`) stops syncing and is marked +//! [`SpaceRun::NoAccess`]. Nothing in its folder is touched. Its sync state is dropped once its session is closed, so if +//! access comes back the folder starts afresh and is merged with the space like a first sync: no local file is deleted +//! or replaced (a file that differs on both sides keeps both, one as a conflict copy), what was deleted here meanwhile is +//! not deleted on the server, and what was deleted on the server meanwhile is not deleted here. An archived space is not +//! lost: its members keep read access, so it stays listed and its engine is read-only. (The server lists it with the stored +//! role, an editor stays `editor`, although access is capped at viewer: the agent goes by `archived`.) +//! - **Mass-delete approval**: a team space's held deletions are approved through `spaces//approval.json`, which +//! `sync --approve-deletes --space` and `POST /v1/spaces/{id}/approve-deletes` write. +//! - **Read-only**: a role of viewer (or an archived space) puts the space's engine in read-only mode before its first +//! write; the status says so ([`SpaceStatus::read_only`]). + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use craft_cloud_proto::api::{SpaceListItem, SpaceRole}; +use craft_cloud_proto::local::{SpaceInfo, SpaceRun, SpaceStatus, SpaceSync, SpaceSyncRequest}; +use craft_cloud_proto::name::FORBIDDEN; +use craft_cloud_proto::{Name, NodeId, SpaceId}; +use craft_cloud_sync::TransportError; +use tokio::sync::{mpsc, oneshot, watch}; + +use crate::account::{Account, DataDir, LastPass, Outcome, PassSummary, SpaceSetting}; +use crate::cli::{AgentError, Exit, Session}; +use crate::daemon::{RunOptions, counts_as_change, pass_end, take_due_peeks}; +use crate::http::{CallError, DeviceToken, HttpTransport, PollError, ServerUrl}; +use crate::local_api::Control; +use crate::root::{self, RootMarker}; +use crate::schedule::{Ms, Next, Phase, Scheduler}; +use crate::settle::Stability; +use crate::watch::{FolderWatcher, WatchEvent}; + +/// The folder beside the personal one that holds the team spaces' folders. +pub const SHARED_FOLDER: &str = "Craft Cloud Shared"; +/// The first HTTP protocol with team spaces (`craft-space`, `/spaces`). +pub const TEAM_PROTOCOL: u32 = 2; +/// What a team space that is not synced here is told when the server is too old. +pub const NEEDS_UPDATE: &str = "the server needs an update for team spaces"; +/// The longest a local API call waits for the spaces task to answer. +const ANSWER_WAIT: Duration = Duration::from_secs(60); +/// The longest the task waits for a space's session to be closed before it changes the folder (shorter than +/// [`ANSWER_WAIT`], so the caller is answered). +const CLOSE_WAIT: Duration = Duration::from_secs(50); +/// What a change of a space's sync is told when another change of the same space waits for its session to close (the local +/// API answers `409`). +pub const CHANGING: &str = "this space is being changed, try again in a moment"; +/// What turning a space on is told when the space's last pass did not end within [`CLOSE_WAIT`]. +const STILL_FINISHING: &str = "this space is still finishing its last pass, try again in a moment"; +/// What a space that is held back until its raised generation is saved shows as its state. +const UNSAVED: &str = "this space's settings could not be saved yet; it starts again as soon as they can"; +/// How soon a discovery that failed is tried again while someone waits for it. +const DISCOVERY_RETRY: Duration = Duration::from_secs(3); +/// The longest those waiting for a discovery are kept waiting for one that works (they are released after that, with the old +/// list). +const REFRESH_BOUND: Duration = Duration::from_secs(15); +/// The longest folder name made from a space's name, in characters. +const MAX_FOLDER_CHARS: usize = 100; + +/// Takes turns between the personal space's passes and the team spaces' ones: whoever holds it runs a pass. +#[derive(Clone, Default)] +pub struct PassGate(Arc>); + +impl PassGate { + /// Waits for the turn; the pass runs while the guard lives. + pub fn hold(&self) -> MutexGuard<'_, ()> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Folders + +/// The folder name for a team space: its name with what a file name can't hold (and control characters) made `_`, no dots +/// or spaces at the ends, cut to a sane length, and never a reserved name. +pub fn folder_name(space_name: &str) -> String { + let cleaned: String = space_name.chars().map(|c| if c.is_control() || FORBIDDEN.contains(&c) { '_' } else { c }).collect(); + let mut name: String = cleaned.trim().trim_end_matches(['.', ' ']).chars().take(MAX_FOLDER_CHARS).collect(); + // A leading dot would hide the folder (and `.craft-cloud` is the agent's own). + if name.starts_with('.') { + name.replace_range(..1, "_"); + } + if name.is_empty() { + return "Team space".to_owned(); + } + for candidate in [name.clone(), format!("{name} (team space)")] { + if let Ok(valid) = Name::new(&candidate) { + return valid.as_str().to_owned(); + } + } + "Team space".to_owned() +} + +/// How two folder names are told apart: as the node rules do, so names that differ only by case or by normalisation are one. +fn key(name: &str) -> String { + Name::new(name).map_or_else(|_| name.to_lowercase(), |n| n.fold_key().as_str().to_owned()) +} + +/// A folder in `shared` for a space called `space_name` that nothing uses: not in `taken` (the folders of the spaces this +/// computer knows) and not already on disk. A name that is taken gets ` (2)`, ` (3)` and so on. +pub fn free_folder(shared: &Path, space_name: &str, taken: &[PathBuf]) -> PathBuf { + let base = folder_name(space_name); + let on_disk: BTreeSet = std::fs::read_dir(shared) + .map(|entries| entries.filter_map(Result::ok).map(|e| key(&e.file_name().to_string_lossy())).collect()) + .unwrap_or_default(); + let used = |candidate: &str| { + let wanted = key(candidate); + on_disk.contains(&wanted) + || taken.iter().any(|t| t.parent() == Some(shared) && t.file_name().is_some_and(|n| key(&n.to_string_lossy()) == wanted)) + }; + for n in 1..=1_000u32 { + let candidate = if n == 1 { base.clone() } else { format!("{base} ({n})") }; + if !used(&candidate) { + return shared.join(candidate); + } + } + shared.join(format!("{base} ({})", SpaceId::generate())) +} + +/// Where the team spaces' folders go: `Craft Cloud Shared` beside the personal folder. +pub fn shared_root(personal: &Path) -> Option { + personal.parent().filter(|p| p.parent().is_some() || p.file_name().is_some()).map(|p| p.join(SHARED_FOLDER)) +} + +/// What a discovery changed in the settings. +#[derive(Debug, Clone, PartialEq)] +pub struct Reconciled { + /// The settings after: one per team space the server lists as active (new ones with syncing off and a folder), and the + /// ones for spaces it no longer lists, unchanged. + pub settings: Vec, + /// Spaces that were renamed on the server (their folders keep their names). + pub renamed: Vec<(SpaceId, String, String)>, +} + +/// Settings after the server listed the account's spaces. `shared` is where new folders go (`None`: no place for them; a +/// space without a folder is given one when it is turned on). +pub fn reconcile(settings: &[SpaceSetting], listed: &[SpaceListItem], shared: Option<&Path>) -> Reconciled { + let mut settings = settings.to_vec(); + let mut renamed = Vec::new(); + for item in listed.iter().filter(|i| i.kind == "team" && i.state == "active") { + let name = item.name.clone().unwrap_or_default(); + if let Some(setting) = settings.iter_mut().find(|s| s.space == item.id) { + if let Some(old) = setting.name.replace(name.clone()).filter(|old| *old != name) { + renamed.push((item.id, old, name)); + } + if item.root.is_some() { + setting.root = item.root; + } + } else if let Some(shared) = shared { + let taken: Vec = settings.iter().map(|s| s.folder.clone()).collect(); + settings.push(SpaceSetting { + space: item.id, + folder: free_folder(shared, &name, &taken), + sync: false, + name: Some(name), + root: item.root, + generation: 0, + }); + } + } + Reconciled { settings, renamed } +} + +// --------------------------------------------------------------------------------------------------------------------- +// The handle the local API uses + +enum Msg { + /// Discover now; answered once it is done. + Refresh(Option>), + Sync(SpaceId, SpaceSyncRequest, oneshot::Sender>), + Paused(bool), + /// No new pass starts (the agent is stopping). + Quiesce, + Stop, + /// A second stop: no queued pass starts and the pass thread is not waited for. + StopNow, +} + +/// What the local API calls on the spaces task. Cheap to clone. +#[derive(Clone)] +pub struct SpacesHandle { + tx: mpsc::UnboundedSender, +} + +impl SpacesHandle { + /// Looks at the server's list of spaces now, and returns once that is done (or the wait ran out). + pub async fn refresh(&self) { + let (tx, rx) = oneshot::channel(); + if self.tx.send(Msg::Refresh(Some(tx))).is_ok() { + let _ = tokio::time::timeout(ANSWER_WAIT, rx).await; + } + } + + /// Turns the sync of a team space on this computer on or off ([`SpaceSyncRequest`]). + pub async fn sync(&self, space: SpaceId, request: SpaceSyncRequest) -> Result { + let gone = || AgentError::Usage("the agent is stopping".into()); + let (tx, rx) = oneshot::channel(); + self.tx.send(Msg::Sync(space, request, tx)).map_err(|_| gone())?; + match tokio::time::timeout(ANSWER_WAIT, rx).await { + Ok(Ok(answer)) => answer, + _ => Err(gone()), + } + } + + /// Syncing is paused by the user (or resumed): no pass of a team space starts meanwhile. + pub fn set_paused(&self, paused: bool) { + let _ = self.tx.send(Msg::Paused(paused)); + } + + /// The agent is stopping: no new pass starts. + pub fn quiesce(&self) { + let _ = self.tx.send(Msg::Quiesce); + } +} + +/// What starts the spaces task. +pub struct SpacesConfig { + pub data: DataDir, + pub account: Account, + pub options: RunOptions, + pub control: Control, + pub gate: PassGate, + pub paused: bool, +} + +/// The running spaces task and the thread that runs the team spaces' passes. +pub struct Spaces { + handle: SpacesHandle, + task: tokio::task::JoinHandle<()>, +} + +impl Spaces { + /// Starts looking after the account's team spaces. The first discovery runs at once. + pub fn start(config: SpacesConfig) -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + let handle = SpacesHandle { tx }; + let task = tokio::spawn(Manager::new(config).run(rx)); + Self { handle, task } + } + + pub fn handle(&self) -> SpacesHandle { + self.handle.clone() + } + + /// Stops the task: a running pass finishes first, then every folder is let go of. With `at_once` (the second stop) no + /// queued pass starts and the thread of the passes is not waited for: it ends with the pass it runs. + pub async fn stop(self, at_once: bool) { + let _ = self.handle.tx.send(if at_once { Msg::StopNow } else { Msg::Stop }); + let _ = self.task.await; + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// The passes' thread + +/// One pass of a team space, asked of the thread. +struct Job { + setting: SpaceSetting, + stability: Stability, + /// The role is viewer (or the space is archived): the engine is told before its first write. + read_only: bool, + approve: Vec, + /// The server's stamp of the account's membership (see [`SpaceListItem::joined_ms`]) that the saved state must have been + /// made under. + joined_ms: Option, + /// Which running of the space's sync asked (see [`Active::run`]), so a result that comes after the sync was stopped or + /// moved to another folder is not taken for the current one's. + run: u64, +} + +enum ToThread { + Pass(Box), + /// Let go of the space's folder (its session ends), and with `forget` remove its saved state. `ack` is told once that + /// is done: no pass of the space runs or writes its state after that. + Close { + space: SpaceId, + forget: bool, + ack: Option>>, + }, +} + +/// A finished pass, from the thread. +struct Done { + space: SpaceId, + run: u64, + result: Result, + last_pass: Option, + seq: Option, + recheck_in: Option, +} + +/// What reaches the spaces task besides its messages. +enum Ev { + Watch(SpaceId, u64, WatchEvent), + WatcherGone(SpaceId, u64), + /// The space's change feed moved, or can't continue (a pass sorts it out). + Remote(SpaceId), + /// The long-poll got an answer after failing. + Back(SpaceId), + /// The long-poll was told `404`: the space may be gone for this account (a discovery finds out). + Gone, + /// A discovery ended. + Listed(Looked), + Pass(Box), + /// The wait for a space's session to close ended, for the change asked under this number. + Closed(u64, Closure), + /// The pass thread found that a space's folder leads somewhere else than its session was opened on, and dropped the + /// session: the space's generation is raised and saved before its next pass. + FolderMoved(SpaceId), +} + +/// How the wait for a space's session to close ended. +enum Closure { + /// The session is closed (and the state forgotten if that was asked). + Closed, + /// The pass thread did not let go within [`CLOSE_WAIT`]. + TimedOut, + /// The session is closed but its saved state could not be removed. + NotForgotten(String), +} + +/// The runnings of the sync ([`Active::run`]) that are current: a queued pass of any other is not started. +type Live = Arc>>; + +fn live_set(live: &Live) -> MutexGuard<'_, BTreeSet> { + live.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// The thread of the team spaces' passes: a session per space, each opened by its first pass, ended when it needs the user +/// or the space is closed. Runs a pass while it holds the turn ([`PassGate`]); a pass whose space was closed since it was +/// queued (turned off, moved, lost, or the agent is stopping) is not run. +fn pass_thread( + data: DataDir, + account: Account, + gate: PassGate, + live: Live, + jobs: std::sync::mpsc::Receiver, + events: mpsc::UnboundedSender, +) { + let Ok(runtime) = tokio::runtime::Builder::new_current_thread().enable_all().build() else { return }; + let mut sessions: Vec<(SpaceId, Session)> = Vec::new(); + while let Ok(job) = jobs.recv() { + let job = match job { + ToThread::Close { space, forget, ack } => { + sessions.retain(|(id, _)| *id != space); + let forgotten = if forget { data.space_dir(space).remove_state().map_err(|e| e.to_string()) } else { Ok(()) }; + if let Err(e) = &forgotten { + crate::runlog::note(format_args!("can't forget the saved state of team space {space}: {e}")); + } + if let Some(ack) = ack { + let _ = ack.send(forgotten); + } + continue; + } + ToThread::Pass(job) => *job, + }; + let space = job.setting.space; + let _turn = gate.hold(); + // The space was closed (turned off, moved to another folder, lost) after this pass was queued: it is not run. The + // result is reported all the same, so the task knows the thread is free. + if !live_set(&live).contains(&job.run) { + let done = Done { space, run: job.run, result: Ok(PassSummary::default()), last_pass: None, seq: None, recheck_in: None }; + if events.send(Ev::Pass(Box::new(done))).is_err() { + return; + } + continue; + } + // A session is for one folder and one membership: the space's folder may have changed since it opened, or the account + // may have been removed from the space and invited again (its state is then not used, see `load_state_for`). + if sessions.iter().any(|(id, s)| { + *id == space && (s.account().folder != job.setting.folder || s.joined_ms().zip(job.joined_ms).is_some_and(|(a, b)| a != b)) + }) { + sessions.retain(|(id, _)| *id != space); + } + // The folder may lead somewhere else than when the session opened (a link pointed at another folder): the session's + // state is about the old one, so it is dropped and the space starts afresh there, as a first sync. The task is told + // first, so it raises and saves the space's generation: a state that can't be removed is not trusted again either. + // This pass is not run; the next one starts under the new generation. + if sessions.iter().any(|(id, s)| *id == space && s.folder_moved()) { + sessions.retain(|(id, _)| *id != space); + crate::runlog::note(format_args!("the folder of team space {space} leads somewhere else now: starting afresh, nothing is deleted")); + let removed = data.space_dir(space).remove_state(); + let _ = events.send(Ev::FolderMoved(space)); + let why = match removed { + Ok(()) => "the folder of this space leads somewhere else now; the next pass starts afresh there".to_owned(), + Err(e) => format!( + "the folder of this space leads somewhere else now, and its saved state could not be removed ({e}); the next pass starts afresh there" + ), + }; + let done = Done { space, run: job.run, result: Err(AgentError::Usage(why)), last_pass: None, seq: None, recheck_in: None }; + if events.send(Ev::Pass(Box::new(done))).is_err() { + return; + } + continue; + } + let index = match sessions.iter().position(|(id, _)| *id == space) { + Some(index) => Ok(index), + None => Session::open_team(&data, &account, &job.setting, job.joined_ms, Some(job.stability.clone())).map(|session| { + sessions.push((space, session)); + sessions.len() - 1 + }), + }; + let result = match index { + Ok(index) => match sessions.get_mut(index) { + Some((_, session)) => { + if job.read_only { + session.set_read_only(true); + } + runtime.block_on(session.pass(job.approve)) + } + None => Err(AgentError::Usage("the space's session is gone".into())), + }, + Err(e) => Err(e), + }; + // A stop that needs the user (the folder went, say) opens the folder afresh next time. + if result.as_ref().is_err_and(|e| e.exit() == Exit::NeedsYou) { + sessions.retain(|(id, _)| *id != space); + } + let session = sessions.iter().find(|(id, _)| *id == space).map(|(_, s)| s); + let last_pass = session.and_then(Session::last_pass).or_else(|| { + data.space_dir(space) + .load_state_for(space, &job.setting.folder, job.joined_ms, job.setting.generation) + .ok() + .and_then(|(s, _)| s.last_pass) + }); + let seq = session.filter(|s| s.state().space.is_some()).map(|s| s.state().seq); + let recheck_in = session.and_then(Session::recheck_in); + if events.send(Ev::Pass(Box::new(Done { space, run: job.run, result, last_pass, seq, recheck_in }))).is_err() { + return; + } + } +} + +/// The long-poll of a team space: waits for the space's change feed to move past what the passes read, and tells the task. +/// Backs off on errors as the personal long-poll does, and says when the server answered `404` (the account may have +/// lost the space). It keeps time on tokio's clock. +async fn team_poll( + transport: HttpTransport, + space: SpaceId, + mut since: watch::Receiver>, + events: mpsc::UnboundedSender, + options: RunOptions, +) { + let mut known: Option = None; + let mut failures: u32 = 0; + let mut last_start: Option = None; + loop { + if let Some(last) = last_start { + let gap = options.poll_min_spacing.saturating_sub(last.elapsed()); + if !gap.is_zero() { + tokio::time::sleep(gap).await; + } + } + let passes = *since.borrow_and_update(); + let from = match (passes, known) { + (Some(passes), Some(seen)) => passes.max(seen), + (Some(passes), None) => passes, + (None, _) => { + if since.changed().await.is_err() { + return; + } + continue; + } + }; + last_start = Some(tokio::time::Instant::now()); + let tell = |ev: Ev| events.send(ev).is_ok(); + let pause = match transport.wait_for_changes(from, options.poll_wait).await { + Ok(seq) => { + if failures > 0 && !tell(Ev::Back(space)) { + return; + } + failures = 0; + if seq != from { + known = Some(seq); + if !tell(Ev::Remote(space)) { + return; + } + } + None + } + Err(PollError::Busy(after)) => Some(after.unwrap_or(options.poll_busy)), + Err(PollError::Resync) => { + known = None; + if !tell(Ev::Remote(space)) { + return; + } + failures = failures.saturating_add(1); + Some(backoff(&options, failures)) + } + Err(PollError::Failed(error)) => { + if error == TransportError::NotFound && !tell(Ev::Gone) { + return; + } + failures = failures.saturating_add(1); + Some(backoff(&options, failures)) + } + }; + if let Some(pause) = pause { + tokio::time::sleep(pause).await; + } + } +} + +/// The delay after the `failures`-th failed long-poll in a row: doubled each time, up to the cap. +fn backoff(options: &RunOptions, failures: u32) -> Duration { + let first = options.poll_retry_first.max(Duration::from_millis(1)); + first.saturating_mul(1u32.checked_shl(failures.saturating_sub(1).min(20)).unwrap_or(u32::MAX)).min(options.poll_retry_max) +} + +// --------------------------------------------------------------------------------------------------------------------- +// The task + +/// What a discovery found. +enum Looked { + /// The server has no team spaces (protocol 1, or no `/spaces`). + Unsupported, + /// The team spaces of the account, as the server lists them (invitations among them). + Listed(Vec), + /// It couldn't be found out (offline, a server error): nothing changes. + Failed(String), +} + +/// Asks the server about team spaces: its version first, then the list. A `404` for the list from a server that said it speaks +/// team spaces (a proxy while the server restarts, say) is a failed look, not the end of team spaces: the version is asked +/// once more, and only a server that now says protocol 1 is taken to have none. +async fn look(server: ServerUrl, token: DeviceToken) -> Looked { + match crate::http::info(&server).await { + Ok(info) if info.protocol_max >= TEAM_PROTOCOL => {} + Ok(_) => return Looked::Unsupported, + Err(e) => return Looked::Failed(e.to_string()), + } + let transport = match HttpTransport::new(server.clone(), &token) { + Ok(transport) => transport, + Err(e) => return Looked::Failed(e.to_string()), + }; + match transport.spaces().await { + Ok(list) => Looked::Listed(list.spaces.into_iter().filter(|i| i.kind == "team").collect()), + Err(e @ (CallError::Transport(TransportError::NotFound) | CallError::Refused(crate::http::Refusal { status: 404, .. }))) => { + match crate::http::info(&server).await { + Ok(info) if info.protocol_max < TEAM_PROTOCOL => Looked::Unsupported, + Ok(_) => Looked::Failed(e.to_string()), + Err(again) => Looked::Failed(again.to_string()), + } + } + Err(e) => Looked::Failed(e.to_string()), + } +} + +/// A synced team space's running parts. +struct Active { + scheduler: Scheduler, + stability: Stability, + roots: Vec, + watcher: Option, + /// Counts the watchers started, so the end of an old one's events isn't taken for the current one stopping. + generation: u64, + /// Which running of the space's sync this is (a new number each time one is started), echoed by its passes' results. + run: u64, + /// The long-poll; `None` while syncing is paused by the user. + poll: Option>, + since: watch::Sender>, + approved: Vec, + /// What the long-poll is started again with. + polling: Polling, + /// Where the pass thread looks up whether [`Active::run`] is still current; this running is taken out when it ends. + live: Live, +} + +/// What a long-poll of a team space is made of. +struct Polling { + transport: HttpTransport, + space: SpaceId, + events: mpsc::UnboundedSender, + options: RunOptions, +} + +impl Active { + /// Starts the long-poll if it isn't running. + fn start_poll(&mut self) { + if self.poll.is_none() { + let Polling { transport, space, events, options } = &self.polling; + self.poll = Some(tokio::spawn(team_poll(transport.clone(), *space, self.since.subscribe(), events.clone(), *options))); + } + } + + /// Ends the long-poll (syncing is paused). + fn stop_poll(&mut self) { + if let Some(poll) = self.poll.take() { + poll.abort(); + } + } +} + +impl Drop for Active { + fn drop(&mut self) { + live_set(&self.live).remove(&self.run); + self.stop_poll(); + } +} + +/// A team space this computer knows: what is saved of it, and how its sync goes. +struct Team { + setting: SpaceSetting, + active: Option, + /// The server no longer lists the space as the account's (or answered `404`). + lost: bool, + read_only: bool, + last_sync_ms: Option, +} + +/// A change of a space's sync that waits for its session to close. +struct Pending { + id: u64, + space: SpaceId, + request: SpaceSyncRequest, + reply: oneshot::Sender>, +} + +/// What applying a change of a space's sync came to. +enum Applied { + Done(Box), + /// The space's session must be closed first (and its saved state forgotten with `forget`). + NeedsClose { + forget: bool, + }, +} + +struct Manager { + data: DataDir, + account: Account, + options: RunOptions, + control: Control, + start: Instant, + teams: Vec, + /// The server's list at the last discovery (team spaces only). + listed: Vec, + /// Whether the server does team spaces (`None` until the first discovery got an answer). + supported: Option, + next_discovery: Ms, + /// The discovery under way (it runs apart from the task, so a server that doesn't answer never holds it up). + discovery: Option>, + /// Who waits for the discovery under way, and for the one after it (asked for while that one ran). + waiting: Vec>, + waiting_next: Vec>, + /// The last discovery failure that was noted, so a server that stays down is noted once. + noted: Option, + events: mpsc::UnboundedReceiver, + events_tx: mpsc::UnboundedSender, + jobs: std::sync::mpsc::Sender, + thread: Option>, + /// The current runnings of the spaces' syncs (see [`Active::live`]). + live: Live, + /// Turning a space off or moving it waits for its session to close, apart from the task: those under way. + pending: Vec, + /// The spaces whose raised generation could not be saved (account.json can't be written): it is saved again at every look at + /// the list, and these spaces are not started meanwhile (the status says why). Other spaces and running syncs go on. Without + /// this, a restart before the save would find the old generation on disk and trust a state that was meant to be dropped. + unsaved: BTreeSet, + /// The last number given to a change in `pending`. + changes: u64, + /// When those waiting for a discovery stop being kept waiting for one that works. + refresh_until: Option, + /// The last number given to a running of a space's sync ([`Active::run`]). + runs: u64, + /// The space whose pass runs now. + in_flight: Option, + /// Where the next pass is looked for first, so spaces take turns. + turn: usize, + paused: bool, + stopping: bool, +} + +/// A random number for jitter and backoff (the spaces' schedulers only use it to spread retries out). +fn random() -> u64 { + let mut bytes = [0u8; 8]; + let _ = getrandom::fill(&mut bytes); + u64::from_le_bytes(bytes) +} + +fn ms(d: Duration) -> Ms { + u64::try_from(d.as_millis()).unwrap_or(u64::MAX) +} + +impl Manager { + fn new(config: SpacesConfig) -> Self { + let SpacesConfig { data, account, options, control, gate, paused } = config; + let (events_tx, events) = mpsc::unbounded_channel(); + let (jobs, jobs_rx) = std::sync::mpsc::channel(); + let (thread_data, thread_account, thread_events) = (data.clone(), account.clone(), events_tx.clone()); + let live = Live::default(); + let thread_live = live.clone(); + let thread = std::thread::Builder::new() + .name("craft-cloud-spaces".into()) + .spawn(move || pass_thread(thread_data, thread_account, gate, thread_live, jobs_rx, thread_events)) + .ok(); + let teams = account + .spaces + .iter() + .map(|setting| Team { setting: setting.clone(), active: None, lost: false, read_only: false, last_sync_ms: None }) + .collect(); + Self { + data, + account, + options, + control, + start: Instant::now(), + teams, + listed: Vec::new(), + supported: None, + next_discovery: 0, + discovery: None, + waiting: Vec::new(), + waiting_next: Vec::new(), + noted: None, + events, + events_tx, + jobs, + thread, + live, + pending: Vec::new(), + unsaved: BTreeSet::new(), + changes: 0, + refresh_until: None, + runs: 0, + in_flight: None, + turn: 0, + paused, + stopping: false, + } + } + + fn now(&self) -> Ms { + ms(self.start.elapsed()) + } + + /// How often the server's list is looked at: every two long-poll waits (ten times less often when the server has no + /// team spaces). + fn discovery_every(&self) -> Ms { + let every = ms(self.options.poll_wait).saturating_mul(2).max(1_000); + if self.supported == Some(false) { every.saturating_mul(10) } else { every } + } + + async fn run(mut self, mut rx: mpsc::UnboundedReceiver) { + self.publish(); + let mut at_once = false; + loop { + let now = self.now(); + if now >= self.next_discovery { + self.start_discovery(); + } + let wake = self.drive(self.now()).min(self.next_discovery); + let sleep = tokio::time::sleep(Duration::from_millis(wake.saturating_sub(self.now()))); + tokio::select! { + msg = rx.recv() => match msg { + None | Some(Msg::Stop) => break, + Some(Msg::StopNow) => { + at_once = true; + break; + } + Some(msg) => self.message(msg), + }, + event = self.events.recv() => { + if let Some(event) = event { + self.event(event); + } + } + () = sleep => {} + } + } + self.finish(at_once).await; + } + + /// Lets go of every folder: a running pass finishes, then the thread ends. Stopping `at_once`, the thread is left to end + /// by itself (with the pass it runs, if any) and no queued pass starts, as the personal pass thread is treated. + async fn finish(mut self, at_once: bool) { + for team in &mut self.teams { + team.active = None; + } + // A space being turned off when the agent stops stays off: its `Close` (which forgets the state) is queued already. + let mut turned_off = false; + for pending in std::mem::take(&mut self.pending).into_iter().filter(|p| !p.request.on) { + if let Some(setting) = self.account.spaces.iter_mut().find(|s| s.space == pending.space) { + setting.sync = false; + turned_off = true; + } + } + if turned_off && let Err(e) = self.data.save_account(&self.account) { + crate::runlog::note(format_args!("can't remember the team spaces: {e}")); + } + if let Some(discovery) = self.discovery.take() { + discovery.abort(); + } + let Self { jobs, thread, .. } = self; + drop(jobs); + if let Some(thread) = thread + && !at_once + { + let _ = tokio::task::spawn_blocking(move || thread.join()).await; + } + } + + fn message(&mut self, msg: Msg) { + match msg { + Msg::Refresh(done) => { + // A discovery already under way may have started before the change that asks for this one: the answer + // waits for the one after it. + if done.is_some() && self.refresh_until.is_none() { + self.refresh_until = Some(self.now().saturating_add(ms(REFRESH_BOUND))); + } + match (self.discovery.is_some(), done) { + (true, Some(done)) => self.waiting_next.push(done), + (false, Some(done)) => { + self.waiting.push(done); + self.start_discovery(); + } + (true, None) => {} + (false, None) => self.start_discovery(), + } + } + Msg::Sync(space, request, reply) => self.set_sync(space, request, reply), + Msg::Paused(paused) => { + self.paused = paused; + // The long-polls rest while syncing is paused, as the personal one does, and start again on resume. + for active in self.teams.iter_mut().filter_map(|t| t.active.as_mut()) { + if paused { + active.stop_poll(); + } else { + active.start_poll(); + active.scheduler.urgent_change(); + } + } + self.publish(); + } + Msg::Quiesce => self.stopping = true, + Msg::Stop | Msg::StopNow => {} + } + } + + fn event(&mut self, event: Ev) { + let now = self.now(); + match event { + Ev::Watch(space, generation, WatchEvent::Changed { paths, closed }) => { + if let Some(active) = self.active_mut(space).filter(|a| a.generation == generation) + && counts_as_change(&active.roots, &active.stability, &paths, closed) + { + active.scheduler.local_change(now); + } + } + Ev::Watch(space, generation, WatchEvent::Lost(why)) => { + crate::runlog::note(format_args!("the folder watcher of a team space lost track ({why}): looking at the whole folder")); + if let Some(active) = self.active_mut(space).filter(|a| a.generation == generation) { + active.scheduler.urgent_change(); + active.watcher = None; + active.scheduler.set_watching(false); + } + } + Ev::WatcherGone(space, generation) => { + if let Some(active) = self.active_mut(space).filter(|a| a.generation == generation && a.watcher.is_some()) { + active.scheduler.urgent_change(); + active.watcher = None; + active.scheduler.set_watching(false); + } + } + Ev::Remote(space) => { + if let Some(active) = self.active_mut(space) { + active.scheduler.urgent_change(); + } + } + Ev::Back(space) => { + if let Some(active) = self.active_mut(space) { + active.scheduler.server_back(); + } + } + // Looked at by a discovery at once: it lists the space or it doesn't. + Ev::Gone => self.next_discovery = 0, + Ev::Listed(looked) => self.listed(looked), + Ev::Pass(done) => self.pass_done(*done, now), + Ev::Closed(id, closed) => self.closed(id, closed), + Ev::FolderMoved(space) => { + // The folder leads somewhere else: what is watched (and what counts as inside it) is looked at again, under a + // new generation so the old watcher's last events are ignored. + if let Some(team) = self.teams.iter_mut().find(|t| t.setting.space == space) + && let Some(active) = team.active.as_mut() + { + active.generation += 1; + active.roots = crate::watch::roots(&team.setting.folder); + active.watcher = start_watcher(&team.setting.folder, space, active.generation, &active.polling.events); + active.scheduler.set_watching(active.watcher.is_some()); + } + if let Err(e) = self.raise_generation(space) { + crate::runlog::note(format_args!( + "can't save the raised generation of team space {space} yet (tried again at the next look): {e}" + )); + } + self.settled(); + } + } + } + + fn active_mut(&mut self, space: SpaceId) -> Option<&mut Active> { + self.teams.iter_mut().find(|t| t.setting.space == space).and_then(|t| t.active.as_mut()) + } + + fn pass_done(&mut self, done: Done, now: Ms) { + if self.in_flight == Some(done.space) { + self.in_flight = None; + } + let Some(team) = self.teams.iter_mut().find(|t| t.setting.space == done.space) else { return }; + // A `404` may mean the account has no access any more: a discovery looks at once, and stops the space if the + // server no longer lists it. Until then the pass counts as failed (retried with a growing delay), so a `404` that + // means something else (a chunk the server lacks) never makes a loop. + if matches!(done.result, Err(AgentError::Server(TransportError::NotFound))) { + self.next_discovery = 0; + } + // The result of a pass of a sync that was stopped (or moved to another folder) since is not the current one's. + let Some(active) = team.active.as_mut().filter(|a| a.run == done.run) else { + self.publish(); + return; + }; + if let Some(seq) = done.seq { + let _ = active.since.send(Some(seq)); + } + let pass = pass_end(&done.result, done.recheck_in); + if let Some(last) = &done.last_pass + && !matches!(last.outcome, Outcome::Failed(_)) + { + team.last_sync_ms = Some(last.finished_ms); + } + if let Err(e) = &done.result { + crate::runlog::note(format_args!("a team space's pass failed: {e}")); + } + active.scheduler.pass_finished(now, pass); + self.publish(); + } + + /// Starts the pass that is due, if none runs; returns when to look again. + fn drive(&mut self, now: Ms) -> Ms { + let mut wake = Ms::MAX; + let count = self.teams.len(); + if count == 0 || self.stopping { + return wake; + } + for offset in 0..count { + let index = (self.turn + offset) % count; + let mut start: Option<(usize, Box)> = None; + let paused = self.paused; + let busy = self.in_flight.is_some(); + let joined_ms = self.teams.get(index).and_then(|t| self.item_of(t.setting.space)).and_then(|i| i.joined_ms); + let Some(team) = self.teams.get_mut(index) else { continue }; + let read_only = team.read_only; + let setting = team.setting.clone(); + let Some(active) = team.active.as_mut() else { continue }; + if take_due_peeks(&setting.folder, &active.stability) { + active.scheduler.local_change(now); + } + if paused { + continue; + } + match active.scheduler.next(now) { + Next::RunPass if !busy => { + active.scheduler.pass_started(); + let approve = std::mem::take(&mut active.approved); + start = + Some((index, Box::new(Job { setting, stability: active.stability.clone(), read_only, approve, joined_ms, run: active.run }))); + } + Next::RunPass => {} + Next::CheckApproval => { + let state_dir = self.data.space_dir(setting.space); + let summary = active_summary(&state_dir, &setting); + let waiting = [summary.as_ref().and_then(PassSummary::pending_set), summary.as_ref().and_then(PassSummary::held_set)]; + if let Ok(Some(approval)) = state_dir.take_approval() + && waiting.contains(&Some(approval.set.clone())) + { + active.approved.extend(approval.nodes); + active.scheduler.urgent_change(); + } + active.scheduler.approval_checked(now); + wake = wake.min(now); + } + Next::Wait(until) => wake = wake.min(until.unwrap_or(Ms::MAX)), + Next::Exit => {} + } + if let Some(peek) = active.stability.next_peek_in() { + wake = wake.min(now.saturating_add(ms(peek))); + } + if let Some((index, job)) = start { + self.in_flight = Some(job.setting.space); + self.turn = index + 1; + if self.jobs.send(ToThread::Pass(job)).is_err() { + self.in_flight = None; + } + self.publish(); + // One pass at a time: the others are looked at again when it ends. + return wake.min(now.saturating_add(1_000)); + } + } + wake + } + + /// Starts asking the server which spaces the account is in (a discovery under way is let be). + fn start_discovery(&mut self) { + if self.discovery.is_some() { + return; + } + self.next_discovery = self.now().saturating_add(self.discovery_every()); + let (server, token, events) = (self.account.server.clone(), self.account.token.clone(), self.events_tx.clone()); + self.discovery = Some(tokio::spawn(async move { + let _ = events.send(Ev::Listed(look(server, token).await)); + })); + } + + /// A discovery ended: the teams are brought in line with what it found, and whoever waited for it is told. + fn listed(&mut self, looked: Looked) { + self.discovery = None; + let failed = matches!(looked, Looked::Failed(_)); + match looked { + Looked::Listed(list) => { + self.noted = None; + self.supported = Some(true); + self.listed = list; + self.settled(); + } + Looked::Unsupported => { + self.listed.clear(); + self.supported = Some(false); + self.settled(); + } + Looked::Failed(why) => self.discovery_failed(&why), + } + // Whoever waits for the list (an accepted invitation is to be found in it) is not told a list that is out of date: a + // discovery that failed is tried again within seconds, and they wait for that, up to a bound. + let now = self.now(); + if failed && self.refresh_until.is_some_and(|until| now < until) && !(self.waiting.is_empty() && self.waiting_next.is_empty()) { + let later = std::mem::take(&mut self.waiting_next); + self.waiting.extend(later); + self.next_discovery = now.saturating_add(ms(DISCOVERY_RETRY)); + return; + } + self.refresh_until = None; + for done in self.waiting.drain(..) { + let _ = done.send(()); + } + if !self.waiting_next.is_empty() { + self.waiting = std::mem::take(&mut self.waiting_next); + self.refresh_until = Some(now.saturating_add(ms(REFRESH_BOUND))); + self.start_discovery(); + } + } + + fn discovery_failed(&mut self, why: &str) { + if self.noted.as_deref() != Some(why) { + crate::runlog::note(format_args!("couldn't look at the account's team spaces: {why}")); + self.noted = Some(why.to_owned()); + } + } + + /// The list (or its absence) is known: settings, runs and the local API's view follow. + fn settled(&mut self) { + if !self.unsaved.is_empty() { + match self.data.save_account(&self.account) { + Ok(()) => self.unsaved.clear(), + Err(e) => crate::runlog::note(format_args!("can't save a raised generation of a team space yet: {e}")), + } + } + let shared = shared_root(&self.account.folder); + let before = self.account.spaces.clone(); + let reconciled = reconcile(&self.account.spaces, &self.listed, shared.as_deref()); + for (space, _, _) in &reconciled.renamed { + crate::runlog::note(format_args!("team space {space} was renamed on the server; its folder keeps its name")); + } + self.account.spaces = reconciled.settings; + if self.account.spaces != before + && let Err(e) = self.data.save_account(&self.account) + { + crate::runlog::note(format_args!("can't remember the team spaces: {e}")); + } + // Teams follow the settings (a new space is a new team). + for setting in &self.account.spaces { + match self.teams.iter_mut().find(|t| t.setting.space == setting.space) { + Some(team) => team.setting = setting.clone(), + None => self.teams.push(Team { setting: setting.clone(), active: None, lost: false, read_only: false, last_sync_ms: None }), + } + } + for index in 0..self.teams.len() { + self.follow(index); + } + self.publish(); + } + + /// The listing of a team's space, if the account has it as an active member. + fn item_of(&self, space: SpaceId) -> Option<&SpaceListItem> { + self.listed.iter().find(|i| i.id == space && i.state == "active") + } + + /// Starts or stops the team's sync to match what the settings, the server's list and the server's version say. + fn follow(&mut self, index: usize) { + let supported = self.supported != Some(false); + let Some(team) = self.teams.get(index) else { return }; + let space = team.setting.space; + let listed = (self.supported == Some(true)).then(|| self.item_of(space).cloned()); + // Only a discovery that got an answer says the space is gone. + let lost = match &listed { + Some(item) => item.is_none(), + None => team.lost, + }; + let read_only = listed.as_ref().and_then(Option::as_ref).is_some_and(|i| i.role == SpaceRole::Viewer || i.archived); + let joined_ms = listed.as_ref().and_then(Option::as_ref).and_then(|i| i.joined_ms); + // A space whose session is being closed (turned off, moved) is not started again before that is done. + let changing = self.pending.iter().any(|p| p.space == space); + let want = team.setting.sync && supported && !lost && !self.stopping && !changing; + // Access was just lost: the saved state goes, once the session is closed, so that if access comes back the folder + // is merged with the space afresh (what was deleted on either side meanwhile is not deleted on the other). + let newly_lost = lost && !team.lost; + if newly_lost && let Err(e) = self.raise_generation(space) { + crate::runlog::note(format_args!("can't save the raised generation of lost team space {space} yet (tried again at the next look): {e}")); + } + let timing = self.options.timing; + let (options, events, now, paused) = (self.options, self.events_tx.clone(), self.now(), self.paused); + let (data, account, live) = (self.data.clone(), self.account.clone(), self.live.clone()); + let may_start = !self.unsaved.contains(&space); + self.runs += 1; + let run = self.runs; + let Some(team) = self.teams.get_mut(index) else { return }; + team.lost = lost; + // A role that changed in either direction is acted on by a pass now: a viewer made an editor uploads what it kept, + // an editor made a viewer stops before its next write. + if team.read_only != read_only + && let Some(active) = team.active.as_mut() + { + active.scheduler.urgent_change(); + } + team.read_only = read_only; + match (want, team.active.is_some()) { + (true, false) if may_start => { + team.active = start_active(&data, &account, &team.setting, joined_ms, timing, options, events, now, (run, live), paused); + } + (false, true) => { + team.active = None; + let _ = self.jobs.send(ToThread::Close { space, forget: newly_lost, ack: None }); + } + (false, false) if newly_lost => { + let _ = self.jobs.send(ToThread::Close { space, forget: true, ack: None }); + } + _ => {} + } + } + + /// Turns the sync of a team space on or off here. Turning it off, or moving it to another folder, first closes the + /// space's session; the task does not wait for that (other spaces, a stop and the local API go on), and the caller is + /// answered once it is done or [`CLOSE_WAIT`] has passed ([`Manager::closed`]). + fn set_sync(&mut self, space: SpaceId, request: SpaceSyncRequest, reply: oneshot::Sender>) { + if self.pending.iter().any(|p| p.space == space) { + let _ = reply.send(Err(AgentError::Usage(CHANGING.into()))); + return; + } + match self.apply_sync(space, &request, None) { + Ok(Applied::Done(info)) => { + let _ = reply.send(Ok(*info)); + } + Ok(Applied::NeedsClose { forget }) => { + self.changes += 1; + let id = self.changes; + self.pending.push(Pending { id, space, request, reply }); + self.close_session(id, space, forget); + } + Err(e) => { + let _ = reply.send(Err(e)); + } + } + } + + /// The wait for a session to close ended: the change that waited for it is applied and its caller answered. + fn closed(&mut self, id: u64, closure: Closure) { + let Some(index) = self.pending.iter().position(|p| p.id == id) else { return }; + let Pending { space, request, reply, .. } = self.pending.remove(index); + // A state that could not be removed is not acknowledged as gone: the change fails, with the error. (The generation + // was raised and saved before, so that state is not used again either way.) + if let Closure::NotForgotten(why) = &closure { + self.settled(); + let _ = reply.send(Err(AgentError::Usage(format!("the saved state of this space could not be removed: {why}")))); + return; + } + let answer = match self.apply_sync(space, &request, Some(matches!(closure, Closure::Closed))) { + Ok(Applied::Done(info)) => Ok(*info), + Ok(Applied::NeedsClose { .. }) => Err(AgentError::Usage("the space's session could not be closed; try again in a moment".into())), + Err(e) => Err(e), + }; + let _ = reply.send(answer); + } + + /// Applies turning a space's sync on or off. `closed` is `None` before the space's session was closed, and afterwards + /// whether it was closed within the wait; a change that needs the session closed first answers + /// [`Applied::NeedsClose`] while it is `None`. + fn apply_sync(&mut self, space: SpaceId, request: &SpaceSyncRequest, closed: Option) -> Result { + let not_found = || AgentError::Server(TransportError::NotFound); + let known = self.account.spaces.iter().position(|s| s.space == space); + let item = self.item_of(space).cloned(); + if request.on && (self.supported == Some(false)) { + return Err(AgentError::Usage(NEEDS_UPDATE.into())); + } + if request.on && item.is_none() { + return Err(not_found()); + } + if !request.on { + let index = known.ok_or_else(not_found)?; + // The session ends before the setting says off, so a pass that is running finishes first (if it takes longer + // than the wait, turning the sync on again waits for it then). Its saved state is forgotten: turned on again, + // in this folder or another, the space starts afresh as a first sync, so what was deleted here or on the server + // while it was off is not deleted on the other side. + if closed.is_none() { + // The old state is made unusable before it is removed: a generation raised and saved first, so a state that + // survives its removal (a file that can't be deleted, a crash in between) is not trusted when the space is + // turned on again. + self.raise_generation(space)?; + return Ok(Applied::NeedsClose { forget: true }); + } + if let Some(setting) = self.account.spaces.get_mut(index) { + setting.sync = false; + } + } else { + let item = item.ok_or_else(not_found)?; + let root = item.root.ok_or_else(|| AgentError::Usage("the server didn't say where the space starts; try again".into()))?; + let chosen = request.folder.as_deref().map(PathBuf::from); + let current = known.and_then(|i| self.account.spaces.get(i)).map(|s| s.folder.clone()); + let shared = shared_root(&self.account.folder); + let taken: Vec = self.account.spaces.iter().filter(|s| s.space != space).map(|s| s.folder.clone()).collect(); + let name = item.name.clone().unwrap_or_default(); + let folder = match (chosen, current, shared) { + (Some(folder), _, _) => folder, + (None, Some(folder), _) if !folder.as_os_str().is_empty() => folder, + (None, _, Some(shared)) => free_folder(&shared, &name, &taken), + _ => return Err(AgentError::Usage("choose a folder for the space (there is no place beside the personal folder)".into())), + }; + if !folder.is_absolute() { + return Err(AgentError::Usage("the folder must be an absolute path".into())); + } + self.check_folder(space, &folder)?; + // A sync that goes on in the same folder is left running. Otherwise its session is closed first, and the pass + // thread has let go of it before the new folder is set up: a pass for the old folder must not write its state + // after that. The wait is only for a space that has a session here or a pass under way: nothing else of the space + // can write its state, any older `Close` of it is ahead in the thread's queue of whatever the thread does for + // another space, and the thread may be busy with that for minutes. + let running = self.teams.iter().any(|t| t.setting.space == space && t.active.is_some()); + let unchanged = running && known.and_then(|i| self.account.spaces.get(i)).is_some_and(|s| s.sync && s.folder == folder); + if !unchanged { + match closed { + None if running || self.in_flight == Some(space) => return Ok(Applied::NeedsClose { forget: false }), + None | Some(true) => {} + Some(false) => return Err(AgentError::Usage(STILL_FINISHING.into())), + } + } + // Setting up a folder that has no marker yet installs one and starts the space's state afresh. The old state is + // made unusable (the generation raised and saved) before the marker is written, so a crash between the marker and + // the removal of the state leaves a state that is never trusted for this folder. + let data = self.data.clone(); + set_up_folder(&data, &folder, space, root, request.adopt, || self.raise_generation(space))?; + let generation = known.and_then(|i| self.account.spaces.get(i)).map_or(0, |s| s.generation); + let setting = SpaceSetting { space, folder, sync: true, name: item.name.clone(), root: Some(root), generation }; + match known { + Some(index) => { + if let Some(slot) = self.account.spaces.get_mut(index) { + *slot = setting; + } + } + None => self.account.spaces.push(setting), + } + } + self.data.save_account(&self.account)?; + self.settled(); + self.info_of(space).map(|info| Applied::Done(Box::new(info))).ok_or_else(not_found) + } + + /// Raises the generation of the space's setting and saves it, so a state made under the old one is not used again. + /// A raise that could not be saved stays in memory and is saved again at the next look at the list; until then no space is + /// started (see `unsaved`), so a state that was meant to be unusable is not used by a session that begins. + fn raise_generation(&mut self, space: SpaceId) -> Result<(), AgentError> { + if let Some(setting) = self.account.spaces.iter_mut().find(|s| s.space == space) { + setting.generation = setting.generation.saturating_add(1); + if let Err(e) = self.data.save_account(&self.account) { + self.unsaved.insert(space); + return Err(e.into()); + } + } + Ok(()) + } + + /// Stops the space's sync here and asks the pass thread to close its session (a pass that was running has ended and + /// nothing of the space writes its state any more once it is closed). The task is told by [`Ev::Closed`], `false` if + /// that didn't happen within [`CLOSE_WAIT`]. + fn close_session(&mut self, id: u64, space: SpaceId, forget: bool) { + if let Some(team) = self.teams.iter_mut().find(|t| t.setting.space == space) { + team.active = None; + } + let (ack, closed) = oneshot::channel(); + let events = self.events_tx.clone(); + if self.jobs.send(ToThread::Close { space, forget, ack: Some(ack) }).is_err() { + let _ = events.send(Ev::Closed(id, Closure::Closed)); + return; + } + tokio::spawn(async move { + // A thread that has ended drops the acknowledgement: nothing runs then either. + let closure = match tokio::time::timeout(CLOSE_WAIT, closed).await { + Ok(Ok(Err(why))) => Closure::NotForgotten(why), + Ok(_) => Closure::Closed, + Err(_) => Closure::TimedOut, + }; + let _ = events.send(Ev::Closed(id, closure)); + }); + } + + /// A team folder must not lie inside another synced folder (or the other way round), nor the agent's data folder. + fn check_folder(&self, space: SpaceId, folder: &Path) -> Result<(), AgentError> { + crate::cli::check_apart(&self.data, folder)?; + let others = + std::iter::once(&self.account.folder).chain(self.account.spaces.iter().filter(|s| s.space != space && s.sync).map(|s| &s.folder)); + for other in others { + if crate::cli::nested(other, folder) { + return Err(AgentError::Usage(format!( + "{} lies inside another synced folder, or holds one ({}); choose a folder of its own", + folder.display(), + other.display() + ))); + } + } + Ok(()) + } + + // The local API's view ----------------------------------------------------------------------------------------- + + fn run_of(&self, team: &Team) -> Option { + if team.lost { + return Some(SpaceRun::NoAccess { + message: format!( + "You no longer have access to {}. Your local copy is kept and no longer syncs.", + team.setting.name.as_deref().unwrap_or("this space") + ), + }); + } + if team.setting.sync && self.supported == Some(false) { + return Some(SpaceRun::Unavailable { message: NEEDS_UPDATE.into() }); + } + if team.active.is_none() && team.setting.sync && self.unsaved.contains(&team.setting.space) { + return Some(SpaceRun::Paused { reason: UNSAVED.into() }); + } + let active = team.active.as_ref()?; + if self.paused { + return Some(SpaceRun::PausedByUser); + } + Some(match active.scheduler.phase() { + Phase::Idle => SpaceRun::Idle, + Phase::Syncing => SpaceRun::Syncing, + Phase::Retrying { message, .. } => SpaceRun::Retrying { message }, + Phase::Paused { reason, .. } => SpaceRun::Paused { reason }, + Phase::Stopping => SpaceRun::Idle, + }) + } + + fn info_of(&self, space: SpaceId) -> Option { + let team = self.teams.iter().find(|t| t.setting.space == space); + let item = self.listed.iter().find(|i| i.id == space); + self.info(team, item) + } + + fn info(&self, team: Option<&Team>, item: Option<&SpaceListItem>) -> Option { + let space = team.map(|t| t.setting.space).or_else(|| item.map(|i| i.id))?; + let name = item.and_then(|i| i.name.clone()).or_else(|| team.and_then(|t| t.setting.name.clone())); + let invited = item.is_some_and(|i| i.state != "active"); + let run_state = team.and_then(|t| self.run_of(t)); + let sync = match team { + Some(t) if t.lost || (t.setting.sync && self.supported == Some(false)) => SpaceSync::Unavailable, + Some(t) if t.setting.sync && !invited => SpaceSync::On, + _ => SpaceSync::Off, + }; + Some(SpaceInfo { + space, + kind: "team".into(), + name, + role: item.map_or(SpaceRole::Viewer, |i| i.role), + state: if invited { "invited" } else { "active" }.into(), + archived: item.is_some_and(|i| i.archived), + folder: team.filter(|t| !invited && !t.setting.folder.as_os_str().is_empty()).map(|t| t.setting.folder.display().to_string()), + sync, + run_state, + last_sync_ms: team.and_then(|t| t.last_sync_ms), + read_only: team.is_some_and(|t| t.read_only && t.active.is_some()), + }) + } + + /// Tells the local API what the spaces look like now. + fn publish(&self) { + let mut infos: Vec = Vec::new(); + for item in &self.listed { + let team = self.teams.iter().find(|t| t.setting.space == item.id); + infos.extend(self.info(team, Some(item))); + } + // Spaces this computer remembers that the server doesn't list: the account lost them (or the server is too old). + for team in self.teams.iter().filter(|t| !self.listed.iter().any(|i| i.id == t.setting.space)) { + infos.extend(self.info(Some(team), None)); + } + let status: Vec = infos + .iter() + .filter(|i| i.sync != SpaceSync::Off) + .map(|i| SpaceStatus { + space: i.space, + run_state: i.run_state.clone().unwrap_or(SpaceRun::Idle), + last_sync_ms: i.last_sync_ms, + read_only: i.read_only, + }) + .collect(); + self.control.set_spaces(infos, status); + } +} + +/// The last pass summary a team space's own `state.json` holds, if that state is for the space's folder. +fn active_summary(state_dir: &DataDir, setting: &SpaceSetting) -> Option { + state_dir.load_state_for(setting.space, &setting.folder, None, setting.generation).ok().and_then(|(s, _)| s.last_pass).and_then(|l| l.summary) +} + +/// Starts watching a team space's folder, forwarding what happens as events of the space under `generation`. +fn start_watcher(folder: &Path, space: SpaceId, generation: u64, events: &mpsc::UnboundedSender) -> Option { + let (wtx, mut wrx) = mpsc::unbounded_channel(); + let watcher = crate::watch::watch(folder, wtx).ok()?; + let forward = events.clone(); + tokio::spawn(async move { + while let Some(event) = wrx.recv().await { + if forward.send(Ev::Watch(space, generation, event)).is_err() { + return; + } + } + let _ = forward.send(Ev::WatcherGone(space, generation)); + }); + Some(watcher) +} + +/// The parts of a synced team space: scheduler, folder watcher, long-poll. +#[allow(clippy::too_many_arguments)] +fn start_active( + data: &DataDir, + account: &Account, + setting: &SpaceSetting, + joined_ms: Option, + timing: crate::schedule::Timing, + options: RunOptions, + events: mpsc::UnboundedSender, + now: Ms, + (run, live): (u64, Live), + paused: bool, +) -> Option { + let space = setting.space; + let transport = HttpTransport::new(account.server.clone(), &account.token).and_then(|t| t.bound_to(space)).ok()?; + let since = data + .space_dir(space) + .load_state_for(space, &setting.folder, joined_ms, setting.generation) + .ok() + .and_then(|(saved, _)| saved.state.space.map(|_| saved.state.seq)); + let (since_tx, _) = watch::channel(since); + let polling = Polling { transport, space, events: events.clone(), options }; + let mut scheduler = Scheduler::new(timing, now, random); + let roots = crate::watch::roots(&setting.folder); + let generation = 1; + let watcher = start_watcher(&setting.folder, space, generation, &events); + scheduler.set_watching(watcher.is_some()); + let mut active = Active { + scheduler, + stability: Stability::new(timing.stability), + roots, + watcher, + generation, + run, + poll: None, + since: since_tx, + approved: Vec::new(), + polling, + live: live.clone(), + }; + live_set(&live).insert(run); + if !paused { + active.start_poll(); + } + Some(active) +} + +/// For tests: the folder at which the next set-up stops right after its marker is written, as if the agent had crashed there. +#[doc(hidden)] +pub static STOP_AFTER_MARKER: Mutex> = Mutex::new(None); + +/// Whether the set-up of `folder` is to stop now (once; see [`STOP_AFTER_MARKER`]). +fn stop_after_marker(folder: &Path) -> bool { + let mut armed = STOP_AFTER_MARKER.lock().unwrap_or_else(PoisonError::into_inner); + if armed.as_deref() == Some(folder) { + *armed = None; + return true; + } + false +} + +/// Makes `folder` the folder of `space`, whose root is `root`: a new or empty folder gets the space's marker; one with +/// files is taken only with `adopt` (what differs from the space's becomes conflict copies, nothing is deleted); one that +/// already has this space's marker is taken as it is; another space's marker refuses it. +fn set_up_folder( + data: &DataDir, + folder: &Path, + space: SpaceId, + root: NodeId, + adopt: bool, + invalidate_state: impl FnOnce() -> Result<(), AgentError>, +) -> Result<(), AgentError> { + let marker = RootMarker { space, root }; + let existing = if folder.is_dir() { root::read(folder)? } else { None }; + match existing { + Some(found) if found == marker => return Ok(()), + Some(_) => return Err(root::RootError::SetUpForOtherSpace(folder.to_owned()).into()), + None => {} + } + if folder.is_dir() && crate::cli::has_files(folder)? && !adopt { + return Err(AgentError::FolderHasFiles(folder.to_owned())); + } + invalidate_state()?; + std::fs::create_dir_all(folder).map_err(|source| AgentError::Io { path: folder.to_owned(), source })?; + root::write(folder, &marker)?; + if stop_after_marker(folder) { + return Err(AgentError::Usage("stopped after the marker was written (test hook)".into())); + } + // A fresh folder starts from a fresh state: what a state remembers of an older folder must not decide what this one + // is missing. + data.space_dir(space).remove_state()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_cloud_proto::Epoch; + + fn item(name: &str, state: &str) -> SpaceListItem { + SpaceListItem { + id: SpaceId::generate(), + kind: "team".into(), + name: Some(name.into()), + role: SpaceRole::Editor, + state: state.into(), + archived: false, + created_ms: 0, + members: 2, + root: (state == "active").then(NodeId::generate), + epoch: (state == "active").then(Epoch::generate), + joined_ms: (state == "active").then_some(1), + } + } + + #[test] + fn folder_names_are_safe_file_names() { + assert_eq!(folder_name("Studio"), "Studio"); + assert_eq!(folder_name("Plans: 2026 / v2"), "Plans_ 2026 _ v2"); + assert_eq!(folder_name("ac|d?e*f\\g\"h"), "a_b_c_d_e_f_g_h"); + assert_eq!(folder_name("Team..."), "Team", "no dots at the end, which Windows drops"); + assert_eq!(folder_name(".hidden"), "_hidden"); + assert_eq!(folder_name(".craft-cloud"), "_craft-cloud"); + assert_eq!(folder_name("CON"), "CON (team space)", "a reserved name"); + assert_eq!(folder_name(" "), "Team space"); + assert_eq!(folder_name("tab\there"), "tab_here"); + assert_eq!(folder_name(&"x".repeat(300)).chars().count(), MAX_FOLDER_CHARS); + assert_eq!(folder_name("Cafe\u{301}"), "Caf\u{e9}", "NFC, as file names are"); + for name in ["Studio", "a/b", "con", "..", "x\u{0}y", "\u{202e}rtl"] { + assert!(Name::new(&folder_name(name)).is_ok(), "{name:?}"); + } + } + + #[test] + fn a_taken_folder_name_is_numbered() { + let dir = tempfile::tempdir().unwrap(); + let shared = dir.path().join(SHARED_FOLDER); + assert_eq!(free_folder(&shared, "Studio", &[]), shared.join("Studio"), "a shared folder that doesn't exist yet"); + std::fs::create_dir_all(shared.join("Studio")).unwrap(); + assert_eq!(free_folder(&shared, "Studio", &[]), shared.join("Studio (2)"), "a folder on disk"); + let taken = vec![shared.join("Studio (2)"), shared.join("studio (3)")]; + assert_eq!(free_folder(&shared, "Studio", &taken), shared.join("Studio (4)"), "folders of other spaces, whatever the case"); + assert_eq!(free_folder(&shared, "Other", &taken), shared.join("Other")); + assert_eq!(shared_root(Path::new("/home/ana/Craft Cloud")), Some(PathBuf::from("/home/ana").join(SHARED_FOLDER))); + } + + #[test] + fn a_discovery_adds_new_spaces_with_syncing_off_and_follows_renames() { + let dir = tempfile::tempdir().unwrap(); + let shared = dir.path().join(SHARED_FOLDER); + let (a, b, invited) = (item("Plans: 1", "active"), item("Plans? 1", "active"), item("Invited", "invited")); + let first = reconcile(&[], &[a.clone(), b.clone(), invited], Some(&shared)); + assert_eq!(first.settings.len(), 2, "an invitation has no setting"); + assert!(first.settings.iter().all(|s| !s.sync && s.root.is_some())); + let folders: Vec<_> = first.settings.iter().map(|s| s.folder.file_name().unwrap().to_string_lossy().into_owned()).collect(); + assert_eq!(folders, ["Plans_ 1", "Plans_ 1 (2)"]); + // The same list again changes nothing; a rename changes the name and keeps the folder. + assert_eq!(reconcile(&first.settings, &[a.clone(), b.clone()], Some(&shared)).settings, first.settings); + let renamed = SpaceListItem { name: Some("Plans two".into()), ..a.clone() }; + let second = reconcile(&first.settings, &[renamed, b.clone()], Some(&shared)); + assert_eq!(second.renamed, vec![(a.id, "Plans: 1".to_owned(), "Plans two".to_owned())]); + assert_eq!( + second.settings.first().map(|s| (s.folder.clone(), s.name.clone())), + Some((first.settings[0].folder.clone(), Some("Plans two".into()))) + ); + // A space that is no longer listed keeps its setting (its folder and name say which it was). + assert_eq!(reconcile(&first.settings, &[b], Some(&shared)).settings, first.settings); + // Without a place for folders, a new space gets no setting yet. + assert!(reconcile(&[], &[a], None).settings.is_empty()); + } +} diff --git a/apps/craft-cloud-agent/src/watch.rs b/apps/craft-cloud-agent/src/watch.rs index 8132de56..a23fe037 100644 --- a/apps/craft-cloud-agent/src/watch.rs +++ b/apps/craft-cloud-agent/src/watch.rs @@ -79,10 +79,24 @@ pub fn watch(folder: &Path, events: UnboundedSender) -> Result PathBuf { + #[cfg(unix)] + if std::fs::symlink_metadata(folder).is_ok_and(|m| m.file_type().is_symlink()) + && let Ok(real) = std::fs::canonicalize(folder) + { + return real; + } + folder.to_owned() +} + /// Whether an event is a change in the synced folder: not a read, and not only inside the agent's own folder. fn counts(event: &Event, own: &[PathBuf]) -> bool { match event.kind { @@ -121,6 +135,35 @@ mod tests { assert!(counts(&event(EventKind::Any, &[]), &own), "an event without paths may be anything"); } + #[cfg(unix)] + #[test] + fn a_linked_folder_is_watched_where_it_leads() { + let dir = tempfile::tempdir().unwrap(); + let (real, link) = (dir.path().join("real"), dir.path().join("link")); + std::fs::create_dir(&real).unwrap(); + std::os::unix::fs::symlink(&real, &link).unwrap(); + assert_eq!(watched(&link), std::fs::canonicalize(&real).unwrap()); + assert_eq!(watched(&real), real, "a folder that is not a link is watched as given"); + let missing = dir.path().join("missing"); + assert_eq!(watched(&missing), missing); + + // A change in the folder behind the link is reported. + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let _watcher = watch(&link, tx).unwrap(); + std::fs::write(real.join("a.txt"), b"x").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + match rx.try_recv() { + Ok(WatchEvent::Changed { paths, .. }) if paths.iter().any(|p| p.ends_with("a.txt")) => break, + Ok(_) => {} + Err(_) => { + assert!(std::time::Instant::now() < deadline, "no event for a change behind the link"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + } + } + } + #[test] fn event_paths_map_into_the_folder() { let roots = vec![PathBuf::from("/var/f"), PathBuf::from("/private/var/f")]; diff --git a/apps/craft-cloud-agent/tests/end_to_end.rs b/apps/craft-cloud-agent/tests/end_to_end.rs index 92d2fd97..4920700a 100644 --- a/apps/craft-cloud-agent/tests/end_to_end.rs +++ b/apps/craft-cloud-agent/tests/end_to_end.rs @@ -49,9 +49,24 @@ impl Backend for CountingBackend { self.inner.get_raw(path).await } + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + if path.starts_with("chunks/") { + self.chunk_reads.fetch_add(1, Ordering::Relaxed); + } + self.inner.get_raw_limited(path, max_bytes).await + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + self.inner.size_raw(path).await + } + async fn delete_raw(&self, path: &str) -> Result<(), StoreError> { self.inner.delete_raw(path).await } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + self.inner.list_raw(prefix, after, limit).await + } } async fn server() -> Server { @@ -163,6 +178,43 @@ async fn a_first_sync_goes_both_ways() { assert_eq!(a.files(), b.files()); } +/// Deleting a folder with files in it and restoring it from the trash: the folder and everything that was in the trash +/// below it, also a file deleted on its own before, come back to the disk of both computers by their normal pass. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_restored_from_the_trash_comes_back_with_its_files() { + let server = server().await; + let a = Agent::signed_in(&server, "desktop").await; + a.write("dir1/child.txt", "deleted with the folder"); + a.write("dir1/early.txt", "deleted first"); + a.write("dir1/sub/deep.txt", "deep"); + a.write("keep.txt", "stays"); + assert_eq!(a.sync().await, Exit::Done); + let b = Agent::signed_in(&server, "laptop").await; + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(b.files().len(), 4); + + std::fs::remove_file(a.path("dir1/early.txt")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + std::fs::remove_dir_all(a.path("dir1")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(a.files(), ["keep.txt"]); + assert_eq!(b.files(), ["keep.txt"]); + + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let listed = server.meta.trash(ana.space, None, 200, 0).await.unwrap().entries; + assert_eq!(listed.iter().map(|e| e.name.as_str()).collect::>(), ["dir1"], "the folder is listed alone"); + let device = server.meta.list_devices(ana.id).await.unwrap().into_iter().find(|d| d.name == "desktop").unwrap().id; + server.meta.restore_from_trash(ana.space, device, listed[0].id, 0).await.unwrap(); + + for agent in [&a, &b] { + assert_eq!(agent.sync().await, Exit::Done); + assert_eq!(agent.files(), ["dir1/child.txt", "dir1/early.txt", "dir1/sub/deep.txt", "keep.txt"]); + assert_eq!(agent.read("dir1/sub/deep.txt").as_deref(), Some("deep")); + assert_eq!(agent.read("dir1/early.txt").as_deref(), Some("deleted first")); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn big_files_travel_in_chunks_and_keep_their_exec_bit() { let server = server().await; @@ -761,6 +813,39 @@ async fn a_failed_pass_keeps_the_deletes_the_user_was_shown() { assert_eq!((exit, summary.removed_remotely), (Exit::Done, 12), "{summary:?}"); } +/// The agent tells the server what it is when it signs in and when it joins, so the account's device list shows it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn login_and_join_send_the_agents_device_details() { + use craft_cloud_agent::cli::JoinArgs; + use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; + let server = server().await; + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + async fn details(server: &Server, user: craft_cloud_meta::UserId, name: &str) -> (String, String, String) { + let devices = server.meta.list_devices(user).await.unwrap(); + let device = devices.into_iter().find(|d| d.name == name).unwrap_or_else(|| panic!("no device {name}")); + (device.client.kind, device.client.platform, device.client.app_version) + } + let expected = ("agent".to_owned(), std::env::consts::OS.to_owned(), env!("CARGO_PKG_VERSION").to_owned()); + + let a = Agent::signed_in(&server, "desk").await; + assert_eq!(details(&server, ana.id, "desk").await, expected, "after login"); + drop(a); + + let invite = Code::generate(INVITE_CHARS).unwrap(); + let now = i64::try_from(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()).unwrap(); + server.meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + let b = Agent::new(); + let args = JoinArgs { + join: JoinString::new(server.url.clone(), invite).link(), + folder: Some(b.folder.clone()), + device_name: Some("laptop".into()), + adopt: false, + autostart: false, + }; + cli::join(&b.data, &args, None, &mut Vec::new()).await.unwrap(); + assert_eq!(details(&server, ana.id, "laptop").await, expected, "after join"); +} + #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn login_makes_the_agent_folder_owner_only() { diff --git a/apps/craft-cloud-agent/tests/local_api.rs b/apps/craft-cloud-agent/tests/local_api.rs index 2839d706..a55f0194 100644 --- a/apps/craft-cloud-agent/tests/local_api.rs +++ b/apps/craft-cloud-agent/tests/local_api.rs @@ -10,7 +10,7 @@ use craft_cloud_agent::cli::{self, AgentError, Exit}; use craft_cloud_agent::daemon::{RunOptions, run_loop}; use craft_cloud_agent::schedule::Timing; use craft_cloud_meta::{Meta, Role}; -use craft_cloud_proto::api::{DeviceInfo, ErrorBody, PairingState, PairingStatus}; +use craft_cloud_proto::api::{DeviceInfo, ErrorBody, HistoryResponse, PairingState, PairingStatus}; use craft_cloud_proto::local::{JoinRequest, LOCAL_API_FILE, LocalApiFile, LoginRequest, PairingInfo, RunState, Status, codes}; use craft_cloud_server::AppState; use craft_cloud_server::auth::hash_password; @@ -24,6 +24,7 @@ const PATIENCE: Duration = Duration::from_secs(30); struct Server { url: String, meta: Meta, + task: tokio::task::JoinHandle>, _dirs: (tempfile::TempDir, tempfile::TempDir), } @@ -34,8 +35,8 @@ async fn server() -> Server { let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]).unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); - Server { url, meta, _dirs: (db, blobs) } + let task = tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, task, _dirs: (db, blobs) } } fn quick() -> RunOptions { @@ -172,8 +173,42 @@ const ROUTES: &[(&str, &str)] = &[ ("GET", "/v1/pairings/0192f0c4-0000-7000-8000-000000000000"), ("DELETE", "/v1/pairings/0192f0c4-0000-7000-8000-000000000000"), ("GET", "/v1/devices"), + ("GET", "/v1/usage"), ("DELETE", "/v1/devices/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/files/history?path=Notes%2Ftodo.txt"), + ("POST", "/v1/files/restore"), + ("GET", "/v1/trash?limit=1"), + ("POST", "/v1/trash/restore"), + ("POST", "/v1/trash/remove"), ("POST", "/v1/autostart"), + ("POST", "/v1/login/second-factor"), + ("GET", "/v1/account/two-factor"), + ("POST", "/v1/account/two-factor/setup"), + ("POST", "/v1/account/two-factor/confirm"), + ("POST", "/v1/account/two-factor/disable"), + ("POST", "/v1/account/two-factor/recovery-codes"), + ("POST", "/v1/account/password"), + ("POST", "/v1/devices/revoke-others"), + ("PATCH", "/v1/devices/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/admin/server"), + ("GET", "/v1/admin/storage"), + ("GET", "/v1/admin/users"), + ("POST", "/v1/admin/users"), + ("PATCH", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/usage"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/disable"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/enable"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/invite"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/invite"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/password-reset"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/reset-two-factor"), + ("GET", "/v1/account"), + ("GET", "/v1/account/activity"), + ("GET", "/v1/admin/audit"), + ("GET", "/v1/admin/audit/export"), + ("GET", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/devices"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/devices/0192f0c4-0000-7000-8000-000000000001"), ]; /// A user session without systemd: Linux's entry is the XDG autostart file. @@ -522,6 +557,53 @@ async fn signed_in(server: &Server, data: &DataDir, folder: &Path) -> (Running, (running, api) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn usage_reads_through_to_the_server_without_caching() { + let server = server().await; + let data = DataDir::new(tempfile::tempdir().unwrap().keep()); + let folder = tempfile::tempdir().unwrap(); + let (_running, api) = signed_in(&server, &data, folder.path()).await; + let account = data.load_account().unwrap().unwrap(); + let transport = craft_cloud_agent::http::HttpTransport::new(account.server.clone(), &account.token).unwrap(); + std::fs::write(folder.path().join("usage-first.bin"), b"first payload").unwrap(); + reaches_the_server(&server, "usage-first.bin").await; + let server_first = transport.usage().await.unwrap(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 200, "{body}"); + let local_first: craft_cloud_proto::api::AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!(local_first, server_first, "the local API passes nonzero numbers through unchanged"); + assert!(local_first.stored_bytes > 0); + + std::fs::write(folder.path().join("usage-second.bin"), b"a different payload for fresh usage").unwrap(); + reaches_the_server(&server, "usage-second.bin").await; + let server_second = transport.usage().await.unwrap(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 200, "{body}"); + let local_second: craft_cloud_proto::api::AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!(local_second, server_second, "a second request sees server changes; no cache"); + assert!(local_second.stored_bytes > local_first.stored_bytes); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn usage_refuses_signed_out_and_reports_server_unreachable() { + let server = server().await; + let data = DataDir::new(tempfile::tempdir().unwrap().keep()); + let running = Running::start(&data); + let api = api_file(data.path()).await; + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 409, "{body}"); + assert_eq!(error_code(&body), codes::NOT_SIGNED_IN); + let _ = running.stop().await; + + let folder = tempfile::tempdir().unwrap(); + let (running, api) = signed_in(&server, &data, folder.path()).await; + server.task.abort(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 503, "{body}"); + assert_eq!(error_code(&body), codes::OFFLINE); + drop(running); +} + async fn reaches_the_server(server: &Server, name: &str) { let deadline = Instant::now() + PATIENCE; while !server_has(server, name).await { @@ -530,6 +612,190 @@ async fn reaches_the_server(server: &Server, name: &str) { } } +async fn history_page(api: &LocalApiFile, path: &str, cursor: Option<&str>, limit: u32) -> HistoryResponse { + let mut query = format!("path={path}&limit={limit}"); + if let Some(cursor) = cursor { + query.push_str("&cursor="); + query.push_str(cursor); + } + let (code, body) = call(api, "GET", &format!("/v1/files/history?{query}"), ""); + assert_eq!(code, 200, "{body}"); + serde_json::from_str(&body).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_file_history_pages_restores_and_syncs_to_disk() { + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + let subdir = folder.join("Notes"); + std::fs::create_dir(&subdir).unwrap(); + let file = subdir.join("story.txt"); + + let mut previous = None; + for content in [b"first version".as_slice(), b"second version".as_slice(), b"third version".as_slice()] { + std::fs::write(&file, content).unwrap(); + let deadline = Instant::now() + PATIENCE; + loop { + let (code, body) = call(&api, "GET", "/v1/files/history?path=Notes%2Fstory.txt&limit=1", ""); + if code == 200 { + let page: HistoryResponse = serde_json::from_str(&body).unwrap(); + if page.versions.first().is_some_and(|version| Some(version.id) != previous) { + previous = page.versions.first().map(|version| version.id); + break; + } + } + assert!(Instant::now() < deadline, "version reaches server history"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + let newest = history_page(&api, "Notes%2Fstory.txt", None, 1).await; + let cursor = newest.next.clone().expect("more than one history page"); + let older = history_page(&api, "Notes%2Fstory.txt", Some(&cursor), 1).await; + let current = newest.versions.first().expect("current version").id; + let restore = older.versions.first().expect("older version").id; + let body = serde_json::json!({ "path": "Notes/story.txt", "version": restore, "expected_version": current }).to_string(); + let (code, answer) = call(&api, "POST", "/v1/files/restore", &body); + assert_eq!(code, 200, "{answer}"); + let restored: craft_cloud_proto::api::RestoreResponse = serde_json::from_str(&answer).unwrap(); + assert_eq!(restored.parent_version, current); + + let stale = serde_json::json!({ "path": "Notes/story.txt", "version": current, "expected_version": current }).to_string(); + let (code, answer) = call(&api, "POST", "/v1/files/restore", &stale); + assert_eq!((code, error_code(&answer).as_str()), (409, "conflict"), "{answer}"); + + let deadline = Instant::now() + PATIENCE; + loop { + if std::fs::read(&file).is_ok_and(|bytes| bytes.as_slice() == b"second version") { + break; + } + assert!(Instant::now() < deadline, "the change feed syncs restored bytes to disk"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let (exit, output) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{output}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_file_history_rejects_unknown_outside_and_directory_paths() { + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + std::fs::create_dir(folder.join("Folder")).unwrap(); + let deadline = Instant::now() + PATIENCE; + loop { + let (code, body) = call(&api, "GET", "/v1/files/history?path=Folder", ""); + if code == 400 { + assert_eq!(error_code(&body), "invalid"); + break; + } + assert!(Instant::now() < deadline, "directory enters sync state"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let (code, body) = call(&api, "GET", "/v1/files/history?path=missing.txt", ""); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = call(&api, "GET", "/v1/files/history?path=..%2Foutside.txt", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let _ = running.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_trash_pages_restores_renames_and_removes_nodes() { + use craft_cloud_proto::local::TrashResponse; + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + + std::fs::write(folder.join("first.txt"), b"first bytes").unwrap(); + std::fs::write(folder.join("second.txt"), b"second bytes").unwrap(); + reaches_the_server(&server, "first.txt").await; + reaches_the_server(&server, "second.txt").await; + std::fs::remove_file(folder.join("first.txt")).unwrap(); + std::fs::remove_file(folder.join("second.txt")).unwrap(); + let deadline = Instant::now() + PATIENCE; + let page = loop { + let (code, body) = call(&api, "GET", "/v1/trash?limit=1", ""); + if code == 200 { + let page: TrashResponse = serde_json::from_str(&body).unwrap(); + if page.items.len() == 1 && page.next.is_some() { + break page; + } + } + assert!(Instant::now() < deadline, "both deleted files enter the trash"); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let first = page.items.first().unwrap(); + assert_eq!(first.path.as_deref(), Some(first.name.as_str()), "live root parent yields a path hint"); + let cursor = page.next.as_deref().unwrap(); + let (code, body) = call(&api, "GET", &format!("/v1/trash?limit=1&cursor={cursor}"), ""); + assert_eq!(code, 200, "{body}"); + let second_page: TrashResponse = serde_json::from_str(&body).unwrap(); + let second = second_page.items.first().unwrap(); + assert_ne!(first.id, second.id); + + let restore_body = serde_json::json!({ "id": first.id }).to_string(); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &restore_body); + assert_eq!(code, 200, "{body}"); + let restored: craft_cloud_proto::api::TrashRestoreResponse = serde_json::from_str(&body).unwrap(); + assert!(!restored.renamed); + let restored_path = folder.join(&restored.name); + let deadline = Instant::now() + PATIENCE; + loop { + if std::fs::read(&restored_path) + .is_ok_and(|bytes| bytes.as_slice() == if first.name == "first.txt" { &b"first bytes"[..] } else { &b"second bytes"[..] }) + { + break; + } + assert!(Instant::now() < deadline, "the agent syncs restored bytes to disk"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // Re-delete the restored ID, create a live node with its old name, then verify conflict rename on restore. + std::fs::remove_file(&restored_path).unwrap(); + let deadline = Instant::now() + PATIENCE; + let mut trashed_again = false; + while !trashed_again { + let (code, body) = call(&api, "GET", "/v1/trash?limit=50", ""); + if code == 200 { + let page: TrashResponse = serde_json::from_str(&body).unwrap(); + trashed_again = page.items.iter().any(|item| item.id == restored.node); + } + assert!(Instant::now() < deadline, "restored node is deleted again"); + if !trashed_again { + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + std::fs::write(&restored_path, b"replacement bytes").unwrap(); + reaches_the_server(&server, &restored.name).await; + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": restored.node }).to_string()); + assert_eq!(code, 200, "{body}"); + let renamed: craft_cloud_proto::api::TrashRestoreResponse = serde_json::from_str(&body).unwrap(); + assert!(renamed.renamed, "{renamed:?}"); + assert_ne!(renamed.name, restored.name); + + let (code, body) = call(&api, "POST", "/v1/trash/remove", &serde_json::json!({ "id": second.id }).to_string()); + assert_eq!(code, 204, "{body}"); + let (code, body) = call(&api, "GET", "/v1/trash?limit=50", ""); + assert_eq!(code, 200, "{body}"); + let remaining: TrashResponse = serde_json::from_str(&body).unwrap(); + assert!(remaining.items.iter().all(|item| item.id != second.id)); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": second.id }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": "invalid" }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let unknown = craft_cloud_proto::NodeId::generate(); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": unknown }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let _ = running.stop().await; +} + /// Paused, nothing syncs and the status says so (not as a pause that needs someone); it stays paused after `run` /// starts again, until it is resumed. Quit then ends `run` cleanly, and its `local-api.json` with it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -694,6 +960,8 @@ impl Machine { device_name: Some("lptp1".into()), adopt, password: password.map(str::to_owned), + totp_code: None, + recovery_code: None, }) .unwrap() } @@ -851,3 +1119,451 @@ async fn an_invite_to_a_new_account_asks_for_a_password_through_the_agent() { assert_eq!(joined.account.as_deref(), Some("cleo")); b.stop().await; } + +// ---- the admin calls (`/v1/admin/…`) ---- + +use craft_cloud_proto::api::{ + AccountUsage, AdminCode, AdminCreatedUser, AdminDevices, AdminServerInfo, AdminStorageOverview, AdminUserInfo, AdminUserList, +}; + +/// The server with `root`, an administrator, beside `ana`, a member. +async fn admin_server() -> Server { + let server = server().await; + server.meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + server +} + +/// Signs `run` in as `user` through its local API and waits until it is idle. +async fn signed_in_as(server: &Server, user: &str, data: &DataDir, folder: &Path) -> (Running, LocalApiFile) { + let running = Running::start(data); + let api = api_file(data.path()).await; + let body = serde_json::to_string(&LoginRequest { + server: server.url.clone(), + user: user.into(), + password: PASSWORD.into(), + folder: Some(folder.display().to_string()), + device_name: Some(format!("{user}-pc")), + adopt: false, + }) + .unwrap(); + let (code, answer) = call(&api, "POST", "/v1/login", &body); + assert_eq!(code, 200, "{answer}"); + eventually("signed in and idle", || status(&api).run_state == RunState::Idle).await; + (running, api) +} + +/// A call with the confirmation `password` and the rest of the body in `fields`. +fn confirmed(api: &LocalApiFile, method: &str, path: &str, password: &str, fields: serde_json::Value) -> (u16, String) { + let mut body = fields; + body["password"] = password.into(); + call(api, method, path, &body.to_string()) +} + +/// Whether an admin disabled the account. +async fn is_disabled(server: &Server, name: &str) -> bool { + let user = server.meta.user_by_name(name).await.unwrap().unwrap(); + server.meta.admin_user(user.id, 0).await.unwrap().unwrap().disabled_ms.is_some() +} + +fn files_under(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + files_under(&path, out); + } else { + out.push(path); + } + } +} + +/// Every route passes the server's answer on with its status and code: reads, the changes of an account, the codes, +/// the devices; the confirmation is the admin's password in the body; a missing or wrong one is the server's `400 +/// invalid` and `403 wrong_password` (never 401), and too many wrong ones its `429`. Nothing of the password or of a +/// code is kept by the agent or printed by it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_calls_reach_the_servers_admin_api_and_pass_its_answers_through() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in_as(&server, "root", &data, &folder).await; + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let ana_path = format!("/v1/admin/users/{}", ana.id.0); + + // Reads. + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!(code, 200, "{body}"); + let info: AdminServerInfo = serde_json::from_str(&body).unwrap(); + assert_eq!((info.users, info.devices_active, info.storage_kind.as_str()), (2, 1, "disk")); + let (code, body) = call(&api, "GET", "/v1/admin/storage", ""); + assert_eq!(code, 200, "{body}"); + let overview: AdminStorageOverview = serde_json::from_str(&body).unwrap(); + assert_eq!(overview.accounts, 2); + assert_eq!(overview.stored_bytes, 0); + let (code, body) = call(&api, "GET", &format!("{ana_path}/usage"), ""); + assert_eq!(code, 200, "{body}"); + let usage: AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!((usage.stored_bytes, usage.quota_bytes, usage.trash_bytes, usage.history_bytes), (0, None, Some(0), Some(0))); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=1", ""); + assert_eq!(code, 200, "{body}"); + let page: AdminUserList = serde_json::from_str(&body).unwrap(); + assert_eq!(page.users.iter().map(|u| u.username.as_str()).collect::>(), ["ana"]); + let next = page.next.expect("a second page"); + let (code, body) = call(&api, "GET", &format!("/v1/admin/users?limit=1&cursor={next}"), ""); + assert_eq!(code, 200, "{body}"); + let second: AdminUserList = serde_json::from_str(&body).unwrap(); + assert_eq!((second.users.first().map(|u| u.username.as_str()), second.next), (Some("root"), None)); + let (code, body) = call(&api, "GET", "/v1/admin/users?query=AN", ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().users.len(), 1); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=0", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "the server's own refusal: {body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=x", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = call(&api, "GET", &format!("{ana_path}/devices"), ""); + assert_eq!(code, 200, "{body}"); + assert!(serde_json::from_str::(&body).unwrap().devices.is_empty()); + + // A missing confirmation is the server's `invalid`; a wrong one its `wrong_password`, a 403: never a 401. + let (code, body) = call(&api, "POST", &format!("{ana_path}/disable"), "{}"); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), "not the password", serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (403, "wrong_password"), "{body}"); + let (code, body) = call(&api, "POST", &format!("{ana_path}/disable"), "not json"); + assert_eq!(code, 400, "{body}"); + assert!(!is_disabled(&server, "ana").await, "nothing changed without a right confirmation"); + assert!(status(&api).signed_in && !status(&api).removed, "a 403 doesn't sign the agent out"); + + // The changes: with the confirmation, each is the server's answer. + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!(code, 200, "{body}"); + let created: AdminCreatedUser = serde_json::from_str(&body).unwrap(); + assert_eq!((created.user.username.as_str(), created.user.role.as_str(), created.user.invite_pending), ("bo", "member", true)); + let invite_code = created.invite.code.clone(); + assert!(!invite_code.is_empty()); + let bo_path = format!("/v1/admin/users/{}", created.user.id); + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, "username_taken"), "{body}"); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "quota_bytes": 50_000_000 })); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().quota_bytes, Some(50_000_000)); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "quota_bytes": null, "role": "admin" })); + assert_eq!(code, 200, "{body}"); + let changed: AdminUserInfo = serde_json::from_str(&body).unwrap(); + assert_eq!((changed.quota_bytes, changed.role.as_str()), (None, "admin")); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "role": "member" })); + assert_eq!(code, 200, "{body}"); + // The last administrator can't be demoted: the server's 409. + let root = server.meta.user_by_name("root").await.unwrap().unwrap(); + let (code, body) = confirmed(&api, "PATCH", &format!("/v1/admin/users/{}", root.id.0), PASSWORD, serde_json::json!({ "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, "last_admin"), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + assert!(is_disabled(&server, "ana").await); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/enable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/password-reset"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 200, "{body}"); + let reset: AdminCode = serde_json::from_str(&body).unwrap(); + assert!(!reset.code.is_empty()); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (409, "conflict"), "ana has a password: the server says so: {body}"); + let (code, body) = confirmed(&api, "POST", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 200, "{body}"); + let second_invite: AdminCode = serde_json::from_str(&body).unwrap(); + let (code, body) = confirmed(&api, "DELETE", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + let (code, body) = confirmed(&api, "DELETE", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "nothing left to revoke: {body}"); + let unknown_device = craft_cloud_proto::DeviceId::generate(); + let (code, body) = confirmed(&api, "DELETE", &format!("{ana_path}/devices/{unknown_device}"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = confirmed(&api, "DELETE", &bo_path, PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (202, ""), "the server's own status: {body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users", ""); + assert_eq!(code, 200, "{body}"); + assert!(serde_json::from_str::(&body).unwrap().users.iter().all(|u| u.username != "bo"), "removed accounts are out of the list"); + let (code, body) = call(&api, "GET", "/v1/admin/users/not-an-id/devices", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + + // Wrong passwords are counted by the server: it says when it has had enough, and that stays a 429. + let mut limited = None; + for _ in 0..8 { + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), "still wrong", serde_json::json!({})); + if code == 429 { + limited = Some(error_code(&body)); + break; + } + assert_eq!((code, error_code(&body).as_str()), (403, "wrong_password"), "{body}"); + } + assert_eq!(limited.as_deref(), Some("rate_limited")); + assert!(status(&api).signed_in && !status(&api).removed); + + let (exit, out) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + // Nothing of the confirmation or the codes was kept or printed (the files are read once the agent has stopped: it + // holds the data folder's lock while it runs). + let mut written = Vec::new(); + files_under(data.path(), &mut written); + for file in written.iter().filter(|f| f.file_name().is_some_and(|n| n != LOCAL_API_FILE)) { + let text = String::from_utf8_lossy(&std::fs::read(file).unwrap()).into_owned(); + for secret in [PASSWORD, "still wrong", invite_code.as_str(), reset.code.as_str(), second_invite.code.as_str()] { + assert!(!text.contains(secret), "{} holds a secret", file.display()); + } + } + for secret in [PASSWORD, "still wrong", invite_code.as_str(), reset.code.as_str(), second_invite.code.as_str()] { + assert!(!out.contains(secret), "the agent printed a secret: {out}"); + } +} + +/// A member's calls are refused by the server as `403 forbidden`, and that is what the app is told; the agent is +/// not signed out by it (a 401 would mean "this device was revoked"). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_member_is_told_forbidden_and_stays_signed_in() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (running, api) = signed_in_as(&server, "ana", &data, &dir.path().join("Craft Cloud")).await; + let some_id = "0192f0c4-0000-7000-8000-000000000000"; + for (method, path) in + [("GET", "/v1/admin/server".to_owned()), ("GET", "/v1/admin/users".to_owned()), ("GET", format!("/v1/admin/users/{some_id}/devices"))] + { + let (code, body) = call(&api, method, &path, ""); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{method} {path}: {body}"); + } + for (method, path) in [ + ("POST", "/v1/admin/users".to_owned()), + ("PATCH", format!("/v1/admin/users/{some_id}")), + ("POST", format!("/v1/admin/users/{some_id}/disable")), + ("DELETE", format!("/v1/admin/users/{some_id}")), + ] { + let fields = serde_json::json!({ "username": "x", "role": "admin", "quota_bytes": 5_000_000 }); + let (code, body) = confirmed(&api, method, &path, PASSWORD, fields); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{method} {path}: {body}"); + } + let now = status(&api); + assert!(now.signed_in && !now.removed, "{now:?}"); + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + assert_eq!(ana.role, Role::Member); + let (exit, out) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); +} + +/// The server's `401` for a revoked admin device is not a local `401` (which says the token is wrong): it is told as +/// every signed-in call tells it, as something that needs the person. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_revoked_admin_device_is_told_as_needs_you() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (running, api) = signed_in_as(&server, "root", &data, &dir.path().join("Craft Cloud")).await; + let root = server.meta.user_by_name("root").await.unwrap().unwrap(); + for device in server.meta.list_devices(root.id).await.unwrap() { + server.meta.revoke_device(root.id, device.id).await.unwrap(); + } + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NEEDS_YOU), "{body}"); + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "x", "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NEEDS_YOU), "{body}"); + running.stop().await.0.unwrap(); +} + +/// Signed out there is no server to ask; the bodies are small; the checks on every call are the other routes'. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_calls_need_a_sign_in_and_small_bodies() { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let running = Running::start(&data); + let api = api_file(data.path()).await; + let id = "0192f0c4-0000-7000-8000-000000000000"; + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NOT_SIGNED_IN), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("/v1/admin/users/{id}/disable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NOT_SIGNED_IN), "{body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=9999999999", ""); + assert_eq!(code, 400, "a limit that is no number: {body}"); + let long = serde_json::json!({ "username": "a".repeat(9 * 1024), "role": "member", "password": PASSWORD }).to_string(); + let (code, body) = call(&api, "POST", "/v1/admin/users", &long); + assert_eq!((code, error_code(&body).as_str()), (413, "too_large"), "{body}"); + let (code, _) = call(&api, "POST", "/v1/admin/users", "{\"username\": 5}"); + assert_eq!(code, 422, "the shape of the body is checked here too"); + running.stop().await.0.unwrap(); +} + +// ---- the account, its activity and the audit log ---- + +use craft_cloud_proto::api::{AccountInfo, AuditEvent, AuditPage}; +use craft_cloud_server::audit::AuditExportLimits; + +/// [`admin_server`] with the export cut as `limits` say. +async fn admin_server_limited(limits: AuditExportLimits) -> Server { + let (db, blobs) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]).unwrap().with_audit_export(limits); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, task, _dirs: (db, blobs) } +} + +fn page_of(body: &str) -> AuditPage { + serde_json::from_str(body).unwrap_or_else(|e| panic!("not an audit page ({e}): {body}")) +} + +/// The account, its activity, the audit list and its export pass through the agent with the server's statuses and codes: a +/// member's `403 forbidden` is never a signed-out agent; filters are checked before they are sent; the export is the server's +/// stream, whole, and `truncated` when the server cuts it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_account_its_activity_and_the_audit_log_pass_through() { + let server = admin_server_limited(AuditExportLimits::default()).await; + let dir = tempfile::tempdir().unwrap(); + let root_data = DataDir::new(dir.path().join("root")); + let ana_data = DataDir::new(dir.path().join("ana")); + let (root_run, root) = signed_in_as(&server, "root", &root_data, &dir.path().join("Root Cloud")).await; + let (ana_run, ana) = signed_in_as(&server, "ana", &ana_data, &dir.path().join("Ana Cloud")).await; + let ana_user = server.meta.user_by_name("ana").await.unwrap().unwrap(); + + // Some more events: the admin creates an account, disables it and enables it again (not ana: that would sign her out). + let (code, body) = confirmed(&root, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!(code, 200, "{body}"); + let bo = serde_json::from_str::(&body).unwrap().user.id; + for what in ["disable", "enable"] { + let (code, body) = confirmed(&root, "POST", &format!("/v1/admin/users/{bo}/{what}"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 204, "{body}"); + } + + // The signed-in account, as the server says it. + let (code, body) = call(&root, "GET", "/v1/account", ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().role, "admin"); + let (code, body) = call(&ana, "GET", "/v1/account", ""); + assert_eq!((code, serde_json::from_str::(&body).unwrap().role.as_str()), (200, "member")); + + // The admin's list: pages with a cursor given back as it came, filters, the server's own refusals. + let (code, body) = call(&root, "GET", "/v1/admin/audit?limit=2", ""); + assert_eq!(code, 200, "{body}"); + let first = page_of(&body); + assert_eq!(first.events.len(), 2); + assert!(first.events[0].at_ms >= first.events[1].at_ms, "newest first"); + let next = first.next.clone().expect("more events than a page"); + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?limit=2&cursor={next}"), ""); + assert_eq!(code, 200, "{body}"); + let second = page_of(&body); + assert!(second.events.iter().all(|e| first.events.iter().all(|f| f.id != e.id)), "no event twice"); + let (code, body) = call(&root, "GET", "/v1/admin/audit?action=admin.", ""); + assert_eq!(code, 200, "{body}"); + let admin_events = page_of(&body).events; + assert!(admin_events.len() >= 3 && admin_events.iter().all(|e| e.action.starts_with("admin.")), "{admin_events:?}"); + let (code, body) = call(&root, "GET", "/v1/admin/audit?action=auth.login", ""); + let logins = page_of(&body).events; + assert!(code == 200 && logins.len() == 2 && logins.iter().all(|e| e.action == "auth.login"), "{body}"); + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?user={}", ana_user.id.0), ""); + assert_eq!(code, 200, "{body}"); + let about_ana = page_of(&body).events; + assert!(!about_ana.is_empty() && about_ana.iter().all(|e| e.actor_user == Some(ana_user.id.0) || e.target_id == Some(ana_user.id.0))); + let (code, body) = call(&root, "GET", "/v1/admin/audit?before_ms=1", ""); + assert!(code == 200 && page_of(&body).events.is_empty(), "{body}"); + // What is wrong is said before anything is sent, or by the server (a cursor it didn't make). + for query in [ + "action=nope.", + "action=auth.log", + "action=x%27%20OR%201=1", + "user=not-an-id", + "limit=0", + "limit=201", + "limit=x", + "before_ms=-1", + "cursor=a%20b", + "cursor=AAAA", + ] { + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?{query}"), ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{query}: {body}"); + } + + // A member's own activity: the events about her, whatever the admin's filters say; a member can't read the log. + let (code, body) = call(&ana, "GET", "/v1/account/activity", ""); + assert_eq!(code, 200, "{body}"); + let own = page_of(&body).events; + assert!(!own.is_empty(), "{own:?}"); + assert!(own.iter().all(|e| e.actor_user == Some(ana_user.id.0) || e.target_id == Some(ana_user.id.0)), "{own:?}"); + let (code, body) = + call(&ana, "GET", &format!("/v1/account/activity?user={}&action=admin.", server.meta.user_by_name("root").await.unwrap().unwrap().id.0), ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(page_of(&body).events.len(), own.len(), "an account's activity has no admin filters"); + let (code, body) = call(&ana, "GET", "/v1/account/activity?limit=101", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + for path in ["/v1/admin/audit", "/v1/admin/audit/export"] { + let (code, body) = call(&ana, "GET", path, ""); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{path}: {body}"); + } + let now = status(&ana); + assert!(now.signed_in && !now.removed, "a 403 doesn't sign the agent out: {now:?}"); + + // The export: the server's lines, then its last line; whole through the typed client, which writes as it reads. + let client = craft_cloud_app::Client::connect_in(root_data.path()).unwrap(); + let all = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + let outcome = client.admin_audit_export(&craft_cloud_app::AuditQuery::default(), &mut out); + (outcome, out) + }) + .await + .unwrap(); + let (outcome, out) = all; + let outcome = outcome.unwrap(); + assert!(!outcome.truncated); + let lines: Vec = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len() as u64, outcome.events); + let (code, body) = call(&root, "GET", "/v1/admin/audit?limit=200", ""); + assert_eq!(code, 200, "{body}"); + assert!(page_of(&body).events.iter().all(|e| lines.iter().any(|l| l.id == e.id)), "the export holds every event of the list"); + // A filtered export; a refusal before the stream; a filter that is wrong. + let client = craft_cloud_app::Client::connect_in(root_data.path()).unwrap(); + let filtered = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + let query = craft_cloud_app::AuditQuery { action: Some("auth.".into()), ..Default::default() }; + let outcome = client.admin_audit_export(&query, &mut out).unwrap(); + (outcome, out) + }) + .await + .unwrap(); + assert_eq!(filtered.0.events, 2, "the two sign-ins"); + let ana_client = craft_cloud_app::Client::connect_in(ana_data.path()).unwrap(); + let refused = tokio::task::spawn_blocking(move || ana_client.admin_audit_export(&Default::default(), &mut Vec::new())).await.unwrap(); + assert_eq!(refused.unwrap_err().code(), Some("forbidden")); + let (code, body) = call(&root, "GET", "/v1/admin/audit/export?action=nope.", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + + // Nothing of any of it was printed. + let (exit, out) = root_run.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + assert!(!out.contains(PASSWORD), "{out}"); + ana_run.stop().await.0.unwrap(); +} + +/// A server that holds more events than an export may: the last line is `truncated`, and the client says so (not whole). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_truncated_export_is_said_to_be() { + let server = admin_server_limited(AuditExportLimits { page: 1, channel: 1, max_rows: 2 }).await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("root")); + let (running, api) = signed_in_as(&server, "root", &data, &dir.path().join("Root Cloud")).await; + for name in ["bo", "cy", "di"] { + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": name, "role": "member" })); + assert_eq!(code, 200, "{body}"); + } + let client = craft_cloud_app::Client::connect_in(data.path()).unwrap(); + let (outcome, out) = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + (client.admin_audit_export(&Default::default(), &mut out), out) + }) + .await + .unwrap(); + let outcome = outcome.unwrap(); + assert_eq!((outcome.events, outcome.truncated), (2, true)); + assert_eq!(String::from_utf8(out).unwrap().lines().count(), 2, "the newest two, and no last line in what is written"); + running.stop().await.0.unwrap(); +} diff --git a/apps/craft-cloud-agent/tests/old_agent_blobs.rs b/apps/craft-cloud-agent/tests/old_agent_blobs.rs index 0aab2154..456a4cf7 100644 --- a/apps/craft-cloud-agent/tests/old_agent_blobs.rs +++ b/apps/craft-cloud-agent/tests/old_agent_blobs.rs @@ -23,6 +23,7 @@ async fn an_old_agent_is_too_old_for_blobs_of_any_size() { autostart: None, last_sync_ms: None, removed: false, + spaces: Vec::new(), }; let app = Router::new() .route("/v1/status", get(move || async move { axum::Json(status.clone()) })) diff --git a/apps/craft-cloud-agent/tests/read_only.rs b/apps/craft-cloud-agent/tests/read_only.rs new file mode 100644 index 00000000..887fe0d3 --- /dev/null +++ b/apps/craft-cloud-agent/tests/read_only.rs @@ -0,0 +1,133 @@ +//! The agent's HTTP transport, bound to a team space, against a real server on a loopback port: a viewer's engine edits a +//! file, the server's copy stays as it was, and the server's `403 read_only` reaches the engine as `TransportError::ReadOnly` +//! on the wire. Raising the role resumes the uploads. + +use craft_cloud_agent::cli::{self, LoginArgs}; +use craft_cloud_agent::http::{HttpTransport, ServerUrl}; +use craft_cloud_meta::{Meta, Role, SpaceActor, UserRecord}; +use craft_cloud_proto::api::SpaceRole; +use craft_cloud_proto::{ChunkId, ConflictStamp, DeviceId, Op, SpaceId, SpaceName}; +use craft_cloud_server::AppState; +use craft_cloud_server::auth::hash_password; +use craft_cloud_store::Store; +use craft_cloud_store::disk::DiskBackend; +use craft_cloud_sync::{Config, Engine, Fs, MemFs, READ_ONLY_EDITED, READ_ONLY_NEW, State, Transport, TransportError, rel}; + +const PASSWORD: &str = "correct horse battery"; +const STAMP: ConflictStamp = ConflictStamp { year: 2026, month: 11, day: 8, hour: 9, minute: 0, utc: false }; + +struct World { + url: ServerUrl, + meta: Meta, + space: SpaceId, + ben: UserRecord, + ana_token: craft_cloud_agent::http::DeviceToken, + ben_token: craft_cloud_agent::http::DeviceToken, + _shutdown: tokio::sync::oneshot::Sender<()>, + _dirs: Vec, +} + +/// A device token of `name`, made as the agent's `login` makes it. +async fn token_of(url: &str, name: &str) -> (craft_cloud_agent::http::DeviceToken, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let data = craft_cloud_agent::account::DataDir::new(dir.path().join("data")); + let args = LoginArgs { + url: url.to_owned(), + user: name.to_owned(), + folder: Some(dir.path().join("Craft Cloud")), + device_name: Some(format!("{name}-pc")), + adopt: false, + }; + cli::login(&data, &args, PASSWORD, &mut Vec::new()).await.unwrap(); + (data.load_account().unwrap().unwrap().token, dir) +} + +async fn world() -> World { + let db = tempfile::tempdir().unwrap(); + let blobs = tempfile::tempdir().unwrap(); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(craft_cloud_server::serve(state, listener, async move { + let _ = rx.await; + })); + let hash = hash_password(PASSWORD).unwrap(); + let ana = meta.create_user("ana", &hash, Role::Member, 0).await.unwrap(); + let ben = meta.create_user("ben", &hash, Role::Member, 0).await.unwrap(); + let space = meta.create_team_space(&SpaceActor::Cli, &SpaceName::new("Studio").unwrap(), ana.id, None, 0).await.unwrap(); + meta.invite_member(&SpaceActor::Cli, space, ben.id, SpaceRole::Viewer, 0).await.unwrap(); + let (ana_token, ana_dir) = token_of(&url, "ana").await; + let (ben_token, ben_dir) = token_of(&url, "ben").await; + World { url: ServerUrl::parse(&url).unwrap(), meta, space, ben, ana_token, ben_token, _shutdown: tx, _dirs: vec![db, blobs, ana_dir, ben_dir] } +} + +impl World { + fn transport(&self, token: &craft_cloud_agent::http::DeviceToken) -> HttpTransport { + HttpTransport::new(self.url.clone(), token).unwrap().bound_to(self.space).unwrap() + } + + fn engine(&self, token: &craft_cloud_agent::http::DeviceToken, name: &str) -> Engine { + Engine::new(MemFs::new(), self.transport(token), State::default(), Config::new(name, DeviceId::generate()), || STAMP).unwrap() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_viewer_edits_a_file_and_the_servers_copy_is_unchanged() { + let w = world().await; + let mut ana = w.engine(&w.ana_token, "ana-pc"); + let mut ben = w.engine(&w.ben_token, "ben-pc"); + ana.fs_mut().write(&rel("doc.txt"), b"base", false, 0).unwrap(); + assert_eq!(ana.sync_once().await.unwrap().uploaded.len(), 1, "an editor uploads"); + assert_eq!(ben.sync_once().await.unwrap().downloaded.len(), 1, "a viewer downloads"); + let before = w.transport(&w.ana_token).tree(None).await.unwrap(); + assert!(before.nodes.iter().any(|n| !n.deleted && n.name.as_ref().is_some_and(|name| name.as_str() == "doc.txt"))); + + // The server's answer on the wire. + let viewer = w.transport(&w.ben_token); + assert_eq!( + viewer.put_chunk(&ChunkId::from_bytes(*blake3::hash(b"x").as_bytes()), bytes::Bytes::from_static(b"x")).await, + Err(TransportError::ReadOnly) + ); + assert_eq!(viewer.commit(&[] as &[Op]).await, Err(TransportError::ReadOnly)); + assert_eq!(viewer.missing_chunks(&[ChunkId::from_bytes([1; 32])]).await, Err(TransportError::ReadOnly)); + assert!(viewer.tree(None).await.is_ok(), "reading is allowed"); + + // The viewer edits the file and adds one: both stay here, nothing is sent, nothing fails. + ben.fs_mut().write(&rel("doc.txt"), b"viewer edit", false, 0).unwrap(); + ben.fs_mut().write(&rel("mine.txt"), b"mine", false, 0).unwrap(); + let report = ben.sync_once().await.unwrap(); + assert!(report.read_only && ben.is_read_only(), "{report:?}"); + assert!(report.uploaded.is_empty() && report.refused.is_empty(), "{report:?}"); + assert!(report.skipped.contains(&("doc.txt".to_owned(), READ_ONLY_EDITED.to_owned())), "{report:?}"); + assert!(report.skipped.contains(&("mine.txt".to_owned(), READ_ONLY_NEW.to_owned())), "{report:?}"); + assert_eq!(ben.fs().read(&rel("doc.txt")).unwrap(), b"viewer edit"); + assert!(ben.sync_once().await.unwrap().skipped.is_empty(), "quiet once reported"); + + // The server's copy is the version it was: same tree, and the owner's engine sees no change. + let after = w.transport(&w.ana_token).tree(None).await.unwrap(); + assert_eq!(after.nodes, before.nodes); + assert!(!ana.sync_once().await.unwrap().changed()); + assert_eq!(ana.fs().read(&rel("doc.txt")).unwrap(), b"base"); + + // Raised to editor: the same files go up as ordinary changes. + w.meta.change_member_role(&SpaceActor::Cli, w.space, w.ben.id, SpaceRole::Editor, 1).await.unwrap(); + let report = ben.sync_once().await.unwrap(); + assert!(!report.read_only, "{report:?}"); + assert_eq!(report.uploaded.len(), 2, "{report:?}"); + assert_eq!(ana.sync_once().await.unwrap().downloaded.len(), 2); + assert_eq!(ana.fs().read(&rel("doc.txt")).unwrap(), b"viewer edit"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_personal_space_is_not_read_only_and_needs_no_header() { + let w = world().await; + // Without `bound_to` the transport works on the account's personal space, where the account is the owner. + let own = HttpTransport::new(w.url.clone(), &w.ben_token).unwrap(); + let space = own.space().await.unwrap(); + assert_ne!(space.space, w.space); + assert!(own.tree(None).await.is_ok()); + assert_eq!(w.transport(&w.ben_token).space().await.unwrap().role, Some(SpaceRole::Viewer)); +} diff --git a/apps/craft-cloud-agent/tests/team_spaces.rs b/apps/craft-cloud-agent/tests/team_spaces.rs new file mode 100644 index 00000000..a3be2f13 --- /dev/null +++ b/apps/craft-cloud-agent/tests/team_spaces.rs @@ -0,0 +1,1828 @@ +//! The agent with several spaces, against a real server on a loopback port: an invitation reaches the local API, is +//! accepted there, the team folder appears beside the personal one and syncs both ways between two accounts; a role +//! lowered to viewer makes the folder read-only; a removed member keeps the local copy, which stops syncing and is reported; +//! a renamed space keeps its folder; folder names clash and are made safe; an answer without the space's echo is refused; +//! an old server shows no team spaces and is never sent the `craft-space` header; and a Phase 1 data folder starts unchanged. +//! Review round: a state belongs to its folder (moving a space, turning it off and on, signing out), regained access merges +//! instead of resuming, held deletions of a team space are approved from the command line and the local API, a second stop +//! ends at once with a silent server, an archived space stays readable, pausing rests the long-polls, and one `404` for the +//! list does not end team spaces. +//! Second review round: turning a space off forgets its state, access lost and regained while the agent is stopped merges, a +//! space being turned off does not hold the spaces task up (a second stop, other calls, a queued pass), and a failed discovery +//! is retried for those waiting for the list. +//! Every wait is for a condition, with a timeout; the loops get short timings so the tests stay quick. + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderName, Request, Response, StatusCode}; +use craft_cloud_agent::account::DataDir; +use craft_cloud_agent::cli::{self, AgentError, Exit, LoginArgs}; +use craft_cloud_agent::daemon::{RunOptions, run_loop}; +use craft_cloud_agent::http::{HttpTransport, ServerUrl}; +use craft_cloud_agent::schedule::Timing; +use craft_cloud_app::{Client, SpaceInfo, SpaceList, SpaceRole, SpaceRun, SpaceSync, SpaceSyncRequest}; +use craft_cloud_meta::{Meta, Role, SpaceActor, UserRecord}; +use craft_cloud_proto::{SpaceId, SpaceName}; +use craft_cloud_server::AppState; +use craft_cloud_server::auth::hash_password; +use craft_cloud_store::Store; +use craft_cloud_store::disk::DiskBackend; +use craft_cloud_sync::Transport; +use tokio::sync::mpsc; + +const PASSWORD: &str = "correct horse battery"; +/// Longest wait for anything to show up. +const PATIENCE: Duration = Duration::from_secs(40); + +/// A server on a loopback port with two accounts, `ana` and `ben`. +struct World { + url: String, + meta: Meta, + ana: UserRecord, + ben: UserRecord, + _shutdown: tokio::sync::oneshot::Sender<()>, + _dirs: Vec, +} + +async fn world() -> World { + let (db, blobs) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = craft_cloud_server::serve(state, listener, async move { + let _ = rx.await; + }) + .await; + }); + let hash = hash_password(PASSWORD).unwrap(); + let ana = meta.create_user("ana", &hash, Role::Member, 0).await.unwrap(); + let ben = meta.create_user("ben", &hash, Role::Member, 0).await.unwrap(); + World { url, meta, ana, ben, _shutdown: tx, _dirs: vec![db, blobs] } +} + +impl World { + /// A team space `name` owned by ana. + async fn space(&self, name: &str) -> SpaceId { + self.meta.create_team_space(&SpaceActor::Cli, &SpaceName::new(name).unwrap(), self.ana.id, None, 0).await.unwrap() + } + + /// Ana (the owner) invites ben: a pending invitation that gives no access until he accepts it. (The command line's + /// invitation is accepted at once.) + async fn invite_ben(&self, space: SpaceId, role: SpaceRole) { + let ana = SpaceActor::User { user: self.ana.id, device: None }; + self.meta.invite_member(&ana, space, self.ben.id, role, 0).await.unwrap(); + } +} + +/// Timings for tests: everything in fractions of a second. +fn quick() -> RunOptions { + RunOptions { + timing: Timing { + debounce: Duration::from_millis(100), + max_delay: Duration::from_secs(1), + stability: Duration::from_millis(200), + periodic: Duration::from_secs(600), + unwatched_periodic: Duration::from_millis(500), + backoff_first: Duration::from_millis(200), + backoff_max: Duration::from_secs(1), + recheck: Duration::from_millis(300), + recheck_slow: Duration::from_secs(5), + approval_check: Duration::from_millis(100), + held_check: Duration::from_millis(300), + }, + heartbeat: Duration::from_millis(200), + // The spaces are looked at every two of these. + poll_wait: Duration::from_millis(500), + poll_retry_first: Duration::from_millis(100), + poll_retry_max: Duration::from_secs(1), + poll_min_spacing: Duration::from_millis(100), + poll_busy: Duration::from_millis(200), + fail_watch: false, + record_delay: Duration::ZERO, + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, + autostart: craft_cloud_agent::autostart::Target::Off, + } +} + +type LoopResult = (Result, String); + +struct Running { + stop: mpsc::UnboundedSender<()>, + thread: Option>, +} + +impl Running { + async fn stop(mut self) -> LoopResult { + let _ = self.stop.send(()); + let thread = self.thread.take().unwrap(); + tokio::task::spawn_blocking(move || thread.join().unwrap()).await.unwrap() + } +} + +impl Drop for Running { + fn drop(&mut self) { + // A test that ends (or failed) midway: two stops end the loop at once, and it is waited for, so no loop is left + // writing into a folder the test removes. + let _ = self.stop.send(()); + let _ = self.stop.send(()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// An account on a computer: its data folder, its personal folder, and the loop that runs there. +struct Person { + /// First, so the loop is stopped before the folders go. + running: Option, + data: DataDir, + personal: PathBuf, + dir: tempfile::TempDir, +} + +impl Person { + async fn signed_in(url: &str, user: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + let person = Self { running: None, data: DataDir::new(dir.path().join("data")), personal: dir.path().join("Craft Cloud"), dir }; + let args = LoginArgs { + url: url.to_owned(), + user: user.to_owned(), + folder: Some(person.personal.clone()), + device_name: Some(format!("{user}-pc")), + adopt: false, + }; + cli::login(&person.data, &args, PASSWORD, &mut Vec::new()).await.unwrap(); + person + } + + fn run(&mut self) { + self.run_with(quick()); + } + + /// Waits until the loop's local API answers. + async fn ready(&self) { + eventually("the local API answers", || async { self.api(|c| c.status()).await.is_ok() }).await; + } + + fn run_with(&mut self, options: RunOptions) { + let (stop, stops) = mpsc::unbounded_channel(); + let data = self.data.clone(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let mut out = Vec::new(); + let exit = runtime.block_on(run_loop(&data, options, stops, &mut out)); + (exit, String::from_utf8_lossy(&out).into_owned()) + }); + self.running = Some(Running { stop, thread: Some(thread) }); + } + + async fn stop(&mut self) -> LoopResult { + self.running.take().unwrap().stop().await + } + + /// The shared place of team folders: beside the personal folder. + fn shared(&self) -> PathBuf { + self.dir.path().join("Craft Cloud Shared") + } + + /// A call of the local API, off the async threads (the client blocks). + async fn api( + &self, + call: impl FnOnce(Client) -> Result + Send + 'static, + ) -> Result { + let dir = self.data.path().to_owned(); + tokio::task::spawn_blocking(move || call(Client::connect_in(&dir)?)).await.unwrap() + } + + async fn spaces(&self) -> Option { + self.api(|c| c.spaces()).await.ok() + } + + /// The team space `id` as the local API lists it. + async fn space(&self, id: SpaceId) -> Option { + self.spaces().await?.spaces.into_iter().find(|s| s.space == id) + } + + async fn accept(&self, id: SpaceId) { + self.api(move |c| c.accept_space(id)).await.unwrap(); + // The agent looks at its list again before the call answers, but a look that failed under load leaves the old list. + eventually("the accepted space is listed as active", || async { self.space(id).await.is_some_and(|s| s.state == "active") }).await; + } + + /// Turns the sync of a team space on and returns where it syncs. + async fn sync_on(&self, id: SpaceId) -> PathBuf { + let info = self.api(move |c| c.set_space_sync(id, &SpaceSyncRequest { on: true, folder: None, adopt: false })).await.unwrap(); + assert_eq!(info.sync, SpaceSync::On, "{info:?}"); + PathBuf::from(info.folder.unwrap()) + } + + async fn status(&self) -> craft_cloud_app::Status { + self.api(|c| c.status()).await.unwrap() + } + + /// Turns the sync of a team space on, in `folder`. + async fn sync_on_in(&self, id: SpaceId, folder: &Path) -> SpaceInfo { + let folder = folder.display().to_string(); + let info = self.api(move |c| c.set_space_sync(id, &SpaceSyncRequest { on: true, folder: Some(folder), adopt: false })).await.unwrap(); + assert_eq!(info.sync, SpaceSync::On, "{info:?}"); + info + } + + /// Turns the sync of a team space off. + async fn sync_off(&self, id: SpaceId) { + let info = self.api(move |c| c.set_space_sync(id, &SpaceSyncRequest { on: false, folder: None, adopt: false })).await.unwrap(); + assert_eq!(info.sync, SpaceSync::Off, "{info:?}"); + } + + /// The saved state file of a team space, as JSON, if there is one. + fn space_state(&self, id: SpaceId) -> Option { + let text = std::fs::read_to_string(self.data.path().join("spaces").join(id.to_string()).join("state.json")).ok()?; + serde_json::from_str(&text).ok() + } +} + +/// The files directly in `folder`, by name (hidden ones, the agent's own, left out). +fn names(folder: &Path) -> Vec { + let mut found: Vec = std::fs::read_dir(folder) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| !n.starts_with('.')) + .collect() + }) + .unwrap_or_default(); + found.sort(); + found +} + +/// Waits until `cond` holds, failing after [`PATIENCE`]. +async fn eventually>(what: &str, mut cond: impl FnMut() -> F) { + let deadline = Instant::now() + PATIENCE; + while !cond().await { + assert!(Instant::now() < deadline, "timed out waiting until {what}"); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn write(folder: &Path, rel: &str, data: &str) { + let path = folder.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, data).unwrap(); +} + +fn read(folder: &Path, rel: &str) -> Option { + std::fs::read_to_string(folder.join(rel)).ok() +} + +async fn has_file(folder: PathBuf, rel: &'static str, data: &'static str) -> bool { + read(&folder, rel).as_deref() == Some(data) +} + +/// Ana and ben, each with a computer, both running, and a team space `Studio` that ben was invited to as `role`. +struct Team { + world: World, + ana: Person, + ben: Person, + space: SpaceId, +} + +async fn team(role: SpaceRole) -> Team { + let world = world().await; + let (mut ana, mut ben) = (Person::signed_in(&world.url, "ana").await, Person::signed_in(&world.url, "ben").await); + let space = world.space("Studio").await; + world.invite_ben(space, role).await; + ana.run(); + ben.run(); + Team { world, ana, ben, space } +} + +/// As [`team`], with ben's computer reaching the server through a front that can misbehave. +async fn team_via_front(role: SpaceRole, behaviour: Front) -> (Team, Arc, tokio::task::JoinHandle<()>) { + let world = world().await; + let (url, front, task) = front(&world.url, behaviour).await; + let (mut ana, mut ben) = (Person::signed_in(&world.url, "ana").await, Person::signed_in(&url, "ben").await); + let space = world.space("Studio").await; + world.invite_ben(space, role).await; + ana.run(); + ben.run(); + (Team { world, ana, ben, space }, front, task) +} + +/// Ben accepts, and both turn the space's sync on. Returns ana's and ben's team folders. +async fn both_syncing(t: &Team) -> (PathBuf, PathBuf) { + let space = t.space; + eventually("the invitation is listed", || async move { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + // Ana's agent may not have looked at the server's list yet (the space was made just before it started). + eventually("ana's agent lists the space", || async move { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let (ana_folder, ben_folder) = (t.ana.sync_on(space).await, t.ben.sync_on(space).await); + (ana_folder, ben_folder) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_invitation_is_accepted_in_the_local_api_and_the_team_folder_syncs_both_ways() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + + // The invitation is listed, with no folder; the personal space comes first. + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + let list = t.ben.spaces().await.unwrap(); + assert_eq!(list.spaces.first().map(|s| s.kind.as_str()), Some("personal"), "{list:?}"); + let invited = t.ben.space(space).await.unwrap(); + assert_eq!((invited.sync, invited.folder.clone(), invited.name.as_deref()), (SpaceSync::Off, None, Some("Studio")), "{invited:?}"); + + // Accepted: it is an active member, syncing off, with its folder chosen beside the personal one but not made yet. + t.ben.accept(space).await; + let accepted = t.ben.space(space).await.unwrap(); + assert_eq!((accepted.state.as_str(), accepted.sync, accepted.role), ("active", SpaceSync::Off, SpaceRole::Editor), "{accepted:?}"); + let planned = PathBuf::from(accepted.folder.unwrap()); + assert_eq!(planned, t.ben.shared().join("Studio")); + assert!(!planned.exists(), "nothing is made until the person turns the sync on"); + assert!(t.ben.status().await.spaces.is_empty(), "a space that doesn't sync has no status entry"); + + // Turned on: the folder appears with the root marker, and the team space syncs both ways. + eventually("ana's agent lists the space", || async { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let (ana_folder, ben_folder) = (t.ana.sync_on(space).await, t.ben.sync_on(space).await); + assert_eq!(ben_folder, planned); + assert!(ben_folder.join(".craft-cloud/root.json").is_file()); + write(&ana_folder, "brief.txt", "from ana"); + eventually("ana's file reaches ben", || has_file(ben_folder.clone(), "brief.txt", "from ana")).await; + write(&ben_folder, "Shots/010.psd", "layers by ben"); + write(&ben_folder, "brief.txt", "edited by ben"); + eventually("ben's work reaches ana", || async { + read(&ana_folder, "Shots/010.psd").as_deref() == Some("layers by ben") && read(&ana_folder, "brief.txt").as_deref() == Some("edited by ben") + }) + .await; + + // The status names the space with its state; the personal folder is not mixed in. + eventually("the status says idle for the space", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Idle && s.last_sync_ms.is_some() && !s.read_only) + }) + .await; + assert!(!t.ben.personal.join("Studio").exists() && !t.ben.personal.join("brief.txt").exists()); + assert_eq!(std::fs::read_dir(&t.ben.personal).unwrap().filter_map(Result::ok).filter(|e| e.file_name() != ".craft-cloud").count(), 0); + // Its own state, apart from the personal one. + assert!(t.ben.data.path().join("spaces").join(space.to_string()).join("state.json").is_file()); + + let (ana, mut ben) = (t.ana, t.ben); + let (exit, out) = ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + // Started again: the team folder is still synced, without turning it on again. + ben.run(); + write(&ana_folder, "after.txt", "after the restart"); + eventually("the restarted agent still syncs the team folder", || has_file(ben_folder.clone(), "after.txt", "after the restart")).await; + let _ = ana; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn declining_an_invitation_removes_it() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.api(move |c| c.decline_space(space)).await.unwrap(); + assert!(t.ben.space(space).await.is_none(), "declined: no longer listed"); + // A second answer to the same invitation is the server's `not_found`, passed through (not an old agent's). + let again = t.ben.api(move |c| c.decline_space(space)).await.unwrap_err(); + assert!(matches!(&again, craft_cloud_app::Error::Refused { code, .. } if code == "not_found"), "{again:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_role_lowered_to_viewer_makes_the_folder_read_only() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ana_folder, "base.txt", "base"); + eventually("ben has the file", || has_file(ben_folder.clone(), "base.txt", "base")).await; + + t.world.meta.change_member_role(&SpaceActor::Cli, space, t.world.ben.id, SpaceRole::Viewer, 1).await.unwrap(); + eventually("the local API says read-only", || async { t.ben.space(space).await.is_some_and(|s| s.role == SpaceRole::Viewer && s.read_only) }) + .await; + eventually("the status says read-only", || async { t.ben.status().await.spaces.iter().any(|s| s.space == space && s.read_only) }).await; + + // A viewer's edit and new file stay on the viewer's disk; the server's changes still arrive. + write(&ben_folder, "base.txt", "edited by the viewer"); + write(&ben_folder, "mine.txt", "viewer's own"); + write(&ana_folder, "from-ana.txt", "still arrives"); + eventually("ana's later file reaches the viewer", || has_file(ben_folder.clone(), "from-ana.txt", "still arrives")).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ana_folder, "base.txt").as_deref(), Some("base"), "the viewer's edit never went up"); + assert_eq!(read(&ana_folder, "mine.txt"), None, "nor the viewer's new file"); + assert_eq!(read(&ben_folder, "base.txt").as_deref(), Some("edited by the viewer"), "and is not undone here"); + assert_eq!(read(&ben_folder, "mine.txt").as_deref(), Some("viewer's own")); + + // Raised again: what was only here goes up. + t.world.meta.change_member_role(&SpaceActor::Cli, space, t.world.ben.id, SpaceRole::Editor, 2).await.unwrap(); + eventually("ben's files go up once he may write", || async { + read(&ana_folder, "mine.txt").as_deref() == Some("viewer's own") && read(&ana_folder, "base.txt").as_deref() == Some("edited by the viewer") + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_removed_member_keeps_the_local_copy_which_stops_syncing_and_is_reported() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ana_folder, "plans/a.txt", "plan a"); + write(&ben_folder, "notes.txt", "ben's notes"); + eventually("both have everything", || async { read(&ben_folder, "plans/a.txt").is_some() && read(&ana_folder, "notes.txt").is_some() }).await; + + t.world.meta.remove_member(&SpaceActor::Cli, space, t.world.ben.id, 1).await.unwrap(); + eventually("the space reports no access", || async { + t.ben.status().await.spaces.iter().any(|s| { + s.space == space && matches!(&s.run_state, SpaceRun::NoAccess { message } if message.contains("Studio") && message.contains("kept")) + }) + }) + .await; + let info = t.ben.space(space).await.unwrap(); + assert_eq!(info.sync, SpaceSync::Unavailable, "{info:?}"); + assert!(matches!(info.run_state, Some(SpaceRun::NoAccess { .. })), "{info:?}"); + + // Nothing was deleted here, the folder and its marker stay, and nothing moves either way any more. + assert_eq!(read(&ben_folder, "plans/a.txt").as_deref(), Some("plan a")); + assert_eq!(read(&ben_folder, "notes.txt").as_deref(), Some("ben's notes")); + assert!(ben_folder.join(".craft-cloud/root.json").is_file()); + write(&ana_folder, "later.txt", "after the removal"); + write(&ben_folder, "local-only.txt", "written after the removal"); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ben_folder, "later.txt"), None, "no further changes arrive"); + assert_eq!(read(&ana_folder, "local-only.txt"), None, "and none leave"); + assert_eq!(read(&ana_folder, "notes.txt").as_deref(), Some("ben's notes"), "the server still has ben's earlier work"); + + // The personal space is unaffected, and a re-invited member syncs again without losing the local files. + assert!(t.ben.status().await.signed_in); + t.world.invite_ben(space, SpaceRole::Editor).await; + eventually("the invitation is listed again", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("the sync goes on and catches up", || async { + read(&ben_folder, "later.txt").as_deref() == Some("after the removal") + && read(&ana_folder, "local-only.txt").as_deref() == Some("written after the removal") + }) + .await; + assert_eq!(read(&ben_folder, "notes.txt").as_deref(), Some("ben's notes")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_renamed_space_keeps_its_folder_and_shows_the_new_name() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ana_folder, "one.txt", "1"); + eventually("synced", || has_file(ben_folder.clone(), "one.txt", "1")).await; + + let owner = SpaceActor::User { user: t.world.ana.id, device: None }; + t.world.meta.rename_team_space(&owner, space, &SpaceName::new("Studio Two").unwrap(), 1).await.unwrap(); + eventually("the new name is shown", || async { t.ben.space(space).await.is_some_and(|s| s.name.as_deref() == Some("Studio Two")) }).await; + let info = t.ben.space(space).await.unwrap(); + assert_eq!(info.folder.as_deref().map(PathBuf::from), Some(ben_folder.clone()), "the folder keeps its name: {info:?}"); + assert!(ben_folder.is_dir() && !t.ben.shared().join("Studio Two").exists()); + write(&ana_folder, "two.txt", "2"); + eventually("the sync goes on in the same folder", || has_file(ben_folder.clone(), "two.txt", "2")).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn spaces_whose_names_make_the_same_folder_get_numbered_and_a_folder_in_the_way_is_not_taken() { + let world = world().await; + let first = world.space("Plans: 2026").await; + let second = world.space("Plans? 2026").await; + let third = world.space("Studio").await; + for space in [first, second, third] { + world.invite_ben(space, SpaceRole::Editor).await; + } + let mut ben = Person::signed_in(&world.url, "ben").await; + // A folder of that name is already there, with files in it. + write(&ben.shared().join("Studio"), "mine.txt", "not a team space's"); + ben.run(); + for space in [first, second, third] { + let ben = &ben; + eventually("listed", || async move { ben.space(space).await.is_some() }).await; + ben.accept(space).await; + } + let folder = |space: SpaceId| { + let ben = &ben; + async move { ben.space(space).await.unwrap().folder.map(PathBuf::from).unwrap() } + }; + let (a, b, c) = (folder(first).await, folder(second).await, folder(third).await); + let mut names = [a.clone(), b.clone()].map(|p| p.file_name().unwrap().to_string_lossy().into_owned()); + names.sort(); + assert_eq!(names, ["Plans_ 2026", "Plans_ 2026 (2)"], "the unsafe characters are replaced, and the clash is numbered"); + assert_eq!(c.file_name().unwrap().to_string_lossy(), "Studio (2)", "a folder already there is not taken"); + // Turning on a folder that is free works; the occupied one is never adopted without consent. + ben.sync_on(first).await; + let in_the_way = ben.shared().join("Studio").display().to_string(); + let occupied = ben.api(move |cl| cl.set_space_sync(third, &SpaceSyncRequest { on: true, folder: Some(in_the_way), adopt: false })).await; + assert!(matches!(&occupied, Err(craft_cloud_app::Error::Refused { code, .. }) if code == "folder_has_files"), "{occupied:?}"); + assert_eq!(read(&ben.shared().join("Studio"), "mine.txt").as_deref(), Some("not a team space's")); + // A folder inside the personal folder is refused. + let inside = ben.personal.join("Team").display().to_string(); + let refused = ben.api(move |cl| cl.set_space_sync(third, &SpaceSyncRequest { on: true, folder: Some(inside), adopt: false })).await; + assert!(matches!(&refused, Err(craft_cloud_app::Error::Refused { code, .. }) if code == "invalid"), "{refused:?}"); +} + +// --------------------------------------------------------------------------------------------------------------------- +// A server in front of the real one that can misbehave like an old one + +/// What the front does to the requests it forwards. +#[derive(Default)] +struct Front { + /// Answers `/api/v1/info` with protocol 1 and `/api/v1/spaces` with `404`, as a server from before team spaces does. + old: bool, + /// Leaves the `craft-space` header out of every answer, as a server that ignores the header does. + no_echo: bool, + /// Every path asked, and whether it carried `craft-space`. + seen: Mutex>, + /// Answers `/api/v1/spaces` with `404` this many more times, then goes on as usual (a proxy during a restart). + spaces_404: Mutex, + /// Does not answer while set: the connection stays open and silent, and the request goes on once it is cleared. + stall: AtomicBool, + /// Waits this long (milliseconds) before forwarding each request. + delay_ms: AtomicU64, + /// The long-polls of a team space asked so far. + polls: AtomicUsize, +} + +async fn front(upstream: &str, behaviour: Front) -> (String, Arc, tokio::task::JoinHandle<()>) { + let behaviour = Arc::new(behaviour); + let state = (upstream.to_owned(), behaviour.clone(), reqwest::Client::new()); + let app = axum::Router::new().fallback(forward).with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (url, behaviour, task) +} + +type FrontState = (String, Arc, reqwest::Client); + +async fn forward(State((upstream, front, client)): State, request: Request) -> Response { + let (parts, body) = request.into_parts(); + let path = parts.uri.path().to_owned(); + let space_header = HeaderName::from_static("craft-space"); + let named = parts.headers.contains_key(&space_header); + front.seen.lock().unwrap().push((path.clone(), named)); + if named && parts.uri.query().is_some_and(|q| q.contains("wait=")) { + front.polls.fetch_add(1, Ordering::SeqCst); + } + // A request that arrives while `stall` is set (or is held by it) waits until it is cleared. + while front.stall.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(50)).await; + } + let delay = front.delay_ms.load(Ordering::SeqCst); + if delay > 0 { + tokio::time::sleep(Duration::from_millis(delay)).await; + } + let refuse = path == "/api/v1/spaces" && { + let mut left = front.spaces_404.lock().unwrap(); + let refuse = *left > 0; + *left = left.saturating_sub(1); + refuse + }; + if refuse { + return Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("{\"error\":\"not_found\",\"message\":\"no such route\"}")) + .unwrap(); + } + if front.old && path == "/api/v1/spaces" { + return Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("{\"error\":\"not_found\",\"message\":\"no such route\"}")) + .unwrap(); + } + let bytes = axum::body::to_bytes(body, 64 * 1024 * 1024).await.unwrap(); + let target = format!("{upstream}{}", parts.uri.path_and_query().map_or("", |p| p.as_str())); + let mut call = client.request(parts.method.clone(), target).body(bytes); + for (name, value) in &parts.headers { + if name != "host" && name != "content-length" { + call = call.header(name, value); + } + } + let answer = call.send().await.unwrap(); + let status = answer.status(); + let headers = answer.headers().clone(); + let mut body = answer.bytes().await.unwrap().to_vec(); + if front.old && path == "/api/v1/info" { + let mut info: serde_json::Value = serde_json::from_slice(&body).unwrap(); + info["protocol_max"] = 1.into(); + body = serde_json::to_vec(&info).unwrap(); + } + let mut response = Response::builder().status(status); + for (name, value) in &headers { + if name == "content-length" || name == "transfer-encoding" || (front.no_echo && name == "craft-space") { + continue; + } + response = response.header(name, value); + } + response.body(Body::from(body)).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_server_that_answers_without_the_echo_is_refused() { + let world = world().await; + let space = world.space("Studio").await; + world.invite_ben(space, SpaceRole::Editor).await; + world.meta.accept_invitation(world.ben.id, None, space, 1).await.unwrap(); + let ben = Person::signed_in(&world.url, "ben").await; + let account = ben.data.load_account().unwrap().unwrap(); + let (url, front, _task) = front(&world.url, Front { no_echo: true, ..Front::default() }).await; + let transport = HttpTransport::new(ServerUrl::parse(&url).unwrap(), &account.token).unwrap(); + + // Without the space, an answer needs no echo (it is the personal space, as ever). + assert!(transport.space().await.is_ok()); + // Bound to a team space, the same answer is refused: it could be about the personal space. + let bound = transport.clone().bound_to(space).unwrap(); + let refused = bound.space().await.unwrap_err(); + assert!(refused.to_string().contains("too old for team spaces"), "{refused}"); + assert!(bound.tree(None).await.is_err_and(|e| e.to_string().contains("too old for team spaces"))); + assert!(front.seen.lock().unwrap().iter().any(|(path, named)| path.ends_with("/space") && *named)); + + // The real server's answers carry the echo and are accepted; so is a refusal of the layer (`404`). + let direct = HttpTransport::new(ServerUrl::parse(&world.url).unwrap(), &account.token).unwrap(); + assert_eq!(direct.clone().bound_to(space).unwrap().space().await.unwrap().space, space); + let other = direct.bound_to(SpaceId::generate()).unwrap().space().await.unwrap_err(); + assert_eq!(other, craft_cloud_sync::TransportError::NotFound); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_old_server_shows_no_team_spaces_and_is_never_sent_the_space_header() { + let world = world().await; + let space = world.space("Studio").await; + world.invite_ben(space, SpaceRole::Editor).await; + let (url, front, _task) = front(&world.url, Front { old: true, ..Front::default() }).await; + let mut ben = Person::signed_in(&url, "ben").await; + ben.run(); + ben.ready().await; + write(&ben.personal, "mine.txt", "personal as ever"); + eventually("the personal folder syncs", || async { ben.status().await.last_sync_ms.is_some() }).await; + // Discovery runs more than once in this time; it asks about the version only. + tokio::time::sleep(Duration::from_secs(5)).await; + let list = ben.spaces().await.unwrap(); + assert_eq!(list.spaces.len(), 1, "personal only: {list:?}"); + assert_eq!(list.spaces[0].kind, "personal"); + assert!(ben.status().await.spaces.is_empty()); + { + let seen = front.seen.lock().unwrap(); + assert!(seen.iter().all(|(_, named)| !named), "no request names a space: {seen:?}"); + assert!(seen.iter().all(|(path, _)| path != "/api/v1/spaces"), "/spaces is not even asked: {seen:?}"); + let infos = seen.iter().filter(|(path, _)| path == "/api/v1/info").count(); + assert!(infos < 40, "no loop of version checks: {infos}"); + } + // Turning a team space on is refused with the reason. + let refused = ben.api(move |c| c.set_space_sync(space, &SpaceSyncRequest { on: true, folder: None, adopt: false })).await; + assert!(refused.is_err(), "{refused:?}"); + let (exit, out) = ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_phase_1_data_folder_starts_unchanged() { + let world = world().await; + let mut ben = Person::signed_in(&world.url, "ben").await; + let account_file = ben.data.path().join("account.json"); + let before = std::fs::read(&account_file).unwrap(); + assert!(!String::from_utf8_lossy(&before).contains("spaces"), "an account from before team spaces has no such field"); + write(&ben.personal, "doc.txt", "as before"); + ben.run(); + ben.ready().await; + eventually("a pass finished", || async { ben.status().await.last_sync_ms.is_some() }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + let (exit, out) = ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + assert_eq!(std::fs::read(&account_file).unwrap(), before, "account.json is byte for byte as it was"); + assert!(!ben.data.path().join("spaces").exists(), "no folder for team spaces' state without a team space"); + assert!(ben.data.path().join("state.json").is_file()); + let _: SocketAddr = "127.0.0.1:1".parse().unwrap(); +} + +// --------------------------------------------------------------------------------------------------------------------- +// The review round of P1-5 + +/// H3: a state made for one folder is never used for another. A space that synced to A, then to an empty B (which also +/// received what was added on the server later), turned back to A: A has the marker of the space, but B's state must not +/// decide what A is missing. The files B got and A never had are not deleted on the server. Signing out forgets the states. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_state_made_for_one_folder_is_not_used_for_another() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = both_syncing(&t).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben's first folder has the files", || has_file(a.clone(), "f5.txt", "content 5")).await; + + // Off, and on in an empty folder B: it fills with everything, and with what is added on the server later. + t.ben.sync_off(space).await; + let b = t.ben.dir.path().join("Elsewhere"); + t.ben.sync_on_in(space, &b).await; + eventually("the second folder has the files", || has_file(b.clone(), "f5.txt", "content 5")).await; + write(&ana_folder, "later.txt", "added later"); + eventually("the second folder has what came later", || has_file(b.clone(), "later.txt", "added later")).await; + assert_eq!(read(&a, "later.txt"), None, "the first folder is not synced meanwhile"); + + // Off, and on in A again: A has the space's marker, and B's state is not used for it. + t.ben.sync_off(space).await; + t.ben.sync_on_in(space, &a).await; + eventually("A catches up", || has_file(a.clone(), "later.txt", "added later")).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 7, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert_eq!(read(&ana_folder, "later.txt").as_deref(), Some("added later")); + assert_eq!(names(&a).len(), 7, "nor in A: {:?}", names(&a)); + assert_eq!(names(&b).len(), 7, "nor in B: {:?}", names(&b)); + let state = t.ben.space_state(space).unwrap(); + assert_eq!(state["folder"].as_str().map(PathBuf::from), std::fs::canonicalize(&a).ok(), "the state names the folder it is for"); + + // Signing out forgets the states of the team spaces, as it does the personal one; the folders and files stay. + let (_ana, mut ben) = (t.ana, t.ben); + let (exit, out) = ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + cli::logout(&ben.data, true, &mut Vec::new()).await.unwrap(); + assert!(ben.space_state(space).is_none(), "the team space's state is gone"); + assert!(!ben.data.path().join("state.json").exists()); + assert_eq!(names(&a).len(), 7); + assert_eq!(names(&b).len(), 7); + assert!(a.join(".craft-cloud/root.json").is_file() && b.join(".craft-cloud/root.json").is_file(), "the markers stay"); +} + +/// H1: moving a space that is syncing to another folder stops syncing the old one at once and starts the new folder from +/// a fresh state: nothing the old folder held is taken for a deletion in the new one, before or after a restart. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn moving_a_syncing_space_to_another_folder_syncs_the_new_folder_and_deletes_nothing() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = both_syncing(&t).await; + for i in 0..8 { + write(&ana_folder, &format!("doc{i}.txt"), &format!("document {i}")); + } + eventually("ben has the eight files", || has_file(a.clone(), "doc7.txt", "document 7")).await; + + // The person moves the space to an empty folder while it syncs. + let b = t.ben.dir.path().join("Moved"); + let info = t.ben.sync_on_in(space, &b).await; + assert_eq!(info.folder.as_deref().map(PathBuf::from), Some(b.clone())); + eventually("the new folder fills", || async { names(&b).len() == 8 }).await; + + // From now on the new folder is the one that syncs; the old one is left as it is. + write(&a, "left-behind.txt", "written in the old folder"); + write(&ana_folder, "new.txt", "written by ana"); + eventually("the new folder gets ana's later file", || has_file(b.clone(), "new.txt", "written by ana")).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ana_folder, "left-behind.txt"), None, "the old folder no longer goes up"); + assert_eq!(read(&a, "new.txt"), None, "and gets nothing"); + + // A restart opens the new folder with the state made for it. + let (_ana, mut ben) = (t.ana, t.ben); + let (exit, out) = ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + ben.run(); + write(&ana_folder, "after.txt", "after the restart"); + eventually("the restarted agent syncs the new folder", || has_file(b.clone(), "after.txt", "after the restart")).await; + tokio::time::sleep(Duration::from_secs(1)).await; + assert_eq!(names(&ana_folder).len(), 10, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert_eq!(names(&b).len(), 10, "{:?}", names(&b)); +} + +/// H2: turning a space off waits for its running pass to end, so the pass cannot write the old folder's state after the +/// space is set up for another folder. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn turning_a_space_off_waits_for_its_pass_and_the_old_state_is_not_written_after() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (ana_folder, a) = both_syncing(&t).await; + for i in 0..4 { + write(&ana_folder, &format!("f{i}.txt"), "content"); + } + eventually("ben has the files", || async { names(&a).len() == 4 }).await; + + // Every request takes a while now, so a pass lasts several seconds. + front.delay_ms.store(400, Ordering::SeqCst); + write(&a, "busy.txt", "ben's new file"); + eventually("a pass of the space is running", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) + }) + .await; + t.ben.sync_off(space).await; + // Off forgets the state, and the call is answered only when the running pass has ended: the pass cannot write the state + // after it (a call answered early would leave a state behind, or see one written afterwards). + assert!(t.ben.space_state(space).is_none(), "the state is gone when the call is answered"); + tokio::time::sleep(Duration::from_millis(2500)).await; + assert!(t.ben.space_state(space).is_none(), "no pass of the old folder writes its state after the space is off"); + + // On in another folder: the state it starts from is not the old folder's. + front.delay_ms.store(0, Ordering::SeqCst); + let b = t.ben.dir.path().join("Elsewhere"); + t.ben.sync_on_in(space, &b).await; + eventually("the new folder fills", || async { names(&b).len() >= 4 }).await; + eventually("the state is the new folder's", || async { + t.ben.space_state(space).is_some_and(|s| s["folder"].as_str().map(PathBuf::from) == std::fs::canonicalize(&b).ok()) + }) + .await; + tokio::time::sleep(Duration::from_secs(1)).await; + for i in 0..4 { + assert_eq!(read(&ana_folder, &format!("f{i}.txt")).as_deref(), Some("content"), "nothing was deleted on the server"); + assert_eq!(read(&b, &format!("f{i}.txt")).as_deref(), Some("content"), "nor in the new folder"); + } +} + +/// H4: access comes back after it was lost. Meanwhile ben deleted a file here and ana deleted one on the server, both +/// edited a third, and ben made a new one. Nothing ben has is deleted or replaced, his delete is not sent, hers does not +/// remove his file, the two edits are both kept, and his new file goes up. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn regained_access_merges_and_never_deletes_for_what_happened_meanwhile() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + for (name, content) in + [("keep.txt", "kept"), ("ben-deletes.txt", "ben removes this"), ("ana-deletes.txt", "ana removes this"), ("both.txt", "base")] + { + write(&ana_folder, name, content); + } + eventually("ben has everything", || async { names(&ben_folder).len() == 4 }).await; + + t.world.meta.remove_member(&SpaceActor::Cli, space, t.world.ben.id, 1).await.unwrap(); + eventually("the space reports no access", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && matches!(s.run_state, SpaceRun::NoAccess { .. })) + }) + .await; + eventually("its saved state is dropped", || async { t.ben.space_state(space).is_none() }).await; + + // While access is lost: deletes on both sides, an edit on both, a new file here. + std::fs::remove_file(ben_folder.join("ben-deletes.txt")).unwrap(); + std::fs::remove_file(ana_folder.join("ana-deletes.txt")).unwrap(); + write(&ana_folder, "both.txt", "ana's edit"); + write(&ben_folder, "both.txt", "ben's edit"); + write(&ben_folder, "ben-new.txt", "ben's new file"); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ana_folder, "ben-deletes.txt").as_deref(), Some("ben removes this"), "ben's delete did not go up"); + + // Access is given again. + t.world.invite_ben(space, SpaceRole::Editor).await; + eventually("the invitation is listed again", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("the folders are merged", || async { + read(&ana_folder, "ben-new.txt").as_deref() == Some("ben's new file") + && read(&ben_folder, "ben-deletes.txt").as_deref() == Some("ben removes this") + }) + .await; + tokio::time::sleep(Duration::from_secs(2)).await; + // Ben's own deleted file came back from the server; ana's delete did not remove ben's file. + assert_eq!(read(&ana_folder, "ben-deletes.txt").as_deref(), Some("ben removes this"), "still on the server"); + assert_eq!(read(&ben_folder, "ana-deletes.txt").as_deref(), Some("ana removes this"), "ana's delete did not remove ben's file"); + assert_eq!(read(&ana_folder, "keep.txt").as_deref(), Some("kept")); + // The two edits are both kept: one under the name, one as a conflict copy. + let both: Vec = names(&ben_folder).into_iter().filter(|n| n.contains("both")).collect(); + let contents: Vec = both.iter().filter_map(|n| read(&ben_folder, n)).collect(); + assert!(contents.contains(&"ana's edit".to_owned()) && contents.contains(&"ben's edit".to_owned()), "{both:?} {contents:?}"); +} + +/// The deletions a team space holds back (a mass delete) are approved from the command line, with the space's name, and +/// from the local API; the running agent then carries them out. (Ana's guard holds her deletes back; once they are on the +/// server, ben's guard holds them back on his side, and he approves too.) +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn held_deletions_of_a_team_space_are_approved_from_the_command_line_and_the_local_api() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + for i in 0..30 { + write(&ana_folder, &format!("f{i:02}.txt"), "content"); + } + eventually("ben has the thirty files", || async { names(&ben_folder).len() == 30 }).await; + let waiting_set = |person: &Person| { + person.data.space_dir(space).load_state().ok().and_then(|s| s.last_pass).and_then(|l| l.summary).and_then(|s| s.pending_set()) + }; + let approve = |person: &Person, args: Vec| { + let data = person.data.path().to_owned(); + async move { + let mut argv = vec!["craft-cloud-agent".to_owned(), "--data-dir".to_owned(), data.display().to_string(), "sync".to_owned()]; + argv.extend(args); + let cli = ::try_parse_from(argv).unwrap(); + let mut out = Vec::new(); + let exit = cli::run(cli, &mut std::io::empty(), &mut out).await; + (exit, String::from_utf8_lossy(&out).into_owned()) + } + }; + let args = |list: &[&str]| list.iter().map(|a| (*a).to_owned()).collect::>(); + + // Ana deletes twenty-five: her guard holds them back, and ben keeps his files. + for i in 0..25 { + std::fs::remove_file(ana_folder.join(format!("f{i:02}.txt"))).unwrap(); + } + eventually("ana's agent holds the deletions", || async { waiting_set(&t.ana).is_some() }).await; + let set = waiting_set(&t.ana).unwrap(); + assert_eq!(names(&ben_folder).len(), 30); + + // A space that isn't there is refused. + let (unknown, _) = approve(&t.ana, args(&["--approve-deletes", "--space", "Nowhere"])).await; + assert!(matches!(unknown, Err(AgentError::Usage(_))), "{unknown:?}"); + // The bare flag lists first (nothing showed the set before), and approves nothing. + let (first, out) = approve(&t.ana, args(&["--approve-deletes", "--space", "studio"])).await; + assert_eq!(first.unwrap(), Exit::NeedsApproval, "{out}"); + assert!(out.contains("nothing approved"), "{out}"); + assert!(!t.ana.data.space_dir(space).path().join("approval.json").exists()); + // A set that isn't the one waiting approves nothing either. + let (wrong, out) = approve(&t.ana, args(&["--approve-deletes=0000000000", "--space", "Studio"])).await; + assert_eq!(wrong.unwrap(), Exit::NeedsApproval, "{out}"); + assert_eq!(names(&ben_folder).len(), 30); + // The set itself, with the space named by its ID. + let (given, out) = approve(&t.ana, vec![format!("--approve-deletes={set}"), "--space".into(), space.to_string()]).await; + assert_eq!(given.unwrap(), Exit::Done, "{out}"); + assert!(out.contains("will carry out these deletions"), "{out}"); + // On the server now; ben's guard holds them, and he approves them his way. + eventually("ben's agent holds the deletions", || async { waiting_set(&t.ben).is_some() }).await; + let ben_set = waiting_set(&t.ben).unwrap(); + assert_eq!(names(&ben_folder).len(), 30); + let (given, out) = approve(&t.ben, vec![format!("--approve-deletes={ben_set}"), "--space".into(), "Studio".into()]).await; + assert_eq!(given.unwrap(), Exit::Done, "{out}"); + eventually("the approved deletions reach ben", || async { names(&ben_folder).len() == 5 }).await; + + // The second round goes through the local API: twenty new files, eighteen of them deleted. + for i in 0..20 { + write(&ana_folder, &format!("g{i:02}.txt"), "more"); + } + eventually("ben has them", || async { names(&ben_folder).len() == 25 }).await; + for i in 0..18 { + std::fs::remove_file(ana_folder.join(format!("g{i:02}.txt"))).unwrap(); + } + eventually("ana's agent holds the next deletions", || async { waiting_set(&t.ana).is_some() }).await; + let set = waiting_set(&t.ana).unwrap(); + let wrong = t.ana.api(move |c| c.approve_space_deletes(space, Some("0000000000"))).await.unwrap(); + assert!(!wrong.approved && wrong.message.contains("nothing approved"), "{wrong:?}"); + let answer = t.ana.api(move |c| c.approve_space_deletes(space, Some(&set))).await.unwrap(); + assert!(answer.approved, "{answer:?}"); + eventually("ben's agent holds them", || async { waiting_set(&t.ben).is_some() }).await; + let ben_set = waiting_set(&t.ben).unwrap(); + assert_eq!(names(&ben_folder).len(), 25); + let answer = t.ben.api(move |c| c.approve_space_deletes(space, Some(&ben_set))).await.unwrap(); + assert!(answer.approved, "{answer:?}"); + eventually("the deletions approved through the local API are carried out", || async { names(&ben_folder).len() == 7 }).await; + // A space that doesn't sync here is not found. + let missing = t.ana.api(|c| c.approve_space_deletes(SpaceId::generate(), None)).await.unwrap_err(); + assert!(matches!(&missing, craft_cloud_app::Error::Refused { code, .. } if code == "not_found"), "{missing:?}"); +} + +/// M1: a second stop ends the loop at once even when the server is silent and a pass of a team space is queued behind +/// another pass: no queued team pass starts, and the thread of the team passes is not waited for. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_stop_does_not_wait_for_the_team_spaces_with_a_silent_server() { + let (mut t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let (_, ben_folder) = both_syncing(&t).await; + write(&ben_folder, "first.txt", "first"); + eventually("the first file went up", || async { t.ben.status().await.spaces.iter().any(|s| s.last_sync_ms.is_some()) }).await; + + front.stall.store(true, Ordering::SeqCst); + write(&ben_folder, "stuck.txt", "never answered"); + write(&t.ben.personal, "stuck-too.txt", "never answered"); + tokio::time::sleep(Duration::from_secs(2)).await; + let mut running = t.ben.running.take().unwrap(); + let _ = running.stop.send(()); + let _ = running.stop.send(()); + let thread = running.thread.take().unwrap(); + let started = Instant::now(); + let ended = tokio::time::timeout(Duration::from_secs(8), tokio::task::spawn_blocking(move || thread.join().unwrap())).await; + front.stall.store(false, Ordering::SeqCst); + let (exit, out) = ended.expect("the loop ends within seconds of the second stop, though the server is silent").unwrap(); + assert!(started.elapsed() < Duration::from_secs(8)); + assert_eq!(exit.unwrap(), Exit::Failed, "{out}"); +} + +/// M2: a member of an archived space keeps read access: the space stays listed (marked archived), syncs read-only and is not +/// reported as lost; nothing of the member's goes up until it is unarchived. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_member_of_an_archived_space_keeps_a_read_only_copy() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ana_folder, "plan.txt", "the plan"); + eventually("ben has the plan", || has_file(ben_folder.clone(), "plan.txt", "the plan")).await; + + let owner = SpaceActor::User { user: t.world.ana.id, device: None }; + t.world.meta.set_space_archived(&owner, space, true, 1).await.unwrap(); + eventually("ben's space is read-only and archived", || async { + t.ben.space(space).await.is_some_and(|s| s.archived && s.read_only && s.sync == SpaceSync::On) + }) + .await; + write(&ben_folder, "mine.txt", "written while archived"); + tokio::time::sleep(Duration::from_secs(2)).await; + let info = t.ben.space(space).await.unwrap(); + assert!(!matches!(info.run_state, Some(SpaceRun::NoAccess { .. })), "not reported as lost: {info:?}"); + assert_eq!((info.sync, info.role), (SpaceSync::On, SpaceRole::Editor), "{info:?}"); + assert_eq!(read(&ana_folder, "mine.txt"), None, "nothing goes up meanwhile"); + assert_eq!(read(&ben_folder, "plan.txt").as_deref(), Some("the plan"), "and nothing is lost here"); + + t.world.meta.set_space_archived(&owner, space, false, 2).await.unwrap(); + eventually("what was kept goes up once it is unarchived", || has_file(ana_folder.clone(), "mine.txt", "written while archived")).await; +} + +/// L3: pausing rests the long-polls of the team spaces, resuming starts them again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pausing_rests_the_team_spaces_long_polls() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let _ = both_syncing(&t).await; + eventually("the team space long-polls", || async { front.polls.load(Ordering::SeqCst) >= 3 }).await; + + t.ben.api(|c| c.pause()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(1500)).await; + let resting = front.polls.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(front.polls.load(Ordering::SeqCst), resting, "no long-poll is asked while paused"); + + t.ben.api(|c| c.resume()).await.unwrap(); + eventually("the long-poll starts again", || async { front.polls.load(Ordering::SeqCst) > resting }).await; +} + +/// L4: one `404` for the list of spaces from a server that speaks team spaces does not end them: the space goes on +/// syncing and is not shown as unavailable. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn one_404_for_the_list_does_not_end_the_team_spaces() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ana_folder, "one.txt", "1"); + eventually("synced", || has_file(ben_folder.clone(), "one.txt", "1")).await; + + *front.spaces_404.lock().unwrap() = 1; + eventually("the 404 was answered", || async { *front.spaces_404.lock().unwrap() == 0 }).await; + // Looked at for a while after it: the space keeps syncing. + for _ in 0..8 { + let info = t.ben.space(space).await.unwrap(); + assert_eq!(info.sync, SpaceSync::On, "{info:?}"); + assert!(!matches!(info.run_state, Some(SpaceRun::Unavailable { .. } | SpaceRun::NoAccess { .. })), "{info:?}"); + tokio::time::sleep(Duration::from_millis(300)).await; + } + write(&ana_folder, "two.txt", "2"); + eventually("the sync goes on", || has_file(ben_folder.clone(), "two.txt", "2")).await; +} + +// --------------------------------------------------------------------------------------------------------------------- +// The second review round of P1-5 + +/// N1: turning a space off forgets its state, so turning it on again in the same folder is a first sync: what was deleted +/// here, or on the server, while it was off is not deleted on the other side. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn turning_a_space_off_forgets_its_state_so_turning_it_on_again_deletes_nothing() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben has the files", || async { names(&ben_folder).len() == 6 }).await; + + t.ben.sync_off(space).await; + assert!(t.ben.space_state(space).is_none(), "turning the sync off forgets the saved state"); + // While it is off, ben deletes one file here as from a local copy, and ana deletes another one on the server. + std::fs::remove_file(ben_folder.join("f0.txt")).unwrap(); + std::fs::remove_file(ana_folder.join("f1.txt")).unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ben_folder, "f1.txt").as_deref(), Some("content 1"), "ana's delete does not reach a space that is off"); + + // On again in the same folder: nothing is deleted on either side. What each side deleted comes back from the other. + let again = t.ben.sync_on(space).await; + assert_eq!(again, ben_folder); + eventually("ben's deleted file comes back from the server", || has_file(ben_folder.clone(), "f0.txt", "content 0")).await; + eventually("ana's deleted file is put back on the server", || has_file(ana_folder.clone(), "f1.txt", "content 1")).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 6, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert_eq!(names(&ben_folder).len(), 6, "nor here: {:?}", names(&ben_folder)); +} + +/// H4, the agent was down for the whole loss of access: the member is removed and invited again while it is stopped, a file +/// is deleted on each side, and the agent is started. The server's stamp of the membership differs from the one the saved state +/// was made under, so the state is not used: nothing is deleted on either side. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn access_lost_and_regained_while_the_agent_is_stopped_merges_and_deletes_nothing() { + let mut t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + for i in 0..4 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben has the files", || async { names(&ben_folder).len() == 4 }).await; + let (exit, out) = t.ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + + // Ben is removed and invited again while his agent is not running. (The stamps given here differ from the one the server + // gave when he first accepted, which is the real time.) + t.world.meta.remove_member(&SpaceActor::Cli, space, t.world.ben.id, 1).await.unwrap(); + t.world.invite_ben(space, SpaceRole::Editor).await; + t.world.meta.accept_invitation(t.world.ben.id, None, space, 2).await.unwrap(); + std::fs::remove_file(ben_folder.join("f0.txt")).unwrap(); + std::fs::remove_file(ana_folder.join("f1.txt")).unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + + t.ben.run(); + eventually("the file ben deleted comes back from the server", || has_file(ben_folder.clone(), "f0.txt", "content 0")).await; + eventually("the file ana deleted is put back on the server", || has_file(ana_folder.clone(), "f1.txt", "content 1")).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 4, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert_eq!(names(&ben_folder).len(), 4, "nor here: {:?}", names(&ben_folder)); +} + +/// N2: turning a space off while its pass is queued behind a slow pass of the personal space does not hold the spaces task up: +/// a second stop ends the loop at once. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_stop_does_not_wait_for_a_space_that_is_being_turned_off() { + let (mut t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (_, ben_folder) = both_syncing(&t).await; + write(&ben_folder, "first.txt", "first"); + eventually("the first file went up", || async { t.ben.status().await.spaces.iter().any(|s| s.last_sync_ms.is_some()) }).await; + + // The personal pass hangs, and the team space's pass waits for its turn behind it. + front.stall.store(true, Ordering::SeqCst); + write(&t.ben.personal, "stuck-too.txt", "never answered"); + eventually("the personal pass is running", || async { t.ben.status().await.run_state == craft_cloud_app::RunState::Syncing }).await; + write(&ben_folder, "stuck.txt", "never answered"); + eventually("the team space's pass is queued", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) + }) + .await; + // Turning the space off waits for that pass to let go of it: the call stays open. + let dir = t.ben.data.path().to_owned(); + let off = tokio::task::spawn_blocking(move || { + Client::connect_in(&dir).and_then(|c| c.set_space_sync(space, &SpaceSyncRequest { on: false, folder: None, adopt: false })) + }); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(!off.is_finished(), "the change waits for the space's session to close"); + + let mut running = t.ben.running.take().unwrap(); + let _ = running.stop.send(()); + let _ = running.stop.send(()); + let thread = running.thread.take().unwrap(); + let started = Instant::now(); + let ended = tokio::time::timeout(Duration::from_secs(8), tokio::task::spawn_blocking(move || thread.join().unwrap())).await; + front.stall.store(false, Ordering::SeqCst); + let (exit, out) = ended.expect("the loop ends within seconds of the second stop, though a space is being turned off").unwrap(); + assert!(started.elapsed() < Duration::from_secs(8)); + assert_eq!(exit.unwrap(), Exit::Failed, "{out}"); + drop(off); +} + +/// N2: a pass of a space that was queued before the space was turned off does not run afterwards, and turning the space off +/// does not hold up the answer to anything else. Here the pass waits behind a slow personal pass. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_queued_pass_of_a_space_that_was_turned_off_does_not_run() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ben_folder, "first.txt", "first"); + eventually("the first file reached ana", || has_file(ana_folder.clone(), "first.txt", "first")).await; + + // Nothing is running (a pass after the first file looks at it again after the stability window), so the personal pass + // takes the first turn. + eventually("nothing runs", || async { + let status = t.ben.status().await; + status.run_state == craft_cloud_app::RunState::Idle && status.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) + }) + .await; + // The personal pass hangs, and the team space's pass waits behind it. + tokio::time::sleep(Duration::from_millis(1500)).await; + eventually("still nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + front.stall.store(true, Ordering::SeqCst); + write(&t.ben.personal, "slow.txt", "slow"); + // The personal pass runs and no pass of the team space does: the personal pass holds the turn. + eventually("the personal pass is running and the team space's is not", || async { + let status = t.ben.status().await; + status.run_state == craft_cloud_app::RunState::Syncing && status.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) + }) + .await; + write(&ben_folder, "queued.txt", "written before the space was turned off"); + eventually("the team space's pass is queued", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) + }) + .await; + // The change waits for the pass to let go; other calls are answered meanwhile. + let dir = t.ben.data.path().to_owned(); + let off = tokio::task::spawn_blocking(move || { + Client::connect_in(&dir).and_then(|c| c.set_space_sync(space, &SpaceSyncRequest { on: false, folder: None, adopt: false })) + }); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(!off.is_finished(), "the change waits for the space's session to close"); + let started = Instant::now(); + assert!(t.ben.spaces().await.is_some()); + assert!(started.elapsed() < Duration::from_secs(5), "the local API is not held up by a space that is being turned off"); + front.stall.store(false, Ordering::SeqCst); + let info = off.await.unwrap().unwrap(); + assert_eq!(info.sync, SpaceSync::Off, "{info:?}"); + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(read(&ana_folder, "queued.txt"), None, "the pass queued before the space was turned off did not run"); + assert_eq!(read(&ben_folder, "queued.txt").as_deref(), Some("written before the space was turned off")); +} + +/// M1, the pass thread: a second stop ends the loop while a team pass is queued behind a slow pass; the queued pass never +/// starts, even when the slow pass ends later. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_team_pass_queued_when_the_second_stop_comes_never_starts() { + let (mut t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (ana_folder, ben_folder) = both_syncing(&t).await; + write(&ben_folder, "first.txt", "first"); + eventually("the first file reached ana", || has_file(ana_folder.clone(), "first.txt", "first")).await; + + // Nothing is running (a pass after the first file looks at it again after the stability window), so the personal pass + // takes the first turn. + eventually("nothing runs", || async { + let status = t.ben.status().await; + status.run_state == craft_cloud_app::RunState::Idle && status.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) + }) + .await; + tokio::time::sleep(Duration::from_millis(1500)).await; + eventually("still nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + front.stall.store(true, Ordering::SeqCst); + write(&t.ben.personal, "slow.txt", "slow"); + // The personal pass runs and no pass of the team space does: the personal pass holds the turn. + eventually("the personal pass is running and the team space's is not", || async { + let status = t.ben.status().await; + status.run_state == craft_cloud_app::RunState::Syncing && status.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) + }) + .await; + write(&ben_folder, "queued.txt", "never to be uploaded"); + eventually("the team space's pass is queued", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) + }) + .await; + let mut running = t.ben.running.take().unwrap(); + let _ = running.stop.send(()); + let _ = running.stop.send(()); + let thread = running.thread.take().unwrap(); + let (exit, out) = tokio::time::timeout(Duration::from_secs(8), tokio::task::spawn_blocking(move || thread.join().unwrap())) + .await + .expect("the loop ends within seconds of the second stop") + .unwrap(); + assert_eq!(exit.unwrap(), Exit::Failed, "{out}"); + // The slow pass ends by itself later; the queued pass must not take its turn. + front.stall.store(false, Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(8)).await; + assert_eq!(read(&ana_folder, "queued.txt"), None, "the queued pass did not start after the second stop"); +} + +/// A discovery that fails while someone waits for the list is tried again within seconds, and they wait for that: an +/// invitation accepted here and the sync turned on at once finds the space in the list. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn accepting_and_turning_the_sync_on_at_once_works_after_one_failed_discovery() { + // Ben's agent looks at the list only when asked (the periodic look is far off), so the one failure is the look the accept + // asks for. + let world = world().await; + let (url, front, _task) = front(&world.url, Front::default()).await; + let mut ben = Person::signed_in(&url, "ben").await; + let space = world.space("Studio").await; + world.invite_ben(space, SpaceRole::Editor).await; + ben.run_with(RunOptions { poll_wait: Duration::from_secs(30), ..quick() }); + let t = Team { ana: Person::signed_in(&world.url, "ana").await, ben, world, space }; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + *front.spaces_404.lock().unwrap() = 1; + t.ben.api(move |c| c.accept_space(space)).await.unwrap(); + // No wait for the list in between: the agent has found the space, or the answer to the accept waited until it did. + let info = t.ben.api(move |c| c.set_space_sync(space, &SpaceSyncRequest { on: true, folder: None, adopt: false })).await; + assert!(info.is_ok_and(|i| i.sync == SpaceSync::On), "the sync could be turned on right after the accept"); + assert_eq!(*front.spaces_404.lock().unwrap(), 0, "the failed discovery happened"); +} + +// --------------------------------------------------------------------------------------------------------------------- +// The third review round of P1-5 + +/// F1: turning a space on does not wait for another space's long pass: the pass thread is busy with space A, and space B, +/// which has no session and no pass, is turned on at once. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn turning_on_a_space_is_not_held_up_by_a_long_pass_of_another_space() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let a = t.space; + let b = t.world.space("Archive").await; + t.world.invite_ben(b, SpaceRole::Editor).await; + for space in [a, b] { + let ben = &t.ben; + eventually("the invitation is listed", || async move { ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + } + let a_folder = t.ben.sync_on(a).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // A's pass hangs; the pass thread is busy with it. + front.stall.store(true, Ordering::SeqCst); + write(&a_folder, "big.txt", "a long first sync, say"); + eventually("A's pass is running", || async { t.ben.status().await.spaces.iter().any(|s| s.space == a && s.run_state == SpaceRun::Syncing) }) + .await; + let dir = t.ben.data.path().to_owned(); + let on = tokio::task::spawn_blocking(move || { + Client::connect_in(&dir).and_then(|c| c.set_space_sync(b, &SpaceSyncRequest { on: true, folder: None, adopt: false })) + }); + let answered = tokio::time::timeout(Duration::from_secs(10), on).await; + front.stall.store(false, Ordering::SeqCst); + let info = answered.expect("space B is turned on while A's pass runs").unwrap().unwrap(); + assert_eq!(info.sync, SpaceSync::On, "{info:?}"); +} + +/// F2: the agent stops while a space is being turned off (its pass has not let go yet): the space stays off at the next start. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_space_being_turned_off_when_the_agent_stops_stays_off() { + let (mut t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + let (_, ben_folder) = both_syncing(&t).await; + write(&ben_folder, "first.txt", "first"); + eventually("the first file went up", || async { t.ben.status().await.spaces.iter().any(|s| s.last_sync_ms.is_some()) }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + front.stall.store(true, Ordering::SeqCst); + write(&ben_folder, "busy.txt", "never answered"); + eventually("the space's pass is running", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) + }) + .await; + let dir = t.ben.data.path().to_owned(); + let off = tokio::task::spawn_blocking(move || { + Client::connect_in(&dir).and_then(|c| c.set_space_sync(space, &SpaceSyncRequest { on: false, folder: None, adopt: false })) + }); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(!off.is_finished(), "turning the space off waits for its pass"); + + // A stop waits for the running pass (the personal space is idle, so the loop does not wait for a second stop); the + // space's turn-off is applied when the spaces task ends, before the pass lets go. + let mut running = t.ben.running.take().unwrap(); + let _ = running.stop.send(()); + let thread = running.thread.take().unwrap(); + let ended = tokio::spawn(async move { tokio::task::spawn_blocking(move || thread.join().unwrap()).await.unwrap() }); + tokio::time::sleep(Duration::from_secs(2)).await; + front.stall.store(false, Ordering::SeqCst); + let (exit, out) = tokio::time::timeout(Duration::from_secs(30), ended).await.expect("the loop ends once the pass has").unwrap(); + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + drop(off); + + t.ben.run(); + eventually("the space is listed", || async { t.ben.space(space).await.is_some() }).await; + let info = t.ben.space(space).await.unwrap(); + assert_eq!(info.sync, SpaceSync::Off, "the space that was being turned off stays off: {info:?}"); +} + +// --------------------------------------------------------------------------------------------------------------------- +// The fourth review round of P1-5 (items 3 to 5 of the independent review) + +/// Makes `link` a symlink (Unix) or junction (Windows) to the folder `target`. +fn link_dir(link: &Path, target: &Path) { + #[cfg(unix)] + std::os::unix::fs::symlink(target, link).unwrap(); + #[cfg(windows)] + { + let made = std::process::Command::new("cmd.exe").args(["/d", "/c", "mklink", "/J"]).arg(link).arg(target).output().unwrap(); + assert!(made.status.success(), "{made:?}"); + } +} + +/// Removes the link `link` made by [`link_dir`], not what it leads to. +fn unlink_dir(link: &Path) { + #[cfg(unix)] + std::fs::remove_file(link).unwrap(); + #[cfg(windows)] + std::fs::remove_dir(link).unwrap(); +} + +/// Item 3: a team space syncs through a link to folder A. While its session is open, the link is pointed at folder B, which +/// carries the same space's marker but not A's files. The session notices before the next pass and starts afresh in B (a +/// first sync): the files A had are not taken for deleted in B, and nothing is deleted on the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_link_pointed_at_another_folder_while_syncing_deletes_nothing() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("ana's agent lists the space", || async { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let ana_folder = t.ana.sync_on(space).await; + let (a, b, link) = (t.ben.dir.path().join("A"), t.ben.dir.path().join("B"), t.ben.dir.path().join("Link")); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + link_dir(&link, &a); + t.ben.sync_on_in(space, &link).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("the folder behind the link has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // Folder B carries the space's marker (as an older folder of the space would) but none of the files. + std::fs::create_dir_all(b.join(".craft-cloud")).unwrap(); + std::fs::copy(a.join(".craft-cloud/root.json"), b.join(".craft-cloud/root.json")).unwrap(); + unlink_dir(&link); + link_dir(&link, &b); + write(&ana_folder, "poke.txt", "asks for a pass"); + eventually("the folder behind the link now has everything", || async { names(&b).len() == 7 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 7, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert_eq!(names(&a).len(), 6, "the old folder is left as it is"); +} + +/// After a link is pointed at another folder (and the space starts afresh there), a local change in the new folder wakes a pass: +/// the folder watcher is made again for where the link leads now. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_local_change_behind_a_repointed_link_is_noticed() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("ana's agent lists the space", || async { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let ana_folder = t.ana.sync_on(space).await; + let (a, b, link) = (t.ben.dir.path().join("A"), t.ben.dir.path().join("B"), t.ben.dir.path().join("Link")); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + link_dir(&link, &a); + t.ben.sync_on_in(space, &link).await; + write(&ana_folder, "first.txt", "the first file"); + eventually("the folder behind the link has the file", || async { names(&a).len() == 1 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // The link is pointed at B (with the space's marker); a change on the server asks for the pass that notices. + std::fs::create_dir_all(b.join(".craft-cloud")).unwrap(); + std::fs::copy(a.join(".craft-cloud/root.json"), b.join(".craft-cloud/root.json")).unwrap(); + unlink_dir(&link); + link_dir(&link, &b); + write(&ana_folder, "poke.txt", "asks for a pass"); + eventually("the folder behind the link now has everything", || async { names(&b).len() == 2 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // A change in B reaches the server although nothing else asks for a pass; one in the old folder A is not synced. + write(&a, "old.txt", "in the folder the link left"); + write(&link, "late.txt", "written behind the link"); + eventually("the local change reaches the server", || async { read(&ana_folder, "late.txt").as_deref() == Some("written behind the link") }).await; + assert_eq!(read(&ana_folder, "old.txt"), None, "the old folder is not synced any more"); +} + +/// A file that can't be deleted while this lives: held open without delete sharing (Windows), or in a read-only folder (Unix). +struct Held { + _file: std::fs::File, + #[cfg(unix)] + dir: PathBuf, +} + +impl Held { + fn new(path: &Path) -> Self { + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + // Share read and write, not delete. + Self { _file: std::fs::OpenOptions::new().read(true).share_mode(1 | 2).open(path).unwrap() } + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let dir = path.parent().unwrap().to_owned(); + let file = std::fs::File::open(path).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + Self { _file: file, dir } + } + } +} + +#[cfg(unix)] +impl Drop for Held { + fn drop(&mut self) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&self.dir, std::fs::Permissions::from_mode(0o755)); + } +} + +/// Item 4: turning a space off when its saved state can't be removed (something holds the file) is not acknowledged as done: +/// the call fails with the error. The setting's generation was raised and saved first, so the state that survived is never +/// trusted again: after a restart, files deleted here meanwhile are not deleted on the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_state_that_cannot_be_removed_fails_the_turn_off_and_is_never_trusted_again() { + let mut t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = both_syncing(&t).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + let state_path = t.ben.data.path().join("spaces").join(space.to_string()).join("state.json"); + assert!(state_path.is_file()); + + let generation = |t: &Team| t.ben.data.load_account().unwrap().unwrap().spaces.iter().find(|s| s.space == space).map(|s| s.generation); + let before = generation(&t).unwrap(); + let held = Held::new(&state_path); + let off = t.ben.api(move |c| c.set_space_sync(space, &SpaceSyncRequest { on: false, folder: None, adopt: false })).await; + assert!(off.is_err(), "the state could not be removed: the turn-off is not acknowledged: {off:?}"); + assert!(state_path.is_file(), "the state survived"); + assert_eq!(generation(&t), Some(before + 1), "the generation was raised and saved"); + + // The agent stops while the file is still held (nothing replaces it), then files are deleted here. + let (exit, out) = t.ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + drop(held); + assert!(state_path.is_file(), "the old state is still there"); + for i in 0..3 { + std::fs::remove_file(a.join(format!("f{i}.txt"))).unwrap(); + } + t.ben.run(); + eventually("the files deleted here come back from the server", || async { names(&a).len() == 6 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 6, "nothing was deleted on the server: {:?}", names(&ana_folder)); +} + +/// Item 5: a team folder is wiped and recreated at the same path; setting it up installs the space's marker and stops right +/// there (as a crash would), before the old state is removed. The old state must not be trusted for the new folder after a +/// restart: the files it knew are not taken for deleted, and the folder is filled from the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_crash_after_the_marker_is_installed_does_not_revive_the_old_state() { + let mut t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = both_syncing(&t).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + t.ben.api(|c| c.pause()).await.unwrap(); + + // The folder is wiped and made again, empty; the person turns the sync on for it, and the set-up stops after the marker. + std::fs::remove_dir_all(&a).unwrap(); + std::fs::create_dir_all(&a).unwrap(); + *craft_cloud_agent::spaces::STOP_AFTER_MARKER.lock().unwrap() = Some(a.clone()); + let again = t.ben.api(move |c| c.set_space_sync(space, &SpaceSyncRequest { on: true, folder: None, adopt: false })).await; + assert!(again.is_err(), "the set-up stopped after the marker: {again:?}"); + assert!(a.join(".craft-cloud/root.json").is_file(), "the marker was written"); + assert!(t.ben.space_state(space).is_some(), "the old state was not removed yet"); + + // The agent restarts; the old state must not decide what the new folder is missing. + let (exit, out) = t.ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + t.ben.run(); + t.ben.ready().await; + t.ben.api(|c| c.resume()).await.unwrap(); + eventually("the folder is filled from the server", || async { names(&a).len() == 6 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 6, "nothing was deleted on the server: {:?}", names(&ana_folder)); +} + +// --------------------------------------------------------------------------------------------------------------------- +// The fifth review round of P1-5 (the re-review of the fourth) + +/// Replaces `folder` at the same path as a backup restore would: it is wiped and made again with the space's marker and only +/// the files in `keep` (with the content they had). The new folder has another ID. +fn restore_backup(folder: &Path, keep: &[&str]) { + let marker = std::fs::read(folder.join(".craft-cloud/root.json")).unwrap(); + let kept: Vec<(String, Vec)> = keep.iter().map(|n| ((*n).to_owned(), std::fs::read(folder.join(n)).unwrap())).collect(); + std::fs::remove_dir_all(folder).unwrap(); + std::fs::create_dir_all(folder.join(".craft-cloud")).unwrap(); + std::fs::write(folder.join(".craft-cloud/root.json"), marker).unwrap(); + for (name, data) in kept { + std::fs::write(folder.join(name), data).unwrap(); + } +} + +/// Ben syncs a team folder with six files; returns ana's folder and his. +async fn six_files(t: &Team) -> (PathBuf, PathBuf) { + let (ana_folder, a) = both_syncing(t).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("ben has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + (ana_folder, a) +} + +/// F1, same path, running: the folder is replaced by a backup that holds the marker but not the newer files, while the session is +/// open, and the state file can't be removed. The old state is not trusted when the session reopens: nothing is deleted on the +/// server and the folder is filled from it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_replaced_at_the_same_path_with_a_state_that_cannot_be_removed_deletes_nothing() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = six_files(&t).await; + let state_path = t.ben.data.path().join("spaces").join(space.to_string()).join("state.json"); + let generation = |t: &Team| t.ben.data.load_account().unwrap().unwrap().spaces.iter().find(|s| s.space == space).map(|s| s.generation); + let before = generation(&t).unwrap(); + let held = Held::new(&state_path); + restore_backup(&a, &["f0.txt", "f1.txt"]); + write(&ana_folder, "poke.txt", "asks for a pass"); + eventually("the folder is filled from the server", || async { names(&a).len() == 7 }).await; + drop(held); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 7, "nothing was deleted on the server: {:?}", names(&ana_folder)); + assert!(generation(&t) > Some(before), "the generation was raised when the folder was found replaced"); +} + +/// F1, same path, across a restart: the agent is stopped while the folder is replaced by a backup with the marker but without the +/// newer files. The state records the old folder's ID, so it is not used for the new one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_replaced_at_the_same_path_while_the_agent_is_stopped_deletes_nothing() { + let mut t = team(SpaceRole::Editor).await; + let (ana_folder, a) = six_files(&t).await; + let (exit, out) = t.ben.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + restore_backup(&a, &["f0.txt", "f1.txt"]); + t.ben.run(); + eventually("the folder is filled from the server", || async { names(&a).len() == 6 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 6, "nothing was deleted on the server: {:?}", names(&ana_folder)); +} + +/// F1, same path, across a pass that needs the user: the folder goes (the pass stops, the session is dropped) and comes back +/// at the same path as a backup with the marker but without the newer files. Nothing is deleted on the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_that_goes_and_comes_back_replaced_deletes_nothing() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = six_files(&t).await; + let marker = std::fs::read(a.join(".craft-cloud/root.json")).unwrap(); + let kept: Vec> = ["f0.txt", "f1.txt"].iter().map(|n| std::fs::read(a.join(n)).unwrap()).collect(); + std::fs::remove_dir_all(&a).unwrap(); + write(&ana_folder, "poke.txt", "asks for a pass"); + eventually("the pass needs the user", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && matches!(s.run_state, SpaceRun::Paused { .. })) + }) + .await; + std::fs::create_dir_all(a.join(".craft-cloud")).unwrap(); + std::fs::write(a.join(".craft-cloud/root.json"), marker).unwrap(); + std::fs::write(a.join("f0.txt"), &kept[0]).unwrap(); + std::fs::write(a.join("f1.txt"), &kept[1]).unwrap(); + eventually("the folder is filled from the server", || async { names(&a).len() == 7 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 7, "nothing was deleted on the server: {:?}", names(&ana_folder)); +} + +/// F3: a link is pointed at another folder (with the same marker, without the files) in the middle of a pass, after the pass began +/// and before it sends its deletions. The pass stops without deleting anything on the server, and the next one starts afresh in the +/// new folder. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_link_pointed_at_another_folder_during_a_pass_deletes_nothing() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("ana's agent lists the space", || async { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let ana_folder = t.ana.sync_on(space).await; + let (a, b, link) = (t.ben.dir.path().join("A"), t.ben.dir.path().join("B"), t.ben.dir.path().join("Link")); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + link_dir(&link, &a); + t.ben.sync_on_in(space, &link).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("the folder behind the link has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // A pass begins (a local change) and waits at the server; meanwhile the link is pointed at folder B. + front.stall.store(true, Ordering::SeqCst); + write(&link, "local.txt", "a local change"); + eventually("the pass is running", || async { t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) }) + .await; + std::fs::create_dir_all(b.join(".craft-cloud")).unwrap(); + std::fs::copy(a.join(".craft-cloud/root.json"), b.join(".craft-cloud/root.json")).unwrap(); + unlink_dir(&link); + link_dir(&link, &b); + front.stall.store(false, Ordering::SeqCst); + eventually("the folder behind the link is filled from the server", || async { names(&b).len() == 6 }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(names(&ana_folder).len(), 6, "nothing was deleted on the server: {:?}", names(&ana_folder)); +} + +/// F4: losing access raises the space's generation (and saves it); when account.json cannot be written at that moment, the raise +/// is kept and saved at a later look at the list, so a restart still finds it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn losing_access_raises_the_generation_even_when_it_cannot_be_saved_at_once() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (_, a) = six_files(&t).await; + let generation = |t: &Team| t.ben.data.load_account().unwrap().unwrap().spaces.iter().find(|s| s.space == space).map(|s| s.generation); + let before = generation(&t).unwrap(); + let account_path = t.ben.data.path().join("account.json"); + + // account.json can't be replaced while it is held: the raise is made in memory but not saved. + let held = Held::new(&account_path); + t.world.meta.remove_member(&SpaceActor::Cli, space, t.world.ben.id, 1).await.unwrap(); + eventually("the space reports no access", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == space && matches!(s.run_state, SpaceRun::NoAccess { .. })) + }) + .await; + assert_eq!(generation(&t), Some(before), "account.json could not be written"); + drop(held); + eventually("the raised generation is saved later", || async { generation(&t) > Some(before) }).await; + assert_eq!(names(&a).len(), 6, "the local copy is kept"); +} + +// --------------------------------------------------------------------------------------------------------------------- +// The sixth review round of P1-5 (N1 to N4 of the re-review of the fifth) + +/// N1: a link is pointed at another folder in the middle of a pass; the pass takes in the server's newest file there, stops at +/// its first checkpoint, and the link is pointed back before the next pass. The session that took in part of the other folder is +/// not used again (nor its state saved): the file is not taken for deleted in the first folder and deleted on the server. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_link_pointed_elsewhere_and_back_around_a_pass_deletes_nothing() { + let (t, front, _task) = team_via_front(SpaceRole::Editor, Front::default()).await; + let space = t.space; + eventually("the invitation is listed", || async { t.ben.space(space).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(space).await; + eventually("ana's agent lists the space", || async { t.ana.space(space).await.is_some_and(|s| s.state == "active") }).await; + let ana_folder = t.ana.sync_on(space).await; + let (a, b, link) = (t.ben.dir.path().join("A"), t.ben.dir.path().join("B"), t.ben.dir.path().join("Link")); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + link_dir(&link, &a); + t.ben.sync_on_in(space, &link).await; + for i in 0..6 { + write(&ana_folder, &format!("f{i}.txt"), &format!("content {i}")); + } + eventually("the folder behind the link has the files", || async { names(&a).len() == 6 }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + + // A pass begins and waits at the server. The server gets a new file; the link is pointed at B; syncing is paused so no pass + // follows at once; the pass goes on, takes the new file into B and stops at its first checkpoint. + front.stall.store(true, Ordering::SeqCst); + write(&link, "local.txt", "a local change"); + eventually("the pass is running", || async { t.ben.status().await.spaces.iter().any(|s| s.space == space && s.run_state == SpaceRun::Syncing) }) + .await; + write(&ana_folder, "newest.txt", "from the server"); + tokio::time::sleep(Duration::from_secs(3)).await; + std::fs::create_dir_all(b.join(".craft-cloud")).unwrap(); + std::fs::copy(a.join(".craft-cloud/root.json"), b.join(".craft-cloud/root.json")).unwrap(); + unlink_dir(&link); + link_dir(&link, &b); + t.ben.api(|c| c.pause()).await.unwrap(); + front.stall.store(false, Ordering::SeqCst); + eventually("the pass took the new file into the other folder", || async { read(&b, "newest.txt").as_deref() == Some("from the server") }).await; + tokio::time::sleep(Duration::from_secs(1)).await; + + // Pointed back before the next pass: that pass must not trust what the first one took in. + unlink_dir(&link); + link_dir(&link, &a); + t.ben.api(|c| c.resume()).await.unwrap(); + eventually("the first folder gets the server's file", || async { read(&a, "newest.txt").as_deref() == Some("from the server") }).await; + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(read(&ana_folder, "newest.txt").as_deref(), Some("from the server"), "the server's file was not deleted"); + for i in 0..6 { + assert_eq!(read(&ana_folder, &format!("f{i}.txt")).as_deref(), Some(format!("content {i}").as_str()), "f{i} is still on the server"); + } +} + +/// N3: the nonce in a team folder's `.craft-cloud` is rewritten at every checkpoint and recorded in the state saved with it, so a +/// folder restored from an older copy is told apart even where a file system reuses folder IDs. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_folder_nonce_is_rewritten_and_recorded_in_the_state() { + let t = team(SpaceRole::Editor).await; + let space = t.space; + let (ana_folder, a) = six_files(&t).await; + let nonce_of = |folder: &Path| std::fs::read_to_string(folder.join(".craft-cloud/nonce")).ok(); + let before = nonce_of(&a).expect("the folder has a nonce once a pass has run"); + write(&ana_folder, "more.txt", "another file"); + eventually("the nonce was rewritten", || async { nonce_of(&a).is_some_and(|n| n != before) }).await; + eventually("nothing runs", || async { t.ben.status().await.spaces.iter().all(|s| s.run_state == SpaceRun::Idle) }).await; + let nonce = nonce_of(&a).unwrap(); + let recorded = t.ben.space_state(space).and_then(|s| s["folder_nonce"].as_str().map(str::to_owned)); + assert_eq!(recorded, Some(nonce), "the state records the nonce the folder carries"); +} + +/// N4: while a raised generation of one space cannot be saved, only that space is held back from starting, and its state says so; +/// another space goes on syncing. Once account.json can be written the space starts. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn only_the_space_with_an_unsaved_generation_is_held_back_and_says_so() { + let t = team(SpaceRole::Editor).await; + let a = t.space; + let b = t.world.space("Archive").await; + t.world.invite_ben(b, SpaceRole::Editor).await; + let (_, _) = both_syncing(&t).await; + eventually("the second invitation is listed", || async { t.ben.space(b).await.is_some_and(|s| s.state == "invited") }).await; + t.ben.accept(b).await; + eventually("ana's agent lists the second space", || async { t.ana.space(b).await.is_some_and(|s| s.state == "active") }).await; + let (ana_b, ben_b) = (t.ana.sync_on(b).await, t.ben.sync_on(b).await); + let account_path = t.ben.data.path().join("account.json"); + + // account.json can't be written: the first space loses access (its raise can't be saved), then gets it back. + let held = Held::new(&account_path); + t.world.meta.remove_member(&SpaceActor::Cli, a, t.world.ben.id, 1).await.unwrap(); + eventually("the first space reports no access", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == a && matches!(s.run_state, SpaceRun::NoAccess { .. })) + }) + .await; + t.world.invite_ben(a, SpaceRole::Editor).await; + t.world.meta.accept_invitation(t.world.ben.id, None, a, 2).await.unwrap(); + eventually("the first space is held back and says why", || async { + t.ben + .status() + .await + .spaces + .iter() + .any(|s| s.space == a && matches!(&s.run_state, SpaceRun::Paused { reason } if reason.contains("could not be saved"))) + }) + .await; + // The second space goes on. + write(&ana_b, "meanwhile.txt", "the other space is not held back"); + eventually("the second space still syncs", || has_file(ben_b.clone(), "meanwhile.txt", "the other space is not held back")).await; + + drop(held); + eventually("the first space starts once the settings can be saved", || async { + t.ben.status().await.spaces.iter().any(|s| s.space == a && !matches!(s.run_state, SpaceRun::Paused { .. } | SpaceRun::NoAccess { .. })) + }) + .await; +} diff --git a/apps/craft-cloud-agent/tests/two_factor.rs b/apps/craft-cloud-agent/tests/two_factor.rs new file mode 100644 index 00000000..8cda152b --- /dev/null +++ b/apps/craft-cloud-agent/tests/two_factor.rs @@ -0,0 +1,795 @@ +//! Two-factor sign-in through the agent: the second step of a sign-in and the code of a join, the account's two-factor calls +//! passed on to the server, and the command line's prompts, against a real server (with a key for two-factor secrets) on a +//! loopback port and the server's clock held still, so that codes don't change under a test. +//! +//! What they pin down: a challenge, its expiry and a lock's `Retry-After` reach the app typed; a wrong code, a wrong +//! password or a lock never sign the agent out (a 403 or 429 on the account's calls, a 401 at the second step), while a real +//! 401 still does; nothing secret is kept on disk or printed. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::{Duration, Instant}; + +use clap::Parser; +use craft_cloud_agent::account::DataDir; +use craft_cloud_agent::cli::{self, AgentError, Exit}; +use craft_cloud_agent::daemon::{RunOptions, run_loop}; +use craft_cloud_agent::schedule::Timing; +use craft_cloud_app::{Client, Error, JoinRequest, LoginRequest, SecondFactorLoginRequest, TwoFactorProofRequest}; +use craft_cloud_meta::{Meta, Role}; +use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS}; +use craft_cloud_proto::local::{LOCAL_API_FILE, LocalApiFile}; +use craft_cloud_server::AppState; +use craft_cloud_server::auth::hash_password; +use craft_cloud_server::secretbox::{SecretBox, TwoFactorKey}; +use craft_cloud_server::totp; +use craft_cloud_store::Store; +use craft_cloud_store::disk::DiskBackend; +use tokio::sync::mpsc; + +const PASSWORD: &str = "correct horse battery"; +const PATIENCE: Duration = Duration::from_secs(30); + +struct Server { + url: String, + meta: Meta, + clock: Arc, + _dirs: (tempfile::TempDir, tempfile::TempDir), +} + +impl Server { + async fn start() -> Self { + let (db, blobs) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + // The clock stands still at the start of a TOTP step, until a test moves it. + let clock = Arc::new(AtomicI64::new(totp::step_at(craft_cloud_server::state::now_ms()) * 30_000 + 1_000)); + let ticking = clock.clone(); + let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]) + .unwrap() + .with_two_factor(Some(SecretBox::new(&TwoFactorKey::from_bytes([9; 32]), None).unwrap())) + .with_clock(Arc::new(move || ticking.load(Ordering::SeqCst))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, clock, _dirs: (db, blobs) } + } + + fn now(&self) -> i64 { + self.clock.load(Ordering::SeqCst) + } + + fn advance(&self, ms: i64) { + self.clock.fetch_add(ms, Ordering::SeqCst); + } +} + +fn quick() -> RunOptions { + RunOptions { + timing: Timing { + debounce: Duration::from_millis(100), + max_delay: Duration::from_secs(1), + stability: Duration::from_millis(200), + periodic: Duration::from_secs(600), + unwatched_periodic: Duration::from_millis(500), + backoff_first: Duration::from_millis(200), + backoff_max: Duration::from_secs(1), + recheck: Duration::from_millis(300), + recheck_slow: Duration::from_secs(5), + approval_check: Duration::from_millis(100), + held_check: Duration::from_millis(300), + }, + heartbeat: Duration::from_millis(200), + poll_wait: Duration::from_secs(1), + poll_retry_first: Duration::from_millis(100), + poll_retry_max: Duration::from_secs(1), + poll_min_spacing: Duration::from_millis(100), + poll_busy: Duration::from_millis(200), + fail_watch: false, + record_delay: Duration::ZERO, + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, + autostart: craft_cloud_agent::autostart::Target::Off, + } +} + +/// A running agent (`run` on a thread of its own), signed out, and a client of its local API. +struct Machine { + data: DataDir, + folder: PathBuf, + stop: mpsc::UnboundedSender<()>, + thread: Option, String)>>, + _dir: tempfile::TempDir, +} + +impl Machine { + async fn start() -> Self { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (stop, stops) = mpsc::unbounded_channel(); + let loop_data = data.clone(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let mut out = Vec::new(); + let exit = runtime.block_on(run_loop(&loop_data, quick(), stops, &mut out)); + (exit, String::from_utf8_lossy(&out).into_owned()) + }); + let file = data.path().join(LOCAL_API_FILE); + let deadline = Instant::now() + PATIENCE; + while !file.exists() { + assert!(Instant::now() < deadline, "the local API never came up"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + Self { folder: dir.path().join("Craft Cloud"), data, stop, thread: Some(thread), _dir: dir } + } + + fn client(&self) -> Client { + Client::connect_in(self.data.path()).unwrap() + } + + fn api(&self) -> LocalApiFile { + serde_json::from_slice(&std::fs::read(self.data.path().join(LOCAL_API_FILE)).unwrap()).unwrap() + } + + fn login_request(&self, server: &Server) -> LoginRequest { + LoginRequest { + server: server.url.clone(), + user: "ana".into(), + password: PASSWORD.into(), + folder: Some(self.folder.display().to_string()), + device_name: Some("desktop".into()), + adopt: true, + } + } + + fn second_step(&self, server: &Server, challenge: &str, totp_code: Option<&str>, recovery_code: Option<&str>) -> SecondFactorLoginRequest { + SecondFactorLoginRequest { + server: server.url.clone(), + user: "ana".into(), + folder: Some(self.folder.display().to_string()), + device_name: Some("desktop".into()), + adopt: true, + challenge: challenge.into(), + totp_code: totp_code.map(str::to_owned), + recovery_code: recovery_code.map(str::to_owned), + } + } + + async fn stop(mut self) -> (Result, String) { + let _ = self.stop.send(()); + let thread = self.thread.take().unwrap(); + tokio::task::spawn_blocking(move || thread.join().unwrap()).await.unwrap() + } +} + +impl Drop for Machine { + fn drop(&mut self) { + if self.thread.is_some() { + let _ = self.stop.send(()); + let _ = self.stop.send(()); + } + } +} + +/// A blocking call of the app client, off the runtime. +async fn off(f: impl FnOnce() -> Result + Send + 'static) -> Result { + tokio::task::spawn_blocking(f).await.unwrap() +} + +fn code_for(secret: &[u8], at_ms: i64) -> String { + totp::code_for(secret, totp::step_at(at_ms)).unwrap() +} + +fn wrong_code(code: &str) -> String { + format!("{:06}", (code.parse::().unwrap() + 1) % 1_000_000) +} + +/// Signs the machine in and turns two-factor sign-in on through the agent. The secret and the recovery codes come back. +async fn protect(server: &Server, machine: &Machine) -> (Vec, Vec) { + let client = machine.client(); + let request = machine.login_request(server); + let c = client.clone(); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + let c = client.clone(); + let setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let code = code_for(&secret, server.now()); + let c = client.clone(); + let codes = off(move || c.two_factor_confirm(&code)).await.unwrap(); + assert_eq!(codes.recovery_codes.len(), 10); + (secret, codes.recovery_codes) +} + +async fn sign_out(machine: &Machine) { + let c = machine.client(); + assert!(!off(move || c.logout(false)).await.unwrap().signed_in); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_second_step_of_a_sign_in_goes_through_the_agent_typed() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, recovery) = protect(&server, &machine).await; + sign_out(&machine).await; + let client = machine.client(); + + // The first step: a challenge, typed, with its expiry; the agent is still signed out and keeps nothing. + let request = machine.login_request(&server); + let c = client.clone(); + let refused = off(move || c.login(&request)).await.unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required")); + let challenge = refused.challenge().cloned().expect("a challenge"); + assert_eq!(challenge.challenge.len(), 43); + assert_eq!(challenge.expires_ms, server.now() + 5 * 60_000); + assert!(!client.status().unwrap().signed_in); + // A wrong password is still a wrong password, with nothing about two-factor sign-in. + let wrong = LoginRequest { password: "wrong password".into(), ..machine.login_request(&server) }; + let c = client.clone(); + let refused = off(move || c.login(&wrong)).await.unwrap_err(); + assert!(matches!(&refused, Error::Refused { code, .. } if code == "wrong_password"), "{refused:?}"); + + // A wrong code is `two_factor_invalid` and signs nothing in or out; a challenge takes five. + let right = code_for(&secret, server.now() + 30_000); + server.advance(30_000); + let step = machine.second_step(&server, &challenge.challenge, Some(&wrong_code(&right)), None); + let c = client.clone(); + let invalid = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + assert!(!client.status().unwrap().signed_in); + // The right code finishes it, and the agent syncs as after a password sign-in. + let step = machine.second_step(&server, &challenge.challenge, Some(&right), None); + let c = client.clone(); + let status = off(move || c.login_second_factor(&step)).await.unwrap(); + assert!(status.signed_in && status.account.as_deref() == Some("ana"), "{status:?}"); + // The challenge is used up: the same step again is `two_factor_expired`, never a signed-out agent. + let step = machine.second_step(&server, &challenge.challenge, Some(&right), None); + let c = client.clone(); + let again = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(client.status().unwrap().signed_in); + assert!(matches!(&again, Error::Refused { .. } | Error::RefusedWith { .. }), "{again:?}"); + + // A recovery code, once. + sign_out(&machine).await; + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + let step = machine.second_step(&server, &challenge.challenge, None, Some(&recovery[0].to_lowercase().replace('-', " "))); + let c = client.clone(); + assert!(off(move || c.login_second_factor(&step)).await.unwrap().signed_in); + sign_out(&machine).await; + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + let step = machine.second_step(&server, &challenge.challenge, None, Some(&recovery[0])); + let c = client.clone(); + let used = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&used, Error::Refused { code, .. } if code == "two_factor_invalid"), "{used:?}"); + + // A challenge that ran out is `two_factor_expired` (a 401 here would read as a revoked device). + server.advance(5 * 60_000 + 1); + let step = machine.second_step(&server, &challenge.challenge, Some(&code_for(&secret, server.now())), None); + let c = client.clone(); + let expired = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&expired, Error::Refused { code, .. } if code == "two_factor_expired"), "{expired:?}"); + let (exit, out) = machine.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_accounts_calls_are_passed_on_and_a_refusal_never_signs_the_agent_out() { + let server = Server::start().await; + let machine = Machine::start().await; + let client = machine.client(); + let request = machine.login_request(&server); + let c = client.clone(); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + + let c = client.clone(); + let off_state = off(move || c.two_factor_status()).await.unwrap(); + assert_eq!((off_state.enabled, off_state.recovery_codes_left, off_state.pending), (false, 0, false)); + // A wrong password is 403 `wrong_password`, and the agent stays signed in, not even "removed". + let c = client.clone(); + let wrong = off(move || c.two_factor_setup("wrong password!")).await.unwrap_err(); + assert!(matches!(&wrong, Error::Refused { code, .. } if code == "wrong_password"), "{wrong:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + let c = client.clone(); + let setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + assert_eq!(setup.secret.len(), 32); + assert!(setup.otpauth_uri.starts_with("otpauth://totp/Craft%20Cloud:ana?secret="), "{}", setup.otpauth_uri); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let c = client.clone(); + assert!(off(move || c.two_factor_status()).await.unwrap().pending); + // A wrong first code: 403 `two_factor_invalid`; the agent is not signed out. + let right = code_for(&secret, server.now()); + let bad = wrong_code(&right); + let c = client.clone(); + let invalid = off(move || c.two_factor_confirm(&bad)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + assert!(client.status().unwrap().signed_in); + let c = client.clone(); + let codes = off(move || c.two_factor_confirm(&right)).await.unwrap(); + assert_eq!(codes.recovery_codes.len(), 10); + let c = client.clone(); + let on = off(move || c.two_factor_status()).await.unwrap(); + assert_eq!((on.enabled, on.recovery_codes_left), (true, 10)); + // Already on: a conflict, said as one. + let c = client.clone(); + let conflict = off(move || c.two_factor_setup(PASSWORD)).await.unwrap_err(); + assert!(matches!(&conflict, Error::Refused { code, .. } if code == "conflict"), "{conflict:?}"); + + // New recovery codes with a fresh step; the old ones die. + server.advance(30_000); + let fresh = code_for(&secret, server.now()); + let c = client.clone(); + let renewed = off(move || c.two_factor_recovery_codes(PASSWORD, &fresh)).await.unwrap(); + assert!(renewed.recovery_codes.iter().all(|code| !codes.recovery_codes.contains(code))); + + // Password change and the other computers: passed on too. + let c = client.clone(); + let weak = off(move || c.change_password(PASSWORD, "short")).await.unwrap_err(); + assert!(matches!(&weak, Error::Refused { code, .. } if code == "weak_password"), "{weak:?}"); + let c = client.clone(); + assert_eq!(off(move || c.revoke_other_devices()).await.unwrap().revoked, 0); + let c = client.clone(); + let devices = off(move || c.devices()).await.unwrap(); + let this = devices.iter().find(|d| d.current).unwrap().id; + let c = client.clone(); + let renamed = off(move || c.rename_device(this, "my desktop")).await.unwrap(); + assert_eq!(renamed.name, "my desktop"); + let c = client.clone(); + let missing = off(move || c.rename_device(craft_cloud_app::DeviceId::generate(), "x")).await.unwrap_err(); + assert!(matches!(&missing, Error::Refused { code, .. } if code == "not_found"), "{missing:?}"); + + // Ten wrong codes lock it: 429 `two_factor_locked`, typed with its wait, and still no sign-out. + for _ in 0..10 { + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(bad_code(&secret, &server)), recovery_code: None }; + let c = client.clone(); + let invalid = off(move || c.two_factor_disable(&proof)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + } + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(code_for(&secret, server.now())), recovery_code: None }; + let c = client.clone(); + let locked = off(move || c.two_factor_disable(&proof)).await.unwrap_err(); + assert_eq!((locked.code(), locked.retry_after_secs()), (Some("two_factor_locked"), Some(900)), "{locked:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + + // Turning it off with a recovery code works once the lock is over. + server.advance(900_001); + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: None, recovery_code: Some(renewed.recovery_codes[0].clone()) }; + let c = client.clone(); + off(move || c.two_factor_disable(&proof)).await.unwrap(); + let c = client.clone(); + assert!(!off(move || c.two_factor_status()).await.unwrap().enabled); + + // A real 401 (the device was signed out on the server) is still what it was: the agent can't go on. + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + server.meta.revoke_all_devices(ana.id, craft_cloud_meta::RevokeReason::Admin, server.now()).await.unwrap(); + let c = client.clone(); + let revoked = off(move || c.two_factor_status()).await.unwrap_err(); + assert!(matches!(&revoked, Error::Refused { code, .. } if code == "needs_you"), "{revoked:?}"); + let _ = machine.stop().await; +} + +fn bad_code(secret: &[u8], server: &Server) -> String { + wrong_code(&code_for(secret, server.now())) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn joining_an_account_with_two_factor_takes_the_code_in_a_second_call() { + let server = Server::start().await; + let first = Machine::start().await; + let (secret, recovery) = protect(&server, &first).await; + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + + let second = Machine::start().await; + let client = second.client(); + let join = JoinRequest { + join: format!("{}/{}", server.url, invite.display()), + folder: Some(second.folder.display().to_string()), + device_name: Some("laptop".into()), + adopt: true, + password: None, + totp_code: None, + recovery_code: None, + }; + // No second factor: `two_factor_required` (no challenge to give: the same join goes again), and the invite is not used up. + let c = client.clone(); + let j = join.clone(); + let refused = off(move || c.join(&j)).await.unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required"), "{refused:?}"); + assert!(!client.status().unwrap().signed_in); + // A wrong code is `two_factor_invalid`; the invite still works. + server.advance(30_000); + let right = code_for(&secret, server.now()); + let c = client.clone(); + let j = JoinRequest { totp_code: Some(wrong_code(&right)), ..join.clone() }; + let invalid = off(move || c.join(&j)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + let c = client.clone(); + let j = JoinRequest { totp_code: Some(right), ..join.clone() }; + assert!(off(move || c.join(&j)).await.unwrap().signed_in); + // A recovery code does it too, on a new invite. + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + let third = Machine::start().await; + let c = third.client(); + let j = JoinRequest { + join: format!("{}/{}", server.url, invite.display()), + folder: Some(third.folder.display().to_string()), + recovery_code: Some(recovery[2].clone()), + ..join + }; + assert!(off(move || c.join(&j)).await.unwrap().signed_in); + let _ = (first.stop().await, second.stop().await, third.stop().await); +} + +/// The new calls take small bodies only, and what is secret is kept nowhere: not in `account.json`, not in what `run` prints. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nothing_secret_is_stored_or_printed_and_bodies_are_small() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, recovery) = protect(&server, &machine).await; + sign_out(&machine).await; + let client = machine.client(); + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + server.advance(30_000); + let used = code_for(&secret, server.now()); + let step = machine.second_step(&server, &challenge.challenge, Some(&used), None); + let c = client.clone(); + off(move || c.login_second_factor(&step)).await.unwrap(); + + // Big bodies are refused before they are read. + let api = machine.api(); + let big = format!(r#"{{"password":"{}"}}"#, "x".repeat(6000)); + for path in ["/v1/account/two-factor/setup", "/v1/login/second-factor", "/v1/account/password"] { + let (status, _) = raw( + api.port, + &format!("POST {path} HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nAuthorization: Bearer {}\r\nContent-Type: application/json", api.port, api.token), + &big, + ); + assert_eq!(status, 413, "{path}"); + } + let account_json = std::fs::read_to_string(machine.data.path().join("account.json")).unwrap(); + let mut every_file = String::new(); + collect(machine.data.path(), &mut every_file); + let (exit, out) = machine.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + for secret_text in std::iter::once(used).chain(std::iter::once(challenge.challenge)).chain(recovery).chain(std::iter::once(PASSWORD.to_owned())) { + assert!(!account_json.contains(&secret_text), "{secret_text} in account.json"); + assert!(!every_file.contains(&secret_text), "{secret_text} in the data folder"); + assert!(!out.contains(&secret_text), "{secret_text} in what run printed"); + } +} + +fn collect(dir: &Path, into: &mut String) { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + collect(&path, into); + } else if let Ok(text) = std::fs::read_to_string(&path) { + // The local API's own token is this folder's business, not a second factor. + if path.file_name().is_some_and(|n| n != LOCAL_API_FILE) { + into.push_str(&text); + } + } + } +} + +/// One raw HTTP/1.1 request, so a test controls every header. +fn raw(port: u16, head: &str, body: &str) -> (u16, String) { + use std::io::{Read, Write}; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(PATIENCE)).unwrap(); + let request = format!("{head}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + stream.write_all(request.as_bytes()).unwrap(); + let mut answer = String::new(); + let _ = stream.read_to_string(&mut answer); + let status = answer.split(' ').nth(1).and_then(|s| s.parse().ok()).unwrap_or(0); + (status, answer.split_once("\r\n\r\n").map(|(_, b)| b.to_owned()).unwrap_or_default()) +} + +// ---- the command line ---- + +fn cli_login(url: &str, data: &Path, folder: &Path) -> cli::Cli { + cli::Cli::try_parse_from([ + "craft-cloud-agent", + "--data-dir", + data.to_str().unwrap(), + "login", + url, + "--user", + "ana", + "--folder", + folder.to_str().unwrap(), + "--adopt", + ]) + .unwrap() +} + +/// A signed-in, protected account from the command line's point of view: the secret and a recovery code. Another agent's data +/// folder made it, through the server's own routes. +async fn protected_account(server: &Server) -> (Vec, Vec) { + let machine = Machine::start().await; + let made = protect(server, &machine).await; + let _ = machine.stop().await; + made +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_command_line_asks_for_the_code_on_standard_input() { + let server = Server::start().await; + let (secret, recovery) = protected_account(&server).await; + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + + // A wrong code, then the right one: two lines after the password, and the code is never an argument. + server.advance(30_000); + let right = code_for(&secret, server.now()); + let mut out = Vec::new(); + let stdin = format!("{PASSWORD}\n{}\n{right}\n", wrong_code(&right)); + let exit = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("two-factor sign-in") && printed.contains("signed in as ana"), "{printed}"); + assert!(!printed.contains(&right) && !printed.contains(PASSWORD), "nothing secret is echoed: {printed}"); + + // Three wrong codes: it gives up with the server's message, and nothing is signed in. + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let wrong = wrong_code(&right); + let stdin = format!("{PASSWORD}\n{wrong}\n{wrong}\n{wrong}\n"); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + assert!(matches!(err, AgentError::TwoFactorInvalid(_)), "{err:?}"); + assert!(DataDir::new(data.clone()).load_account().unwrap().is_none()); + + // A recovery code instead of a code. + let stdin = format!("{PASSWORD}\n{}\n", recovery[0].to_lowercase()); + let mut out = Vec::new(); + let exit = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + assert_eq!(exit, Exit::Done, "{}", String::from_utf8_lossy(&out)); + + // Nothing on standard input where the code should be: a plain usage error, no panic. + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let stdin = format!("{PASSWORD}\n"); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + assert!(matches!(err, AgentError::Usage(ref m) if m.contains("no code")), "{err:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_lock_says_how_long_to_wait() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, _) = protect(&server, &machine).await; + let client = machine.client(); + for _ in 0..10 { + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(bad_code(&secret, &server)), recovery_code: None }; + let c = client.clone(); + let _ = off(move || c.two_factor_disable(&proof)).await; + } + let _ = machine.stop().await; + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let stdin = format!("{PASSWORD}\n{}\n", code_for(&secret, server.now() + 30_000)); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + match err { + AgentError::TwoFactorLocked { retry_after_secs, message } => { + assert_eq!(retry_after_secs, Some(900)); + assert!(message.contains("15 minutes"), "{message}"); + } + other => panic!("{other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_command_line_joins_with_a_code_and_through_a_running_agent() { + let server = Server::start().await; + let (secret, _) = protected_account(&server).await; + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + server.advance(30_000); + let right = code_for(&secret, server.now()); + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let join = cli::Cli::try_parse_from([ + "craft-cloud-agent", + "--data-dir", + data.to_str().unwrap(), + "join", + &format!("{}/{}", server.url, invite.display()), + "--folder", + folder.to_str().unwrap(), + "--adopt", + ]) + .unwrap(); + let mut out = Vec::new(); + let stdin = format!("{right}\n"); + let exit = cli::run(join, &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("two-factor sign-in") && printed.contains("joined as ana"), "{printed}"); + + // `login` while `run` holds the data folder goes through the running agent, second step and all. + let machine = Machine::start().await; + server.advance(30_000); + let right = code_for(&secret, server.now()); + let data = machine.data.path().to_path_buf(); + let stdin = format!("{PASSWORD}\n{right}\n"); + let mut out = Vec::new(); + let exit = cli::run(cli_login(&server.url, &data, &machine.folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("through the running agent"), "{printed}"); + assert!(machine.client().status().unwrap().signed_in); + let _ = machine.stop().await; +} + +// ---- the admin pages' confirmation with a second factor ---- + +use craft_cloud_app::{AdminConfirm, AdminPasswordReset, UserQuery}; + +impl Server { + /// The same server with `root`, an administrator with the same password. + async fn start_with_admin() -> Self { + let server = Self::start().await; + server.meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + server + } +} + +impl Machine { + fn login_as(&self, server: &Server, user: &str) -> LoginRequest { + LoginRequest { user: user.into(), device_name: Some(format!("{user}-pc")), ..self.login_request(server) } + } +} + +/// A confirmation with the password and, if given, a code (six digits) or recovery code. +fn confirm_with(code: Option<&str>) -> AdminConfirm { + let mut confirm = AdminConfirm { password: Some(PASSWORD.into()), ..AdminConfirm::default() }; + if let Some(code) = code { + if code.len() == 6 && code.bytes().all(|b| b.is_ascii_digit()) { + confirm.totp_code = Some(code.into()); + } else { + confirm.recovery_code = Some(code.into()); + } + } + confirm +} + +/// An admin whose account has two-factor sign-in confirms each change with a code too: the agent passes the code, and the +/// server's `two_factor_invalid` (403) and `two_factor_locked` (429, with the wait) come back typed, never as a signed-out +/// agent. A reset code with `reset_two_factor` and the reset of an account's two-factor sign-in carry their fields. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_admin_with_two_factor_confirms_changes_with_a_code_through_the_agent() { + let server = Server::start_with_admin().await; + let admin = Machine::start().await; + let client = admin.client(); + let c = client.clone(); + let request = admin.login_as(&server, "root"); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + + // The account says it is an admin, without two-factor sign-in yet. + let c = client.clone(); + let me = off(move || c.account()).await.unwrap(); + assert_eq!((me.username.as_str(), me.role.as_str(), me.two_factor_enabled), ("root", "admin", false)); + // Without it, a password is enough. + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let ana_id = ana.id.0; + let c = client.clone(); + off(move || c.admin_disable_user(ana_id, &confirm_with(None))).await.unwrap(); + let c = client.clone(); + off(move || c.admin_enable_user(ana_id, &confirm_with(None))).await.unwrap(); + + // Two-factor sign-in on for the admin. + let setup = { + let c = client.clone(); + off(move || c.two_factor_setup(PASSWORD)).await.unwrap() + }; + let secret = totp::base32_decode(&setup.secret).unwrap(); + let first = code_for(&secret, server.now()); + let c = client.clone(); + let recovery = off(move || c.two_factor_confirm(&first)).await.unwrap().recovery_codes; + let c = client.clone(); + assert!(off(move || c.account()).await.unwrap().two_factor_enabled); + + // A password alone, or with a wrong code: 403 `two_factor_invalid`; the agent is as it was. + let c = client.clone(); + let missing = off(move || c.admin_disable_user(ana_id, &confirm_with(None))).await.unwrap_err(); + assert_eq!(missing.code(), Some("two_factor_invalid"), "{missing:?}"); + server.advance(30_000); + let right = code_for(&secret, server.now()); + let wrong = wrong_code(&right); + let c = client.clone(); + let bad = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&wrong)))).await.unwrap_err(); + assert_eq!(bad.code(), Some("two_factor_invalid"), "{bad:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + assert!(!server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_some(), "nothing changed"); + // The right code: done. + let c = client.clone(); + let fresh = right.clone(); + off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&fresh)))).await.unwrap(); + assert!(server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_some()); + // A recovery code, in the other field, once. + let c = client.clone(); + let code = recovery[0].clone(); + off(move || c.admin_enable_user(ana_id, &confirm_with(Some(&code)))).await.unwrap(); + assert!(server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_none()); + let c = client.clone(); + let code = recovery[0].clone(); + let used = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&code)))).await.unwrap_err(); + assert_eq!(used.code(), Some("two_factor_invalid"), "a recovery code works once: {used:?}"); + + // A reset code that also turns two-factor off when it is used: the fields travel. + server.advance(30_000); + let reset = AdminPasswordReset { + password: Some(PASSWORD.into()), + totp_code: Some(code_for(&secret, server.now())), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + let c = client.clone(); + let issued = off(move || c.admin_issue_reset(ana_id, &reset)).await.unwrap(); + assert!(!issued.code.is_empty()); + + // The list shows who has two-factor sign-in; resetting it for an account that has it. + let ana_machine = Machine::start().await; + let ana_client = ana_machine.client(); + let c = ana_client.clone(); + let ana_request = ana_machine.login_as(&server, "ana"); + // (Ana's password was not changed: the reset code is only issued.) + assert!(off(move || c.login(&ana_request)).await.unwrap().signed_in); + let c = ana_client.clone(); + let ana_setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + let ana_secret = totp::base32_decode(&ana_setup.secret).unwrap(); + let ana_first = code_for(&ana_secret, server.now()); + let c = ana_client.clone(); + off(move || c.two_factor_confirm(&ana_first)).await.unwrap(); + let c = client.clone(); + let listed = off(move || c.admin_users(&UserQuery { query: Some("ana".into()), ..UserQuery::default() })).await.unwrap(); + assert!(listed.users.iter().any(|u| u.username == "ana" && u.two_factor_enabled), "{listed:?}"); + server.advance(30_000); + let c = client.clone(); + let fresh = code_for(&secret, server.now()); + off(move || c.admin_reset_two_factor(ana_id, &confirm_with(Some(&fresh)))).await.unwrap(); + let c = ana_client.clone(); + let off_state = off(move || c.two_factor_status()).await; + // Ana's device was signed out with it: her agent is told so, as a revoked device. + assert!(off_state.is_err() || !off_state.unwrap().enabled); + let c = client.clone(); + let listed = off(move || c.admin_users(&UserQuery { query: Some("ana".into()), ..UserQuery::default() })).await.unwrap(); + assert!(listed.users.iter().any(|u| u.username == "ana" && !u.two_factor_enabled), "{listed:?}"); + + // Too many wrong codes stop the confirmation: the server's per-account limit says `429 rate_limited` first (wrong codes + // count like wrong passwords), and its lock `429 two_factor_locked` with the wait comes after more; either is kept as the + // server said it (the code, the status, the wait if there is one), and the agent stays signed in. + let mut stopped = None; + for _ in 0..12 { + let c = client.clone(); + let wrong = wrong_code(&code_for(&secret, server.now())); + let refused = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&wrong)))).await.unwrap_err(); + if matches!(refused.code(), Some("rate_limited" | "two_factor_locked")) { + stopped = Some(refused); + break; + } + assert_eq!(refused.code(), Some("two_factor_invalid"), "{refused:?}"); + } + let stopped = stopped.expect("too many wrong codes are stopped"); + if stopped.code() == Some("two_factor_locked") { + assert!(stopped.retry_after_secs().is_some_and(|secs| secs > 0), "{stopped:?}"); + } + assert!(client.status().unwrap().signed_in); + + let (exit, out) = admin.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + for secret in [PASSWORD, issued.code.as_str()] { + assert!(!out.contains(secret), "the agent printed a secret: {out}"); + } + let _ = ana_machine.stop().await; +} diff --git a/apps/craft-cloud-desktop/src/admin.rs b/apps/craft-cloud-desktop/src/admin.rs new file mode 100644 index 00000000..3eaf17b4 --- /dev/null +++ b/apps/craft-cloud-desktop/src/admin.rs @@ -0,0 +1,1700 @@ +//! The admin pages' state, as a pure function of what happened, like [`crate::model`] (which hosts it: the pages are +//! [`Screen::Admin`], and [`Event::Admin`] and the admin [`Effect`]s go through the same [`Model::update`]). +//! +//! - **Who sees them**: the pages are offered only when the server answers the admin counters +//! ([`Effect::AdminServer`], asked at every sign-in; [`Access::Admin`]); a `forbidden` makes it [`Access::Member`]. +//! That is a convenience, not a check: the server decides every call again, and a refusal is shown as the server's. +//! - **Pages**: the users (a paged list with search), one user (role and quota, disable, enable, remove, invite and +//! reset code, the user's devices), and a new account (name and role; its quota is set afterwards, on the user's +//! page, as a change of its own). +//! - **Every change asks for the admin's password afresh** in a confirmation ([`Dialog`], holding a [`Confirmation`]): +//! its text goes into the call as the body's `password` and is cleared from the model at once. Whatever else the +//! server comes to want of a confirmation (a second factor's code) is another field of [`Confirmation`] and of +//! [`Confirmation::take`]: the dialog, the events and the calls stay as they are. +//! - **A code is shown once** ([`ShownCode`]): an invite or a reset code is kept in the model only until the admin +//! dismisses it (or leaves the pages), never written anywhere, and left out of `Debug` output. + +use craft_cloud_app::{ + AccountInfo, AccountUsage, AdminCode, AdminCreatedUser, AdminServerInfo, AdminStorageOverview, AdminUserInfo, AdminUserList, DeviceId, + DeviceInfo, Error, UserQuery, Uuid, +}; +use craft_cloud_proto::api::{AdminConfirm, codes}; + +use crate::model::{Effect, Event, Model, Overview, Screen}; +use crate::security::{digits, explain, is_code}; + +/// A number no earlier call of the admin pages had, for matching an answer to its call: the numbers never restart, so an +/// answer that comes late (after the pages were left and opened again, or the account signed out and in) matches +/// nothing and is ignored. +pub(crate) fn fresh() -> u64 { + static LAST: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + LAST.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) +} + +/// Accounts on one page of the list. +pub const PAGE_SIZE: u32 = 20; +/// The most a quota may be, in GiB (a petabyte: a typo, not a quota, past it). +pub const MAX_QUOTA_GIB: f64 = 1_048_576.0; +const GIB: f64 = 1_073_741_824.0; + +/// Whether the signed-in account is an admin, as the server describes the account (`GET /account`). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum Access { + /// Not asked yet, or the server couldn't be reached: no admin pages. + #[default] + Unknown, + /// The account is a member (or the server refused the admin calls as `forbidden`). + Member, + /// The server says the account is an admin: the admin pages are offered. `two_factor`: whether the account has two-factor + /// sign-in, so that its confirmations ask for a code too. + Admin { two_factor: bool }, +} + +/// An account's role. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum RoleChoice { + #[default] + Member, + Admin, +} + +impl RoleChoice { + /// The server's name for it. + pub fn as_str(self) -> &'static str { + match self { + Self::Member => "member", + Self::Admin => "admin", + } + } + + /// The role the server names (anything but `admin` reads as a member). + pub fn of(role: &str) -> Self { + if role == "admin" { Self::Admin } else { Self::Member } + } + + pub fn label(self) -> &'static str { + match self { + Self::Member => "Member", + Self::Admin => "Admin", + } + } +} + +/// The admin's confirmation of a change: their current password and, when their account has two-factor sign-in, a code from +/// the authenticator app or a recovery code (one field: six digits are a code, anything else a recovery code, as on the Security +/// page). Its `Debug` output hides both. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Confirmation { + pub password: String, + pub code: String, +} + +impl std::fmt::Debug for Confirmation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Confirmation").field("password", &"").field("code", &"").finish() + } +} + +impl Confirmation { + /// Whether enough was typed to send: the password, and the code where the page asks for one. + pub fn ready(&self, needs_code: bool) -> bool { + !self.password.is_empty() && (!needs_code || !self.code.trim().is_empty()) + } + + /// The confirmation as the calls send it, with everything typed cleared from here. + pub fn take(&mut self) -> AdminConfirm { + let password = std::mem::take(&mut self.password); + let code = std::mem::take(&mut self.code); + let code = code.trim(); + let (totp_code, recovery_code) = if code.is_empty() { + (None, None) + } else if is_code(code) { + (Some(digits(code)), None) + } else { + (None, Some(code.to_owned())) + }; + AdminConfirm { password: Some(password), totp_code, recovery_code } + } +} + +/// A change the admin can make, with what its confirmation page says about it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + CreateUser { + username: String, + role: RoleChoice, + }, + /// A new role and/or quota (`quota`: `Some(None)` removes it). + UpdateUser { + id: Uuid, + username: String, + role: Option, + quota: Option>, + }, + Disable { + id: Uuid, + username: String, + }, + Enable { + id: Uuid, + username: String, + }, + Remove { + id: Uuid, + username: String, + }, + IssueInvite { + id: Uuid, + username: String, + }, + RevokeInvite { + id: Uuid, + username: String, + }, + /// A password reset code; `reset_two_factor`: redeeming it also turns the account's two-factor sign-in off. + IssueReset { + id: Uuid, + username: String, + reset_two_factor: bool, + }, + /// Takes the account's two-factor sign-in away and signs its computers out. + ResetTwoFactor { + id: Uuid, + username: String, + }, + RevokeDevice { + id: Uuid, + username: String, + device: DeviceId, + device_name: String, + }, +} + +impl Action { + /// The confirmation page's title. + pub fn title(&self) -> String { + match self { + Self::CreateUser { username, .. } => format!("Create {username}"), + Self::UpdateUser { username, .. } => format!("Change {username}"), + Self::Disable { username, .. } => format!("Disable {username}"), + Self::Enable { username, .. } => format!("Enable {username}"), + Self::Remove { username, .. } => format!("Remove {username}"), + Self::IssueInvite { username, .. } => format!("New invite for {username}"), + Self::RevokeInvite { username, .. } => format!("Revoke the invite of {username}"), + Self::IssueReset { username, .. } => format!("Password reset code for {username}"), + Self::ResetTwoFactor { username, .. } => format!("Turn two-factor sign-in off for {username}"), + Self::RevokeDevice { username, device_name, .. } => format!("Sign {device_name} of {username} out"), + } + } + + /// What the change does, in a sentence or two. + pub fn explain(&self) -> String { + match self { + Self::CreateUser { role, .. } => { + format!("A new {} account without a password. You get an invite code to give them, shown once.", role.as_str()) + } + Self::UpdateUser { role, quota, .. } => { + let mut parts = Vec::new(); + if let Some(role) = role { + parts.push(format!("the role becomes {}", role.as_str())); + } + match quota { + Some(Some(bytes)) => parts.push(format!("the quota becomes {}", format_bytes(*bytes))), + Some(None) => parts.push("the quota is removed".to_owned()), + None => {} + } + sentence(&parts.join(" and ")) + } + Self::Disable { .. } => "Their computers are signed out and they can't sign in until you enable the account again.".into(), + Self::Enable { .. } => "They can sign in again. Computers that were signed out stay signed out.".into(), + Self::Remove { .. } => "The account is marked for removal and its computers are signed out. Nothing it stored is deleted.".into(), + Self::IssueInvite { .. } => "A new invite code for an account that has no password yet, shown once. It replaces the open one.".into(), + Self::RevokeInvite { .. } => "The account's open invite or password reset code stops working.".into(), + Self::IssueReset { reset_two_factor, .. } => { + let mut text = "A code good for an hour that sets a new password. Using it signs every computer of the account out. Shown once.".to_owned(); + if *reset_two_factor { + text.push_str(" Using it also turns the account's two-factor sign-in off."); + } + text + } + Self::ResetTwoFactor { .. } => { + "The account no longer asks for a code at sign-in, and every computer of it is signed out. It can turn it on again on its Security page.".into() + } + Self::RevokeDevice { .. } => "That computer can't sync any more; its files stay there.".into(), + } + } + + /// The label of the confirm button. + pub fn button(&self) -> &'static str { + match self { + Self::CreateUser { .. } => "Create account", + Self::UpdateUser { .. } => "Save changes", + Self::Disable { .. } => "Disable", + Self::Enable { .. } => "Enable", + Self::Remove { .. } => "Remove account", + Self::IssueInvite { .. } => "Make an invite", + Self::RevokeInvite { .. } => "Revoke", + Self::IssueReset { .. } => "Make a reset code", + Self::ResetTwoFactor { .. } => "Turn two-factor off", + Self::RevokeDevice { .. } => "Sign out", + } + } +} + +/// What a change came to. +#[derive(Debug, Clone, PartialEq)] +pub enum Outcome { + Created(AdminCreatedUser), + Updated(AdminUserInfo), + Code(AdminCode), + Done, +} + +/// The password page of one change. +#[derive(Debug, Clone, PartialEq)] +pub struct Dialog { + pub action: Action, + pub confirmation: Confirmation, + /// Whether the page asks for a code too (the admin's account has two-factor sign-in, or the server said so). + pub needs_code: bool, + /// What went wrong with the last try, for people. + pub error: Option, + /// The call is with the server: nothing can be sent twice. + pub busy: bool, +} + +/// What kind of code is shown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CodeKind { + Invite, + PasswordReset, +} + +/// A code the server made, shown once. Its `Debug` output hides the code and the link. +#[derive(Clone, PartialEq, Eq)] +pub struct ShownCode { + pub username: String, + pub kind: CodeKind, + pub code: String, + /// The whole join string, when the server knows its public address. + pub join_url: Option, + pub expires_ms: i64, +} + +impl std::fmt::Debug for ShownCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ShownCode") + .field("username", &self.username) + .field("kind", &self.kind) + .field("code", &"") + .field("join_url", &self.join_url.as_ref().map(|_| "")) + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// The list of accounts: one page, the search that made it, and the cursors back to the first. +#[derive(Debug, Clone, PartialEq)] +pub struct UserList { + /// The search field, as typed. + pub search: String, + /// The search the shown page was asked with. + pub applied: String, + pub users: Vec, + /// How each page up to this one was asked for: `cursors[0]` is `None`, the last is this page's. + pub cursors: Vec>, + /// The cursor of the page after this one, if there is one. + pub next: Option, + pub loading: bool, + pub loaded: bool, + pub error: Option, + /// The number of the call whose answer is wanted; older answers are ignored. + ticket: u64, +} + +impl UserList { + fn new() -> Self { + Self { + search: String::new(), + applied: String::new(), + users: Vec::new(), + cursors: vec![None], + next: None, + loading: false, + loaded: false, + error: None, + ticket: 0, + } + } + + /// The page number shown (from 1). + pub fn page(&self) -> usize { + self.cursors.len() + } + + pub fn has_previous(&self) -> bool { + self.cursors.len() > 1 + } + + /// Asks for the current page. + fn load(&mut self) -> Effect { + self.loading = true; + self.error = None; + self.ticket = fresh(); + let query = UserQuery { + cursor: self.cursors.last().cloned().flatten(), + limit: Some(PAGE_SIZE), + query: (!self.applied.is_empty()).then(|| self.applied.clone()), + }; + Effect::AdminUsers { query, ticket: self.ticket } + } + + fn find(&self, id: Uuid) -> Option<&AdminUserInfo> { + self.users.iter().find(|u| u.id == id) + } +} + +/// The form for a new account. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CreateForm { + pub username: String, + pub role: RoleChoice, + pub error: Option, +} + +/// One account's page: its role and quota to change, and its devices. +#[derive(Debug, Clone, PartialEq)] +pub struct UserView { + pub id: Uuid, + pub username: String, + /// The role field. + pub role: RoleChoice, + /// The quota field, in GiB (empty: none). + pub quota: String, + /// What the quota field showed when the page was opened or last saved, to tell whether it was changed. + pub quota_shown: String, + pub error: Option, + pub devices: Option>, + pub devices_error: Option, + /// The number of the devices call whose answer is wanted. + devices_ticket: u64, + pub usage: Option, + pub usage_error: Option, + pub usage_loading: bool, + usage_ticket: u64, +} + +impl UserView { + fn new(user: &AdminUserInfo) -> Self { + let quota = user.quota_bytes.map(quota_text).unwrap_or_default(); + Self { + id: user.id, + username: user.username.clone(), + role: RoleChoice::of(&user.role), + quota_shown: quota.clone(), + quota, + error: None, + devices: None, + devices_error: None, + devices_ticket: 0, + usage: None, + usage_error: None, + usage_loading: false, + usage_ticket: 0, + } + } +} + +/// Which page of the admin pages is shown. +#[derive(Debug, Clone, PartialEq)] +pub enum View { + List, + Create(CreateForm), + User(Box), + /// The security audit log ([`crate::audit`]). + Audit(Box), +} + +/// The admin pages. +#[derive(Debug, Clone, PartialEq)] +pub struct AdminScreen { + /// The signed-in view the pages came from and go back to. + pub overview: Overview, + pub list: UserList, + /// The server's counters, once they came. + pub server: Option, + pub storage: Option, + pub storage_loading: bool, + pub storage_error: Option, + storage_ticket: u64, + pub view: View, + /// A change waiting for the admin's password. + pub dialog: Option, + /// A code just made, until it is dismissed. + pub code: Option, +} + +/// What happened on the admin pages: clicks and the answers of the admin calls. +#[derive(Debug, Clone, PartialEq)] +pub enum AdminEvent { + /// [`Effect::Account`]'s answer: the role of the signed-in account. + Account { + session: u64, + result: Result, + }, + /// [`Effect::AdminServer`]'s answer: the counters. + Server { + session: u64, + result: Result, + }, + /// [`Effect::AdminStorage`]'s answer. + Storage { + session: u64, + ticket: u64, + result: Result, + }, + /// [`Effect::AdminUsers`]'s answer. + Users { + ticket: u64, + result: Result, + }, + /// [`Effect::AdminDevices`]'s answer. + Devices { + user: Uuid, + ticket: u64, + result: Result, Error>, + }, + /// [`Effect::AdminUserUsage`]'s answer. + Usage { + user: Uuid, + ticket: u64, + result: Result, + }, + /// [`Effect::AdminDo`]'s answer. + Done(Result), + + /// The Administration button. + Open, + /// Back to the signed-in view. + Close, + /// Search for the text in the search field. + Search, + ClearSearch, + NextPage, + PreviousPage, + Reload, + NewUser, + /// Go on from the new account's form to the password. + SubmitCreate, + OpenUser(Uuid), + /// Back to the list from an account's page or the new account's form. + BackToList, + /// Go on from the account's role and quota to the password. + SubmitEdit, + /// Ask for the password of a change. + Ask(Action), + /// Send the change with the password typed. + Confirm, + CancelDialog, + /// The shown code was noted: forget it. + DismissCode, + RefreshDevices, + RefreshUsage, +} + +impl Model { + /// The admin pages, if shown. + pub fn admin_screen(&self) -> Option<&AdminScreen> { + match &self.screen { + Screen::Admin(admin) => Some(admin), + _ => None, + } + } + + /// Whether the admin pages are offered: the server said the signed-in account is an admin. + pub fn is_admin(&self) -> bool { + self.overview().is_some_and(|o| matches!(o.access, Access::Admin { .. })) + } + + pub(crate) fn admin(&mut self, event: AdminEvent) -> Vec { + match event { + AdminEvent::Account { session, result } => self.admin_probed(session, result), + AdminEvent::Open => self.open_admin(), + AdminEvent::Close => { + self.leave_admin(); + Vec::new() + } + event => { + let Screen::Admin(admin) = &mut self.screen else { return Vec::new() }; + admin.update(event) + } + } + } + + /// The account came (an admin or not), or its role can't be told. + fn admin_probed(&mut self, session: u64, result: Result) -> Vec { + // An answer for an earlier sign-in (another account, maybe) says nothing about this one. + if self.overview().map(|o| o.session) != Some(session) { + return Vec::new(); + } + let access = match result { + Ok(account) if account.role == "admin" => Access::Admin { two_factor: account.two_factor_enabled }, + Ok(_) => Access::Member, + Err(Error::Refused { code, .. }) if code == codes::FORBIDDEN => Access::Member, + // Unreachable, or an agent or server that doesn't know the call: asked again at the next sign-in or refresh. + Err(_) => return Vec::new(), + }; + if let Some(overview) = self.overview_mut() { + overview.access = access.clone(); + } + if access == Access::Member { + self.leave_admin(); + } + Vec::new() + } + + fn open_admin(&mut self) -> Vec { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + if !matches!(overview.access, Access::Admin { .. }) { + return Vec::new(); + } + let mut admin = AdminScreen { + overview: overview.clone(), + list: UserList::new(), + server: None, + storage: None, + storage_loading: false, + storage_error: None, + storage_ticket: 0, + view: View::List, + dialog: None, + code: None, + }; + let effects = vec![admin.list.load(), Effect::AdminServer { session: admin.overview.session }, admin.load_storage()]; + self.screen = Screen::Admin(Box::new(admin)); + effects + } + + /// Back to the signed-in view; a code still shown, and anything typed, is dropped. + fn leave_admin(&mut self) { + if let Screen::Admin(admin) = &self.screen { + self.screen = Screen::SignedIn(admin.overview.clone()); + } + } +} + +impl AdminScreen { + fn update(&mut self, event: AdminEvent) -> Vec { + match event { + AdminEvent::Account { .. } | AdminEvent::Open | AdminEvent::Close => Vec::new(), + AdminEvent::Server { session, result } => { + if session != self.overview.session { + return Vec::new(); + } + match result { + Ok(info) => { + self.server = Some(info); + Vec::new() + } + // Not an admin any more, or not told: the account is looked at again. + Err(e) if is_forbidden(&e) => vec![Effect::Account { session: self.overview.session }], + Err(_) => Vec::new(), + } + } + AdminEvent::Storage { session, ticket, result } => { + if session != self.overview.session || ticket != self.storage_ticket { + return Vec::new(); + } + self.storage_loading = false; + match result { + Ok(storage) => { + self.storage = Some(storage); + self.storage_error = None; + Vec::new() + } + Err(error) => { + self.storage_error = Some(error_text(&error)); + if is_forbidden(&error) { vec![Effect::Account { session: self.overview.session }] } else { Vec::new() } + } + } + } + AdminEvent::Users { ticket, result } => self.users_came(ticket, result), + AdminEvent::Devices { user, ticket, result } => { + let mut effects = Vec::new(); + if let View::User(view) = &mut self.view + && view.id == user + && view.devices_ticket == ticket + { + match result { + Ok(list) => { + view.devices = Some(list); + view.devices_error = None; + } + Err(e) => { + view.devices_error = Some(error_text(&e)); + if is_forbidden(&e) { + effects.push(Effect::Account { session: self.overview.session }); + } + } + } + } + effects + } + AdminEvent::Usage { user, ticket, result } => { + let mut effects = Vec::new(); + if let View::User(view) = &mut self.view + && view.id == user + && view.usage_ticket == ticket + { + view.usage_loading = false; + match result { + Ok(usage) => { + view.usage = Some(usage); + view.usage_error = None; + } + Err(error) => { + view.usage_error = Some(error_text(&error)); + if is_forbidden(&error) { + effects.push(Effect::Account { session: self.overview.session }); + } + } + } + } + effects + } + AdminEvent::Done(result) => self.done(result), + AdminEvent::Search => { + self.list.applied = self.list.search.trim().to_owned(); + self.list.search = self.list.applied.clone(); + self.list.cursors = vec![None]; + vec![self.list.load()] + } + AdminEvent::ClearSearch => { + self.list.search.clear(); + self.list.applied.clear(); + self.list.cursors = vec![None]; + vec![self.list.load()] + } + AdminEvent::NextPage => match self.list.next.clone() { + Some(next) if !self.list.loading => { + self.list.cursors.push(Some(next)); + vec![self.list.load()] + } + _ => Vec::new(), + }, + AdminEvent::PreviousPage => { + if self.list.has_previous() && !self.list.loading { + self.list.cursors.pop(); + vec![self.list.load()] + } else { + Vec::new() + } + } + AdminEvent::Reload => { + if self.list.loading { + return Vec::new(); + } + vec![self.list.load(), self.load_storage()] + } + AdminEvent::NewUser => { + if self.dialog.is_none() { + self.view = View::Create(CreateForm::default()); + } + Vec::new() + } + AdminEvent::SubmitCreate => self.submit_create(), + AdminEvent::OpenUser(id) => { + let Some(user) = self.list.find(id) else { return Vec::new() }; + if self.dialog.is_some() { + return Vec::new(); + } + let mut view = UserView::new(user); + view.devices_ticket = fresh(); + view.usage_ticket = fresh(); + view.usage_loading = true; + let effects = vec![ + Effect::AdminDevices { user: id, ticket: view.devices_ticket }, + Effect::AdminUserUsage { user: id, ticket: view.usage_ticket }, + ]; + self.view = View::User(Box::new(view)); + effects + } + AdminEvent::BackToList => { + if self.dialog.is_none() { + self.view = View::List; + } + Vec::new() + } + AdminEvent::SubmitEdit => { + self.submit_edit(); + Vec::new() + } + AdminEvent::Ask(action) => { + if self.dialog.is_none() { + self.dialog = Some(self.dialog(action)); + } + Vec::new() + } + AdminEvent::Confirm => self.confirm(), + AdminEvent::CancelDialog => { + if self.dialog.as_ref().is_some_and(|d| !d.busy) { + self.dialog = None; + } + Vec::new() + } + AdminEvent::DismissCode => { + self.code = None; + Vec::new() + } + AdminEvent::RefreshDevices => match &mut self.view { + View::User(view) => { + view.devices_ticket = fresh(); + vec![Effect::AdminDevices { user: view.id, ticket: view.devices_ticket }] + } + _ => Vec::new(), + }, + AdminEvent::RefreshUsage => match &mut self.view { + View::User(view) => { + view.usage_ticket = fresh(); + view.usage_loading = true; + view.usage_error = None; + vec![Effect::AdminUserUsage { user: view.id, ticket: view.usage_ticket }] + } + _ => Vec::new(), + }, + } + } + + fn load_storage(&mut self) -> Effect { + self.storage_ticket = fresh(); + self.storage_loading = true; + self.storage_error = None; + Effect::AdminStorage { session: self.overview.session, ticket: self.storage_ticket } + } + + fn users_came(&mut self, ticket: u64, result: Result) -> Vec { + if ticket != self.list.ticket { + return Vec::new(); + } + self.list.loading = false; + match result { + Ok(list) => { + self.list.loaded = true; + self.list.error = None; + self.list.users = list.users; + self.list.next = list.next; + Vec::new() + } + Err(e) => { + self.list.error = Some(error_text(&e)); + if is_forbidden(&e) { vec![Effect::Account { session: self.overview.session }] } else { Vec::new() } + } + } + } + + fn submit_create(&mut self) -> Vec { + let View::Create(form) = &mut self.view else { return Vec::new() }; + if self.dialog.is_some() { + return Vec::new(); + } + let username = form.username.trim().to_owned(); + if !valid_username(&username) { + form.error = Some(BAD_USERNAME.to_owned()); + return Vec::new(); + } + form.error = None; + let action = Action::CreateUser { username, role: form.role }; + self.dialog = Some(self.dialog(action)); + Vec::new() + } + + fn submit_edit(&mut self) { + let View::User(view) = &mut self.view else { return }; + if self.dialog.is_some() { + return; + } + let Some(user) = self.list.users.iter().find(|u| u.id == view.id) else { + view.error = Some("That account isn't in the list any more. Go back and look again.".to_owned()); + return; + }; + let role = (view.role.as_str() != user.role).then_some(view.role); + let quota_changed = view.quota.trim() != view.quota_shown.trim(); + let quota = if quota_changed { + match parse_quota(&view.quota) { + Ok(quota) => Some(quota), + Err(why) => { + view.error = Some(why); + return; + } + } + } else { + None + }; + if role.is_none() && quota.is_none() { + view.error = Some("Nothing to change.".to_owned()); + return; + } + view.error = None; + let action = Action::UpdateUser { id: view.id, username: view.username.clone(), role, quota }; + self.dialog = Some(self.dialog(action)); + } + + fn confirm(&mut self) -> Vec { + let Some(dialog) = &mut self.dialog else { return Vec::new() }; + if dialog.busy { + return Vec::new(); + } + if dialog.confirmation.password.is_empty() { + dialog.error = Some(NEEDS_PASSWORD.to_owned()); + return Vec::new(); + } + if !dialog.confirmation.ready(dialog.needs_code) { + dialog.error = Some(NEEDS_CODE.to_owned()); + return Vec::new(); + } + dialog.error = None; + dialog.busy = true; + // The typed password leaves the model here: it travels in the call and nowhere else. + let confirm = dialog.confirmation.take(); + vec![Effect::AdminDo { action: dialog.action.clone(), confirm }] + } + + fn done(&mut self, result: Result) -> Vec { + let Some(dialog) = &mut self.dialog else { return Vec::new() }; + if !dialog.busy { + return Vec::new(); + } + let outcome = match result { + Ok(outcome) => outcome, + Err(e) => { + dialog.busy = false; + dialog.confirmation = Confirmation::default(); + // The server wants a code: ask for it (the account's two-factor state may have changed since sign-in). + if e.code() == Some(codes::TWO_FACTOR_INVALID) { + dialog.needs_code = true; + } + dialog.error = Some(match (&dialog.action, &e) { + (Action::RevokeInvite { .. }, Error::Refused { code, .. }) if code == codes::NOT_FOUND => NO_OPEN_CODE.to_owned(), + _ => error_text(&e), + }); + let none_open = matches!(&dialog.action, Action::RevokeInvite { .. }) && dialog.error.as_deref() == Some(NO_OPEN_CODE); + return if is_forbidden(&e) { + vec![Effect::Account { session: self.overview.session }] + } else if none_open { + // The list said an invite was open, or didn't say: look again. + vec![self.list.load()] + } else { + Vec::new() + }; + } + }; + let action = dialog.action.clone(); + self.dialog = None; + let mut effects = Vec::new(); + match (&action, outcome) { + (Action::CreateUser { username, .. }, Outcome::Created(created)) => { + self.show_code(username, CodeKind::Invite, created.invite); + self.view = View::List; + } + (Action::IssueInvite { username, .. }, Outcome::Code(code)) => self.show_code(username, CodeKind::Invite, code), + (Action::IssueReset { username, .. }, Outcome::Code(code)) => self.show_code(username, CodeKind::PasswordReset, code), + (Action::UpdateUser { .. }, Outcome::Updated(info)) => { + if let View::User(view) = &mut self.view + && view.id == info.id + { + **view = UserView { + devices: view.devices.take(), + devices_error: view.devices_error.take(), + devices_ticket: view.devices_ticket, + ..UserView::new(&info) + }; + view.usage_ticket = fresh(); + view.usage_loading = true; + effects.push(Effect::AdminUserUsage { user: view.id, ticket: view.usage_ticket }); + } + } + (Action::Remove { .. }, _) => self.view = View::List, + (Action::RevokeDevice { id, .. }, _) => { + if let View::User(view) = &mut self.view + && view.id == *id + { + view.devices_ticket = fresh(); + effects.push(Effect::AdminDevices { user: *id, ticket: view.devices_ticket }); + } + } + _ => {} + } + // What changed shows in the list (and the counters). + effects.push(self.list.load()); + effects.push(Effect::AdminServer { session: self.overview.session }); + effects.push(self.load_storage()); + effects + } + + /// A page for the confirmation of `action`: it asks for a code too when the account has two-factor sign-in. + fn dialog(&self, action: Action) -> Dialog { + let needs_code = matches!(self.overview.access, Access::Admin { two_factor: true }); + Dialog { action, confirmation: Confirmation::default(), needs_code, error: None, busy: false } + } + + fn show_code(&mut self, username: &str, kind: CodeKind, code: AdminCode) { + self.code = Some(ShownCode { username: username.to_owned(), kind, code: code.code, join_url: code.join_url, expires_ms: code.expires_ms }); + } +} + +/// The words when a revoke finds no invite or reset code open (the server's `not_found`). +pub const NO_OPEN_CODE: &str = "This account has no open invite or password reset code."; +/// The words the confirmation page says when the code is missing. +pub const NEEDS_CODE: &str = "Type the code from your authenticator app (or a recovery code) to confirm"; +/// The words the confirmation page says when no password was typed. +pub const NEEDS_PASSWORD: &str = "Type your own password to confirm"; +/// The words when a new account's name breaks the server's rules. +pub const BAD_USERNAME: &str = "A name is 1 to 64 letters, digits, dots, underscores or dashes"; + +/// Whether `name` is a username the server takes (1-64 ASCII letters, digits, `.`, `_` or `-`). +pub fn valid_username(name: &str) -> bool { + (1..=64).contains(&name.len()) && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +pub(crate) fn is_forbidden(e: &Error) -> bool { + matches!(e, Error::Refused { code, .. } if code == codes::FORBIDDEN) +} + +/// An admin call's error, for people: the server's refusals the admin can act on in plain words (a wrong password, a wrong or +/// locked second factor and a rate limit as the Security page says them). +pub fn error_text(e: &Error) -> String { + match e.code() { + Some(codes::WRONG_PASSWORD | codes::TWO_FACTOR_INVALID | codes::TWO_FACTOR_LOCKED | codes::RATE_LIMITED) => explain(e), + Some(codes::LAST_ADMIN) => "That would leave the server without an enabled admin. Make another account an admin first.".to_owned(), + Some(codes::ACCOUNT_DISABLED) => "Your account is disabled, so the server refuses this.".to_owned(), + Some(codes::FORBIDDEN) => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => format!("The server doesn't allow this: {}", lower_first(message)), + other => other.to_string(), + }, + _ => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), + other => other.to_string(), + }, + } +} + +fn lower_first(text: &str) -> String { + let mut chars = text.chars(); + chars.next().map(|first| first.to_lowercase().chain(chars).collect()).unwrap_or_default() +} + +/// `text` as a sentence: a capital first, a full stop last (nothing for an empty text). +fn sentence(text: &str) -> String { + let text = text.trim().trim_end_matches('.'); + let mut chars = text.chars(); + match chars.next() { + Some(first) => format!("{}.", first.to_uppercase().chain(chars).collect::()), + None => String::new(), + } +} + +/// A quota typed in GiB (`50`, `0.5`; empty: none). +pub fn parse_quota(text: &str) -> Result, String> { + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let bad = || "A quota is a number of GiB, like 50 or 0.5 (empty: no quota)".to_owned(); + let gib: f64 = text.parse().map_err(|_| bad())?; + if !gib.is_finite() || gib < 0.0 { + return Err(bad()); + } + if gib > MAX_QUOTA_GIB { + return Err(format!("A quota is at most {MAX_QUOTA_GIB:.0} GiB")); + } + // At most 2^20 GiB: the product is far under u64's range and rounds to a whole number of bytes. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = (gib * GIB).round() as u64; + Ok(Some(bytes)) +} + +/// A quota's bytes as the quota field shows them: GiB, to 3 places, without trailing zeros. +pub fn quota_text(bytes: u64) -> String { + let gib = bytes as f64 / GIB; + let text = format!("{gib:.3}"); + text.trim_end_matches('0').trim_end_matches('.').to_owned() +} + +/// A size for people: `512 B`, `1.5 MiB`, `2 GiB`. +pub fn format_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut value = bytes as f64; + let mut unit = 0; + while value >= 1024.0 && unit + 1 < UNITS.len() { + value /= 1024.0; + unit += 1; + } + let number = if unit == 0 || value >= 100.0 { format!("{value:.0}") } else { format!("{value:.1}") }; + let number = if number.contains('.') { number.trim_end_matches('0').trim_end_matches('.').to_owned() } else { number }; + format!("{number} {}", UNITS.get(unit).copied().unwrap_or("B")) +} + +impl From for Event { + fn from(event: AdminEvent) -> Self { + Event::Admin(event) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Arc; + + use super::*; + use crate::agent::fake::{FakeAgent, sample_user}; + use crate::agent::{Starter, call}; + use crate::model::{JoinDefaults, tests::status}; + + /// An agent signed in as `fabio`, an admin (or not), with `others` more accounts after fabio's own. + pub(crate) fn agent(admin: bool, others: usize) -> FakeAgent { + let agent = FakeAgent::new(status(true)); + agent.admin.store(admin, std::sync::atomic::Ordering::SeqCst); + let mut users = vec![AdminUserInfo { role: "admin".into(), ..sample_user("fabio", Uuid::now_v7()) }]; + users.extend((1..=others).map(|n| sample_user(&format!("user{n:02}"), Uuid::now_v7()))); + *agent.users.lock().unwrap() = users; + agent + } + + /// Feeds `events` to the model, each effect answered by the fake until nothing is left (as the link does). + pub(crate) fn run(model: &mut Model, agent: &FakeAgent, events: Vec) { + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err("not started in tests".to_owned())); + for event in events { + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Some(answer) = call(agent, &start, effect) { + pending.extend(model.update(answer)); + } + } + } + } + + pub(crate) fn signed_in(agent: &FakeAgent) -> Model { + let mut model = Model::new(JoinDefaults::default(), true, "run.log".into()); + run(&mut model, agent, vec![Event::Start]); + model + } + + /// The model on the admin pages. + pub(crate) fn on_the_pages(agent: &FakeAgent) -> Model { + let mut model = signed_in(agent); + run(&mut model, agent, vec![Event::Admin(AdminEvent::Open)]); + assert!(model.admin_screen().is_some(), "{:?}", model.screen); + model + } + + pub(crate) fn admin(model: &Model) -> &AdminScreen { + model.admin_screen().unwrap_or_else(|| panic!("not on the admin pages: {:?}", model.screen)) + } + + pub(crate) fn admin_mut(model: &mut Model) -> &mut AdminScreen { + match &mut model.screen { + Screen::Admin(admin) => admin, + other => panic!("not on the admin pages: {other:?}"), + } + } + + pub(crate) fn user_named(model: &Model, name: &str) -> AdminUserInfo { + admin(model).list.users.iter().find(|u| u.username == name).cloned().unwrap_or_else(|| panic!("no {name} in the list")) + } + + pub(crate) fn ev(event: AdminEvent) -> Vec { + vec![Event::Admin(event)] + } + + pub(crate) fn type_password(model: &mut Model, password: &str) { + admin_mut(model).dialog.as_mut().unwrap().confirmation.password = password.to_owned(); + } + + pub(crate) fn refused(code: &str, message: &str) -> Error { + Error::Refused { code: code.into(), message: message.into(), name_taken: None } + } + + /// The pages are for admins: the server's answer to the counters decides, and a member never gets in. + #[test] + fn only_an_admin_is_offered_the_pages() { + let member = agent(false, 3); + let mut model = signed_in(&member); + assert_eq!(model.overview().unwrap().access, Access::Member); + assert!(!model.is_admin()); + assert!(model.update(Event::Admin(AdminEvent::Open)).is_empty()); + assert!(matches!(model.screen, Screen::SignedIn(_))); + assert_eq!(member.asked.lock().unwrap().iter().filter(|a| *a == "account").count(), 1); + + let boss = agent(true, 3); + let mut model = signed_in(&boss); + assert!(model.is_admin()); + assert_eq!(model.overview().unwrap().access, Access::Admin { two_factor: false }); + let effects = model.update(Event::Admin(AdminEvent::Open)); + assert!(matches!(effects.as_slice(), [Effect::AdminUsers { .. }, Effect::AdminServer { .. }, Effect::AdminStorage { .. }]), "{effects:?}"); + assert!(model.admin_screen().is_some()); + // The counters come with the pages, not with the sign-in. + assert!(model.admin_screen().unwrap().server.is_none()); + let mut model = on_the_pages(&boss); + let info = model.admin_screen().unwrap().server.clone().expect("the counters"); + assert_eq!((info.users, info.storage_kind.as_str()), (4, "disk")); + let _ = &mut model; + + // A server that can't be reached says nothing about the role: no pages, asked again at the next refresh. + let mut model = Model::new(JoinDefaults::default(), true, String::new()); + model.update(Event::Start); + model.update(Event::Found(Ok(status(true)))); + let session = model.overview().unwrap().session; + model.update(Event::Admin(AdminEvent::Account { session, result: Err(Error::Io("reset".into())) })); + assert_eq!(model.overview().unwrap().access, Access::Unknown); + assert!(model.update(Event::RefreshDevices).contains(&Effect::Account { session })); + } + + /// An admin who is demoted meanwhile: the server's `forbidden` is checked against the counters, and the pages close. + #[test] + fn a_demoted_admin_is_taken_off_the_pages() { + let boss = agent(true, 2); + let mut model = on_the_pages(&boss); + boss.admin.store(false, std::sync::atomic::Ordering::SeqCst); + run(&mut model, &boss, ev(AdminEvent::Reload)); + assert!(matches!(model.screen, Screen::SignedIn(_)), "{:?}", model.screen); + assert_eq!(model.overview().unwrap().access, Access::Member); + } + + /// Pages of 20, a cursor each way, and a search that starts over from the first page; an old answer is ignored. + #[test] + fn the_list_is_paged_and_searched() { + let boss = agent(true, 44); + let mut model = on_the_pages(&boss); + assert_eq!(admin(&model).list.users.len(), PAGE_SIZE as usize); + assert_eq!((admin(&model).list.page(), admin(&model).list.next.as_deref()), (1, Some("20"))); + assert!(model.update(Event::Admin(AdminEvent::PreviousPage)).is_empty(), "no page before the first"); + run(&mut model, &boss, ev(AdminEvent::NextPage)); + assert_eq!(admin(&model).list.page(), 2); + assert_eq!(admin(&model).list.users[0].username, "user20"); + run(&mut model, &boss, ev(AdminEvent::NextPage)); + assert_eq!((admin(&model).list.users.len(), admin(&model).list.next.clone()), (5, None)); + assert!(model.update(Event::Admin(AdminEvent::NextPage)).is_empty(), "no page after the last"); + run(&mut model, &boss, ev(AdminEvent::PreviousPage)); + assert_eq!((admin(&model).list.page(), admin(&model).list.users[0].username.as_str()), (2, "user20")); + + admin_mut(&mut model).list.search = " USER4 ".into(); + let effects = model.update(Event::Admin(AdminEvent::Search)); + let [Effect::AdminUsers { query, ticket }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((query.cursor.as_deref(), query.query.as_deref(), query.limit), (None, Some("USER4"), Some(PAGE_SIZE))); + let old = ticket - 1; + model.update(Event::Admin(AdminEvent::Users { ticket: old, result: Ok(AdminUserList { users: Vec::new(), next: None }) })); + assert!(admin(&model).list.loading, "an answer to an older call is ignored"); + for effect in effects { + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + if let Some(answer) = call(&boss, &start, effect) { + model.update(answer); + } + } + let names: Vec<&str> = admin(&model).list.users.iter().map(|u| u.username.as_str()).collect(); + assert_eq!(names, ["user40", "user41", "user42", "user43", "user44"]); + assert_eq!(admin(&model).list.page(), 1); + run(&mut model, &boss, ev(AdminEvent::ClearSearch)); + assert_eq!((admin(&model).list.applied.as_str(), admin(&model).list.users.len()), ("", PAGE_SIZE as usize)); + } + + /// Every change asks for the password afresh; it leaves the model when it is sent, and shows in no `Debug` output. + #[test] + fn a_change_sends_the_password_once_and_clears_it() { + let boss = agent(true, 3); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + model.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(admin(&model).dialog.is_some()); + // Nothing typed: nothing is sent. + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_PASSWORD)); + type_password(&mut model, "admin-secret-pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let [Effect::AdminDo { action, confirm }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!(action, &Action::Disable { id: ana.id, username: "user01".into() }); + assert_eq!(confirm.password.as_deref(), Some("admin-secret-pw")); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.busy && dialog.confirmation.password.is_empty(), "{dialog:?}"); + let shown = format!("{model:?} {effects:?}"); + assert!(!shown.contains("admin-secret-pw"), "{shown}"); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty(), "not twice"); + assert!(model.update(Event::Admin(AdminEvent::CancelDialog)).is_empty()); + assert!(admin(&model).dialog.is_some(), "not while it is with the server"); + // The call is made with the password, and done: the dialog goes, the list is asked again. + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + assert_eq!(boss.passwords.lock().unwrap().as_slice(), [Some("admin-secret-pw".to_owned())]); + let effects = model.update(answer); + assert!(admin(&model).dialog.is_none()); + assert!(effects.iter().any(|e| matches!(e, Effect::AdminUsers { .. })), "{effects:?}"); + } + + /// A wrong password is the server's `wrong_password`: the dialog stays, empty, to try again. + #[test] + fn a_wrong_password_keeps_the_dialog_open() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + boss.admin_fails.lock().unwrap().push(refused(codes::WRONG_PASSWORD, "the password is wrong")); + run( + &mut model, + &boss, + vec![ + Event::Admin(AdminEvent::Ask(Action::Remove { id: ana.id, username: "user01".into() })), + Event::Admin(AdminEvent::Confirm), // nothing typed + ], + ); + type_password(&mut model, "not-my-password"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(!dialog.busy && dialog.confirmation.password.is_empty()); + assert_eq!(dialog.error.as_deref(), Some("That password isn't right.")); + type_password(&mut model, "my-password"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.passwords.lock().unwrap().iter().flatten().collect::>(), ["not-my-password", "my-password"]); + } + + #[test] + fn the_servers_refusals_are_said_clearly() { + assert!(error_text(&refused(codes::LAST_ADMIN, "x")).contains("without an enabled admin")); + assert!(error_text(&refused(codes::ACCOUNT_DISABLED, "x")).contains("disabled")); + assert_eq!(error_text(&refused(codes::FORBIDDEN, "Only admins can do this")), "The server doesn't allow this: only admins can do this"); + let slow = Error::RefusedWith { + code: codes::RATE_LIMITED.into(), + message: "too many wrong passwords; wait a few minutes".into(), + two_factor: None, + retry_after_secs: Some(90), + }; + let text = error_text(&slow); + assert!(text.contains("Too many tries") && text.contains("90 seconds"), "{text}"); + let busy = Error::RefusedWith { + code: codes::RATE_LIMITED.into(), + message: "the server is busy".into(), + two_factor: None, + retry_after_secs: Some(600), + }; + assert!(error_text(&busy).contains("10 minutes")); + assert_eq!(error_text(&refused("invalid", "role is admin or member")), "role is admin or member"); + assert!(error_text(&Error::NotRunning("x".into())).contains("isn't running")); + + // In the dialog, as the user reads it. + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + for (fail, wanted) in [ + (refused(codes::LAST_ADMIN, "x"), "without an enabled admin"), + (slow, "Too many tries"), + (refused(codes::ACCOUNT_DISABLED, "x"), "disabled"), + ] { + boss.admin_fails.lock().unwrap().push(fail); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let error = admin(&model).dialog.as_ref().unwrap().error.clone().unwrap(); + assert!(error.contains(wanted), "{error}"); + run(&mut model, &boss, ev(AdminEvent::CancelDialog)); + assert!(admin(&model).dialog.is_none()); + } + // A `forbidden` on a change asks the server again whether this is still an admin. + boss.admin_fails.lock().unwrap().push(refused(codes::FORBIDDEN, "You can't do that to your own account")); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + assert_eq!(model.update(answer), vec![Effect::Account { session: model.overview().unwrap().session }]); + assert!(admin(&model).dialog.as_ref().unwrap().error.as_deref().is_some_and(|e| e.starts_with("The server doesn't allow this"))); + assert!(model.admin_screen().is_some(), "still an admin: the pages stay"); + } + + /// A new account has a name and a role, and no quota (that is a change of its own, on its page); the invite it + /// makes is shown once. + #[test] + fn a_new_account_shows_its_invite_once() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + model.update(Event::Admin(AdminEvent::NewUser)); + assert!(matches!(admin(&model).view, View::Create(_))); + let form = |model: &mut Model| match &mut admin_mut(model).view { + View::Create(form) => form.clone(), + other => panic!("{other:?}"), + }; + let long = "x".repeat(65); + for bad in ["", " ", "a b", "ana/../root", long.as_str()] { + if let View::Create(f) = &mut admin_mut(&mut model).view { + f.username = bad.into(); + } + assert!(model.update(Event::Admin(AdminEvent::SubmitCreate)).is_empty()); + assert_eq!(form(&mut model).error.as_deref(), Some(BAD_USERNAME), "{bad:?}"); + assert!(admin(&model).dialog.is_none()); + } + if let View::Create(f) = &mut admin_mut(&mut model).view { + f.username = " new.user_1 ".into(); + f.role = RoleChoice::Admin; + } + model.update(Event::Admin(AdminEvent::SubmitCreate)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::CreateUser { username: "new.user_1".into(), role: RoleChoice::Admin }); + type_password(&mut model, "admin-secret-pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_create_user").count(), 1); + let screen = admin(&model); + assert!(screen.dialog.is_none() && matches!(screen.view, View::List)); + let code = screen.code.as_ref().unwrap(); + assert_eq!((code.username.as_str(), code.kind, code.code.as_str()), ("new.user_1", CodeKind::Invite, "K7M4Q9TXK7M4Q9TX")); + let shown = format!("{model:?}"); + assert!(!shown.contains("K7M4Q9TX") && !shown.contains("K7M4-Q9TX") && !shown.contains("admin-secret-pw"), "{shown}"); + // Dismissed, it is gone from the model; leaving the pages drops it too. + model.update(Event::Admin(AdminEvent::DismissCode)); + assert!(admin(&model).code.is_none()); + assert_eq!(format!("{:?}", admin(&model).list.users).matches("K7M4").count(), 0); + model.update(Event::Admin(AdminEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + } + + /// Role and quota are one change, with only what differs sent; an empty quota removes it. + #[test] + fn an_account_page_changes_the_role_and_the_quota() { + let boss = agent(true, 2); + let mut model = on_the_pages(&boss); + let ana = AdminUserInfo { quota_bytes: Some(5 * 1_073_741_824), ..user_named(&model, "user01") }; + boss.users.lock().unwrap()[1] = ana.clone(); + run(&mut model, &boss, ev(AdminEvent::Reload)); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + let view = |model: &Model| match &admin(model).view { + View::User(view) => (**view).clone(), + other => panic!("{other:?}"), + }; + let v = view(&model); + assert_eq!((v.role, v.quota.as_str(), v.devices.as_ref().map(Vec::len)), (RoleChoice::Member, "5", Some(1))); + // Nothing changed: nothing to ask. + model.update(Event::Admin(AdminEvent::SubmitEdit)); + assert_eq!(view(&model).error.as_deref(), Some("Nothing to change.")); + let set = |model: &mut Model, role: RoleChoice, quota: &str| { + if let View::User(v) = &mut admin_mut(model).view { + v.role = role; + v.quota = quota.into(); + } + }; + set(&mut model, RoleChoice::Member, "lots"); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + assert!(view(&model).error.unwrap().starts_with("A quota is a number of GiB")); + assert!(admin(&model).dialog.is_none()); + set(&mut model, RoleChoice::Admin, "5"); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::UpdateUser { id: ana.id, username: "user01".into(), role: Some(RoleChoice::Admin), quota: None }); + run(&mut model, &boss, ev(AdminEvent::CancelDialog)); + set(&mut model, RoleChoice::Member, ""); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::UpdateUser { id: ana.id, username: "user01".into(), role: None, quota: Some(None) }); + assert!(action.explain().contains("quota is removed")); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let v = view(&model); + assert_eq!((v.quota.as_str(), v.error), ("", None), "the page shows what the server answered"); + assert_eq!(v.devices.map(|d| d.len()), Some(1), "the devices stay"); + } + + /// An invite and a reset code are shown once, each; a revoke and a device revoke refresh what they changed. + #[test] + fn invites_reset_codes_and_devices() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + for (action, kind, code) in [ + (Action::IssueInvite { id: ana.id, username: "user01".into() }, CodeKind::Invite, "ABCDEFGHJKMNPQRS"), + (Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: false }, CodeKind::PasswordReset, "TUVWXYZ234567ABC"), + ] { + run(&mut model, &boss, ev(AdminEvent::Ask(action))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let shown = admin(&model).code.clone().unwrap(); + assert_eq!((shown.kind, shown.code.as_str()), (kind, code)); + assert!(matches!(admin(&model).view, View::User(_)), "still on the account's page"); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + } + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).code.is_none() && admin(&model).dialog.is_none()); + + let View::User(view) = &admin(&model).view else { panic!() }; + let device = view.devices.as_ref().unwrap()[0].clone(); + let before = boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_devices").count(); + let revoke = Action::RevokeDevice { id: ana.id, username: "user01".into(), device: device.id, device_name: device.name.clone() }; + assert!(revoke.title().contains("laptop")); + run(&mut model, &boss, ev(AdminEvent::Ask(revoke))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_devices").count(), before + 1, "the devices are asked again"); + + // Removing the account goes back to the list. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Remove { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(matches!(admin(&model).view, View::List)); + } + + #[test] + fn quotas_are_typed_in_gib_and_shown_in_the_largest_unit() { + assert_eq!(parse_quota(""), Ok(None)); + assert_eq!(parse_quota(" 50 "), Ok(Some(50 * 1_073_741_824))); + assert_eq!(parse_quota("0.5"), Ok(Some(536_870_912))); + assert_eq!(parse_quota("0"), Ok(Some(0))); + for bad in ["-1", "abc", "inf", "NaN", "1e99", "2000000", "5 GB"] { + assert!(parse_quota(bad).is_err(), "{bad}"); + } + assert_eq!(quota_text(5 * 1_073_741_824), "5"); + assert_eq!(quota_text(536_870_912), "0.5"); + assert_eq!(quota_text(0), "0"); + assert_eq!(parse_quota("a_text(1_610_612_736)), Ok(Some(1_610_612_736))); + assert_eq!(format_bytes(0), "0 B"); + assert_eq!(format_bytes(1_536), "1.5 KiB"); + assert_eq!(format_bytes(5_000_000_000), "4.7 GiB"); + assert_eq!(format_bytes(2 * 1_073_741_824), "2 GiB"); + assert!(valid_username("a.b_c-9") && !valid_username("") && !valid_username("a b") && !valid_username("é")); + } + + #[test] + fn nothing_secret_is_in_debug_output() { + let confirmation = Confirmation { password: "admin-secret-pw".into(), code: "918273".into() }; + let shown = format!("{confirmation:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + let code = ShownCode { + username: "ana".into(), + kind: CodeKind::Invite, + code: "ABCDEFGH".into(), + join_url: Some("c.example/ABCD".into()), + expires_ms: 1, + }; + let shown = format!("{code:?}"); + assert!(!shown.contains("ABCDEFGH") && !shown.contains("c.example"), "{shown}"); + } + + fn account(role: &str) -> AccountInfo { + AccountInfo { + user_id: Uuid::nil(), + username: "fabio".into(), + role: role.into(), + created_ms: 1, + password_changed_ms: None, + two_factor_enabled: false, + } + } + + /// An answer that comes late (the pages were left and opened again; the devices were asked for twice) matches + /// nothing: it never overwrites newer data. + #[test] + fn answers_that_come_late_are_ignored() { + let boss = agent(true, 3); + let mut model = signed_in(&boss); + let first = model.update(Event::Admin(AdminEvent::Open)); + let Some(Effect::AdminUsers { ticket: old, .. }) = first.first().cloned() else { panic!("{first:?}") }; + model.update(Event::Admin(AdminEvent::Close)); + run(&mut model, &boss, ev(AdminEvent::Open)); + let current = admin(&model).list.users.clone(); + assert_eq!(current.len(), 4); + let ghost = AdminUserList { users: vec![sample_user("ghost", Uuid::now_v7())], next: None }; + model.update(Event::Admin(AdminEvent::Users { ticket: old, result: Ok(ghost) })); + assert_eq!(admin(&model).list.users, current, "the first page's reply is from before the pages were opened again"); + + // The same for a user's devices. + let ana = user_named(&model, "user01"); + let asked = model.update(Event::Admin(AdminEvent::OpenUser(ana.id))); + let [Effect::AdminDevices { ticket: first_devices, .. }, Effect::AdminUserUsage { ticket: first_usage, .. }] = asked.as_slice() else { + panic!("{asked:?}") + }; + run(&mut model, &boss, ev(AdminEvent::RefreshDevices)); + run(&mut model, &boss, ev(AdminEvent::RefreshUsage)); + let View::User(view) = &admin(&model).view else { panic!() }; + let now = view.devices.clone(); + let usage_now = view.usage.clone(); + assert_eq!(now.as_ref().map(Vec::len), Some(1)); + model.update(Event::Admin(AdminEvent::Devices { user: ana.id, ticket: *first_devices, result: Ok(Vec::new()) })); + let View::User(view) = &admin(&model).view else { panic!() }; + assert_eq!(view.devices, now); + model.update(Event::Admin(AdminEvent::Usage { + user: ana.id, + ticket: *first_usage, + result: Ok(craft_cloud_app::AccountUsage { + stored_bytes: 1, + quota_bytes: None, + trash_bytes: Some(2), + history_bytes: Some(3), + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + }), + })); + let View::User(view) = &admin(&model).view else { panic!() }; + assert_eq!(view.usage, usage_now); + } + + /// The admin check of an earlier sign-in (another account, maybe) says nothing about the current one. + #[test] + fn an_admin_check_for_an_earlier_sign_in_is_ignored() { + let mut model = Model::new(JoinDefaults::default(), true, String::new()); + model.update(Event::Start); + let probe = |effects: &[Effect]| { + effects.iter().find_map(|e| if let Effect::Account { session } = e { Some(*session) } else { None }).expect("an admin check") + }; + let first = probe(&model.update(Event::Found(Ok(status(true))))); + model.update(Event::SignedOut(Ok(status(false)))); + let second = probe(&model.update(Event::Polled(Ok(status(true))))); + assert_ne!(first, second); + // The first account was an admin; its late answer doesn't make this one an admin. + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("admin")) })); + assert_eq!(model.overview().unwrap().access, Access::Unknown); + assert!(!model.is_admin()); + model.update(Event::Admin(AdminEvent::Account { session: second, result: Ok(account("member")) })); + assert_eq!(model.overview().unwrap().access, Access::Member); + // And a late "member" answer doesn't take an admin's pages away. + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("member")) })); + model.update(Event::Admin(AdminEvent::Account { session: second, result: Ok(account("admin")) })); + assert!(model.is_admin()); + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("member")) })); + assert!(model.is_admin()); + } + + /// A password reset code can be revoked like an invite: the button doesn't depend on `invite_pending` (which + /// doesn't say there is one), and a revoke that finds nothing open says so and looks at the list again. + #[test] + fn a_revoke_that_finds_no_open_code_says_so() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(!ana.invite_pending); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: false }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + // The reset code is open, though the list doesn't say so: it can be revoked. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_revoke_invite").count(), 1); + // Nothing open (the server's `not_found`): said clearly, and the list is asked again. + boss.admin_fails.lock().unwrap().push(refused(codes::NOT_FOUND, "not found")); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + let effects = model.update(answer); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NO_OPEN_CODE)); + assert!(matches!(effects.as_slice(), [Effect::AdminUsers { .. }]), "{effects:?}"); + } + + /// An admin whose account has two-factor sign-in confirms with a code too: one field, six digits are a code from the app and + /// anything else a recovery code; a missing code is not sent; everything typed leaves the model when it is sent. + #[test] + fn a_confirmation_asks_for_a_code_when_the_account_has_two_factor() { + let boss = agent(true, 1); + boss.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + let mut model = on_the_pages(&boss); + assert_eq!(model.overview().unwrap().access, Access::Admin { two_factor: true }); + let ana = user_named(&model, "user01"); + let type_in = |model: &mut Model, password: &str, code: &str| { + let d = admin_mut(model).dialog.as_mut().unwrap(); + d.confirmation.password = password.into(); + d.confirmation.code = code.into(); + }; + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(admin(&model).dialog.as_ref().unwrap().needs_code); + // A password alone, or a code alone, is not sent. + type_in(&mut model, "admin-secret-pw", ""); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_CODE)); + type_in(&mut model, "", "123456"); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_PASSWORD)); + // Six digits with spaces: a code from the app. + type_in(&mut model, "admin-secret-pw", " 123 456 "); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let [Effect::AdminDo { confirm, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!( + (confirm.password.as_deref(), confirm.totp_code.as_deref(), confirm.recovery_code.as_deref()), + (Some("admin-secret-pw"), Some("123456"), None) + ); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.busy && dialog.confirmation.password.is_empty() && dialog.confirmation.code.is_empty(), "{dialog:?}"); + let shown = format!("{model:?} {effects:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("123456"), "{shown}"); + run(&mut model, &boss, vec![]); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(boss.factors.lock().unwrap().as_slice(), [(Some("123456".to_owned()), None)]); + // Anything else is a recovery code, in the other field. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_in(&mut model, "admin-secret-pw", "abcd-efgh-jkmn-pqrs"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.factors.lock().unwrap().last().unwrap(), &(None, Some("abcd-efgh-jkmn-pqrs".to_owned()))); + // The password dialog of an account without it has no code field. + let plain = agent(true, 1); + let mut model = on_the_pages(&plain); + run(&mut model, &plain, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(!admin(&model).dialog.as_ref().unwrap().needs_code); + } + + /// The server's word on a code: a wrong one (403) is said and the page asks for the code if it didn't; a lock (429) says how + /// long to wait; the typed secrets are gone either way. + #[test] + fn a_wrong_or_locked_code_is_said_and_asked_for_again() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(!{ + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + admin(&model).dialog.as_ref().unwrap().needs_code + }); + boss.admin_fails.lock().unwrap().push(refused(codes::TWO_FACTOR_INVALID, "wrong code")); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.needs_code, "the server wants a code: the page asks for it"); + assert!(dialog.error.as_deref().is_some_and(|e| e.contains("code isn't right")), "{:?}", dialog.error); + assert!(!dialog.busy && dialog.confirmation.password.is_empty() && dialog.confirmation.code.is_empty()); + let locked = + Error::RefusedWith { code: codes::TWO_FACTOR_LOCKED.into(), message: "locked".into(), two_factor: None, retry_after_secs: Some(840) }; + boss.admin_fails.lock().unwrap().push(locked.clone()); + type_password(&mut model, "pw"); + admin_mut(&mut model).dialog.as_mut().unwrap().confirmation.code = "123456".into(); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let error = admin(&model).dialog.as_ref().unwrap().error.clone().unwrap(); + assert!(error.contains("Too many wrong codes") && error.contains("14 minutes"), "{error}"); + assert!(error_text(&locked).contains("14 minutes")); + } + + /// An account's two-factor sign-in is shown, and taken away with a change of its own; a reset code can turn it off when it is + /// used, which the call carries. + #[test] + fn two_factor_is_shown_reset_and_turned_off_by_a_reset_code() { + let boss = agent(true, 1); + boss.users.lock().unwrap()[1].two_factor_enabled = true; + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(ana.two_factor_enabled); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + let reset = Action::ResetTwoFactor { id: ana.id, username: "user01".into() }; + assert!(reset.title().contains("Turn two-factor sign-in off") && reset.explain().contains("signed out")); + run(&mut model, &boss, ev(AdminEvent::Ask(reset))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_reset_two_factor").count(), 1); + for flag in [false, true] { + run(&mut model, &boss, ev(AdminEvent::Ask(Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: flag }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(admin(&model).code.as_ref().map(|c| c.kind), Some(CodeKind::PasswordReset)); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + } + assert_eq!(boss.resets.lock().unwrap().as_slice(), [false, true], "the flag goes with the reset code"); + assert!(Action::IssueReset { id: ana.id, username: "x".into(), reset_two_factor: true }.explain().contains("turns the account's two-factor")); + } +} diff --git a/apps/craft-cloud-desktop/src/agent.rs b/apps/craft-cloud-desktop/src/agent.rs index 2ed7185d..a59cb4aa 100644 --- a/apps/craft-cloud-desktop/src/agent.rs +++ b/apps/craft-cloud-desktop/src/agent.rs @@ -5,14 +5,23 @@ //! link wakes the UI (`wake`), so the UI thread never waits for the agent; a slow call (a join waits for the server) //! holds up nothing else. A clock thread sends [`Event::Tick`] every 2 s, from which the model asks for the status. +use std::io::Write; use std::path::PathBuf; use std::sync::mpsc::{Receiver, Sender, channel}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use craft_cloud_app::{Client, DeviceId, DeviceInfo, Error, JoinRequest, PairingInfo, PairingStatus, Status, Uuid}; +use craft_cloud_app::{ + AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditPage, AuditQuery, Client, DeviceId, DeviceInfo, Error, ExportOutcome, + HistoryResponse, JoinRequest, NodeId, PairingInfo, PairingStatus, RecoveryCodes, RestoreResponse, RevokedOthers, Status, TrashResponse, + TrashRestoreResponse, TwoFactorProofRequest, TwoFactorSetupResponse, TwoFactorStatus, UserQuery, Uuid, VersionId, +}; +use crate::admin::{Action, AdminEvent, Outcome}; +use crate::audit::AuditMsg; use crate::model::{Effect, Event}; +use crate::security::{SecurityAnswer, SecurityCall}; /// How often the clock ticks (and the status is asked for). pub const TICK: Duration = Duration::from_secs(2); @@ -28,11 +37,46 @@ pub trait Agent: Send + Sync { fn pairing_state(&self, id: Uuid) -> Result; fn cancel_pairing(&self, id: Uuid) -> Result<(), Error>; fn devices(&self) -> Result, Error>; + fn usage(&self) -> Result; fn remove_device(&self, id: DeviceId) -> Result, Error>; fn pause(&self) -> Result; fn resume(&self) -> Result; fn quit(&self) -> Result; fn set_autostart(&self, on: bool) -> Result; + /// The Security page's calls: the account's two-factor sign-in, its password and its other computers. + fn two_factor_status(&self) -> Result; + fn two_factor_setup(&self, password: &str) -> Result; + fn two_factor_confirm(&self, totp_code: &str) -> Result; + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error>; + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result; + fn change_password(&self, current: &str, new: &str) -> Result<(), Error>; + fn revoke_other_devices(&self) -> Result; + // The admin calls: each asks the server, which decides every one. + fn account(&self) -> Result; + /// The account's own activity, and for an admin the whole audit log and its export (which is written to `out` as it comes). + fn account_activity(&self, query: &AuditQuery) -> Result; + fn admin_audit(&self, query: &AuditQuery) -> Result; + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result; + fn admin_server(&self) -> Result; + fn admin_storage(&self) -> Result; + fn admin_users(&self, query: &UserQuery) -> Result; + fn admin_devices(&self, user: Uuid) -> Result; + fn admin_user_usage(&self, user: Uuid) -> Result; + fn admin_create_user(&self, request: &AdminCreateUser) -> Result; + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result; + fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result; + fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result; + fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error>; + fn file_history(&self, path: &str, cursor: Option<&str>) -> Result; + fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result; + fn trash(&self, cursor: Option<&str>) -> Result; + fn restore_from_trash(&self, id: NodeId) -> Result; + fn remove_from_trash(&self, id: NodeId) -> Result<(), Error>; } /// The agent through `craft_cloud_app::Client`. It finds the agent again whenever a call finds it gone (the agent @@ -98,6 +142,9 @@ impl Agent for ClientAgent { fn devices(&self) -> Result, Error> { self.with(Client::devices) } + fn usage(&self) -> Result { + self.with(Client::usage) + } fn remove_device(&self, id: DeviceId) -> Result, Error> { self.with(|c| c.remove_device(id)) } @@ -113,6 +160,99 @@ impl Agent for ClientAgent { fn set_autostart(&self, on: bool) -> Result { self.with(|c| c.set_autostart(on)) } + fn two_factor_status(&self) -> Result { + self.with(Client::two_factor_status) + } + fn two_factor_setup(&self, password: &str) -> Result { + self.with(|c| c.two_factor_setup(password)) + } + fn two_factor_confirm(&self, totp_code: &str) -> Result { + self.with(|c| c.two_factor_confirm(totp_code)) + } + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + self.with(|c| c.two_factor_disable(request)) + } + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + self.with(|c| c.two_factor_recovery_codes(password, totp_code)) + } + fn change_password(&self, current: &str, new: &str) -> Result<(), Error> { + self.with(|c| c.change_password(current, new)) + } + fn revoke_other_devices(&self) -> Result { + self.with(Client::revoke_other_devices) + } + fn account(&self) -> Result { + self.with(Client::account) + } + fn account_activity(&self, query: &AuditQuery) -> Result { + self.with(|c| c.account_activity(query)) + } + fn admin_audit(&self, query: &AuditQuery) -> Result { + self.with(|c| c.admin_audit(query)) + } + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + self.with(|c| c.admin_audit_export(query, out)) + } + fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_reset_two_factor(id, confirm)) + } + fn admin_server(&self) -> Result { + self.with(Client::admin_server) + } + fn admin_storage(&self) -> Result { + self.with(Client::admin_storage) + } + fn admin_users(&self, query: &UserQuery) -> Result { + self.with(|c| c.admin_users(query)) + } + fn admin_devices(&self, user: Uuid) -> Result { + self.with(|c| c.admin_devices(user)) + } + fn admin_user_usage(&self, user: Uuid) -> Result { + self.with(|c| c.admin_user_usage(user)) + } + fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.with(|c| c.admin_create_user(request)) + } + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.with(|c| c.admin_update_user(id, request)) + } + fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_disable_user(id, confirm)) + } + fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_enable_user(id, confirm)) + } + fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_remove_user(id, confirm)) + } + fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result { + self.with(|c| c.admin_issue_invite(id, confirm)) + } + fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_revoke_invite(id, confirm)) + } + fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result { + self.with(|c| c.admin_issue_reset(id, request)) + } + fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_revoke_device(id, device, confirm)) + } + fn file_history(&self, path: &str, cursor: Option<&str>) -> Result { + self.with(|c| c.file_history(path, cursor, Some(50))) + } + fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result { + self.with(|c| c.restore_file(path, version, expected_version)) + } + fn trash(&self, cursor: Option<&str>) -> Result { + self.with(|c| c.trash(cursor, Some(50))) + } + fn restore_from_trash(&self, id: NodeId) -> Result { + self.with(|c| c.restore_from_trash(id)) + } + fn remove_from_trash(&self, id: NodeId) -> Result<(), Error> { + self.with(|c| c.remove_from_trash(id)) + } } /// Starts the agent and waits for it ([`crate::start_agent::start_and_wait`]), for [`Effect::StartAgent`]. @@ -162,6 +302,7 @@ impl Link { if !is_call(&effect) { return Some(effect); } + let failed_effect = effect.clone(); let (agent, start, wake, tx) = (self.agent.clone(), self.start.clone(), self.wake.clone(), self.tx.clone()); let spawned = std::thread::Builder::new().name("craft-cloud-call".into()).spawn(move || { if let Some(event) = call(agent.as_ref(), &start, effect) { @@ -170,8 +311,15 @@ impl Link { } }); if let Err(e) = spawned { - // No thread to run it on: said as the call's failure. - let _ = self.tx.send(Event::Polled(Err(Error::Io(format!("can't start a thread: {e}"))))); + let error = Error::Io(format!("can't start a thread: {e}")); + let event = match failed_effect { + Effect::Trash { ticket, .. } => Event::TrashPage { ticket, result: Err(error) }, + Effect::RestoreTrash { ticket, .. } => Event::TrashRestored { ticket, result: Err(error) }, + Effect::RemoveTrash { ticket, .. } => Event::TrashRemoved { ticket, result: Err(error) }, + _ => Event::Polled(Err(error)), + }; + let _ = self.tx.send(event); + (self.wake)(); } None } @@ -179,7 +327,10 @@ impl Link { /// Whether `effect` is a call of the agent. fn is_call(effect: &Effect) -> bool { - !matches!(effect, Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_)) + !matches!( + effect, + Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_) | Effect::PickExportFile { .. } + ) } /// Makes the call `effect` names, blocking, and gives its answer as the event the model expects. @@ -198,19 +349,127 @@ pub fn call(agent: &dyn Agent, start: &Starter, effect: Effect) -> Option return None; } Effect::Devices => Event::Devices(agent.devices()), + Effect::Usage(request_id) => Event::Usage { request_id, result: agent.usage() }, Effect::RemoveDevice(id) => Event::Devices(agent.remove_device(id)), + Effect::History { path, cursor, ticket } => Event::History { ticket, result: agent.file_history(&path, cursor.as_deref()) }, + Effect::Restore { path, version, expected_version, ticket } => { + Event::Restored { ticket, result: agent.restore_file(&path, version, expected_version) } + } + Effect::Trash { cursor, ticket } => Event::TrashPage { ticket, result: agent.trash(cursor.as_deref()) }, + Effect::RestoreTrash { id, ticket } => Event::TrashRestored { ticket, result: agent.restore_from_trash(id) }, + Effect::RemoveTrash { id, ticket } => Event::TrashRemoved { ticket, result: agent.remove_from_trash(id) }, Effect::Logout { local } => Event::SignedOut(agent.logout(local)), Effect::Pause => Event::Answered(agent.pause()), Effect::Resume => Event::Answered(agent.resume()), + Effect::Account { session } => Event::Admin(AdminEvent::Account { session, result: agent.account() }), + Effect::AdminServer { session } => Event::Admin(AdminEvent::Server { session, result: agent.admin_server() }), + Effect::AdminStorage { session, ticket } => Event::Admin(AdminEvent::Storage { session, ticket, result: agent.admin_storage() }), + Effect::AdminUsers { query, ticket } => Event::Admin(AdminEvent::Users { ticket, result: agent.admin_users(&query) }), + Effect::AdminDevices { user, ticket } => { + Event::Admin(AdminEvent::Devices { user, ticket, result: agent.admin_devices(user).map(|d| d.devices) }) + } + Effect::AdminUserUsage { user, ticket } => Event::Admin(AdminEvent::Usage { user, ticket, result: agent.admin_user_usage(user) }), + Effect::AuditPage { own, query, ticket } => { + let result = if own { agent.account_activity(&query) } else { agent.admin_audit(&query) }; + Event::Audit(AuditMsg::Page { own, ticket, result }) + } + Effect::AuditExport { query, path, ticket } => Event::Audit(AuditMsg::Exported { ticket, result: export_to_file(agent, &query, &path) }), + Effect::AdminDo { action, confirm } => Event::Admin(AdminEvent::Done(change(agent, &action, confirm))), // Not running is as good as stopped. Effect::QuitAgent => { let _ = agent.quit(); Event::AgentQuit } - Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_) => return None, + Effect::Security(call) => Event::SecurityAnswer(match call { + SecurityCall::Status => SecurityAnswer::Status(agent.two_factor_status()), + SecurityCall::Setup { password } => SecurityAnswer::Setup(agent.two_factor_setup(&password)), + SecurityCall::Confirm { code } => SecurityAnswer::Confirm(agent.two_factor_confirm(&code)), + SecurityCall::Disable(request) => SecurityAnswer::Disable(agent.two_factor_disable(&request)), + SecurityCall::RecoveryCodes { password, code } => SecurityAnswer::RecoveryCodes(agent.two_factor_recovery_codes(&password, &code)), + SecurityCall::ChangePassword { current, new } => SecurityAnswer::ChangePassword(agent.change_password(¤t, &new)), + SecurityCall::SignOutOthers => SecurityAnswer::SignOutOthers(agent.revoke_other_devices()), + }), + Effect::OpenFolder(_) + | Effect::QuitApp + | Effect::QuitTray + | Effect::AppAutostart(_) + | Effect::PickFolder(_) + | Effect::PickExportFile { .. } => return None, }) } +/// Writes the audit export to `path`: first to a file of its own in the same folder, created for this export alone (it must not +/// exist yet, and no link in its place is followed), which is moved over `path` only when the stream ended properly (the server's +/// last line, the right count) and the file was written to the end. Anything else removes that file, and only that file, and +/// leaves what is at `path` as it was (a file called `.part` is somebody else's), so a broken export is never mistaken for a +/// whole one. A stream the server cut at its limit is a good file, and says so in the outcome. +fn export_to_file(agent: &dyn Agent, query: &AuditQuery, path: &str) -> Result { + let io = |what: &str, e: std::io::Error| Error::Io(format!("{what}: {e}")); + let target = std::path::Path::new(path); + let name = target.file_name().ok_or_else(|| Error::Io(format!("{path} is not a file name")))?.to_string_lossy().into_owned(); + let dir = target.parent().filter(|d| !d.as_os_str().is_empty()).unwrap_or(std::path::Path::new(".")); + let mut made = None; + for _ in 0..16 { + let part = dir.join(format!(".{name}.{}.{}.part", std::process::id(), crate::admin::fresh())); + match std::fs::OpenOptions::new().write(true).create_new(true).open(&part) { + Ok(file) => { + made = Some((part, file)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(io("can't write the export file", e)), + } + } + let Some((part, file)) = made else { return Err(Error::Io("can't make a file for the export".into())) }; + let mut writer = std::io::BufWriter::with_capacity(64 * 1024, file); + let result = agent.admin_audit_export(query, &mut writer); + let finished = writer.flush().and_then(|()| writer.get_ref().sync_all()); + drop(writer); + let outcome = match (result, finished) { + (Ok(outcome), Ok(())) => outcome, + (Ok(_), Err(e)) => { + let _ = std::fs::remove_file(&part); + return Err(io("can't write the export file", e)); + } + (Err(e), _) => { + let _ = std::fs::remove_file(&part); + return Err(e); + } + }; + if let Err(e) = std::fs::rename(&part, target) { + let _ = std::fs::remove_file(&part); + return Err(io("can't save the export file", e)); + } + Ok(outcome) +} + +/// Makes the change `action` names with the admin's `confirm`ation, blocking. This is the one place where the +/// confirmation goes into a call's body (the create and change calls carry it in their own fields). +fn change(agent: &dyn Agent, action: &Action, confirm: AdminConfirm) -> Result { + let AdminConfirm { password, totp_code, recovery_code } = confirm.clone(); + match action { + Action::CreateUser { username, role } => { + let request = AdminCreateUser { username: username.clone(), role: role.as_str().to_owned(), password, totp_code, recovery_code }; + agent.admin_create_user(&request).map(Outcome::Created) + } + Action::UpdateUser { id, role, quota, .. } => { + let request = AdminUpdateUser { role: role.map(|r| r.as_str().to_owned()), quota_bytes: *quota, password, totp_code, recovery_code }; + agent.admin_update_user(*id, &request).map(Outcome::Updated) + } + Action::Disable { id, .. } => agent.admin_disable_user(*id, &confirm).map(|()| Outcome::Done), + Action::Enable { id, .. } => agent.admin_enable_user(*id, &confirm).map(|()| Outcome::Done), + Action::Remove { id, .. } => agent.admin_remove_user(*id, &confirm).map(|()| Outcome::Done), + Action::IssueInvite { id, .. } => agent.admin_issue_invite(*id, &confirm).map(Outcome::Code), + Action::RevokeInvite { id, .. } => agent.admin_revoke_invite(*id, &confirm).map(|()| Outcome::Done), + Action::IssueReset { id, reset_two_factor, .. } => { + let request = AdminPasswordReset { password, totp_code, recovery_code, reset_two_factor: *reset_two_factor }; + agent.admin_issue_reset(*id, &request).map(Outcome::Code) + } + Action::ResetTwoFactor { id, .. } => agent.admin_reset_two_factor(*id, &confirm).map(|()| Outcome::Done), + Action::RevokeDevice { id, device, .. } => agent.admin_revoke_device(*id, *device, &confirm).map(|()| Outcome::Done), + } +} + #[cfg(test)] pub(crate) mod fake { //! A fake agent: a signed-in or signed-out status, answers set per call, and a switch that makes it go away. @@ -218,17 +477,176 @@ pub(crate) mod fake { use super::*; use std::sync::atomic::{AtomicBool, Ordering}; + /// The password, code and recovery code the fake's account takes. + pub const PASSWORD: &str = "correct horse battery"; + pub const CODE: &str = "123456"; + pub const RECOVERY: &str = "ABCD-EFGH-JKMN-PQRS"; + + /// What the fake agent's server has of the account's two-factor sign-in. + #[derive(Debug, Clone, Default)] + pub struct FakeFactor { + pub enabled: bool, + pub pending: bool, + /// A server from before two-factor sign-in: its calls are `not_found`. + pub unsupported: bool, + /// Every check of a code is refused as locked (840 s to wait). + pub locked: bool, + pub other_computers: u32, + } + pub struct FakeAgent { pub status: Mutex, pub gone: AtomicBool, /// What `join` answers, in turn (then the status). pub joins: Mutex>>, + pub usage_result: Mutex>>, pub asked: Mutex>, + pub factor: Mutex, + /// Whether the account is an admin (else every admin call is refused as `forbidden`). + pub admin: AtomicBool, + /// The accounts the admin calls list. + pub users: Mutex>, + /// What the admin changes answer instead of succeeding, taken from the end. + pub admin_fails: Mutex>, + /// The password each admin change was sent with. + pub passwords: Mutex>>, + /// The code (`totp_code`, `recovery_code`) each admin change was sent with. + pub factors: Mutex, Option)>>, + /// Whether the admin account has two-factor sign-in (what `account` says). + pub two_factor: AtomicBool, + /// The `reset_two_factor` flag of each password reset code asked for. + pub resets: Mutex>, + /// The audit events the audit calls and the activity list (newest first, as given), and what was asked. + pub audit_events: Mutex>, + pub audit_queries: Mutex>, + /// An export that the server cuts at its limit; and failures an export answers instead of succeeding. + pub export_truncated: Mutex, + pub export_fails: Mutex>, + } + + fn refused(code: &str, message: &str) -> Error { + Error::Refused { code: code.into(), message: message.into(), name_taken: None } } impl FakeAgent { pub fn new(status: Status) -> Self { - Self { status: Mutex::new(status), gone: AtomicBool::new(false), joins: Mutex::new(Vec::new()), asked: Mutex::new(Vec::new()) } + Self { + status: Mutex::new(status), + gone: AtomicBool::new(false), + joins: Mutex::new(Vec::new()), + usage_result: Mutex::new(None), + asked: Mutex::new(Vec::new()), + factor: Mutex::new(FakeFactor::default()), + admin: AtomicBool::new(false), + users: Mutex::new(Vec::new()), + admin_fails: Mutex::new(Vec::new()), + passwords: Mutex::new(Vec::new()), + factors: Mutex::new(Vec::new()), + two_factor: AtomicBool::new(false), + resets: Mutex::new(Vec::new()), + audit_events: Mutex::new(Vec::new()), + audit_queries: Mutex::new(Vec::new()), + export_truncated: Mutex::new(false), + export_fails: Mutex::new(Vec::new()), + } + } + + /// A refusal or nothing: whether a call of the account's two-factor sign-in gets through at all. + fn factor_gate(&self, what: &str) -> Result<(), Error> { + self.answer(what, |_| ())?; + if self.factor.lock().unwrap().unsupported { + return Err(refused("not_found", "no such endpoint")); + } + Ok(()) + } + + /// A code is checked: locked, wrong or right. + fn check_code(&self, code: Option<&str>) -> Result<(), Error> { + if self.factor.lock().unwrap().locked { + return Err(Error::RefusedWith { + code: "two_factor_locked".into(), + message: "locked".into(), + two_factor: None, + retry_after_secs: Some(840), + }); + } + if code == Some(CODE) || code == Some(RECOVERY) { Ok(()) } else { Err(refused("two_factor_invalid", "the code is wrong")) } + } + + fn codes() -> RecoveryCodes { + RecoveryCodes { recovery_codes: (0..10).map(|n| format!("AAAA-BBBB-CCCC-{n:04}")).collect() } + } + + /// An admin call: refused as a member's is unless the account is an admin. + fn admin_call(&self, what: &str, ok: impl FnOnce() -> T) -> Result { + self.asked.lock().unwrap().push(what.to_owned()); + if self.gone.load(Ordering::SeqCst) { + return Err(Error::NotRunning("can't reach it: connection refused".into())); + } + if !self.admin.load(Ordering::SeqCst) { + return Err(Error::Refused { code: "forbidden".into(), message: "this needs an administrator".into(), name_taken: None }); + } + Ok(ok()) + } + + /// An admin change: the password it was sent with is kept, and a failure set up is answered first. + fn admin_change( + &self, + what: &str, + password: Option, + factor: (Option, Option), + ok: impl FnOnce() -> T, + ) -> Result { + self.passwords.lock().unwrap().push(password); + self.factors.lock().unwrap().push(factor); + self.admin_call(what, || ())?; + if let Some(e) = self.admin_fails.lock().unwrap().pop() { + return Err(e); + } + Ok(ok()) + } + + /// An event of the audit log, `n` seconds after the epoch. + pub fn sample_event(n: u64, action: &str) -> craft_cloud_proto::api::AuditEvent { + craft_cloud_proto::api::AuditEvent { + id: Uuid::from_u128(u128::from(n) + 1), + at_ms: 1_790_000_000_000 - i64::try_from(n).unwrap_or(0) * 1_000, + actor_user: None, + actor_device: None, + action: action.to_owned(), + target_kind: String::new(), + target_id: None, + outcome: "success".into(), + details: Default::default(), + count: 1, + } + } + + /// The events a query lets through, from the cursor (an index) on: one page, and the cursor of the next. + fn audit_page(&self, query: &AuditQuery) -> AuditPage { + self.audit_queries.lock().unwrap().push(query.clone()); + let all: Vec<_> = self + .audit_events + .lock() + .unwrap() + .iter() + .filter(|e| match query.action.as_deref() { + Some(a) if a.ends_with('.') => e.action.starts_with(a), + Some(a) => e.action == a, + None => true, + }) + .filter(|e| query.user.is_none_or(|u| e.actor_user == Some(u) || e.target_id == Some(u))) + .cloned() + .collect(); + let from = query.cursor.as_deref().and_then(|c| c.parse::().ok()).unwrap_or(0); + let limit = query.limit.unwrap_or(50) as usize; + let events: Vec<_> = all.iter().skip(from).take(limit).cloned().collect(); + let next = (from + limit < all.len()).then(|| (from + limit).to_string()); + AuditPage { events, next } + } + + fn user(&self, id: Uuid) -> AdminUserInfo { + self.users.lock().unwrap().iter().find(|u| u.id == id).cloned().unwrap_or_else(|| sample_user("someone", id)) } fn answer(&self, what: &str, ok: impl FnOnce(&Status) -> T) -> Result { @@ -272,6 +690,19 @@ pub(crate) mod fake { fn devices(&self) -> Result, Error> { self.answer("devices", |_| Vec::new()) } + fn usage(&self) -> Result { + self.answer("usage", |_| ())?; + if let Some(result) = self.usage_result.lock().unwrap().clone() { + return result; + } + Ok(AccountUsage { + stored_bytes: 0, + quota_bytes: None, + trash_bytes: Some(0), + history_bytes: Some(0), + estimates: craft_cloud_proto::api::UsageEstimates { index_complete: true, reason: None }, + }) + } fn remove_device(&self, _id: DeviceId) -> Result, Error> { self.answer("remove_device", |_| Vec::new()) } @@ -287,6 +718,280 @@ pub(crate) mod fake { fn set_autostart(&self, on: bool) -> Result { self.answer("set_autostart", |s| Status { autostart: Some(on), ..s.clone() }) } + fn two_factor_status(&self) -> Result { + self.factor_gate("two_factor_status")?; + let f = self.factor.lock().unwrap(); + Ok(TwoFactorStatus { + enabled: f.enabled, + enabled_ms: f.enabled.then_some(1_000), + recovery_codes_left: if f.enabled { 10 } else { 0 }, + pending: f.pending, + }) + } + fn two_factor_setup(&self, password: &str) -> Result { + self.factor_gate("two_factor_setup")?; + if password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + let mut f = self.factor.lock().unwrap(); + if f.enabled { + return Err(refused("conflict", "on already")); + } + f.pending = true; + Ok(TwoFactorSetupResponse { + secret: "JBSWY3DPEHPK3PXP".into(), + otpauth_uri: "otpauth://totp/Craft%20Cloud:fabio?secret=JBSWY3DPEHPK3PXP&issuer=Craft%20Cloud".into(), + expires_ms: 1_790_000_600_000, + }) + } + fn two_factor_confirm(&self, totp_code: &str) -> Result { + self.factor_gate("two_factor_confirm")?; + if !self.factor.lock().unwrap().pending { + return Err(refused("conflict", "no set-up to confirm")); + } + if totp_code != CODE { + return Err(refused("two_factor_invalid", "the code is wrong")); + } + let mut f = self.factor.lock().unwrap(); + f.enabled = true; + f.pending = false; + Ok(Self::codes()) + } + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + self.factor_gate("two_factor_disable")?; + if request.password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + self.check_code(request.totp_code.as_deref().or(request.recovery_code.as_deref()))?; + self.factor.lock().unwrap().enabled = false; + Ok(()) + } + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + self.factor_gate("two_factor_recovery_codes")?; + if password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + self.check_code(Some(totp_code))?; + Ok(Self::codes()) + } + fn change_password(&self, current: &str, new: &str) -> Result<(), Error> { + self.answer("change_password", |_| ())?; + if current != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + if new.len() < 8 { + return Err(refused("weak_password", "passwords are 8 to 1024 bytes")); + } + Ok(()) + } + fn revoke_other_devices(&self) -> Result { + self.answer("revoke_other_devices", |_| ())?; + let mut f = self.factor.lock().unwrap(); + let revoked = std::mem::take(&mut f.other_computers); + Ok(RevokedOthers { revoked }) + } + fn account(&self) -> Result { + self.answer("account", |s| AccountInfo { + user_id: Uuid::nil(), + username: s.account.clone().unwrap_or_default(), + role: if self.admin.load(Ordering::SeqCst) { "admin" } else { "member" }.into(), + created_ms: 1_780_000_000_000, + password_changed_ms: None, + two_factor_enabled: self.two_factor.load(Ordering::SeqCst), + }) + } + fn account_activity(&self, query: &AuditQuery) -> Result { + self.answer("account_activity", |_| ())?; + Ok(self.audit_page(&AuditQuery { user: None, action: None, ..query.clone() })) + } + fn admin_audit(&self, query: &AuditQuery) -> Result { + self.admin_call("admin_audit", || self.audit_page(query)) + } + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + self.admin_call("admin_audit_export", || ())?; + if let Some(e) = self.export_fails.lock().unwrap().pop() { + // Some lines came before the stream broke. + let _ = out.write_all(b"{\"partial\":true}\n"); + return Err(e); + } + let page = self.audit_page(&AuditQuery { cursor: None, limit: Some(u32::MAX), ..query.clone() }); + for event in &page.events { + let _ = out.write_all(format!("{{\"id\":\"{}\",\"action\":\"{}\"}}", event.id, event.action).as_bytes()); + let _ = out.write_all(b"\n"); + } + Ok(ExportOutcome { events: page.events.len() as u64, truncated: *self.export_truncated.lock().unwrap() }) + } + fn admin_reset_two_factor(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_reset_two_factor", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_server(&self) -> Result { + self.admin_call("admin_server", || AdminServerInfo { + version: "0.1.0".into(), + users: self.users.lock().unwrap().len() as u64, + devices_active: 3, + stored_bytes: 5_000_000_000, + live_bytes: 120_000, + storage_kind: "disk".into(), + database_kind: "sqlite".into(), + }) + } + fn admin_storage(&self) -> Result { + self.admin_call("admin_storage", || { + use craft_cloud_app::{AdminManifestIndex, AdminStorageAccount}; + let users = self.users.lock().unwrap(); + let top_accounts = users + .iter() + .take(3) + .map(|user| AdminStorageAccount { + id: user.id, + username: user.username.clone(), + stored_bytes: user.stored_bytes, + quota_bytes: user.quota_bytes.or(Some(10 * 1_073_741_824)), + }) + .collect(); + // The counts follow the fake accounts; the default quota applies where an account has no + // override of its own, and "near" includes accounts at or over their quota, as on the server. + let effective = |user: &AdminUserInfo| user.quota_bytes.unwrap_or(10 * 1_073_741_824); + AdminStorageOverview { + accounts: users.len() as u64, + stored_bytes: 5_000_000_000, + default_quota_bytes: Some(10 * 1_073_741_824), + accounts_with_override: users.iter().filter(|user| user.quota_bytes.is_some()).count() as u64, + accounts_near_quota: users.iter().filter(|user| user.stored_bytes >= effective(user) - effective(user) / 10).count() as u64, + accounts_over_quota: users.iter().filter(|user| user.stored_bytes >= effective(user)).count() as u64, + top_accounts, + manifest_index: AdminManifestIndex { complete: 7, building: 1, not_indexed: 2 }, + } + }) + } + fn admin_users(&self, query: &UserQuery) -> Result { + self.admin_call("admin_users", || { + let needle = query.query.as_deref().unwrap_or_default().to_lowercase(); + let all: Vec = + self.users.lock().unwrap().iter().filter(|u| u.username.to_lowercase().contains(&needle)).cloned().collect(); + // The cursor is the index of the page's first account. + let from = query.cursor.as_deref().and_then(|c| c.parse::().ok()).unwrap_or(0); + let limit = query.limit.unwrap_or(50) as usize; + let users: Vec = all.iter().skip(from).take(limit).cloned().collect(); + let next = (from + limit < all.len()).then(|| (from + limit).to_string()); + AdminUserList { users, next } + }) + } + fn admin_devices(&self, user: Uuid) -> Result { + self.admin_call("admin_devices", || AdminDevices { + devices: vec![DeviceInfo { + id: DeviceId::generate(), + name: format!("{} laptop", self.user(user).username), + created_ms: 0, + last_used_ms: 1_789_990_000_000, + revoked: false, + current: false, + client_kind: "agent".into(), + platform: "linux".into(), + app_version: "0.1.0".into(), + revoked_ms: None, + revoke_reason: None, + }], + }) + } + fn admin_user_usage(&self, user: Uuid) -> Result { + self.admin_call("admin_user_usage", || { + let account = self.user(user); + AccountUsage { + stored_bytes: account.stored_bytes, + quota_bytes: account.quota_bytes.or(Some(10 * 1_073_741_824)), + trash_bytes: Some(80_000_000), + history_bytes: Some(240_000_000), + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + } + }) + } + fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.admin_change("admin_create_user", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + AdminCreatedUser { + user: AdminUserInfo { role: request.role.clone(), invite_pending: true, ..sample_user(&request.username, Uuid::now_v7()) }, + invite: AdminCode { + code: "K7M4Q9TXK7M4Q9TX".into(), + join_url: Some("cloud.shadows.ch/K7M4-Q9TX-K7M4-Q9TX".into()), + expires_ms: 1_790_600_000_000, + }, + } + }) + } + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.admin_change("admin_update_user", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + let mut user = self.user(id); + if let Some(role) = &request.role { + user.role = role.clone(); + } + if let Some(quota) = request.quota_bytes { + user.quota_bytes = quota; + } + user + }) + } + fn admin_disable_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_disable_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_enable_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_enable_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_remove_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_remove_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_issue_invite(&self, _id: Uuid, confirm: &AdminConfirm) -> Result { + self.admin_change("admin_issue_invite", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || { + AdminCode { + code: "ABCDEFGHJKMNPQRS".into(), + join_url: Some("cloud.shadows.ch/ABCD-EFGH-JKMN-PQRS".into()), + expires_ms: 1_790_600_000_000, + } + }) + } + fn admin_revoke_invite(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_revoke_invite", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_issue_reset(&self, _id: Uuid, request: &AdminPasswordReset) -> Result { + self.resets.lock().unwrap().push(request.reset_two_factor); + self.admin_change("admin_issue_reset", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + AdminCode { code: "TUVWXYZ234567ABC".into(), join_url: None, expires_ms: 1_790_003_600_000 } + }) + } + fn admin_revoke_device(&self, _id: Uuid, _device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_revoke_device", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + + fn file_history(&self, _path: &str, _cursor: Option<&str>) -> Result { + self.answer("file_history", |_| HistoryResponse { versions: Vec::new(), next: None }) + } + fn restore_file(&self, _path: &str, _version: VersionId, _expected_version: VersionId) -> Result { + Err(Error::Refused { code: "invalid".into(), message: "not configured".into(), name_taken: None }) + } + fn trash(&self, _cursor: Option<&str>) -> Result { + self.answer("trash", |_| TrashResponse { items: Vec::new(), next: None }) + } + fn restore_from_trash(&self, _id: NodeId) -> Result { + Err(Error::Refused { code: "invalid".into(), message: "not configured".into(), name_taken: None }) + } + fn remove_from_trash(&self, _id: NodeId) -> Result<(), Error> { + self.answer("remove_from_trash", |_| ()) + } + } + + /// An enabled member account that stores nothing. + pub fn sample_user(username: &str, id: Uuid) -> AdminUserInfo { + AdminUserInfo { + id, + username: username.to_owned(), + role: "member".into(), + created_ms: 1_780_000_000_000, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 1, + stored_bytes: 0, + quota_bytes: None, + invite_pending: false, + } } } diff --git a/apps/craft-cloud-desktop/src/app.rs b/apps/craft-cloud-desktop/src/app.rs index 391c8d24..b200f7ce 100644 --- a/apps/craft-cloud-desktop/src/app.rs +++ b/apps/craft-cloud-desktop/src/app.rs @@ -36,6 +36,9 @@ pub trait Host { fn set_autostart(&self, on: bool) -> Result<(), String>; /// Shows the OS's folder picker at `start`, off the UI thread; the answer is sent to `answer` (then `wake`). fn pick_folder(&self, start: String, answer: Sender, wake: Arc); + /// Shows the OS's save dialog with `suggested` as the file name, off the UI thread; the answer (the path, or `None` when + /// cancelled) is sent to `answer` as [`crate::audit::AuditMsg::ExportPicked`] (then `wake`). + fn save_file(&self, suggested: String, ticket: u64, answer: Sender, wake: Arc); /// Ends the tray process (Linux, where the tray is a process of its own). fn quit_tray(&self) {} /// Called every frame with the model, to show its state (the tray icon); events it has go to the model. @@ -84,6 +87,10 @@ impl App { let done = self.host.set_autostart(on); self.handle(Event::AppAutostartSet(done), ctx); } + Some(Effect::PickExportFile { suggested, ticket }) => { + let wake_ctx = ctx.clone(); + self.host.save_file(suggested, ticket, self.to_self.clone(), Arc::new(move || wake_ctx.request_repaint())); + } Some(Effect::PickFolder(start)) => { let wake_ctx = ctx.clone(); self.host.pick_folder(start, self.to_self.clone(), Arc::new(move || wake_ctx.request_repaint())); @@ -167,6 +174,15 @@ impl Host for System { crate::autostart::set_for_this_app(on) } + fn save_file(&self, suggested: String, ticket: u64, answer: Sender, wake: Arc) { + let _ = std::thread::Builder::new().name("craft-cloud-saver".into()).spawn(move || { + let dialog = rfd::AsyncFileDialog::new().set_title("Save the audit log").set_file_name(&suggested); + let picked = block_on(dialog.save_file()).map(|handle| handle.path().display().to_string()); + let _ = answer.send(Event::Audit(crate::audit::AuditMsg::ExportPicked { ticket, path: picked })); + wake(); + }); + } + fn pick_folder(&self, start: String, answer: Sender, wake: Arc) { let _ = std::thread::Builder::new().name("craft-cloud-picker".into()).spawn(move || { let mut dialog = rfd::AsyncFileDialog::new().set_title("The folder to sync"); diff --git a/apps/craft-cloud-desktop/src/audit.rs b/apps/craft-cloud-desktop/src/audit.rs new file mode 100644 index 00000000..aeeb06ea --- /dev/null +++ b/apps/craft-cloud-desktop/src/audit.rs @@ -0,0 +1,824 @@ +//! The security audit log's pages, as a pure state like [`crate::admin`] (which hosts the admin's view of it as +//! [`View::Audit`]; the account's own activity is [`Screen::Activity`], open to every signed-in account). +//! +//! - **What is shown** is the server's: events newest first, a cursor given back exactly as it came, filters that are only +//! what the server's contract allows (an action from the closed list [`AUDIT_ACTIONS`] or a prefix of it, an account the +//! admin picked on its page). The server decides who reads what; a `forbidden` makes the app ask the account's role again. +//! - **The export** is the agent's stream of the server's NDJSON, written by the app straight to a file the person picked +//! ([`Effect::AuditExport`], which writes to `.part` and moves it into place only when the stream ended properly). It +//! is never held in memory. Whole, or cut by the server at its limit (said so, not as a success), or failed (nothing kept). +//! - Answers that come late (a page asked for before the filter changed) match nothing: the numbers are [`crate::admin::fresh`]. + +use craft_cloud_app::{AuditPage, AuditQuery, Error, ExportOutcome, Uuid}; +use craft_cloud_proto::api::AuditEvent; + +use crate::admin::{AdminScreen, View, error_text, fresh, is_forbidden}; +use crate::model::{Effect, Model, Overview, Screen}; + +/// Events on one page. +pub const PAGE_SIZE: u32 = 25; +/// The file name an export is offered under. +pub const EXPORT_NAME: &str = "craft-cloud-audit.ndjson"; + +/// A page of events with the cursors back to the first, and the call whose answer is wanted. +#[derive(Debug, Clone, PartialEq)] +pub struct AuditList { + pub events: Vec, + /// How each page up to this one was asked for: `cursors[0]` is `None`, the last is this page's. The cursors are the + /// server's, kept as they came. + pub cursors: Vec>, + /// The cursor of the (older) page after this one, if there is one. + pub next: Option, + pub loading: bool, + pub loaded: bool, + pub error: Option, + ticket: u64, +} + +impl AuditList { + pub fn new() -> Self { + Self { events: Vec::new(), cursors: vec![None], next: None, loading: false, loaded: false, error: None, ticket: 0 } + } + + /// The page number shown (from 1). + pub fn page(&self) -> usize { + self.cursors.len() + } + + pub fn has_previous(&self) -> bool { + self.cursors.len() > 1 + } + + /// Asks for the current page with the filters of `filter` (none for the account's own activity). + fn load(&mut self, own: bool, filter: &AuditFilter) -> Effect { + self.loading = true; + self.error = None; + self.ticket = fresh(); + let query = AuditQuery { + cursor: self.cursors.last().cloned().flatten(), + before_ms: None, + limit: Some(PAGE_SIZE), + user: filter.user.as_ref().map(|(id, _)| *id), + action: filter.action.clone(), + }; + Effect::AuditPage { own, query, ticket: self.ticket } + } + + fn came(&mut self, ticket: u64, result: Result) -> Option { + if ticket != self.ticket { + return None; + } + self.loading = false; + match result { + Ok(page) => { + self.loaded = true; + self.error = None; + self.events = page.events; + self.next = page.next; + None + } + Err(e) => { + self.error = Some(error_text(&e)); + Some(e) + } + } + } + + fn start_over(&mut self) { + self.cursors = vec![None]; + } +} + +impl Default for AuditList { + fn default() -> Self { + Self::new() + } +} + +/// What the admin's audit view is narrowed to. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditFilter { + /// An event name from [`AUDIT_ACTIONS`], or a prefix of them ending in a dot (`auth.`); none: every event. + pub action: Option, + /// The account the events are about or by (its ID and name, for the label). + pub user: Option<(Uuid, String)>, +} + +/// Where an export is. +#[derive(Debug, Clone, PartialEq)] +pub enum ExportState { + Idle, + /// The file picker is open (the call `ticket` is the one whose answer is wanted). + Picking { + ticket: u64, + }, + /// The stream is being written to this file (the call `ticket` is the one whose answer is wanted). + Running { + path: String, + ticket: u64, + }, + /// Written: how many events, and whether the server cut it at its limit (then it is not the whole log). + Done { + path: String, + outcome: ExportOutcome, + }, + /// Nothing was kept. + Failed(String), +} + +/// The admin's audit view. +#[derive(Debug, Clone, PartialEq)] +pub struct AuditView { + pub list: AuditList, + pub filter: AuditFilter, + pub export: ExportState, +} + +/// The account's own activity. +#[derive(Debug, Clone, PartialEq)] +pub struct ActivityScreen { + /// The signed-in view the page came from and goes back to. + pub overview: Overview, + pub list: AuditList, +} + +/// What happened on the audit pages: clicks and answers. +#[derive(Debug, Clone, PartialEq)] +pub enum AuditMsg { + /// [`Effect::AuditPage`]'s answer (`own`: the account's own activity). + Page { + own: bool, + ticket: u64, + result: Result, + }, + /// [`Effect::AuditExport`]'s answer. + Exported { + ticket: u64, + result: Result, + }, + + /// The admin's audit view, for one account or all of them. + OpenAudit(Option<(Uuid, String)>), + /// Narrow to an event name or prefix of the closed list; `None`: every event. + SetAction(Option), + /// Every account again. + ClearUser, + Next, + Previous, + Reload, + /// Export what the filters say to a file (the person picks it). + Export, + /// The file picker's answer (`None`: cancelled). + ExportPicked { + ticket: u64, + path: Option, + }, + /// The note about the last export was read. + DismissExport, + + /// The signed-in account's own activity. + OpenActivity, + CloseActivity, + ActivityNext, + ActivityPrevious, + ActivityReload, +} + +impl Model { + /// The account's own activity page, if shown. + pub fn activity_screen(&self) -> Option<&ActivityScreen> { + match &self.screen { + Screen::Activity(a) => Some(a), + _ => None, + } + } + + pub(crate) fn audit(&mut self, msg: AuditMsg) -> Vec { + match msg { + AuditMsg::OpenActivity => { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + let mut screen = ActivityScreen { overview: overview.clone(), list: AuditList::new() }; + let effect = screen.list.load(true, &AuditFilter::default()); + self.screen = Screen::Activity(Box::new(screen)); + vec![effect] + } + AuditMsg::CloseActivity => { + if let Screen::Activity(a) = &self.screen { + self.screen = Screen::SignedIn(a.overview.clone()); + } + Vec::new() + } + AuditMsg::Page { own: true, ticket, result } => { + let Screen::Activity(a) = &mut self.screen else { return Vec::new() }; + a.list.came(ticket, result); + Vec::new() + } + AuditMsg::ActivityNext | AuditMsg::ActivityPrevious | AuditMsg::ActivityReload => { + let Screen::Activity(a) = &mut self.screen else { return Vec::new() }; + let list = &mut a.list; + let own = AuditFilter::default(); + match msg { + AuditMsg::ActivityNext => match list.next.clone() { + Some(next) if !list.loading => { + list.cursors.push(Some(next)); + vec![list.load(true, &own)] + } + _ => Vec::new(), + }, + AuditMsg::ActivityPrevious if list.has_previous() && !list.loading => { + list.cursors.pop(); + vec![list.load(true, &own)] + } + AuditMsg::ActivityReload if !list.loading => vec![list.load(true, &own)], + _ => Vec::new(), + } + } + msg => self.admin_audit(msg), + } + } + + fn admin_audit(&mut self, msg: AuditMsg) -> Vec { + let Screen::Admin(admin) = &mut self.screen else { return Vec::new() }; + admin.audit(msg) + } +} + +impl AdminScreen { + fn audit_view(&mut self) -> Option<&mut AuditView> { + match &mut self.view { + View::Audit(view) => Some(view), + _ => None, + } + } + + pub(crate) fn audit(&mut self, msg: AuditMsg) -> Vec { + let session = self.overview.session; + match msg { + AuditMsg::OpenAudit(user) => { + if self.dialog.is_some() || self.code.is_some() { + return Vec::new(); + } + let mut view = AuditView { list: AuditList::new(), filter: AuditFilter { action: None, user }, export: ExportState::Idle }; + let effect = view.list.load(false, &view.filter); + self.view = View::Audit(Box::new(view)); + vec![effect] + } + AuditMsg::Page { own: false, ticket, result } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + match view.list.came(ticket, result) { + Some(e) if is_forbidden(&e) => vec![Effect::Account { session }], + _ => Vec::new(), + } + } + AuditMsg::SetAction(action) => { + let Some(view) = self.audit_view() else { return Vec::new() }; + // Only what the server's contract allows is ever asked for. + if action.as_deref().is_some_and(|a| !craft_cloud_app::valid_audit_action(a)) { + return Vec::new(); + } + view.filter.action = action; + view.list.start_over(); + vec![view.list.load(false, &view.filter)] + } + AuditMsg::ClearUser => { + let Some(view) = self.audit_view() else { return Vec::new() }; + view.filter.user = None; + view.list.start_over(); + vec![view.list.load(false, &view.filter)] + } + AuditMsg::Next => { + let Some(view) = self.audit_view() else { return Vec::new() }; + match view.list.next.clone() { + Some(next) if !view.list.loading => { + view.list.cursors.push(Some(next)); + vec![view.list.load(false, &view.filter)] + } + _ => Vec::new(), + } + } + AuditMsg::Previous => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.list.has_previous() && !view.list.loading { + view.list.cursors.pop(); + vec![view.list.load(false, &view.filter)] + } else { + Vec::new() + } + } + AuditMsg::Reload => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.list.loading { Vec::new() } else { vec![view.list.load(false, &view.filter)] } + } + AuditMsg::Export => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if matches!(view.export, ExportState::Picking { .. } | ExportState::Running { .. }) { + return Vec::new(); + } + // The answers of the picker and of the export carry this number: one that comes after the pages were left and + // opened again, or for an earlier export, matches nothing. + let ticket = fresh(); + view.export = ExportState::Picking { ticket }; + vec![Effect::PickExportFile { suggested: EXPORT_NAME.to_owned(), ticket }] + } + AuditMsg::ExportPicked { ticket, path: picked } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.export != (ExportState::Picking { ticket }) { + return Vec::new(); + } + match picked { + None => { + view.export = ExportState::Idle; + Vec::new() + } + Some(path) => { + view.export = ExportState::Running { path: path.clone(), ticket }; + // Every event the filters say, not one page: no cursor, no limit. + let query = AuditQuery { + cursor: None, + before_ms: None, + limit: None, + user: view.filter.user.as_ref().map(|(id, _)| *id), + action: view.filter.action.clone(), + }; + vec![Effect::AuditExport { query, path, ticket }] + } + } + } + AuditMsg::Exported { ticket, result } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + let ExportState::Running { path, ticket: running } = &view.export else { return Vec::new() }; + if *running != ticket { + return Vec::new(); + } + let path = path.clone(); + match result { + Ok(outcome) => { + view.export = ExportState::Done { path, outcome }; + Vec::new() + } + Err(e) => { + let forbidden = is_forbidden(&e); + view.export = ExportState::Failed(error_text(&e)); + if forbidden { vec![Effect::Account { session }] } else { Vec::new() } + } + } + } + AuditMsg::DismissExport => { + if let Some(view) = self.audit_view() + && !matches!(view.export, ExportState::Running { .. } | ExportState::Picking { .. }) + { + view.export = ExportState::Idle; + } + Vec::new() + } + // Handled by the model. + AuditMsg::Page { own: true, .. } + | AuditMsg::OpenActivity + | AuditMsg::CloseActivity + | AuditMsg::ActivityNext + | AuditMsg::ActivityPrevious + | AuditMsg::ActivityReload => Vec::new(), + } + } +} + +/// What an event name says, in words; a name that isn't in the list (a newer server's) is shown as it is. +pub fn action_label(action: &str) -> &str { + match action { + "auth.login" => "Signed in", + "auth.login_failed" => "Sign-in failed (wrong password)", + "auth.second_factor_failed" => "Sign-in failed (wrong code)", + "auth.recovery_code_used" => "Recovery code used", + "auth.two_factor_locked" => "Two-factor sign-in locked after wrong codes", + "device.revoked" => "Computer signed out", + "device.revoked_others" => "Other computers signed out", + "account.password_changed" => "Password changed", + "account.two_factor_enabled" => "Two-factor sign-in turned on", + "account.two_factor_disabled" => "Two-factor sign-in turned off", + "account.recovery_codes_regenerated" => "New recovery codes made", + "admin.user_created" => "Account created by an admin", + "admin.user_disabled" => "Account disabled by an admin", + "admin.user_enabled" => "Account enabled by an admin", + "admin.user_removed" => "Account removed by an admin", + "admin.role_changed" => "Role changed by an admin", + "admin.quota_changed" => "Quota changed by an admin", + "admin.invite_issued" => "Invite made by an admin", + "admin.invite_revoked" => "Invite or reset code revoked by an admin", + "admin.password_reset_issued" => "Password reset code made by an admin", + "admin.two_factor_reset" => "Two-factor sign-in turned off by an admin", + "admin.device_revoked" => "Computer signed out by an admin", + "join.invite_redeemed" => "Joined with an invite", + "join.reset_redeemed" => "Joined with a password reset code", + "join.pairing_redeemed" => "Computer added with a code", + "cli.password_reset" => "Password reset on the server", + "cli.two_factor_reset" => "Two-factor sign-in turned off on the server", + "cli.device_revoked" => "Computer signed out on the server", + "cli.devices_revoked_all" => "Every computer signed out on the server", + "cli.user_removed" => "Account removed on the server", + other => other, + } +} + +/// A filter choice's label: a prefix as a group, a name as its words. +pub fn filter_label(filter: &str) -> String { + match filter { + "auth." => "Sign-ins".to_owned(), + "device." => "Computers signed out".to_owned(), + "account." => "Account changes".to_owned(), + "admin." => "Admin changes".to_owned(), + "join." => "Joining".to_owned(), + "cli." => "Server command line".to_owned(), + name => action_label(name).to_owned(), + } +} + +/// The groups of the filter, then every name: the closed list the server filters by. +pub fn filter_choices() -> Vec<&'static str> { + let mut choices = vec!["auth.", "device.", "account.", "admin.", "join.", "cli."]; + choices.extend(craft_cloud_app::AUDIT_ACTIONS); + choices +} + +/// The first eight characters of an ID, to tell accounts and computers apart in a line (the whole ID is on hover). +pub fn short_id(id: impl std::fmt::Display) -> String { + id.to_string().chars().filter(|c| *c != '-').take(8).collect() +} + +/// Who did it and what it was about, in a line, and the same with the whole IDs for hover: the account that acted (or "command +/// line" for the server's own commands, "unknown" for a failed sign-in, where nobody proved who they were), the account or computer +/// it concerned, and the computer a sign-out or admin act names in its details. +pub fn who_line(event: &AuditEvent) -> (String, String) { + let mut line = Vec::new(); + let mut full = Vec::new(); + match (event.actor_user, event.action.starts_with("cli.")) { + (Some(user), _) => { + line.push(format!("by {}", short_id(user))); + full.push(format!("by account {user}")); + } + (None, true) => { + line.push("by the command line".to_owned()); + full.push("by the server's command line".to_owned()); + } + (None, false) => { + line.push("by unknown".to_owned()); + full.push("by unknown: nobody had proved who they were".to_owned()); + } + } + if let Some(device) = event.actor_device { + line.push(format!("on computer {}", short_id(device))); + full.push(format!("on computer {device}")); + } + if let Some(target) = event.target_id { + let kind = if event.target_kind.is_empty() || event.target_kind == "user" { "account" } else { event.target_kind.as_str() }; + line.push(format!("about {kind} {}", short_id(target))); + full.push(format!("about {kind} {target}")); + } + if let Some(device) = event.details.device { + line.push(format!("computer {}", short_id(device))); + full.push(format!("computer {device}")); + } + (line.join(" · "), full.join("\n")) +} + +/// The typed details of an event in a few words (nothing the server doesn't write from closed lists). +pub fn details_line(event: &AuditEvent) -> String { + let d = &event.details; + let mut parts: Vec = Vec::new(); + if let Some(kind) = &d.client_kind { + parts.push(format!("from {kind}")); + } + if let Some(platform) = &d.platform { + parts.push(platform.clone()); + } + if let Some(step) = &d.step { + parts.push(format!("at {step}")); + } + if let (Some(from), Some(to)) = (&d.role_from, &d.role_to) { + parts.push(format!("{from} to {to}")); + } + match &d.quota_bytes { + Some(Some(bytes)) => parts.push(format!("quota {}", crate::admin::format_bytes(*bytes))), + Some(None) => parts.push("quota removed".to_owned()), + None => {} + } + if let Some(n) = d.revoked { + parts.push(format!("{n} computer{} signed out", if n == 1 { "" } else { "s" })); + } + if d.two_factor_removed == Some(true) { + parts.push("two-factor turned off".to_owned()); + } + if let Some(reason) = &d.reason { + parts.push(format!("reason: {reason}")); + } + if event.count > 1 { + parts.push(format!("{} tries", event.count)); + } + if event.outcome == "failure" && !parts.iter().any(|p| p.starts_with("at ")) { + parts.push("failed".to_owned()); + } + parts.join(" · ") +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Arc; + + use craft_cloud_proto::api::AuditDetails; + + use super::*; + use crate::admin::AdminEvent; + use crate::admin::tests as support; + use crate::agent::fake::FakeAgent; + use crate::agent::{Starter, call}; + use crate::model::Event; + + /// The picker's answer for the export that waits for it. + pub(crate) fn picked(model: &mut Model, path: Option<&str>) -> Vec { + let View::Audit(view) = &model.admin_screen().unwrap().view else { panic!("not the audit view") }; + let ExportState::Picking { ticket } = view.export else { panic!("not picking: {:?}", view.export) }; + model.update(Event::Audit(AuditMsg::ExportPicked { ticket, path: path.map(str::to_owned) })) + } + + /// The answer of the export that runs. + pub(crate) fn finished(model: &mut Model, result: Result) { + let View::Audit(view) = &model.admin_screen().unwrap().view else { panic!("not the audit view") }; + let ExportState::Running { ticket, .. } = view.export else { panic!("not running: {:?}", view.export) }; + model.update(Event::Audit(AuditMsg::Exported { ticket, result })); + } + + fn on_audit(agent: &FakeAgent) -> Model { + let mut model = support::on_the_pages(agent); + support::run(&mut model, agent, vec![Event::Audit(AuditMsg::OpenAudit(None))]); + model + } + + fn view(model: &Model) -> &AuditView { + match &model.admin_screen().unwrap().view { + View::Audit(view) => view, + other => panic!("not the audit view: {other:?}"), + } + } + + fn ev(msg: AuditMsg) -> Vec { + vec![Event::Audit(msg)] + } + + #[test] + fn the_admins_audit_view_pages_with_the_servers_cursors() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..60).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + let v = view(&model); + assert_eq!((v.list.page(), v.list.events.len(), v.list.next.as_deref()), (1, PAGE_SIZE as usize, Some("25"))); + assert!(model.update(Event::Audit(AuditMsg::Previous)).is_empty()); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!(view(&model).list.page(), 2); + // The cursor goes back exactly as it came. + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!((asked.cursor.as_deref(), asked.limit), (Some("25"), Some(PAGE_SIZE))); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!((view(&model).list.events.len(), view(&model).list.next.clone()), (10, None)); + assert!(model.update(Event::Audit(AuditMsg::Next)).is_empty(), "no page after the last"); + support::run(&mut model, &boss, ev(AuditMsg::Previous)); + assert_eq!(view(&model).list.page(), 2); + } + + #[test] + fn filters_are_the_closed_list_and_start_the_list_over() { + let boss = support::agent(true, 1); + boss.audit_events + .lock() + .unwrap() + .extend((0..30).map(|n| FakeAgent::sample_event(n, if n % 2 == 0 { "auth.login" } else { "admin.user_created" }))); + let mut model = on_audit(&boss); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!(view(&model).list.page(), 2); + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("admin.".into())))); + assert_eq!(view(&model).list.page(), 1, "a filter starts over from the first page"); + assert!(view(&model).list.events.iter().all(|e| e.action.starts_with("admin."))); + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!((asked.action.as_deref(), asked.cursor), (Some("admin."), None)); + // Anything that is not in the closed list (or a prefix of it) is never asked for. + for bad in ["nope.", "auth.log", "x' OR 1=1", ""] { + let before = boss.audit_queries.lock().unwrap().len(); + assert!(model.update(Event::Audit(AuditMsg::SetAction(Some(bad.into())))).is_empty(), "{bad:?}"); + assert_eq!(boss.audit_queries.lock().unwrap().len(), before); + } + assert_eq!(view(&model).filter.action.as_deref(), Some("admin.")); + support::run(&mut model, &boss, ev(AuditMsg::SetAction(None))); + assert_eq!(view(&model).filter.action, None); + // Every choice the page offers is one the server takes. + assert!(filter_choices().iter().all(|c| craft_cloud_app::valid_audit_action(c))); + // The six groups, then every name of the closed list. + assert_eq!(filter_choices().len(), 6 + craft_cloud_app::AUDIT_ACTIONS.len()); + assert_eq!(action_label("auth.login"), "Signed in"); + assert_eq!(action_label("something.new"), "something.new"); + assert_eq!(filter_label("auth."), "Sign-ins"); + } + + #[test] + fn an_account_filter_comes_from_its_page_and_is_cleared() { + let boss = support::agent(true, 2); + let mut model = support::on_the_pages(&boss); + let ana = support::user_named(&model, "user01"); + support::run(&mut model, &boss, ev(AuditMsg::OpenAudit(Some((ana.id, "user01".into()))))); + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!(asked.user, Some(ana.id)); + assert_eq!(view(&model).filter.user.as_ref().map(|(_, n)| n.as_str()), Some("user01")); + support::run(&mut model, &boss, ev(AuditMsg::ClearUser)); + assert_eq!(boss.audit_queries.lock().unwrap().last().unwrap().user, None); + // Not from a password dialog's page. + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::BackToList)]); + let ask = crate::admin::Action::Disable { id: ana.id, username: "user01".into() }; + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::Ask(ask))]); + assert!(model.update(Event::Audit(AuditMsg::OpenAudit(None))).is_empty()); + } + + #[test] + fn a_page_that_comes_late_is_ignored_and_a_refusal_is_said() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..5).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = support::on_the_pages(&boss); + let first = model.update(Event::Audit(AuditMsg::OpenAudit(None))); + let Some(Effect::AuditPage { ticket: old, .. }) = first.first().cloned() else { panic!("{first:?}") }; + // Another filter asks again while the first answer is still on its way. + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("auth.".into())))); + let current = view(&model).list.events.clone(); + assert_eq!(current.len(), 5); + let page = AuditPage { events: vec![FakeAgent::sample_event(99, "auth.login")], next: None }; + model.update(Event::Audit(AuditMsg::Page { own: false, ticket: old, result: Ok(page) })); + assert_eq!(view(&model).list.events, current, "the first call's answer matches nothing"); + // A refusal is said; `forbidden` asks the account's role again. + let forbidden = Error::Refused { code: "forbidden".into(), message: "Only admins".into(), name_taken: None }; + let Some(Effect::AuditPage { ticket, .. }) = model.update(Event::Audit(AuditMsg::Reload)).first().cloned() else { panic!() }; + let effects = model.update(Event::Audit(AuditMsg::Page { own: false, ticket, result: Err(forbidden) })); + assert!(matches!(effects.as_slice(), [Effect::Account { .. }]), "{effects:?}"); + assert!(view(&model).list.error.as_deref().is_some_and(|e| e.starts_with("The server doesn't allow this"))); + } + + #[test] + fn an_export_is_picked_run_and_whole_truncated_or_failed() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..3).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + // Cancelling the picker is nothing. + let asked = model.update(Event::Audit(AuditMsg::Export)); + assert!(matches!(asked.as_slice(), [Effect::PickExportFile { suggested, .. }] if suggested == EXPORT_NAME), "{asked:?}"); + assert!(model.update(Event::Audit(AuditMsg::Export)).is_empty(), "one at a time"); + picked(&mut model, None); + assert_eq!(view(&model).export, ExportState::Idle); + // Whole. + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("auth.".into())))); + model.update(Event::Audit(AuditMsg::Export)); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("out.ndjson").display().to_string(); + let effects = picked(&mut model, Some(&file)); + let [Effect::AuditExport { query, path, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((path.as_str(), query.action.as_deref(), query.cursor.clone(), query.limit), (file.as_str(), Some("auth."), None, None)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(view(&model).export, ExportState::Done { path: file.clone(), outcome: ExportOutcome { events: 3, truncated: false } }); + assert_eq!(std::fs::read_to_string(&file).unwrap().lines().count(), 3); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1, "no temporary file is left"); + // Truncated: the file is the newest events and the page says it is not the whole log. + *boss.export_truncated.lock().unwrap() = true; + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert!(matches!(&view(&model).export, ExportState::Done { outcome, .. } if outcome.truncated), "{:?}", view(&model).export); + // Failed (the stream broke): said, and no file of ours is left, and an older one, and a `.part` that is somebody else's, + // are not touched. + *boss.export_truncated.lock().unwrap() = false; + std::fs::write(&file, "old").unwrap(); + let theirs = format!("{file}.part"); + std::fs::write(&theirs, "not ours").unwrap(); + boss.export_fails.lock().unwrap().push(Error::Io("the export broke off: reset".into())); + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert!(matches!(&view(&model).export, ExportState::Failed(m) if m.contains("broke off")), "{:?}", view(&model).export); + assert_eq!(std::fs::read_to_string(&file).unwrap(), "old", "a failed export leaves what was there"); + assert_eq!(std::fs::read_to_string(&theirs).unwrap(), "not ours", "a file called .part is not ours to truncate or delete"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2, "only those two files"); + // A whole export replaces the destination and still leaves the other file alone. + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(std::fs::read_to_string(&file).unwrap().lines().count(), 3); + assert_eq!(std::fs::read_to_string(&theirs).unwrap(), "not ours"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2); + // An answer when no export runs is ignored. + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Exported { ticket: 1, result: Ok(ExportOutcome { events: 1, truncated: false }) })); + assert_eq!(view(&model).export, ExportState::Idle); + } + + /// An export's answer belongs to the export that asked: leave the pages and open them again, start another, and the first one's + /// answer (or its picker's) finishes nothing of the second. + #[test] + fn an_old_exports_answers_finish_nothing_of_a_newer_one() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..3).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + model.update(Event::Audit(AuditMsg::Export)); + let a = picked(&mut model, Some("a.ndjson")); + let [Effect::AuditExport { ticket: a_ticket, .. }] = a.as_slice() else { panic!("{a:?}") }; + let a_ticket = *a_ticket; + // Leave and come back, and start B (its picker open, then running). + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::BackToList)]); + support::run(&mut model, &boss, vec![Event::Audit(AuditMsg::OpenAudit(None))]); + let asked = model.update(Event::Audit(AuditMsg::Export)); + let [Effect::PickExportFile { ticket: b_picker, .. }] = asked.as_slice() else { panic!("{asked:?}") }; + let b_picker = *b_picker; + assert_ne!(a_ticket, b_picker); + // A's answer arrives while B is still picking, and A's old picker answer arrives too. + model.update(Event::Audit(AuditMsg::Exported { ticket: a_ticket, result: Ok(ExportOutcome { events: 3, truncated: false }) })); + assert_eq!(view(&model).export, ExportState::Picking { ticket: b_picker }); + model.update(Event::Audit(AuditMsg::ExportPicked { ticket: a_ticket, path: Some("a.ndjson".into()) })); + assert_eq!(view(&model).export, ExportState::Picking { ticket: b_picker }, "A's picker answer is not B's"); + let b = model.update(Event::Audit(AuditMsg::ExportPicked { ticket: b_picker, path: Some("b.ndjson".into()) })); + assert!(matches!(b.as_slice(), [Effect::AuditExport { path, .. }] if path == "b.ndjson"), "{b:?}"); + // B runs; A's answer comes late and finishes nothing; B's own does. + model.update(Event::Audit(AuditMsg::Exported { ticket: a_ticket, result: Err(Error::Io("the export broke off".into())) })); + assert!(matches!(&view(&model).export, ExportState::Running { path, .. } if path == "b.ndjson"), "{:?}", view(&model).export); + model.update(Event::Audit(AuditMsg::Exported { ticket: b_picker, result: Ok(ExportOutcome { events: 3, truncated: false }) })); + assert!(matches!(&view(&model).export, ExportState::Done { path, .. } if path == "b.ndjson"), "{:?}", view(&model).export); + } + + #[test] + fn who_did_it_and_what_it_was_about_is_in_every_line() { + let user = Uuid::from_u128(0x0123_4567_89ab_cdef_0123_4567_89ab_cdef); + let other = Uuid::from_u128(0xfedc_ba98_7654_3210_fedc_ba98_7654_3210); + let mut e = FakeAgent::sample_event(1, "admin.user_disabled"); + e.actor_user = Some(user); + e.target_kind = "user".into(); + e.target_id = Some(other); + let (line, full) = who_line(&e); + assert_eq!(line, "by 01234567 · about account fedcba98"); + assert!(full.contains(&user.to_string()) && full.contains(&other.to_string()), "{full}"); + // A computer in the details; a computer that acted. + let device = craft_cloud_app::DeviceId::generate(); + e.details.device = Some(device); + e.actor_device = Some(device); + let (line, full) = who_line(&e); + assert!(line.contains(&format!("on computer {}", short_id(device))) && line.contains(&format!("computer {}", short_id(device))), "{line}"); + assert!(full.contains(&device.to_string())); + // Nobody: the command line, or an unproven sign-in. + let cli = FakeAgent::sample_event(2, "cli.password_reset"); + assert_eq!(who_line(&cli).0, "by the command line"); + let failed = FakeAgent::sample_event(3, "auth.login_failed"); + assert_eq!(who_line(&failed).0, "by unknown"); + assert!(who_line(&failed).1.contains("nobody had proved")); + // A target of another kind keeps its kind. + let mut gone = FakeAgent::sample_event(4, "device.revoked"); + gone.actor_user = Some(user); + gone.target_kind = "device".into(); + gone.target_id = Some(other); + assert_eq!(who_line(&gone).0, "by 01234567 · about device fedcba98"); + } + + #[test] + fn every_account_has_its_own_activity() { + // A member: no admin pages, but the activity. + let member = support::agent(false, 1); + member.audit_events.lock().unwrap().extend((0..30).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = support::signed_in(&member); + assert!(!model.is_admin()); + let effects = model.update(Event::Audit(AuditMsg::OpenActivity)); + let [Effect::AuditPage { own: true, query, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((query.user, query.action.clone(), query.limit), (None, None, Some(PAGE_SIZE))); + support::run(&mut model, &member, vec![]); + let again = vec![Effect::AuditPage { own: true, query: query.clone(), ticket: model.activity_screen().unwrap().list.ticket }]; + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&member, &start, again.into_iter().next().unwrap()).unwrap(); + model.update(answer); + let a = model.activity_screen().unwrap(); + assert_eq!((a.list.events.len(), a.list.next.as_deref()), (PAGE_SIZE as usize, Some("25"))); + // Back to the signed-in view. + model.update(Event::Audit(AuditMsg::CloseActivity)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + // Only from the signed-in view. + let mut out = Model::new(crate::model::JoinDefaults::default(), true, String::new()); + assert!(out.update(Event::Audit(AuditMsg::OpenActivity)).is_empty()); + } + + #[test] + fn events_are_told_in_words_without_anything_the_server_did_not_write() { + let mut e = FakeAgent::sample_event(1, "admin.role_changed"); + e.details = AuditDetails { role_from: Some("member".into()), role_to: Some("admin".into()), ..AuditDetails::default() }; + assert_eq!(details_line(&e), "member to admin"); + e.details = AuditDetails { quota_bytes: Some(None), revoked: Some(2), ..AuditDetails::default() }; + assert_eq!(details_line(&e), "quota removed · 2 computers signed out"); + let mut failed = FakeAgent::sample_event(2, "auth.login_failed"); + failed.outcome = "failure".into(); + failed.count = 4; + assert_eq!(details_line(&failed), "4 tries · failed"); + } +} diff --git a/apps/craft-cloud-desktop/src/lib.rs b/apps/craft-cloud-desktop/src/lib.rs index 8f876736..00f26d6c 100644 --- a/apps/craft-cloud-desktop/src/lib.rs +++ b/apps/craft-cloud-desktop/src/lib.rs @@ -2,7 +2,9 @@ //! another computer, pauses and quits. It never syncs or touches the synced files itself: all of that is the agent's //! (`craft-cloud-agent run`), which it reaches through `craft-cloud-app` and starts when none answers. //! -//! - [`model`]: the state and every way between its screens, pure (unit-tested against a fake agent). +//! - [`model`]: the state and every way between its screens, pure (unit-tested against a fake agent); [`admin`]: the +//! admin pages' part of it. +//! - [`security`]: the Security page and the code a join asks for (two-factor sign-in). //! - [`agent`]: the agent's calls behind a trait, run off the UI thread. //! - [`start_agent`]: starting the agent from this app's folder. //! - [`single_instance`]: one app per user; a second start brings the first window forward. @@ -10,10 +12,13 @@ //! - [`tray`]: the tray icon where there is one; [`autostart`]: the app's own start-at-login entry. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod admin; pub mod agent; pub mod app; +pub mod audit; pub mod autostart; pub mod model; +pub mod security; pub mod single_instance; pub mod start_agent; pub mod tray; diff --git a/apps/craft-cloud-desktop/src/model.rs b/apps/craft-cloud-desktop/src/model.rs index 83c03578..00bc8d55 100644 --- a/apps/craft-cloud-desktop/src/model.rs +++ b/apps/craft-cloud-desktop/src/model.rs @@ -7,10 +7,18 @@ use std::path::Path; -use craft_cloud_app::{DeviceId, DeviceInfo, Error, JoinRequest, PairingInfo, PairingState, PairingStatus, RunState, Status, Uuid, codes}; +use craft_cloud_app::{ + AccountUsage, DeviceId, DeviceInfo, Error, HistoryResponse, JoinRequest, NodeId, PairingInfo, PairingState, PairingStatus, RestoreResponse, + RunState, Status, TrashResponse, TrashRestoreResponse, UserQuery, Uuid, VersionId, codes, +}; +use craft_cloud_proto::api::AdminConfirm; use craft_cloud_proto::api::codes as api_codes; use craft_cloud_proto::join::JoinString; +use crate::admin::{Access, Action, AdminEvent, AdminScreen}; +use crate::audit::{ActivityScreen, AuditMsg}; +use crate::security::{SecondFactorForm, SecurityAnswer, SecurityCall, SecurityEvent, SecurityPage}; + /// A code that is wrong, expired or used (the server never says which). pub const INVALID_CODE: &str = "This code doesn't work any more. Make a new one on a signed-in computer."; /// The agent can't be started. @@ -48,10 +56,22 @@ pub enum Screen { Joining { form: JoinForm, password: Option }, /// An invite to a new account: its password, twice. PasswordNeeded(PasswordForm), + /// The account has two-factor sign-in: the join asks for the code ([`SecondFactorForm`]). + SecondFactor(SecondFactorForm), /// Signed in: the account, the sync, the computers. SignedIn(Overview), + /// The Security page ([`SecurityPage`]): two-factor sign-in, the password, the other computers. Boxed: it is large. + Security(Box), + /// File version history for a path inside the sync root. + Versions(Versions), + /// Deleted files and folders awaiting retention expiry. + Trash(Trash), /// Showing an add-a-computer code. AddingComputer(Adding), + /// The admin pages (an admin's only): the users, one user, a new account, the audit log. + Admin(Box), + /// The signed-in account's own activity: its security events ([`crate::audit`]). + Activity(Box), /// The server refused this computer: it was removed from the account. Removed, /// Something the agent can't put right by itself (a damaged account file), or that this app can't work with. @@ -110,8 +130,15 @@ impl JoinForm { folder: Some(folder), device_name: (!name.is_empty()).then(|| name.to_owned()), password, + totp_code: None, + recovery_code: None, } } + + /// [`JoinForm::request`] with the second factor of an account that has two-factor sign-in. + fn request_with_factor(&self, password: Option, factor: (Option, Option)) -> JoinRequest { + JoinRequest { totp_code: factor.0, recovery_code: factor.1, ..self.request(password) } + } } /// The new account's password, typed twice. @@ -136,17 +163,72 @@ pub struct Overview { /// The account's computers, once asked. pub devices: Option>, pub devices_error: Option, + pub usage: Option, + pub usage_loading: bool, + pub usage_error: Option, /// The computer being removed. pub removing: Option, /// What went wrong with the last action (sign out, pause), for people. pub error: Option, /// Signing out failed because the server can't be reached: offer to forget the account here only. pub offer_local_sign_out: bool, + /// Whether the server lets this account use the admin pages (asked at sign-in). + pub access: Access, + /// Which sign-in this view is of: answers to the admin check made for another one are ignored. + pub session: u64, +} + +/// File history and restore state. +#[derive(Debug, Clone, PartialEq)] +pub struct Versions { + pub path: String, + pub page: Option, + pub current_version: Option, + pub loading: bool, + pub restoring: bool, + pub error: Option, + pub notice: Option, + pub confirm: Option, + pub ticket: u64, +} + +/// Trash list and action state. +#[derive(Debug, Clone, PartialEq)] +pub struct Trash { + pub page: Option, + pub loading: bool, + pub action: Option, + pub error: Option, + pub notice: Option, + pub confirm: Option, + pub ticket: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TrashAction { + /// `parent_path` is `None` without a path hint and empty for an item whose hint is a file in the sync root. + Restore { + path: Option, + parent_path: Option, + }, + Remove, } impl Overview { fn new(status: Status) -> Self { - Self { status, devices: None, devices_error: None, removing: None, error: None, offer_local_sign_out: false } + Self { + status, + devices: None, + devices_error: None, + usage: None, + usage_loading: true, + usage_error: None, + removing: None, + error: None, + offer_local_sign_out: false, + access: Access::Unknown, + session: crate::admin::fresh(), + } } } @@ -201,6 +283,30 @@ pub enum Event { PairingMade(Result), PairingPolled(Result), Devices(Result, Error>), + History { + ticket: u64, + result: Result, + }, + Restored { + ticket: u64, + result: Result, + }, + TrashPage { + ticket: u64, + result: Result, + }, + TrashRestored { + ticket: u64, + result: Result, + }, + TrashRemoved { + ticket: u64, + result: Result<(), Error>, + }, + Usage { + request_id: u64, + result: Result, + }, SignedOut(Result), /// [`Effect::Pause`]'s or [`Effect::Resume`]'s answer. Answered(Result), @@ -209,6 +315,16 @@ pub enum Event { /// [`Effect::AppAutostart`]'s outcome. AppAutostartSet(Result<(), String>), + /// The Security button of the signed-in view. + OpenSecurity, + /// A button of the Security page. + Security(SecurityEvent), + /// An answer to one of the page's calls. + SecurityAnswer(SecurityAnswer), + /// The code of a join that asked for one (the button, or Enter). + SubmitSecondFactor, + /// Back from the code field to the join form. + CancelSecondFactor, /// Join (the form's button, or Enter). SubmitJoin, /// The answer to "this folder has files in it: sync them too?". @@ -216,12 +332,29 @@ pub enum Event { SubmitPassword, CancelPassword, OpenFolder, + OpenVersions, + OpenTrash, + VersionsPath(String), + LoadVersions, + NextVersions, + BackVersions, + ConfirmRestore(VersionId), + CancelRestore, + RestoreVersion, + LoadTrash, + NextTrash, + BackTrash, + RestoreTrash(NodeId), + ConfirmRemoveTrash(NodeId), + CancelRemoveTrash, + RemoveTrash, AddComputer, /// Done or Cancel on the add-a-computer view. CloseAdding, RemoveDevice(DeviceId), /// Ask for the computer list again. RefreshDevices, + RefreshUsage, SignOut, /// Sign out on this computer only (the server can't be reached). SignOutHere, @@ -246,6 +379,10 @@ pub enum Event { CloseRequested, /// Linux: the tray process quit Craft Cloud (it stopped the agent): the window ends too. Exit, + /// Something on the admin pages, or the answer to one of their calls ([`crate::admin`]). + Admin(AdminEvent), + /// Something on the audit pages (the admin's log, or the account's own activity), or an answer to one of their calls. + Audit(AuditMsg), } /// What to do. @@ -255,6 +392,8 @@ pub enum Effect { Connect, /// Start the agent hidden from this app's folder and wait up to 5 s for it ([`Event::AgentStarted`]). StartAgent, + /// A call of the Security page ([`Event::SecurityAnswer`]). + Security(SecurityCall), Status, Join(JoinRequest), SetAutostart(bool), @@ -262,6 +401,30 @@ pub enum Effect { PairingState(Uuid), CancelPairing(Uuid), Devices, + History { + path: String, + cursor: Option, + ticket: u64, + }, + Restore { + path: String, + version: VersionId, + expected_version: VersionId, + ticket: u64, + }, + Trash { + cursor: Option, + ticket: u64, + }, + RestoreTrash { + id: NodeId, + ticket: u64, + }, + RemoveTrash { + id: NodeId, + ticket: u64, + }, + Usage(u64), RemoveDevice(DeviceId), Logout { local: bool, @@ -279,6 +442,56 @@ pub enum Effect { QuitApp, /// Write (`true`) or remove the app's own start-at-login entry. AppAutostart(bool), + /// The signed-in account, as the server says it: how the app finds out whether it is an admin ([`AdminEvent::Account`]). + Account { + session: u64, + }, + /// The server's counters, for the admin pages ([`AdminEvent::Server`]). + AdminServer { + session: u64, + }, + /// The server-wide storage overview for the admin pages. + AdminStorage { + session: u64, + ticket: u64, + }, + /// A page of the user list ([`AdminEvent::Users`] with the same `ticket`). + AdminUsers { + query: UserQuery, + ticket: u64, + }, + /// A user's devices ([`AdminEvent::Devices`]). + AdminDevices { + user: Uuid, + ticket: u64, + }, + /// One account's effective quota and overlapping storage estimates. + AdminUserUsage { + user: Uuid, + ticket: u64, + }, + /// A page of the audit log or of the account's own activity ([`AuditMsg::Page`] with the same `ticket`). + AuditPage { + own: bool, + query: craft_cloud_app::AuditQuery, + ticket: u64, + }, + /// The audit export, written to `path` ([`AuditMsg::Exported`]). + AuditExport { + query: craft_cloud_app::AuditQuery, + path: String, + ticket: u64, + }, + /// Ask for the file an export goes to, with the OS's save dialog, starting at this name ([`AuditMsg::ExportPicked`]). + PickExportFile { + suggested: String, + ticket: u64, + }, + /// A change, with the admin's confirmation ([`AdminEvent::Done`]). Its `Debug` output hides the password. + AdminDo { + action: Action, + confirm: AdminConfirm, + }, } /// Whether the app should start at login, in the tray, given the agent's status: while signed in with the agent @@ -319,6 +532,9 @@ pub struct Model { failed_polls: u32, /// The app's start-at-login entry as last asked for. app_autostart: Option, + versions_ticket: u64, + trash_ticket: u64, + usage_request_id: u64, } impl Model { @@ -339,6 +555,9 @@ impl Model { polling_pairing: false, failed_polls: 0, app_autostart: None, + versions_ticket: 0, + trash_ticket: 0, + usage_request_id: 0, } } @@ -346,15 +565,39 @@ impl Model { pub fn overview(&self) -> Option<&Overview> { match &self.screen { Screen::SignedIn(overview) => Some(overview), + Screen::Security(page) => Some(&page.overview), Screen::AddingComputer(adding) => Some(&adding.overview), + Screen::Admin(admin) => Some(&admin.overview), + Screen::Activity(activity) => Some(&activity.overview), _ => None, } } - fn overview_mut(&mut self) -> Option<&mut Overview> { + /// Leaves a page for a fresh overview, which asks again for everything it shows: the computers, the admin + /// check and the storage usage. Without the last two the view would wait for answers nobody asked for. + fn back_to_overview(&mut self) -> Vec { + let Some(status) = self.status.clone().filter(|s| s.signed_in) else { + self.screen = Screen::Starting; + return Vec::new(); + }; + let overview = Overview::new(status); + let session = overview.session; + self.screen = Screen::SignedIn(overview); + vec![Effect::Devices, Effect::Account { session }, self.next_usage_request()] + } + + fn next_usage_request(&mut self) -> Effect { + self.usage_request_id = self.usage_request_id.wrapping_add(1); + Effect::Usage(self.usage_request_id) + } + + pub(crate) fn overview_mut(&mut self) -> Option<&mut Overview> { match &mut self.screen { Screen::SignedIn(overview) => Some(overview), + Screen::Security(page) => Some(&mut page.overview), Screen::AddingComputer(adding) => Some(&mut adding.overview), + Screen::Admin(admin) => Some(&mut admin.overview), + Screen::Activity(activity) => Some(&mut activity.overview), _ => None, } } @@ -432,6 +675,142 @@ impl Model { } Vec::new() } + Event::History { ticket, result } => { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.ticket != ticket { + return Vec::new(); + } + versions.loading = false; + match result { + Ok(page) => { + if versions.current_version.is_none() { + versions.current_version = page.versions.first().map(|row| row.id); + } + versions.page = Some(page); + versions.error = None; + } + Err(e) => versions.error = Some(message(&e)), + } + Vec::new() + } + Event::Restored { ticket, result } => { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.ticket != ticket { + return Vec::new(); + } + versions.restoring = false; + versions.confirm = None; + match result { + Ok(_) => { + versions.notice = Some("Version restored. Sync will update this file.".into()); + versions.current_version = None; + self.request_history(None) + } + Err(Error::Refused { code, .. }) if code == api_codes::CONFLICT => { + versions.notice = Some("The file changed meanwhile. Version list reloaded.".into()); + versions.current_version = None; + self.request_history(None) + } + Err(Error::Refused { code, .. }) if code == api_codes::INVALID => { + versions.error = Some("This file can't be restored: it may be a live document or deleted.".into()); + Vec::new() + } + Err(e) => { + versions.error = Some(message(&e)); + Vec::new() + } + } + } + Event::TrashPage { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + trash.loading = false; + match result { + Ok(page) => { + trash.page = Some(page); + trash.error = None; + } + Err(e) => trash.error = Some(message(&e)), + } + Vec::new() + } + Event::TrashRestored { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + let prior = trash.action.take(); + match result { + Ok(response) => { + let (path, parent_path) = prior + .and_then(|action| match action { + TrashAction::Restore { path, parent_path, .. } => Some((path, parent_path)), + _ => None, + }) + .unwrap_or((None, None)); + trash.notice = Some(if response.renamed { + match parent_path { + Some(parent) if parent.is_empty() => format!("Restored as {} in the sync folder.", response.name), + Some(parent) => format!("Restored as {} in {}.", response.name, parent), + None => format!("Restored as {}. The destination path is unavailable.", response.name), + } + } else { + match path { + Some(path) => format!("Restored to {}.", path), + None => format!("Restored {}. The destination path is unavailable.", response.name), + } + }); + self.request_trash(None) + } + Err(Error::Refused { code, name_taken, .. }) if code == api_codes::CONFLICT => { + trash.error = Some(if name_taken == Some(true) { + "This item couldn't be restored because another item already has that name.".into() + } else { + "This item couldn't be restored because it conflicts with the current folder contents.".into() + }); + Vec::new() + } + Err(e) => { + trash.error = Some(message(&e)); + Vec::new() + } + } + } + Event::TrashRemoved { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + trash.action = None; + match result { + Ok(()) => { + trash.notice = Some("Removed from trash. It can no longer be restored from the trash.".into()); + self.request_trash(None) + } + Err(e) => { + trash.error = Some(message(&e)); + Vec::new() + } + } + } + Event::Usage { request_id, result } => { + if request_id != self.usage_request_id { + return Vec::new(); + } + if let Some(overview) = self.overview_mut() { + overview.usage_loading = false; + match result { + Ok(usage) => { + overview.usage = Some(usage); + overview.usage_error = None; + } + Err(e) => overview.usage_error = Some(message(&e)), + } + } + Vec::new() + } Event::SignedOut(result) => self.signed_out(result), Event::Answered(result) => { self.busy = self.busy.filter(|b| *b != Busy::Pausing); @@ -451,6 +830,28 @@ impl Model { Vec::new() } + Event::OpenSecurity => { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + let (page, calls) = SecurityPage::open(overview.clone()); + self.screen = Screen::Security(Box::new(page)); + calls.into_iter().map(Effect::Security).collect() + } + Event::Security(event) => self.security(event), + Event::SecurityAnswer(answer) => { + let now_ms = self.now_ms; + let Screen::Security(page) = &mut self.screen else { return Vec::new() }; + page.answer(answer, now_ms).into_iter().map(Effect::Security).collect() + } + Event::SubmitSecondFactor => self.submit_second_factor(), + Event::CancelSecondFactor => { + if let Screen::SecondFactor(form) = &self.screen { + self.screen = match &form.password { + Some(pw) => Screen::PasswordNeeded(pw.clone()), + None => Screen::Join(form.form.clone()), + }; + } + Vec::new() + } Event::SubmitJoin => self.submit_join(), Event::SyncExistingFiles(yes) => { let Screen::Join(form) = &mut self.screen else { return Vec::new() }; @@ -469,6 +870,138 @@ impl Model { Vec::new() } Event::OpenFolder => self.status.as_ref().and_then(|s| s.folder.clone()).map(Effect::OpenFolder).into_iter().collect(), + Event::OpenVersions => { + if !matches!(self.screen, Screen::SignedIn(_)) { + return Vec::new(); + } + self.screen = Screen::Versions(Versions { + path: String::new(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + Vec::new() + } + Event::OpenTrash => { + if !matches!(self.screen, Screen::SignedIn(_)) { + return Vec::new(); + } + self.screen = Screen::Trash(Trash { page: None, loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + self.request_trash(None) + } + Event::LoadTrash => self.request_trash(None), + Event::NextTrash => { + let cursor = match &self.screen { + Screen::Trash(t) if t.action.is_none() && !t.loading => t.page.as_ref().and_then(|p| p.next.clone()), + _ => None, + }; + cursor.map(|cursor| self.request_trash(Some(cursor))).unwrap_or_default() + } + Event::BackTrash => { + if matches!(self.screen, Screen::Trash(_)) { + self.trash_ticket = self.trash_ticket.saturating_add(1); + return self.back_to_overview(); + } + Vec::new() + } + Event::RestoreTrash(id) => { + let Screen::Trash(t) = &mut self.screen else { return Vec::new() }; + if t.loading || t.action.is_some() { + return Vec::new(); + } + let Some(item) = t.page.as_ref().and_then(|p| p.items.iter().find(|item| item.id == id)) else { return Vec::new() }; + let path = item.path.clone(); + let parent_path = path.as_deref().map(|p| p.rsplit_once('/').map(|(parent, _)| parent.to_owned()).unwrap_or_default()); + self.begin_trash_action(id, TrashAction::Restore { path, parent_path }) + } + Event::ConfirmRemoveTrash(id) => { + if let Screen::Trash(t) = &mut self.screen + && !t.loading + && t.action.is_none() + && t.page.as_ref().is_some_and(|p| p.items.iter().any(|i| i.id == id)) + { + t.confirm = Some(id); + } + Vec::new() + } + Event::CancelRemoveTrash => { + if let Screen::Trash(t) = &mut self.screen { + t.confirm = None; + } + Vec::new() + } + Event::RemoveTrash => { + let Screen::Trash(t) = &mut self.screen else { return Vec::new() }; + if t.loading || t.action.is_some() { + return Vec::new(); + } + let Some(id) = t.confirm.take() else { return Vec::new() }; + self.begin_trash_action(id, TrashAction::Remove) + } + Event::VersionsPath(path) => { + if let Screen::Versions(v) = &mut self.screen { + if v.path != path { + self.versions_ticket = self.versions_ticket.saturating_add(1); + v.ticket = self.versions_ticket; + v.page = None; + v.current_version = None; + v.loading = false; + v.restoring = false; + v.confirm = None; + v.notice = None; + } + v.path = path; + v.error = None; + } + Vec::new() + } + Event::LoadVersions => { + if let Screen::Versions(v) = &mut self.screen { + if v.restoring || v.loading { + return Vec::new(); + } + v.notice = None; + v.current_version = None; + } + self.request_history(None) + } + Event::NextVersions => { + let cursor = match &self.screen { + Screen::Versions(v) => v.page.as_ref().and_then(|p| p.next.clone()), + _ => None, + }; + cursor.map(|c| self.request_history(Some(c))).unwrap_or_default() + } + Event::BackVersions => { + if let Screen::Versions(_) = self.screen { + self.versions_ticket = self.versions_ticket.saturating_add(1); + return self.back_to_overview(); + } + Vec::new() + } + Event::ConfirmRestore(version) => { + if let Screen::Versions(v) = &mut self.screen + && !v.restoring + && !v.loading + && Some(version) != v.current_version + && v.page.as_ref().is_some_and(|page| page.versions.iter().any(|row| row.id == version && !row.pruned)) + { + v.confirm = Some(version); + } + Vec::new() + } + Event::CancelRestore => { + if let Screen::Versions(v) = &mut self.screen { + v.confirm = None; + } + Vec::new() + } + Event::RestoreVersion => self.restore_version(), Event::AddComputer => { let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; self.screen = Screen::AddingComputer(Adding { overview: overview.clone(), info: None, state: AddState::Asking }); @@ -494,7 +1027,23 @@ impl Model { overview.removing = Some(id); vec![Effect::RemoveDevice(id)] } - Event::RefreshDevices => self.overview().map(|_| Effect::Devices).into_iter().collect(), + Event::RefreshDevices => { + let Some(overview) = self.overview() else { return Vec::new() }; + // An admin check that couldn't be made at sign-in is made again. + let again = (overview.access == Access::Unknown).then_some(Effect::Account { session: overview.session }); + std::iter::once(Effect::Devices).chain(again).collect() + } + Event::Admin(event) => self.admin(event), + Event::Audit(msg) => self.audit(msg), + Event::RefreshUsage => { + let Some(overview) = self.overview_mut() else { return Vec::new() }; + if overview.usage_loading { + return Vec::new(); + } + overview.usage_loading = true; + overview.usage_error = None; + vec![self.next_usage_request()] + } Event::SignOut => self.sign_out(false), Event::SignOutHere => self.sign_out(true), Event::Pause | Event::Resume => { @@ -587,6 +1136,12 @@ impl Model { /// A status from the agent: the screen follows it, and the app's start-at-login entry follows the agent's. fn status_came(&mut self, status: Status) -> Vec { + let account_changed = + self.overview().is_some_and(|overview| overview.status.account != status.account || overview.status.server != status.server); + if !status.signed_in { + self.usage_request_id = self.usage_request_id.wrapping_add(1); + } + let new_usage_request = (status.signed_in && account_changed).then(|| self.next_usage_request()); let mut effects = Vec::new(); self.status = Some(status.clone()); if let Some(wanted) = app_autostart_wanted(&status, self.login_entry) @@ -601,15 +1156,41 @@ impl Model { } if status.signed_in { match &mut self.screen { - Screen::SignedIn(overview) => overview.status = status, - Screen::AddingComputer(adding) => adding.overview.status = status, + Screen::SignedIn(overview) => { + overview.status = status; + if account_changed { + overview.usage = None; + overview.usage_loading = true; + overview.usage_error = None; + } + } + Screen::AddingComputer(adding) => { + adding.overview.status = status; + if account_changed { + adding.overview.usage = None; + adding.overview.usage_loading = true; + adding.overview.usage_error = None; + } + } + Screen::Security(page) => page.overview.status = status, + Screen::Admin(admin) => admin.overview.status = status, + Screen::Activity(activity) => activity.overview.status = status, // The join's own answer moves on from there. Screen::Joining { .. } => {} + Screen::Versions(_) | Screen::Trash(_) => {} _ => { - self.screen = Screen::SignedIn(Overview::new(status)); + let overview = Overview::new(status); + let session = overview.session; + self.screen = Screen::SignedIn(overview); effects.push(Effect::Devices); + // Whether the account is an admin, for the admin pages. + effects.push(Effect::Account { session }); + effects.push(self.next_usage_request()); } } + if let Some(effect) = new_usage_request { + effects.push(effect); + } return effects; } match (&status.run_state, &self.screen) { @@ -619,7 +1200,7 @@ impl Model { self.screen = Screen::Error(problem); } } - (_, Screen::Join(_) | Screen::Joining { .. } | Screen::PasswordNeeded(_)) => {} + (_, Screen::Join(_) | Screen::Joining { .. } | Screen::PasswordNeeded(_) | Screen::SecondFactor(_)) => {} _ => self.screen = Screen::Join(JoinForm::new(&self.defaults)), } effects @@ -650,7 +1231,7 @@ impl Model { self.version_problem(*v); return; } - Error::Refused { .. } => false, + Error::Refused { .. } | Error::RefusedWith { .. } => false, }; if !gone || matches!(self.screen, Screen::Starting | Screen::NotRunning(_)) || self.busy == Some(Busy::Quitting) { return; @@ -678,6 +1259,34 @@ impl Model { vec![Effect::Join(request)] } + /// The code of a join that asked for one: the same join, with the code, goes to the agent again. + fn submit_second_factor(&mut self) -> Vec { + let Screen::SecondFactor(second) = &mut self.screen else { return Vec::new() }; + let factor = match second.factor() { + Ok(factor) => factor, + Err(why) => { + second.error = Some(why); + return Vec::new(); + } + }; + let request = second.form.request_with_factor(second.password.as_ref().map(|pw| pw.password.clone()), factor); + let (form, password) = (second.form.clone(), second.password.clone()); + self.screen = Screen::Joining { form, password }; + vec![Effect::Join(request)] + } + + /// A button of the Security page. + fn security(&mut self, event: SecurityEvent) -> Vec { + let Screen::Security(page) = &mut self.screen else { return Vec::new() }; + if event == SecurityEvent::Close { + // Whatever the page showed (a secret, recovery codes) goes with it. + let overview = page.overview.clone(); + self.screen = Screen::SignedIn(overview); + return vec![Effect::Devices]; + } + page.update(event).into_iter().map(Effect::Security).collect() + } + fn submit_password(&mut self) -> Vec { let Screen::PasswordNeeded(pw) = &mut self.screen else { return Vec::new() }; pw.error = if pw.password.is_empty() { @@ -704,12 +1313,14 @@ impl Model { let mut effects = self.status_came(status); // Start at login: the agent's entry (the app's follows it). effects.extend([Effect::SetAutostart(true), Effect::Devices]); + effects.extend(self.overview().map(|o| Effect::Account { session: o.session })); + effects.push(self.next_usage_request()); return effects; } Err(e) => e, }; let (code, text) = match &e { - Error::Refused { code, message } => (code.as_str(), message.clone()), + Error::Refused { code, message, .. } | Error::RefusedWith { code, message, .. } => (code.as_str(), message.clone()), Error::NotRunning(_) => { self.status = None; self.screen = Screen::NotRunning(self.not_running(STOPPED, &e.to_string())); @@ -724,6 +1335,11 @@ impl Model { Screen::PasswordNeeded(pw) }; self.screen = match (code, password) { + // The account has two-factor sign-in: the same join goes again with its code (the invite was not used up). + (codes::TWO_FACTOR_REQUIRED, password) => Screen::SecondFactor(SecondFactorForm::new(form, password, None)), + (codes::TWO_FACTOR_INVALID | codes::TWO_FACTOR_LOCKED, password) => { + Screen::SecondFactor(SecondFactorForm::new(form, password, Some(crate::security::explain(&e)))) + } (codes::PASSWORD_REQUIRED, _) => { Screen::PasswordNeeded(PasswordForm { form, password: String::new(), again: String::new(), error: None }) } @@ -785,6 +1401,74 @@ impl Model { vec![Effect::Logout { local }] } + fn request_history(&mut self, cursor: Option) -> Vec { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.restoring || versions.loading { + return Vec::new(); + } + // The path is sent exactly as typed: a leading or trailing space can be part of a real file name. + let path = versions.path.as_str(); + if path.is_empty() + || Path::new(path).is_absolute() + || Path::new(path) + .components() + .any(|part| matches!(part, std::path::Component::ParentDir | std::path::Component::RootDir | std::path::Component::Prefix(_))) + { + versions.error = Some("Enter a file path inside the sync folder.".into()); + return Vec::new(); + } + self.versions_ticket = self.versions_ticket.saturating_add(1); + versions.ticket = self.versions_ticket; + versions.loading = true; + versions.error = None; + vec![Effect::History { path: path.to_owned(), cursor, ticket: versions.ticket }] + } + + fn request_trash(&mut self, cursor: Option) -> Vec { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.loading || trash.action.is_some() { + return Vec::new(); + } + self.trash_ticket = self.trash_ticket.saturating_add(1); + trash.ticket = self.trash_ticket; + trash.loading = true; + trash.error = None; + vec![Effect::Trash { cursor, ticket: trash.ticket }] + } + + fn begin_trash_action(&mut self, id: NodeId, action: TrashAction) -> Vec { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.loading || trash.action.is_some() { + return Vec::new(); + } + self.trash_ticket = self.trash_ticket.saturating_add(1); + trash.ticket = self.trash_ticket; + trash.error = None; + trash.action = Some(action); + match &trash.action { + Some(TrashAction::Restore { .. }) => vec![Effect::RestoreTrash { id, ticket: trash.ticket }], + Some(TrashAction::Remove) => vec![Effect::RemoveTrash { id, ticket: trash.ticket }], + None => Vec::new(), + } + } + + fn restore_version(&mut self) -> Vec { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + // A restore takes a new ticket, which would discard the answer of a page still loading and leave `loading` set. + if versions.restoring || versions.loading { + return Vec::new(); + } + let Some(version) = versions.confirm else { return Vec::new() }; + let Some(expected_version) = versions.current_version else { + versions.error = Some("Current version unavailable. Reload the version list.".into()); + return Vec::new(); + }; + self.versions_ticket = self.versions_ticket.saturating_add(1); + versions.ticket = self.versions_ticket; + versions.restoring = true; + vec![Effect::Restore { path: versions.path.clone(), version, expected_version, ticket: versions.ticket }] + } + fn signed_out(&mut self, result: Result) -> Vec { self.busy = self.busy.filter(|b| *b != Busy::SigningOut); match result { @@ -820,7 +1504,7 @@ impl Model { /// An error's message, for people. fn message(e: &Error) -> String { match e { - Error::Refused { message, .. } => message.clone(), + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), other => other.to_string(), } } @@ -852,6 +1536,40 @@ pub(crate) mod tests { Model::new(defaults(), true, "/home/ana/.local/share/craft-cloud-agent/run.log".into()) } + fn history_page() -> HistoryResponse { + HistoryResponse { + versions: vec![craft_cloud_app::HistoryVersion { + id: VersionId::generate(), + parent_version: None, + epoch: craft_cloud_proto::Epoch::generate(), + seq: 3, + size: 12, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }], + next: Some("opaque-cursor".into()), + } + } + + fn trash_page() -> TrashResponse { + TrashResponse { + items: vec![craft_cloud_app::TrashItem { + id: NodeId::generate(), + parent: NodeId::generate(), + name: "painting.png".into(), + is_dir: false, + size: Some(42_000), + deleted_ms: 1_790_000_000_000, + purge_after_ms: 1_792_000_000_000, + path: Some("Projects/painting.png".into()), + }], + next: Some("trash-cursor".into()), + } + } + pub(crate) fn status(signed_in: bool) -> Status { Status { api: 1, @@ -865,11 +1583,531 @@ pub(crate) mod tests { autostart: Some(signed_in), last_sync_ms: None, removed: false, + spaces: Vec::new(), } } + #[test] + fn versions_load_page_and_restore_with_ticket_guard() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenVersions); + m.update(Event::VersionsPath("art/file.png".into())); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: "art/file.png".into(), cursor: None, ticket: 2 }]); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let mut page = history_page(); + let current = page.versions[0].id; + let old = VersionId::generate(); + page.versions.push(craft_cloud_app::HistoryVersion { + id: old, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + assert!(m.update(Event::History { ticket: 2, result: Ok(page.clone()) }).is_empty()); + assert_eq!( + m.update(Event::NextVersions), + vec![Effect::History { path: "art/file.png".into(), cursor: Some("opaque-cursor".into()), ticket: 3 }] + ); + m.update(Event::History { ticket: 3, result: Ok(page.clone()) }); + m.update(Event::ConfirmRestore(old)); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "art/file.png".into(), version: old, expected_version: current, ticket: 4 }] + ); + assert!(m.update(Event::LoadVersions).is_empty()); + assert!(m.update(Event::NextVersions).is_empty()); + assert!( + m.update(Event::Restored { + ticket: 3, + result: Ok(craft_cloud_app::RestoreResponse { + version: old, + parent_version: current, + seq: 5, + epoch: craft_cloud_proto::Epoch::generate() + }) + }) + .is_empty() + ); + assert!( + m.update(Event::Restored { + ticket: 4, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: None }) + }) + .iter() + .any(|e| matches!(e, Effect::History { cursor: None, .. })) + ); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert_eq!(v.notice.as_deref(), Some("The file changed meanwhile. Version list reloaded.")); + } + + #[test] + fn trash_pages_restore_remove_confirmation_and_stale_tickets() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + assert_eq!(m.update(Event::OpenTrash), vec![Effect::Trash { cursor: None, ticket: 1 }]); + let page = trash_page(); + let id = page.items[0].id; + assert!(m.update(Event::TrashPage { ticket: 0, result: Ok(page.clone()) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.loading && t.page.is_none())); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 2 }]); + assert!(m.update(Event::RestoreTrash(id)).is_empty()); // page load excludes actions + assert!(m.update(Event::ConfirmRemoveTrash(id)).is_empty()); + assert!(m.update(Event::RemoveTrash).is_empty()); + m.update(Event::TrashPage { ticket: 2, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::RestoreTrash(id)), vec![Effect::RestoreTrash { id, ticket: 3 }]); + assert!(m.update(Event::LoadTrash).is_empty()); // action excludes page loads + assert!(m.update(Event::TrashRestored { ticket: 2, result: Err(Error::Io("stale".into())) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.action.is_some() && !t.loading)); + let restored = TrashRestoreResponse { + node: id, + parent: page.items[0].parent, + name: "painting (1).png".into(), + renamed: true, + version: None, + seq: 2, + epoch: craft_cloud_proto::Epoch::generate(), + }; + assert_eq!(m.update(Event::TrashRestored { ticket: 3, result: Ok(restored) }), vec![Effect::Trash { cursor: None, ticket: 4 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png in Projects.")); + m.update(Event::TrashPage { ticket: 4, result: Ok(page.clone()) }); + assert!(m.update(Event::ConfirmRemoveTrash(id)).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.confirm == Some(id))); + assert_eq!(m.update(Event::RemoveTrash), vec![Effect::RemoveTrash { id, ticket: 5 }]); + assert!(m.update(Event::LoadTrash).is_empty()); // removal excludes page loads too + assert!(m.update(Event::TrashRemoved { ticket: 4, result: Ok(()) }).is_empty()); + assert_eq!(m.update(Event::TrashRemoved { ticket: 5, result: Ok(()) }), vec![Effect::Trash { cursor: None, ticket: 6 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Removed from trash. It can no longer be restored from the trash.")); + } + + #[test] + fn trash_restore_name_taken_is_clear_and_returns_to_idle_without_reload() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(m.update(Event::RemoveTrash).is_empty()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert!( + m.update(Event::TrashRestored { + ticket, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: Some(true) }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.as_deref().unwrap_or_default().contains("another item already has that name")); + assert!(!t.loading); + assert!(t.action.is_none()); + } + + #[test] + fn trash_restore_without_path_does_not_invent_a_destination() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = None; + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert_eq!( + m.update(Event::TrashRestored { + ticket, + result: Ok(TrashRestoreResponse { + node: id, + parent: NodeId::generate(), + name: "painting.png".into(), + renamed: false, + version: None, + seq: 1, + epoch: craft_cloud_proto::Epoch::generate(), + }) + }), + vec![Effect::Trash { cursor: None, ticket: ticket + 1 }] + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored painting.png. The destination path is unavailable.")); + } + + #[test] + fn other_trash_conflicts_are_generic_and_do_not_reload() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert!( + m.update(Event::TrashRestored { + ticket, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: Some(false) }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.as_deref().unwrap_or_default().contains("conflicts with the current folder contents")); + assert!(!t.loading); + assert!(t.action.is_none()); + } + + fn trash_restore_response(node: NodeId, name: &str, renamed: bool) -> TrashRestoreResponse { + TrashRestoreResponse { + node, + parent: NodeId::generate(), + name: name.into(), + renamed, + version: None, + seq: 1, + epoch: craft_cloud_proto::Epoch::generate(), + } + } + + fn trash_ticket(m: &Model) -> u64 { + match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + } + } + + #[test] + fn trash_restore_at_the_sync_root_says_so_even_when_renamed() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = Some("painting.png".into()); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = trash_ticket(&m); + m.update(Event::TrashRestored { ticket, result: Ok(trash_restore_response(id, "painting (1).png", true)) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png in the sync folder.")); + } + + #[test] + fn trash_restore_renamed_without_path_keeps_the_unavailable_notice() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = None; + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + m.update(Event::RestoreTrash(id)); + let ticket = trash_ticket(&m); + m.update(Event::TrashRestored { ticket, result: Ok(trash_restore_response(id, "painting (1).png", true)) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png. The destination path is unavailable.")); + } + + #[test] + fn back_from_trash_or_versions_asks_again_for_usage_and_admin_access() { + for (open, back) in [(Event::OpenTrash, Event::BackTrash), (Event::OpenVersions, Event::BackVersions)] { + let mut m = model(); + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(open); + let effects = m.update(back); + let overview = m.overview().expect("back on the overview"); + assert!(overview.usage_loading, "{effects:?}"); + let session = overview.session; + assert!(effects.contains(&Effect::Devices), "{effects:?}"); + assert!(effects.contains(&Effect::Account { session }), "{effects:?}"); + let Some(&Effect::Usage(id)) = effects.iter().find(|e| matches!(e, Effect::Usage(_))) else { panic!("{effects:?}") }; + let usage = craft_cloud_app::AccountUsage { + stored_bytes: 1, + quota_bytes: None, + trash_bytes: None, + history_bytes: None, + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + }; + m.update(Event::Usage { request_id: id, result: Ok(usage) }); + let overview = m.overview().unwrap(); + assert!(!overview.usage_loading && overview.usage.is_some(), "the overview doesn't stay loading"); + } + } + + #[test] + fn trash_late_page_reply_after_back_and_reopen_is_dropped() { + let mut m = model(); + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + assert_eq!(m.update(Event::OpenTrash), vec![Effect::Trash { cursor: None, ticket: 1 }]); + m.update(Event::BackTrash); + assert!(matches!(m.screen, Screen::SignedIn(_))); + let reopened = m.update(Event::OpenTrash); + let Some(Effect::Trash { ticket: fresh, .. }) = reopened.first().cloned() else { panic!("{reopened:?}") }; + assert_ne!(fresh, 1); + assert!(m.update(Event::TrashPage { ticket: 1, result: Ok(trash_page()) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.loading && t.page.is_none())); + m.update(Event::TrashPage { ticket: fresh, result: Ok(trash_page()) }); + assert!(matches!(&m.screen, Screen::Trash(t) if !t.loading && t.page.is_some() && t.error.is_none())); + } + + #[test] + fn trash_late_action_reply_after_back_and_reopen_is_dropped() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenTrash); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::RestoreTrash(id)), vec![Effect::RestoreTrash { id, ticket: 2 }]); + m.update(Event::BackTrash); + let reopened = m.update(Event::OpenTrash); + let Some(Effect::Trash { ticket: fresh, .. }) = reopened.first().cloned() else { panic!("{reopened:?}") }; + assert!(m.update(Event::TrashRestored { ticket: 2, result: Ok(trash_restore_response(id, "painting.png", false)) }).is_empty()); + assert!(m.update(Event::TrashRemoved { ticket: 2, result: Ok(()) }).is_empty()); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.loading && t.notice.is_none()); + m.update(Event::TrashPage { ticket: fresh, result: Ok(page) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(!t.loading && t.action.is_none() && t.page.is_some() && t.error.is_none()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + } + + #[test] + fn trash_remove_failure_shows_the_error_and_frees_the_buttons() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + m.update(Event::ConfirmRemoveTrash(id)); + assert!(matches!(m.update(Event::RemoveTrash).first(), Some(Effect::RemoveTrash { .. }))); + let ticket = trash_ticket(&m); + assert!( + m.update(Event::TrashRemoved { + ticket, + result: Err(Error::Refused { code: api_codes::INTERNAL.into(), message: "server broke".into(), name_taken: None }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.is_some()); + assert!(t.action.is_none() && !t.loading && t.confirm.is_none()); + assert!(t.page.is_some()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + } + + #[test] + fn trash_failed_next_page_keeps_rows_clears_loading_and_can_retry() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenTrash); + let page = trash_page(); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 2 }]); + assert!(m.update(Event::TrashPage { ticket: 2, result: Err(Error::Io("offline".into())) }).is_empty()); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.is_some()); + assert!(!t.loading); + assert_eq!(t.page.as_ref(), Some(&page)); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 3 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.loading && t.error.is_none()); + } + + #[test] + fn versions_refuse_unsafe_paths_and_explain_invalid_restore() { + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: "../secret".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert!(m.update(Event::LoadVersions).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.error.as_deref().unwrap_or_default().contains("inside the sync folder")); + let mut page = history_page(); + let current_version = page.versions.first().map(|version| version.id); + let old_version = VersionId::generate(); + if let Some(current) = current_version { + page.versions.push(craft_cloud_app::HistoryVersion { + id: old_version, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + } + m.screen = Screen::Versions(Versions { + path: "file.txt".into(), + page: Some(page), + current_version, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: Some(old_version), + ticket: 0, + }); + let effect = m.update(Event::RestoreVersion); + let ticket = match effect.first() { + Some(Effect::Restore { ticket, .. }) => *ticket, + other => panic!("{other:?}"), + }; + m.update(Event::Restored { + ticket, + result: Err(Error::Refused { code: api_codes::INVALID.into(), message: "invalid".into(), name_taken: None }), + }); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.error.as_deref().unwrap_or_default().contains("live document or deleted")); + } + + #[test] + fn versions_path_change_discards_answers_for_previous_path() { + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: "old.txt".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: "old.txt".into(), cursor: None, ticket: 1 }]); + m.update(Event::VersionsPath("new.txt".into())); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert_eq!(v.path, "new.txt"); + assert!(v.page.is_none()); + assert!(!v.loading); + } + + /// A versions screen for `path` with two loaded versions, the older one awaiting restore confirmation. + fn versions_ready_to_restore(path: &str) -> (Model, VersionId, VersionId) { + let mut page = history_page(); + let current = page.versions[0].id; + let old = VersionId::generate(); + page.versions.push(craft_cloud_app::HistoryVersion { + id: old, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: path.into(), + page: Some(page), + current_version: Some(current), + loading: false, + restoring: false, + error: None, + notice: None, + confirm: Some(old), + ticket: 0, + }); + (m, current, old) + } + + #[test] + fn versions_restore_is_refused_while_a_page_loads_and_never_sticks() { + let (mut m, current, old) = versions_ready_to_restore("file.txt"); + let more = m.update(Event::NextVersions); + assert_eq!(more, vec![Effect::History { path: "file.txt".into(), cursor: Some("opaque-cursor".into()), ticket: 1 }]); + assert!(m.update(Event::RestoreVersion).is_empty(), "restore must not start while a page loads"); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.loading && !v.restoring); + assert_eq!(v.ticket, 1, "a refused restore must not take a ticket"); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(!v.loading); + // Not stuck: the confirmation is still open and the restore now goes through. + assert_eq!(v.confirm, Some(old)); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "file.txt".into(), version: old, expected_version: current, ticket: 2 }] + ); + } + + #[test] + fn versions_page_request_during_restore_is_ignored_and_reload_follows() { + let (mut m, current, old) = versions_ready_to_restore("file.txt"); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "file.txt".into(), version: old, expected_version: current, ticket: 1 }] + ); + assert!(m.update(Event::NextVersions).is_empty()); + assert!(m.update(Event::LoadVersions).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.restoring && !v.loading); + let restored = + craft_cloud_app::RestoreResponse { version: old, parent_version: current, seq: 5, epoch: craft_cloud_proto::Epoch::generate() }; + let effects = m.update(Event::Restored { ticket: 1, result: Ok(restored) }); + assert_eq!(effects, vec![Effect::History { path: "file.txt".into(), cursor: None, ticket: 2 }]); + assert!(m.update(Event::History { ticket: 2, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(!v.loading && !v.restoring); + } + + #[test] + fn versions_send_the_path_exactly_as_typed() { + let (mut m, current, old) = versions_ready_to_restore(" Projects/file.txt "); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: " Projects/file.txt ".into(), version: old, expected_version: current, ticket: 1 }] + ); + m.update(Event::Restored { ticket: 1, result: Err(refused(api_codes::CONFLICT, "conflict")) }); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.loading, "the conflict reload is under way"); + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: " Projects/file.txt".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: " Projects/file.txt".into(), cursor: None, ticket: 1 }]); + m.update(Event::VersionsPath(" ".into())); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: " ".into(), cursor: None, ticket: 3 }]); + m.update(Event::VersionsPath(String::new())); + assert!(m.update(Event::LoadVersions).is_empty()); + } + fn refused(code: &str, message: &str) -> Error { - Error::Refused { code: code.into(), message: message.into() } + Error::Refused { code: code.into(), message: message.into(), name_taken: None } } /// A model past the start, on the join form. @@ -1105,8 +2343,32 @@ pub(crate) mod tests { let Screen::SignedIn(o) = &m.screen else { panic!() }; assert_eq!(o.status.account.as_deref(), Some("fabio")); let devices = vec![ - DeviceInfo { id: DeviceId::generate(), name: "lptp1".into(), created_ms: 1, last_used_ms: 2, revoked: false, current: true }, - DeviceInfo { id: DeviceId::generate(), name: "BEAST".into(), created_ms: 1, last_used_ms: 3, revoked: false, current: false }, + DeviceInfo { + id: DeviceId::generate(), + name: "lptp1".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: true, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, + DeviceInfo { + id: DeviceId::generate(), + name: "BEAST".into(), + created_ms: 1, + last_used_ms: 3, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, ]; m.update(Event::Devices(Ok(devices.clone()))); // This computer can't be removed from here; another one can, once at a time. @@ -1308,7 +2570,10 @@ pub(crate) mod tests { // At each start, matched once; again only when it changes. let mut m = model(); m.update(Event::Start); - assert_eq!(m.update(Event::Found(Ok(on.clone()))), vec![Effect::AppAutostart(true), Effect::Devices]); + assert_eq!( + m.update(Event::Found(Ok(on.clone()))), + vec![Effect::AppAutostart(true), Effect::Devices, Effect::Account { session: m.overview().unwrap().session }, Effect::Usage(1)] + ); assert!(m.update(Event::Polled(Ok(on.clone()))).is_empty()); assert_eq!(m.update(Event::Polled(Ok(Status { autostart: Some(false), ..on }))), vec![Effect::AppAutostart(false)]); } diff --git a/apps/craft-cloud-desktop/src/security.rs b/apps/craft-cloud-desktop/src/security.rs new file mode 100644 index 00000000..c97e2be4 --- /dev/null +++ b/apps/craft-cloud-desktop/src/security.rs @@ -0,0 +1,532 @@ +//! The Security page and the second step of a join: the signed-in account's two-factor sign-in (status, setting it up with a +//! secret and a QR code, the recovery codes shown once, turning it off, new recovery codes), the password change and signing +//! the other computers out; and the code field a join asks for when the account has two-factor sign-in. +//! +//! Pure like the rest of the model: [`SecurityPage::update`] takes an event and returns the calls to make, answers come back +//! as [`SecurityAnswer`]s. **Nothing secret is kept longer than it is shown**: the password a form held is moved into its call +//! and cleared, the secret and the recovery codes live in the page's step only, and leaving the page (or a signed-out or +//! removed screen) drops them. Every type that holds one hides it in `Debug`. + +use craft_cloud_app::{Error, RecoveryCodes, RevokedOthers, TwoFactorProofRequest, TwoFactorSetupResponse, TwoFactorStatus, codes}; +use craft_cloud_proto::api::codes as api_codes; + +use crate::model::Overview; + +/// The server doesn't know two-factor sign-in (it is older than this app), or the agent doesn't pass it on. +pub const UNSUPPORTED: &str = "This server doesn't support two-factor sign-in yet"; + +/// What the page knows of the account's two-factor sign-in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TwoFactorInfo { + /// Asking the server. + Loading, + /// The server doesn't have it ([`UNSUPPORTED`]). + Unsupported, + Known(TwoFactorStatus), + /// The status couldn't be read (the message). + Failed(String), +} + +/// Why recovery codes are shown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CodesWhy { + /// Two-factor sign-in was just turned on. + TurnedOn, + /// New codes were made; the old ones don't work. + New, +} + +/// What the page is doing. +#[derive(Clone, PartialEq, Eq)] +pub enum Step { + /// The status and the actions. + Home, + /// Set-up, first: the password. + SetupPassword { password: String }, + /// Set-up, second: the secret to put into the authenticator app, and the first code it shows. + SetupCode { secret: String, uri: String, expires_ms: i64, code: String }, + /// The recovery codes, shown once. + Codes { codes: Vec, why: CodesWhy }, + /// Turning it off: the password, and a code or a recovery code. + Disable { password: String, code: String, recovery: bool }, + /// New recovery codes: the password and a code. + Regenerate { password: String, code: String }, + /// A new password: the current one, the new one twice. + Password { current: String, new: String, again: String }, + /// Signing the other computers out: asked once more. + SignOutOthers, +} + +impl std::fmt::Debug for Step { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Which step, never what was typed or shown in it. + f.write_str(match self { + Step::Home => "Home", + Step::SetupPassword { .. } => "SetupPassword()", + Step::SetupCode { .. } => "SetupCode()", + Step::Codes { .. } => "Codes()", + Step::Disable { .. } => "Disable()", + Step::Regenerate { .. } => "Regenerate()", + Step::Password { .. } => "Password()", + Step::SignOutOthers => "SignOutOthers", + }) + } +} + +/// The Security page. +#[derive(Debug, Clone, PartialEq)] +pub struct SecurityPage { + /// The signed-in view the page was opened from and returns to. + pub overview: Overview, + pub info: TwoFactorInfo, + pub step: Step, + /// What went wrong with the last try, for people. + pub error: Option, + /// What the last action did ("Password changed"). + pub note: Option, + /// A call is under way: nothing can be sent twice. + pub busy: bool, +} + +/// What a button of the page asks for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecurityEvent { + /// Back to the signed-in view; whatever the page showed is dropped. + Close, + /// Back to the page's start (the recovery codes shown are dropped). + Home, + StartSetup, + StartDisable, + StartRegenerate, + StartPassword, + StartSignOutOthers, + /// The current step's button: continue, turn on, turn off, make, change, sign out. + Submit, + /// Disable's switch between a code and a recovery code. + ToggleRecovery, + /// Ask the server for the status again. + Refresh, +} + +/// A call of the agent the page makes. +#[derive(Clone, PartialEq, Eq)] +pub enum SecurityCall { + Status, + Setup { password: String }, + Confirm { code: String }, + Disable(TwoFactorProofRequest), + RecoveryCodes { password: String, code: String }, + ChangePassword { current: String, new: String }, + SignOutOthers, +} + +impl std::fmt::Debug for SecurityCall { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SecurityCall::Status => "Status", + SecurityCall::Setup { .. } => "Setup()", + SecurityCall::Confirm { .. } => "Confirm()", + SecurityCall::Disable(_) => "Disable()", + SecurityCall::RecoveryCodes { .. } => "RecoveryCodes()", + SecurityCall::ChangePassword { .. } => "ChangePassword()", + SecurityCall::SignOutOthers => "SignOutOthers", + }) + } +} + +/// An agent's answer to a [`SecurityCall`]. +#[derive(Clone, PartialEq)] +pub enum SecurityAnswer { + Status(Result), + Setup(Result), + Confirm(Result), + Disable(Result<(), Error>), + RecoveryCodes(Result), + ChangePassword(Result<(), Error>), + SignOutOthers(Result), +} + +impl std::fmt::Debug for SecurityAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // An answer may hold a secret or recovery codes: only whether it worked. + let (name, ok) = match self { + SecurityAnswer::Status(r) => ("Status", r.is_ok()), + SecurityAnswer::Setup(r) => ("Setup", r.is_ok()), + SecurityAnswer::Confirm(r) => ("Confirm", r.is_ok()), + SecurityAnswer::Disable(r) => ("Disable", r.is_ok()), + SecurityAnswer::RecoveryCodes(r) => ("RecoveryCodes", r.is_ok()), + SecurityAnswer::ChangePassword(r) => ("ChangePassword", r.is_ok()), + SecurityAnswer::SignOutOthers(r) => ("SignOutOthers", r.is_ok()), + }; + write!(f, "{name}({})", if ok { "ok" } else { "refused" }) + } +} + +/// Six digits (spaces allowed), the shape of a code from the authenticator app. +pub fn is_code(text: &str) -> bool { + let digits: String = text.chars().filter(|c| !c.is_whitespace()).collect(); + digits.len() == 6 && digits.bytes().all(|b| b.is_ascii_digit()) +} + +pub fn digits(text: &str) -> String { + text.chars().filter(|c| !c.is_whitespace()).collect() +} + +/// A wait in words. +fn wait_in_words(secs: u64) -> String { + match secs { + 0..=90 => format!("{secs} seconds"), + 91..=5_400 => format!("{} minutes", secs.div_ceil(60)), + _ => format!("{} hours", secs.div_ceil(3_600)), + } +} + +/// What a refusal says, for people, and what to do about it. +pub fn explain(e: &Error) -> String { + let wait = e.retry_after_secs().map(wait_in_words); + match e.code() { + Some(codes::WRONG_PASSWORD) => "That password isn't right.".into(), + Some(codes::TWO_FACTOR_INVALID) => "That code isn't right, or it was used already. Wait for the app's next code and try again.".into(), + Some(codes::TWO_FACTOR_LOCKED) => match wait { + Some(wait) => format!("Too many wrong codes. Try again in {wait}."), + None => "Too many wrong codes. Try again later.".into(), + }, + Some(codes::TWO_FACTOR_UNAVAILABLE) => "This server can't set up two-factor sign-in yet: its admin has to give it a key first.".into(), + Some(codes::TWO_FACTOR_PENDING) => "Setting it up is open on another computer. Finish it there, or wait ten minutes.".into(), + Some(codes::OFFLINE) => "Can't reach the server. Try again when this computer is online.".into(), + Some(craft_cloud_app::codes::NEEDS_YOU) => "This computer can't use the account any more. Sign out, then join again.".into(), + Some("rate_limited") => match wait { + Some(wait) => format!("Too many tries. Try again in {wait}."), + None => "Too many tries. Try again in a few minutes.".into(), + }, + _ => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), + other => other.to_string(), + }, + } +} + +impl SecurityPage { + /// The page, opened from `overview`; its first call asks the server for the status. + pub fn open(overview: Overview) -> (Self, Vec) { + (Self { overview, info: TwoFactorInfo::Loading, step: Step::Home, error: None, note: None, busy: false }, vec![SecurityCall::Status]) + } + + /// Whether two-factor sign-in is known to be on. + pub fn is_on(&self) -> bool { + matches!(&self.info, TwoFactorInfo::Known(s) if s.enabled) + } + + fn start(&mut self, step: Step) { + self.step = step; + self.error = None; + self.note = None; + } + + /// A button of the page. Returns the calls to make; the page is `busy` while one is under way. + pub fn update(&mut self, event: SecurityEvent) -> Vec { + match event { + SecurityEvent::Close => Vec::new(), + SecurityEvent::Home => { + if !self.busy { + self.start(Step::Home); + } + Vec::new() + } + SecurityEvent::Refresh => { + self.info = TwoFactorInfo::Loading; + vec![SecurityCall::Status] + } + SecurityEvent::StartSetup => { + if !self.busy && !self.is_on() && matches!(self.info, TwoFactorInfo::Known(_)) { + self.start(Step::SetupPassword { password: String::new() }); + } + Vec::new() + } + SecurityEvent::StartDisable => { + if !self.busy && self.is_on() { + self.start(Step::Disable { password: String::new(), code: String::new(), recovery: false }); + } + Vec::new() + } + SecurityEvent::StartRegenerate => { + if !self.busy && self.is_on() { + self.start(Step::Regenerate { password: String::new(), code: String::new() }); + } + Vec::new() + } + SecurityEvent::StartPassword => { + if !self.busy { + self.start(Step::Password { current: String::new(), new: String::new(), again: String::new() }); + } + Vec::new() + } + SecurityEvent::StartSignOutOthers => { + if !self.busy { + self.start(Step::SignOutOthers); + } + Vec::new() + } + SecurityEvent::ToggleRecovery => { + if let Step::Disable { code, recovery, .. } = &mut self.step { + *recovery = !*recovery; + code.clear(); + self.error = None; + } + Vec::new() + } + SecurityEvent::Submit => self.submit(), + } + } + + /// The current step's call. A step that isn't filled in says what is missing and calls nothing; the typed password moves into + /// the call and is cleared from the form. + fn submit(&mut self) -> Vec { + if self.busy { + return Vec::new(); + } + self.error = None; + let call = match &mut self.step { + Step::SetupPassword { password } => { + if password.is_empty() { + self.error = Some("Type your password to start.".into()); + return Vec::new(); + } + SecurityCall::Setup { password: std::mem::take(password) } + } + Step::SetupCode { code, .. } => { + if !is_code(code) { + self.error = Some("Type the six digits the app shows.".into()); + return Vec::new(); + } + SecurityCall::Confirm { code: digits(&std::mem::take(code)) } + } + Step::Disable { password, code, recovery } => { + if password.is_empty() || code.trim().is_empty() { + self.error = Some("Type your password and a code (or a recovery code).".into()); + return Vec::new(); + } + if !*recovery && !is_code(code) { + self.error = Some("A code from the app is six digits.".into()); + return Vec::new(); + } + let typed = std::mem::take(code); + let password = std::mem::take(password); + if *recovery { + SecurityCall::Disable(TwoFactorProofRequest { password, totp_code: None, recovery_code: Some(typed.trim().to_owned()) }) + } else { + SecurityCall::Disable(TwoFactorProofRequest { password, totp_code: Some(digits(&typed)), recovery_code: None }) + } + } + Step::Regenerate { password, code } => { + if password.is_empty() || !is_code(code) { + self.error = Some("Type your password and the six digits the app shows.".into()); + return Vec::new(); + } + SecurityCall::RecoveryCodes { password: std::mem::take(password), code: digits(&std::mem::take(code)) } + } + Step::Password { current, new, again } => { + if current.is_empty() || new.is_empty() { + self.error = Some("Type your current password and the new one.".into()); + return Vec::new(); + } + if new != again { + self.error = Some("The two new passwords aren't the same.".into()); + return Vec::new(); + } + SecurityCall::ChangePassword { current: std::mem::take(current), new: std::mem::take(new) } + } + Step::SignOutOthers => SecurityCall::SignOutOthers, + Step::Home | Step::Codes { .. } => return Vec::new(), + }; + // A form that goes on with a password or a code is emptied for good: the typed ones live in the call now. + if let Step::Password { again, .. } = &mut self.step { + again.clear(); + } + self.busy = true; + vec![call] + } + + /// An answer. May ask for more (the status, after a change). + pub fn answer(&mut self, answer: SecurityAnswer, now_ms: i64) -> Vec { + match answer { + SecurityAnswer::Status(result) => { + self.info = match result { + Ok(status) => TwoFactorInfo::Known(status), + Err(e) if e.code() == Some(api_codes::NOT_FOUND) || e.code() == Some(codes::AGENT_TOO_OLD) => TwoFactorInfo::Unsupported, + Err(e) => TwoFactorInfo::Failed(explain(&e)), + }; + Vec::new() + } + SecurityAnswer::Setup(result) => { + self.busy = false; + match result { + Ok(setup) => { + self.step = + Step::SetupCode { secret: setup.secret, uri: setup.otpauth_uri, expires_ms: setup.expires_ms, code: String::new() }; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::Confirm(result) => { + self.busy = false; + match result { + Ok(codes) => { + self.step = Step::Codes { codes: codes.recovery_codes, why: CodesWhy::TurnedOn }; + self.note = Some("Two-factor sign-in is on. Your other computers were signed out.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => { + // A set-up that ran out or was used up (five wrong codes) has to start again. + if e.code() == Some(codes::CONFLICT) { + self.step = Step::Home; + } + self.error = Some(explain(&e)); + // Past its ten minutes: the secret is of no use. + if matches!(&self.step, Step::SetupCode { expires_ms, .. } if now_ms >= *expires_ms) { + self.step = Step::Home; + } + } + } + Vec::new() + } + SecurityAnswer::Disable(result) => { + self.busy = false; + match result { + Ok(()) => { + self.step = Step::Home; + self.note = Some("Two-factor sign-in is off. Your other computers were signed out.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::RecoveryCodes(result) => { + self.busy = false; + match result { + Ok(codes) => { + self.step = Step::Codes { codes: codes.recovery_codes, why: CodesWhy::New }; + self.note = Some("New recovery codes. The old ones don't work any more.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::ChangePassword(result) => { + self.busy = false; + match result { + Ok(()) => { + self.step = Step::Home; + self.note = Some("Password changed. Your other computers were signed out.".into()); + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::SignOutOthers(result) => { + self.busy = false; + match result { + Ok(done) => { + self.step = Step::Home; + self.note = Some(match done.revoked { + 0 => "No other computer was signed in.".to_owned(), + 1 => "Signed 1 other computer out.".to_owned(), + n => format!("Signed {n} other computers out."), + }); + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + } + } +} + +/// The second step of a join: the account has two-factor sign-in, so the code comes with the same join. +#[derive(Clone, PartialEq, Eq)] +pub struct SecondFactorForm { + pub form: crate::model::JoinForm, + /// The new account's password, when the join came by an invite that needed one (an account that has two-factor sign-in has + /// one already, so this is only ever `None` for it). + pub password: Option, + /// What was typed: a code from the app, or a recovery code. + pub code: String, + /// The field takes a recovery code instead. + pub recovery: bool, + /// What went wrong with the last try, for people. + pub error: Option, +} + +impl std::fmt::Debug for SecondFactorForm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorForm") + .field("form", &self.form) + .field("recovery", &self.recovery) + .field("error", &self.error) + .finish_non_exhaustive() + } +} + +impl SecondFactorForm { + pub fn new(form: crate::model::JoinForm, password: Option, error: Option) -> Self { + Self { form, password, code: String::new(), recovery: false, error } + } + + /// The code as the join takes it: `(totp_code, recovery_code)`, or what is wrong with what was typed. + pub fn factor(&self) -> Result<(Option, Option), String> { + if self.recovery { + let typed = self.code.trim(); + if typed.is_empty() { + return Err("Type a recovery code.".into()); + } + return Ok((None, Some(typed.to_owned()))); + } + if !is_code(&self.code) { + return Err("Type the six digits the app shows.".into()); + } + Ok((Some(digits(&self.code)), None)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_code_is_six_digits_with_spaces_allowed() { + assert!(is_code("123456") && is_code(" 123 456 ")); + for bad in ["12345", "1234567", "12345a", "", "١٢٣٤٥٦"] { + assert!(!is_code(bad), "{bad:?}"); + } + } + + #[test] + fn refusals_are_explained_for_people() { + let refused = |code: &str| Error::Refused { code: code.into(), message: "m".into(), name_taken: None }; + assert!(explain(&refused("wrong_password")).contains("password isn't right")); + assert!(explain(&refused("two_factor_invalid")).contains("next code")); + let locked = Error::RefusedWith { code: "two_factor_locked".into(), message: "m".into(), two_factor: None, retry_after_secs: Some(840) }; + assert_eq!(explain(&locked), "Too many wrong codes. Try again in 14 minutes."); + assert!(explain(&refused("two_factor_unavailable")).contains("admin has to give it a key")); + assert_eq!(explain(&refused("weak_password")), "m", "an unknown code says what the server said"); + } + + #[test] + fn secrets_stay_out_of_debug_output() { + let step = Step::SetupCode { secret: "JBSWY3DPEHPK3PXP".into(), uri: "otpauth://totp/x".into(), expires_ms: 1, code: "918273".into() }; + let codes = Step::Codes { codes: vec!["ABCD-EFGH-JKMN-PQRS".into()], why: CodesWhy::TurnedOn }; + let password = Step::Disable { password: "pw-secret-1".into(), code: "918273".into(), recovery: false }; + for shown in [format!("{step:?}"), format!("{codes:?}"), format!("{password:?}")] { + assert!(!shown.contains("JBSWY") && !shown.contains("ABCD") && !shown.contains("pw-secret") && !shown.contains("918273"), "{shown}"); + } + let call = SecurityCall::Setup { password: "pw-secret-1".into() }; + assert!(!format!("{call:?}").contains("pw-secret")); + let answer = SecurityAnswer::Confirm(Ok(RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into()] })); + assert!(!format!("{answer:?}").contains("ABCD")); + let form = SecondFactorForm { form: Default::default(), password: None, code: "918273".into(), recovery: false, error: None }; + assert!(!format!("{form:?}").contains("918273")); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/add_computer.rs b/apps/craft-cloud-desktop/src/ui/add_computer.rs index 5e4a3c23..230fb672 100644 --- a/apps/craft-cloud-desktop/src/ui/add_computer.rs +++ b/apps/craft-cloud-desktop/src/ui/add_computer.rs @@ -65,7 +65,7 @@ pub(super) fn show(ui: &mut egui::Ui, adding: &Adding, now_ms: i64, events: &mut /// `text` as a QR code `side` points wide: dark modules on white with the standard quiet zone, whatever the theme /// (cameras read it that way round). -fn qr(ui: &mut egui::Ui, text: &str, side: f32) { +pub(super) fn qr(ui: &mut egui::Ui, text: &str, side: f32) { let Ok(code) = QrCode::encode_text(text, QrCodeEcc::Medium) else { return; }; diff --git a/apps/craft-cloud-desktop/src/ui/admin.rs b/apps/craft-cloud-desktop/src/ui/admin.rs new file mode 100644 index 00000000..898e763e --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/admin.rs @@ -0,0 +1,502 @@ +//! The admin pages: the users (search, pages), a new account, one account (role, quota, disable, invites, devices), +//! the password a change asks for, and the code a change made, shown once. + +use egui::RichText; + +use crate::admin::{Action, AdminEvent, AdminScreen, CodeKind, CreateForm, Dialog, RoleChoice, ShownCode, UserList, UserView, View, format_bytes}; +use craft_cloud_app::AdminUserInfo; + +use crate::model::Event; + +struct StorageState<'a> { + overview: Option<&'a craft_cloud_app::AdminStorageOverview>, + loading: bool, + error: Option<&'a str>, +} + +/// The name of the confirmation page's password field, for [`super::security::secret_id`] and the forgetting of its text. +pub(super) const PASSWORD_FIELD: &str = "admin-password"; +/// The name of its field for a code (from the authenticator app, or a recovery code). +pub(super) const CODE_FIELD: &str = "admin-code"; + +pub(super) fn show(ui: &mut egui::Ui, admin: &mut AdminScreen, now_ms: i64, events: &mut Vec) { + let me = admin.overview.status.account.clone().unwrap_or_default(); + // A code just made comes first, then the password a change asks for, then the page. + if let Some(code) = &admin.code { + return shown_code(ui, code, now_ms, events); + } + if let Some(dialog) = &mut admin.dialog { + return confirmation(ui, dialog, events); + } + let server = admin.server.clone(); + match &mut admin.view { + View::List => users( + ui, + server.as_ref(), + StorageState { overview: admin.storage.as_ref(), loading: admin.storage_loading, error: admin.storage_error.as_deref() }, + &mut admin.list, + &me, + events, + ), + View::Create(form) => create(ui, form, events), + View::Audit(view) => super::audit::log(ui, view, now_ms, events), + View::User(view) => { + let user = admin.list.users.iter().find(|u| u.id == view.id).cloned(); + account(ui, view, user.as_ref(), &me, now_ms, events); + } + } +} + +fn send(events: &mut Vec, event: AdminEvent) { + events.push(Event::Admin(event)); +} + +/// "Enter" in the field just edited. +fn entered(response: &egui::Response, ui: &egui::Ui) -> bool { + response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)) +} + +/// What a user is, in a line: role, computers, what they store, and the state of the account. +fn describe(user: &AdminUserInfo) -> String { + let mut parts = vec![if user.role == "admin" { "Admin".to_owned() } else { "Member".to_owned() }]; + parts.push(match user.devices_active { + 1 => "1 computer".to_owned(), + n => format!("{n} computers"), + }); + parts.push(match user.quota_bytes { + Some(quota) => format!("{} of {}", format_bytes(user.stored_bytes), format_bytes(quota)), + None => format_bytes(user.stored_bytes), + }); + if user.disabled_ms.is_some() { + parts.push("disabled".to_owned()); + } + if user.invite_pending { + parts.push("invite open".to_owned()); + } + if user.two_factor_enabled { + parts.push("two-factor on".to_owned()); + } + parts.join(" · ") +} + +fn users( + ui: &mut egui::Ui, + server: Option<&craft_cloud_app::AdminServerInfo>, + storage: StorageState<'_>, + list: &mut UserList, + me: &str, + events: &mut Vec, +) { + super::heading(ui, "Administration"); + if let Some(info) = server { + ui.label( + RichText::new(format!( + "Craft Cloud server {} · {} accounts · {} computers · {} stored ({}, {})", + info.version, + info.users, + info.devices_active, + format_bytes(info.stored_bytes), + info.storage_kind, + info.database_kind + )) + .small() + .weak(), + ); + ui.add_space(8.0); + } + storage_overview(ui, storage); + ui.add_space(8.0); + ui.horizontal(|ui| { + if ui.button("Back").clicked() { + send(events, AdminEvent::Close); + } + if ui.button("New account").clicked() { + send(events, AdminEvent::NewUser); + } + if ui.add_enabled(!list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AdminEvent::Reload); + } + if ui.button("Audit log").on_hover_text("Sign-ins, changes and what admins did").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenAudit(None))); + } + }); + ui.add_space(8.0); + + ui.horizontal(|ui| { + let width = (ui.available_width() - 130.0).max(120.0); + let field = ui.add(egui::TextEdit::singleline(&mut list.search).hint_text("Search accounts").desired_width(width)); + if ui.button("Search").clicked() || entered(&field, ui) { + send(events, AdminEvent::Search); + } + if !list.applied.is_empty() && ui.button("Clear").clicked() { + send(events, AdminEvent::ClearSearch); + } + }); + ui.add_space(8.0); + + if let Some(error) = &list.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if list.loading && !list.loaded { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } else if list.users.is_empty() && list.loaded && list.error.is_none() { + ui.label(if list.applied.is_empty() { "No accounts." } else { "No account has that in its name." }); + } + for user in list.users.clone() { + ui.push_id(user.id, |ui| { + ui.horizontal(|ui| { + ui.vertical(|ui| { + let name = if user.username == me { format!("{} (you)", user.username) } else { user.username.clone() }; + ui.label(RichText::new(name).strong()); + ui.add(egui::Label::new(RichText::new(describe(&user)).small().weak()).wrap()); + }); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if ui.button("Manage").clicked() { + send(events, AdminEvent::OpenUser(user.id)); + } + }); + }); + ui.separator(); + }); + } + if list.has_previous() || list.next.is_some() { + ui.horizontal(|ui| { + if ui.add_enabled(list.has_previous() && !list.loading, egui::Button::new("Previous")).clicked() { + send(events, AdminEvent::PreviousPage); + } + ui.label(format!("Page {}", list.page())); + if ui.add_enabled(list.next.is_some() && !list.loading, egui::Button::new("Next")).clicked() { + send(events, AdminEvent::NextPage); + } + }); + } +} + +fn storage_overview(ui: &mut egui::Ui, state: StorageState<'_>) { + ui.label(RichText::new("Storage").strong()); + if let Some(error) = state.error { + super::error_line(ui, error); + return; + } + let Some(storage) = state.overview else { + if state.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + return; + }; + ui.label(format!( + "{} stored · {} accounts · default quota {}", + format_bytes(storage.stored_bytes), + storage.accounts, + storage.default_quota_bytes.map_or_else(|| "none".to_owned(), format_bytes) + )); + ui.label(format!( + "{} near quota · {} over quota · {} with a quota of their own", + storage.accounts_near_quota, storage.accounts_over_quota, storage.accounts_with_override + )); + ui.label( + RichText::new(format!( + "Manifest index: {} complete · {} building · {} not indexed", + storage.manifest_index.complete, storage.manifest_index.building, storage.manifest_index.not_indexed + )) + .small() + .weak(), + ); + if storage.manifest_index.building > 0 || storage.manifest_index.not_indexed > 0 { + ui.label(RichText::new("Storage estimates are incomplete; run `craft-cloud-server index build`. ").small().weak()); + } + if !storage.top_accounts.is_empty() { + ui.label(RichText::new("Top accounts").small().strong()); + egui::Grid::new("admin-storage-top").num_columns(3).striped(true).spacing([12.0, 4.0]).show(ui, |ui| { + ui.label(RichText::new("Account").small().weak()); + ui.label(RichText::new("Stored").small().weak()); + ui.label(RichText::new("Effective quota").small().weak()); + ui.end_row(); + for account in &storage.top_accounts { + ui.label(&account.username); + ui.label(format_bytes(account.stored_bytes)); + ui.label(account.quota_bytes.map_or_else(|| "none".to_owned(), format_bytes)); + ui.end_row(); + } + }); + } + ui.separator(); +} + +fn role_choice(ui: &mut egui::Ui, role: &mut RoleChoice) { + ui.horizontal(|ui| { + ui.radio_value(role, RoleChoice::Member, RoleChoice::Member.label()); + ui.radio_value(role, RoleChoice::Admin, RoleChoice::Admin.label()); + }); +} + +fn create(ui: &mut egui::Ui, form: &mut CreateForm, events: &mut Vec) { + super::heading(ui, "New account"); + ui.label("Name"); + let name = ui.add(egui::TextEdit::singleline(&mut form.username).hint_text("ana").desired_width(f32::INFINITY)); + let enter = entered(&name, ui); + ui.add_space(8.0); + ui.label("Role"); + role_choice(ui, &mut form.role); + ui.add_space(6.0); + ui.add( + egui::Label::new( + RichText::new( + "The account starts without a password: you get an invite code to give them. Set a quota afterwards, on the account's page.", + ) + .small() + .weak(), + ) + .wrap(), + ); + ui.add_space(10.0); + if let Some(error) = &form.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if super::wide_button(ui, true, "Continue") || enter { + send(events, AdminEvent::SubmitCreate); + } + ui.add_space(4.0); + if ui.button("Cancel").clicked() { + send(events, AdminEvent::BackToList); + } +} + +fn account(ui: &mut egui::Ui, view: &mut UserView, user: Option<&AdminUserInfo>, me: &str, now_ms: i64, events: &mut Vec) { + super::heading(ui, &view.username); + if ui.button("Back to the accounts").clicked() { + send(events, AdminEvent::BackToList); + } + ui.add_space(8.0); + let Some(user) = user else { + super::error_line(ui, "This account isn't in the list any more."); + return; + }; + ui.label(RichText::new(describe(user)).small().weak()); + ui.add_space(10.0); + let is_me = user.username == me; + + ui.horizontal(|ui| { + ui.label(RichText::new("Storage usage").strong()); + if ui.small_button("Refresh").clicked() { + send(events, AdminEvent::RefreshUsage); + } + }); + if let Some(error) = &view.usage_error { + super::error_line(ui, error); + } else if let Some(usage) = &view.usage { + ui.label(format!("Stored: {}", format_bytes(usage.stored_bytes))); + ui.label(format!("Effective quota: {}", usage.quota_bytes.map_or_else(|| "none".to_owned(), format_bytes))); + if usage.estimates.index_complete { + ui.label(format!("Trash estimate (overlapping): {}", usage.trash_bytes.map_or_else(|| "not available".to_owned(), format_bytes))); + ui.label(format!("History estimate (overlapping): {}", usage.history_bytes.map_or_else(|| "not available".to_owned(), format_bytes))); + } else { + ui.label("estimates not available yet"); + } + } else if view.usage_loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + ui.add_space(8.0); + + if ui.button("Activity of this account").on_hover_text("The audit log, for this account").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenAudit(Some((user.id, user.username.clone()))))); + } + ui.add_space(8.0); + + ui.label(RichText::new("Role and quota").strong()); + role_choice(ui, &mut view.role); + ui.horizontal(|ui| { + ui.label("Quota (GiB)"); + ui.add(egui::TextEdit::singleline(&mut view.quota).hint_text("none").desired_width(100.0)); + }); + if let Some(error) = &view.error { + super::error_line(ui, error); + } + if ui.button("Save changes").clicked() { + send(events, AdminEvent::SubmitEdit); + } + ui.add_space(12.0); + + ui.label(RichText::new("Account").strong()); + let id = user.id; + let username = user.username.clone(); + ui.horizontal_wrapped(|ui| { + if is_me { + ui.label(RichText::new("You can't disable or remove your own account here.").small().weak()); + } else if user.disabled_ms.is_some() { + if ui.button("Enable").clicked() { + send(events, AdminEvent::Ask(Action::Enable { id, username: username.clone() })); + } + } else if ui.button("Disable").clicked() { + send(events, AdminEvent::Ask(Action::Disable { id, username: username.clone() })); + } + if !is_me && ui.button("Remove").clicked() { + send(events, AdminEvent::Ask(Action::Remove { id, username: username.clone() })); + } + if !is_me + && user.two_factor_enabled + && ui.button("Turn two-factor off").on_hover_text("They sign in with their password alone; their computers are signed out").clicked() + { + send(events, AdminEvent::Ask(Action::ResetTwoFactor { id, username: username.clone() })); + } + }); + ui.add_space(12.0); + + ui.label(RichText::new("Invites and codes").strong()); + ui.horizontal_wrapped(|ui| { + if ui.button("New invite").on_hover_text("For an account that has no password yet").clicked() { + send(events, AdminEvent::Ask(Action::IssueInvite { id, username: username.clone() })); + } + if ui.button("Password reset code").on_hover_text("Good for an hour; using it signs all their computers out").clicked() { + send(events, AdminEvent::Ask(Action::IssueReset { id, username: username.clone(), reset_two_factor: false })); + } + if user.two_factor_enabled + && ui.button("Reset code, two-factor off").on_hover_text("As a reset code, and using it turns their two-factor sign-in off").clicked() + { + send(events, AdminEvent::Ask(Action::IssueReset { id, username: username.clone(), reset_two_factor: true })); + } + // Always offered: the list says an invite is open (`invite_pending`), but not a password reset code. The server + // answers `not_found` when none is. + if ui.button("Revoke open code").on_hover_text("Ends the account's open invite or password reset code").clicked() { + send(events, AdminEvent::Ask(Action::RevokeInvite { id, username: username.clone() })); + } + }); + ui.add_space(12.0); + + ui.horizontal(|ui| { + ui.label(RichText::new("Computers").strong()); + if ui.small_button("Refresh").clicked() { + send(events, AdminEvent::RefreshDevices); + } + }); + if let Some(error) = &view.devices_error { + super::error_line(ui, error); + } + let Some(devices) = &view.devices else { + if view.devices_error.is_none() { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + return; + }; + if devices.iter().all(|d| d.revoked) { + ui.label(RichText::new("No computers signed in.").weak()); + } + egui::Grid::new("admin-devices").num_columns(3).striped(true).spacing([12.0, 6.0]).show(ui, |ui| { + for device in devices.iter().filter(|d| !d.revoked) { + ui.label(&device.name); + ui.label(RichText::new(format!("last seen {}", super::ago(now_ms, device.last_used_ms))).weak()); + if ui.button("Revoke").on_hover_text("It can't sync any more; its files stay there").clicked() { + let action = Action::RevokeDevice { id, username: username.clone(), device: device.id, device_name: device.name.clone() }; + send(events, AdminEvent::Ask(action)); + } + ui.end_row(); + } + }); +} + +/// The password a change asks for. +fn confirmation(ui: &mut egui::Ui, dialog: &mut Dialog, events: &mut Vec) { + super::heading(ui, &dialog.action.title()); + ui.add(egui::Label::new(dialog.action.explain()).wrap()); + ui.add_space(10.0); + ui.label("Your password"); + let field = ui.add_enabled( + !dialog.busy, + egui::TextEdit::singleline(&mut dialog.confirmation.password) + .id(super::security::secret_id(PASSWORD_FIELD)) + .password(true) + .hint_text("your own current password") + .desired_width(f32::INFINITY), + ); + let mut enter = entered(&field, ui); + if dialog.needs_code { + ui.add_space(6.0); + ui.label("Code from your authenticator app, or a recovery code"); + let code = ui.add_enabled( + !dialog.busy, + egui::TextEdit::singleline(&mut dialog.confirmation.code) + .id(super::security::secret_id(CODE_FIELD)) + .password(true) + .hint_text("123456") + .desired_width(f32::INFINITY), + ); + enter |= entered(&code, ui); + } + ui.add_space(10.0); + if let Some(error) = &dialog.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if dialog.busy { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + return; + } + if super::wide_button(ui, true, dialog.action.button()) || enter { + send(events, AdminEvent::Confirm); + } + ui.add_space(4.0); + if ui.button("Cancel").clicked() { + send(events, AdminEvent::CancelDialog); + } +} + +/// How long until `at_ms`, from `now_ms`: "in 7 days", "in 59 min". +fn until(now_ms: i64, at_ms: i64) -> String { + let secs = at_ms.saturating_sub(now_ms).max(0) / 1000; + match secs { + 0..60 => "in under a minute".into(), + 60..3_600 => format!("in {} min", secs / 60), + 3_600..86_400 => format!("in {} h", secs / 3_600), + _ => format!("in {} days", secs / 86_400), + } +} + +/// A code the server made: shown once, to copy and hand over. +fn shown_code(ui: &mut egui::Ui, code: &ShownCode, now_ms: i64, events: &mut Vec) { + let title = match code.kind { + CodeKind::Invite => format!("Invite for {}", code.username), + CodeKind::PasswordReset => format!("Password reset code for {}", code.username), + }; + super::heading(ui, &title); + super::error_line(ui, "Note it now: it is shown once and can't be shown again."); + ui.add_space(8.0); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(&code.code).monospace().size(18.0).strong()).selectable(true)); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(code.code.clone()); + } + }); + if let Some(url) = &code.join_url { + ui.add_space(8.0); + ui.label("Or the whole join string:"); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(url).monospace()).selectable(true).wrap()); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(url.clone()); + } + }); + } + ui.add_space(8.0); + ui.label(RichText::new(format!("Stops working {}.", until(now_ms, code.expires_ms))).weak()); + ui.add_space(14.0); + if super::wide_button(ui, true, "Done") { + send(events, AdminEvent::DismissCode); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/audit.rs b/apps/craft-cloud-desktop/src/ui/audit.rs new file mode 100644 index 00000000..7416ffb8 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/audit.rs @@ -0,0 +1,175 @@ +//! The audit log's pages: the admin's view (a filter by event and by account, pages, an export to a file) and the account's own +//! activity. Both show events as words, with the typed details the server writes and nothing else. + +use egui::RichText; + +use crate::admin::AdminEvent; +use crate::audit::{ActivityScreen, AuditList, AuditMsg, AuditView, ExportState, action_label, details_line, filter_choices, filter_label, who_line}; +use crate::model::Event; +use craft_cloud_proto::api::AuditEvent; + +fn send(events: &mut Vec, msg: AuditMsg) { + events.push(Event::Audit(msg)); +} + +/// The events of a page: what happened, how long ago, and its details. +fn rows(ui: &mut egui::Ui, list: &AuditList, now_ms: i64) { + if let Some(error) = &list.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if list.loading && !list.loaded { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } else if list.events.is_empty() && list.loaded && list.error.is_none() { + ui.label("Nothing was recorded."); + } + for event in &list.events { + event_row(ui, event, now_ms); + } +} + +fn event_row(ui: &mut egui::Ui, event: &AuditEvent, now_ms: i64) { + ui.push_id(event.id, |ui| { + let label = RichText::new(action_label(&event.action)).strong(); + ui.add(egui::Label::new(if event.outcome == "failure" { label.color(ui.visuals().error_fg_color) } else { label }).wrap()); + let mut line = super::ago(now_ms, event.at_ms); + let details = details_line(event); + if !details.is_empty() { + line.push_str(" · "); + line.push_str(&details); + } + ui.add(egui::Label::new(RichText::new(line).small().weak()).wrap()); + // Who did it and what it was about, with the whole IDs on hover. + let (who, full) = who_line(event); + ui.add(egui::Label::new(RichText::new(who).small().weak()).wrap().sense(egui::Sense::hover())).on_hover_text(full); + ui.separator(); + }); +} + +/// "Previous / Page n / Next", sending `previous` and `next`. +fn paging(ui: &mut egui::Ui, list: &AuditList, previous: AuditMsg, next: AuditMsg, events: &mut Vec) { + if !list.has_previous() && list.next.is_none() { + return; + } + ui.horizontal(|ui| { + if ui.add_enabled(list.has_previous() && !list.loading, egui::Button::new("Previous")).clicked() { + send(events, previous); + } + ui.label(format!("Page {}", list.page())); + if ui.add_enabled(list.next.is_some() && !list.loading, egui::Button::new("Next")).clicked() { + send(events, next); + } + }); +} + +/// The admin's audit log. +pub(super) fn log(ui: &mut egui::Ui, view: &mut AuditView, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Audit log"); + ui.horizontal(|ui| { + if ui.button("Back to the accounts").clicked() { + events.push(Event::Admin(AdminEvent::BackToList)); + } + if ui.add_enabled(!view.list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AuditMsg::Reload); + } + }); + ui.add_space(8.0); + + // The filters: an event (the server's closed list, by group or by name) and the account the page was opened for. + ui.horizontal(|ui| { + ui.label("Show"); + let shown = view.filter.action.as_deref().map_or_else(|| "All events".to_owned(), filter_label); + egui::ComboBox::from_id_salt("audit-action").selected_text(shown).width(220.0).show_ui(ui, |ui| { + if ui.selectable_label(view.filter.action.is_none(), "All events").clicked() && view.filter.action.is_some() { + send(events, AuditMsg::SetAction(None)); + } + for choice in filter_choices() { + let selected = view.filter.action.as_deref() == Some(choice); + let text = if choice.ends_with('.') { format!("{} (all)", filter_label(choice)) } else { filter_label(choice) }; + if ui.selectable_label(selected, text).clicked() && !selected { + send(events, AuditMsg::SetAction(Some(choice.to_owned()))); + } + } + }); + }); + if let Some((_, name)) = &view.filter.user { + ui.horizontal(|ui| { + ui.label(format!("Only events about {name}")); + if ui.small_button("Show everyone").clicked() { + send(events, AuditMsg::ClearUser); + } + }); + } + ui.add_space(6.0); + + // The export. + ui.horizontal(|ui| { + let idle = !matches!(view.export, ExportState::Picking { .. } | ExportState::Running { .. }); + if ui.add_enabled(idle, egui::Button::new("Export…")).on_hover_text("Save what these filters show, all of it, as a file").clicked() { + send(events, AuditMsg::Export); + } + match &view.export { + ExportState::Idle => {} + ExportState::Picking { .. } => { + ui.label(RichText::new("Choose where to save it…").weak()); + } + ExportState::Running { .. } => { + ui.spinner(); + ui.label("Saving…"); + } + ExportState::Done { .. } | ExportState::Failed(_) => { + if ui.small_button("OK").clicked() { + send(events, AuditMsg::DismissExport); + } + } + } + }); + match &view.export { + ExportState::Done { path, outcome } if outcome.truncated => { + super::error_line( + ui, + &format!( + "Only the newest {} events were saved to {path}: the log holds more than an export can. Narrow the filters to get the rest.", + outcome.events + ), + ); + } + ExportState::Done { path, outcome } => { + ui.label(RichText::new(format!("Saved {} events to {path}.", outcome.events)).color(super::good(ui))); + } + ExportState::Failed(why) => super::error_line(ui, &format!("The export failed, and no file was kept: {why}")), + _ => {} + } + ui.add_space(8.0); + + rows(ui, &view.list, now_ms); + paging(ui, &view.list, AuditMsg::Previous, AuditMsg::Next, events); +} + +/// The signed-in account's own activity. +pub(super) fn activity(ui: &mut egui::Ui, screen: &mut ActivityScreen, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Activity"); + ui.add( + egui::Label::new( + RichText::new("What the server recorded for your account: sign-ins, password and two-factor changes, computers signed out.") + .small() + .weak(), + ) + .wrap(), + ); + ui.add_space(6.0); + ui.horizontal(|ui| { + if ui.button("Back").clicked() { + send(events, AuditMsg::CloseActivity); + } + if ui.add_enabled(!screen.list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AuditMsg::ActivityReload); + } + }); + ui.add_space(8.0); + rows(ui, &screen.list, now_ms); + paging(ui, &screen.list, AuditMsg::ActivityPrevious, AuditMsg::ActivityNext, events); +} diff --git a/apps/craft-cloud-desktop/src/ui/mod.rs b/apps/craft-cloud-desktop/src/ui/mod.rs index 6bd7b2cb..499b9302 100644 --- a/apps/craft-cloud-desktop/src/ui/mod.rs +++ b/apps/craft-cloud-desktop/src/ui/mod.rs @@ -3,10 +3,16 @@ //! I/O, so kittest draws every screen against a fake agent. mod add_computer; +mod admin; +mod audit; mod computers; mod join; mod overview; mod problem; +mod second_factor; +mod security; +mod trash; +mod versions; use egui::{Color32, RichText}; @@ -20,6 +26,7 @@ pub const TITLE: &str = "Craft Cloud"; /// Draws the window for `model` at `now_ms`, and returns what its buttons asked for. pub fn show(ui: &mut egui::Ui, model: &mut Model, now_ms: i64) -> Vec { let mut events = Vec::new(); + security::forget_secrets(ui.ctx(), &model.screen); egui::CentralPanel::default().show(ui, |ui| { egui::ScrollArea::vertical().auto_shrink([false, false]).show(ui, |ui| { ui.add_space(8.0); @@ -31,8 +38,14 @@ pub fn show(ui: &mut egui::Ui, model: &mut Model, now_ms: i64) -> Vec { Screen::Join(form) => join::form(ui, form, false, &mut events), Screen::Joining { form, .. } => join::form(ui, form, true, &mut events), Screen::PasswordNeeded(pw) => join::password(ui, pw, &mut events), + Screen::SecondFactor(form) => second_factor::show(ui, form, &mut events), + Screen::Security(page) => security::show(ui, page, &mut events), Screen::SignedIn(o) => overview::show(ui, o, busy, now_ms, autostart_problem.as_deref(), &mut events), Screen::AddingComputer(adding) => add_computer::show(ui, adding, now_ms, &mut events), + Screen::Admin(a) => admin::show(ui, a, now_ms, &mut events), + Screen::Activity(a) => audit::activity(ui, a, now_ms, &mut events), + Screen::Versions(versions) => self::versions::show(ui, versions, now_ms, &mut events), + Screen::Trash(trash) => self::trash::show(ui, trash, now_ms, &mut events), Screen::Removed => problem::removed(ui, busy, &mut events), } if let Some(note) = note { @@ -99,3 +112,6 @@ pub fn ago(now_ms: i64, then_ms: i64) -> String { #[cfg(test)] mod tests; + +#[cfg(test)] +mod tests_security; diff --git a/apps/craft-cloud-desktop/src/ui/overview.rs b/apps/craft-cloud-desktop/src/ui/overview.rs index 92aa3da3..5c15063f 100644 --- a/apps/craft-cloud-desktop/src/ui/overview.rs +++ b/apps/craft-cloud-desktop/src/ui/overview.rs @@ -1,9 +1,10 @@ //! Signed in: "fabio on cloud.shadows.ch", online or offline and the last sync, what the agent is doing, Open folder, //! Pause, Add a computer, the computers, Sign out and Quit. -use craft_cloud_app::{RunState, Status}; +use craft_cloud_app::{AccountUsage, RunState, Status}; use egui::RichText; +use crate::admin::{Access, AdminEvent}; use crate::model::{Busy, Event, Overview}; /// What the sync is doing, in one line, and the Details when it is a problem. @@ -53,6 +54,7 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: events.push(Event::RetryAutostart); } } + storage(ui, o, events); ui.add_space(12.0); ui.horizontal(|ui| { @@ -69,6 +71,28 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: if super::wide_button(ui, true, "Add a computer") { events.push(Event::AddComputer); } + ui.add_space(4.0); + ui.horizontal(|ui| { + if ui.button("Security…").on_hover_text("Two-factor sign-in, your password, your other computers").clicked() { + events.push(Event::OpenSecurity); + } + if ui.button("Activity").on_hover_text("What happened to your account: sign-ins, password changes, computers signed out").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenActivity)); + } + if matches!(o.access, Access::Admin { .. }) + && ui.button("Administration").on_hover_text("Accounts, invites, computers and the audit log").clicked() + { + events.push(Event::Admin(AdminEvent::Open)); + } + }); + ui.horizontal(|ui| { + if ui.button("File versions…").clicked() { + events.push(Event::OpenVersions); + } + if ui.button("Trash…").clicked() { + events.push(Event::OpenTrash); + } + }); ui.add_space(14.0); super::computers::list(ui, o, now_ms, events); @@ -91,3 +115,79 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: } }); } + +fn storage(ui: &mut egui::Ui, o: &Overview, events: &mut Vec) { + ui.group(|ui| { + ui.horizontal(|ui| { + ui.strong("Storage"); + if ui.add_enabled(!o.usage_loading, egui::Button::new("Refresh")).clicked() { + events.push(Event::RefreshUsage); + } + }); + if o.usage_loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading storage usage…"); + }); + } else if let Some(error) = &o.usage_error { + super::error_line(ui, "Storage unavailable"); + ui.label(egui::RichText::new("Details").small().weak()).on_hover_text(error); + } else if let Some(usage) = &o.usage { + usage_lines(ui, usage); + } + }); +} + +fn usage_lines(ui: &mut egui::Ui, usage: &AccountUsage) { + match usage.quota_bytes { + Some(quota) => { + ui.label(format!("{} of {} used", bytes(displayed_total_bytes(usage)), bytes(quota))); + ui.add( + egui::ProgressBar::new(if quota == 0 { 0.0 } else { (displayed_total_bytes(usage) as f64 / quota as f64).min(1.0) as f32 }) + .show_percentage(), + ); + } + None => { + ui.label(format!("{} used · no limit", bytes(displayed_total_bytes(usage)))); + } + }; + if let (true, Some(trash), Some(history)) = (usage.estimates.index_complete, usage.trash_bytes, usage.history_bytes) { + ui.label(format!("about {} in trash", bytes(trash))); + ui.label(format!("about {} in old versions", bytes(history))) + .on_hover_text("Trash and old-version estimates overlap; they are not added to stored bytes."); + } else { + ui.label("Estimates not available yet"); + } +} + +fn displayed_total_bytes(usage: &AccountUsage) -> u64 { + usage.stored_bytes +} + +fn bytes(value: u64) -> String { + const UNITS: [&str; 4] = ["B", "KiB", "MiB", "GiB"]; + let mut amount = value as f64; + let mut unit = 0; + while amount >= 1024.0 && unit < UNITS.len() - 1 { + amount /= 1024.0; + unit += 1; + } + if unit == 0 { format!("{value} B") } else { format!("{amount:.1} {}", UNITS[unit]) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trash_and_history_are_never_added_to_the_stored_total() { + let usage = AccountUsage { + stored_bytes: 11, + quota_bytes: None, + trash_bytes: Some(20), + history_bytes: Some(30), + estimates: craft_cloud_proto::api::UsageEstimates { index_complete: true, reason: None }, + }; + assert_eq!(displayed_total_bytes(&usage), 11); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/second_factor.rs b/apps/craft-cloud-desktop/src/ui/second_factor.rs new file mode 100644 index 00000000..b77d9ccc --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/second_factor.rs @@ -0,0 +1,38 @@ +//! The code a join asks for when the account has two-factor sign-in: the six digits from the authenticator app, or a recovery +//! code instead. + +use egui::RichText; + +use crate::model::Event; +use crate::security::SecondFactorForm; + +pub(super) fn show(ui: &mut egui::Ui, form: &mut SecondFactorForm, events: &mut Vec) { + super::heading(ui, "Enter your code"); + ui.label("This account has two-factor sign-in."); + ui.add_space(8.0); + ui.label(if form.recovery { "Recovery code" } else { "Code from the authenticator app" }); + let hint = if form.recovery { "ABCD-EFGH-JKMN-PQRS" } else { "123456" }; + let response = ui.add( + egui::TextEdit::singleline(&mut form.code).id(super::security::secret_id("second-factor-code")).hint_text(hint).desired_width(f32::INFINITY), + ); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(4.0); + let toggle = if form.recovery { "Use a code from the app" } else { "Use a recovery code" }; + if ui.checkbox(&mut form.recovery, toggle).changed() { + form.code.clear(); + form.error = None; + } + ui.label(RichText::new("A recovery code works once. Your invite or code isn't used up until this works.").small().weak()); + ui.add_space(10.0); + if let Some(error) = &form.error { + super::error_line(ui, error); + ui.add_space(8.0); + } + if super::wide_button(ui, true, "Join") || enter { + events.push(Event::SubmitSecondFactor); + } + ui.add_space(4.0); + if ui.button("Back").clicked() { + events.push(Event::CancelSecondFactor); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/security.rs b/apps/craft-cloud-desktop/src/ui/security.rs new file mode 100644 index 00000000..6a5ee327 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/security.rs @@ -0,0 +1,269 @@ +//! Security: two-factor sign-in (turn it on with a secret and a QR code, the recovery codes shown once, turn it off, new +//! recovery codes), the password, and signing the other computers out. + +use egui::RichText; + +use crate::model::Event; +use crate::security::{CodesWhy, SecurityEvent, SecurityPage, Step, TwoFactorInfo}; + +fn send(events: &mut Vec, event: SecurityEvent) { + events.push(Event::Security(event)); +} + +/// The id of a secret field of the page (a password or a code), by name. +pub(super) fn secret_id(name: &str) -> egui::Id { + egui::Id::new(("security-secret", name)) +} + +/// Every secret field of the page and the second step of a join: its name, and the text the model holds for it in `screen` +/// (empty when the field isn't there). +fn secrets(screen: &crate::model::Screen) -> Vec<(&'static str, &str)> { + use crate::model::Screen; + match screen { + Screen::Security(page) => match &page.step { + Step::SetupPassword { password } => vec![("setup-password", password)], + Step::SetupCode { code, .. } => vec![("setup-code", code)], + Step::Disable { password, code, .. } => vec![("disable-password", password), ("disable-code", code)], + Step::Regenerate { password, code } => vec![("regenerate-password", password), ("regenerate-code", code)], + Step::Password { current, new, again } => vec![("password-current", current), ("password-new", new), ("password-again", again)], + Step::Home | Step::Codes { .. } | Step::SignOutOthers => Vec::new(), + }, + Screen::SecondFactor(form) => vec![("second-factor-code", &form.code)], + // The admin's confirmation: only while it is open to be typed in. + Screen::Admin(a) if a.code.is_none() => match &a.dialog { + Some(d) if !d.busy => { + vec![(super::admin::PASSWORD_FIELD, &d.confirmation.password), (super::admin::CODE_FIELD, &d.confirmation.code)] + } + _ => Vec::new(), + }, + _ => Vec::new(), + } +} + +const SECRET_NAMES: [&str; 12] = [ + "setup-password", + "setup-code", + "disable-password", + "disable-code", + "regenerate-password", + "regenerate-code", + "password-current", + "password-new", + "password-again", + "second-factor-code", + super::admin::PASSWORD_FIELD, + super::admin::CODE_FIELD, +]; + +/// Drops what egui remembers of every secret field (its cursor and, above all, its undo history, which holds the text as it +/// was typed) unless the model holds something typed in that field right now. So once a password or code was sent, a step was +/// cancelled or the page left, Undo can't bring it back; called every frame, before the fields are drawn. egui keeps this state +/// in memory only (this app doesn't enable eframe's persistence, and the undo history is not serialized either). +pub(super) fn forget_secrets(ctx: &egui::Context, screen: &crate::model::Screen) { + let typed = secrets(screen); + for name in SECRET_NAMES { + if !typed.iter().any(|(n, text)| *n == name && !text.is_empty()) { + ctx.data_mut(|d| d.remove::(secret_id(name))); + } + } +} + +/// A password field. +fn password_field(ui: &mut egui::Ui, name: &str, label: &str, text: &mut String) -> egui::Response { + ui.label(label); + let response = ui.add(egui::TextEdit::singleline(text).id(secret_id(name)).password(true).desired_width(f32::INFINITY)); + ui.add_space(6.0); + response +} + +pub(super) fn show(ui: &mut egui::Ui, page: &mut SecurityPage, events: &mut Vec) { + super::heading(ui, "Security"); + let (busy, error, note) = (page.busy, page.error.clone(), page.note.clone()); + let info = page.info.clone(); + match &mut page.step { + Step::Home => home(ui, &info, busy, note.as_deref(), error.as_deref(), events), + Step::SetupPassword { password } => { + ui.label( + "Two-factor sign-in asks for a code from an app on your phone (Aegis, 1Password, Google Authenticator…) as well as your password.", + ); + ui.add_space(8.0); + let response = password_field(ui, "setup-password", "Your password", password); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + footer(ui, busy, error.as_deref(), "Continue", enter, events); + } + Step::SetupCode { secret, uri, expires_ms: _, code } => { + ui.label("1. Scan this with your authenticator app, or type the secret into it."); + ui.add_space(6.0); + super::add_computer::qr(ui, uri, 180.0); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(secret.as_str()).monospace().size(16.0).strong()).selectable(true)); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(secret.clone()); + } + }); + ui.collapsing("The setup link", |ui| { + ui.add(egui::Label::new(RichText::new(uri.as_str()).monospace().small()).selectable(true).wrap()); + if ui.small_button("Copy link").clicked() { + ui.ctx().copy_text(uri.clone()); + } + }); + ui.add_space(10.0); + ui.label("2. Type the six digits the app shows now."); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("setup-code")).hint_text("123456").desired_width(160.0)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Turn on", enter, events); + } + Step::Codes { codes, why } => { + let (line, button) = match why { + CodesWhy::TurnedOn => ("Two-factor sign-in is on.", "I have saved them"), + CodesWhy::New => ("These are your new recovery codes.", "I have saved them"), + }; + ui.label(RichText::new(line).strong().color(super::good(ui))); + ui.add_space(6.0); + ui.label("Save these recovery codes somewhere safe now. Each works once, if you lose your phone, and they are shown only this once."); + ui.add_space(8.0); + egui::Grid::new("recovery-codes").num_columns(2).spacing([16.0, 4.0]).show(ui, |ui| { + for (n, code) in codes.iter().enumerate() { + ui.add(egui::Label::new(RichText::new(code.as_str()).monospace().size(15.0)).selectable(true)); + if n % 2 == 1 { + ui.end_row(); + } + } + }); + ui.add_space(8.0); + if ui.button("Copy all").clicked() { + ui.ctx().copy_text(codes.join("\n")); + } + ui.add_space(10.0); + if super::wide_button(ui, true, button) { + send(events, SecurityEvent::Home); + } + } + Step::Disable { password, code, recovery } => { + ui.label("Turning it off signs your other computers out. Type your password and a code from the app."); + ui.add_space(8.0); + password_field(ui, "disable-password", "Your password", password); + ui.label(if *recovery { "A recovery code" } else { "The six digits from the app" }); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("disable-code")).desired_width(f32::INFINITY)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + if ui.small_button(if *recovery { "Use a code from the app" } else { "Use a recovery code instead" }).clicked() { + send(events, SecurityEvent::ToggleRecovery); + } + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Turn off", enter, events); + } + Step::Regenerate { password, code } => { + ui.label("New recovery codes replace the old ones at once. Type your password and a code from the app."); + ui.add_space(8.0); + password_field(ui, "regenerate-password", "Your password", password); + ui.label("The six digits from the app"); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("regenerate-code")).desired_width(160.0)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Make new codes", enter, events); + } + Step::Password { current, new, again } => { + ui.label("Changing your password signs your other computers out."); + ui.add_space(8.0); + password_field(ui, "password-current", "Current password", current); + password_field(ui, "password-new", "New password", new); + let response = password_field(ui, "password-again", "The new password again", again); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + footer(ui, busy, error.as_deref(), "Change password", enter, events); + } + Step::SignOutOthers => { + ui.label("Every other computer of your account is signed out. Their files stay there; they join again to sync."); + ui.add_space(8.0); + footer(ui, busy, error.as_deref(), "Sign them out", false, events); + } + } +} + +/// The step's error, its main button and the way back. +fn footer(ui: &mut egui::Ui, busy: bool, error: Option<&str>, button: &str, enter: bool, events: &mut Vec) { + if let Some(error) = error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if busy { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Working…"); + }); + } else if super::wide_button(ui, true, button) || enter { + send(events, SecurityEvent::Submit); + } + ui.add_space(4.0); + if ui.add_enabled(!busy, egui::Button::new("Cancel")).clicked() { + send(events, SecurityEvent::Home); + } +} + +fn home(ui: &mut egui::Ui, info: &TwoFactorInfo, busy: bool, note: Option<&str>, error: Option<&str>, events: &mut Vec) { + if let Some(note) = note { + ui.label(RichText::new(note).color(super::good(ui))); + ui.add_space(8.0); + } + if let Some(error) = error { + super::error_line(ui, error); + ui.add_space(8.0); + } + ui.label(RichText::new("Two-factor sign-in").strong()); + match info { + TwoFactorInfo::Loading => { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + TwoFactorInfo::Unsupported => { + ui.label(crate::security::UNSUPPORTED); + ui.label(RichText::new("Ask whoever runs your Craft Cloud server to update it.").small().weak()); + } + TwoFactorInfo::Failed(why) => { + super::error_line(ui, why); + if ui.button("Try again").clicked() { + send(events, SecurityEvent::Refresh); + } + } + TwoFactorInfo::Known(status) if status.enabled => { + ui.label(format!("On. {} recovery codes left.", status.recovery_codes_left)); + ui.add_space(4.0); + ui.horizontal(|ui| { + if ui.add_enabled(!busy, egui::Button::new("Turn off…")).clicked() { + send(events, SecurityEvent::StartDisable); + } + if ui.add_enabled(!busy, egui::Button::new("New recovery codes…")).clicked() { + send(events, SecurityEvent::StartRegenerate); + } + }); + } + TwoFactorInfo::Known(status) => { + ui.label("Off. Anyone who knows your password can sign in."); + if status.pending { + ui.label(RichText::new("A set-up was started and isn't finished.").small().weak()); + } + ui.add_space(4.0); + if ui.add_enabled(!busy, egui::Button::new("Set up two-factor sign-in…")).clicked() { + send(events, SecurityEvent::StartSetup); + } + } + } + ui.add_space(14.0); + ui.separator(); + ui.add_space(6.0); + ui.horizontal(|ui| { + if ui.button("Change password…").clicked() { + send(events, SecurityEvent::StartPassword); + } + if ui.button("Sign out other computers…").clicked() { + send(events, SecurityEvent::StartSignOutOthers); + } + }); + ui.add_space(14.0); + if super::wide_button(ui, true, "Back") { + send(events, SecurityEvent::Close); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/tests.rs b/apps/craft-cloud-desktop/src/ui/tests.rs index e059f6be..77c8aa0f 100644 --- a/apps/craft-cloud-desktop/src/ui/tests.rs +++ b/apps/craft-cloud-desktop/src/ui/tests.rs @@ -6,7 +6,9 @@ //! [`TOLERANCE`] in a channel. use std::path::PathBuf; +use std::sync::OnceLock; +use craft_cloud_app::{AccountUsage, Error}; use craft_cloud_app::{DeviceId, DeviceInfo, PairingInfo, RunState, Status, Uuid}; use egui_kittest::Harness; use egui_kittest::kittest::Queryable; @@ -40,6 +42,7 @@ fn signed_in() -> Status { autostart: Some(true), last_sync_ms: Some(NOW - 120_000), removed: false, + spaces: Vec::new(), } } @@ -77,7 +80,19 @@ fn after(agent: &FakeAgent, events: Vec) -> Model { fn devices() -> Vec { vec![ - DeviceInfo { id: DeviceId::generate(), name: "lptp1".into(), created_ms: 0, last_used_ms: NOW - 5_000, revoked: false, current: true }, + DeviceInfo { + id: DeviceId::generate(), + name: "lptp1".into(), + created_ms: 0, + last_used_ms: NOW - 5_000, + revoked: false, + current: true, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, DeviceInfo { id: DeviceId::generate(), name: "BEAST".into(), @@ -85,13 +100,37 @@ fn devices() -> Vec { last_used_ms: NOW - 3 * 3_600_000, revoked: false, current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, }, ] } +/// The one wgpu instance, adapter, device and queue every test renders with. Tests run on parallel threads, and +/// creating and dropping a D3D12 device (WARP) per harness at the same time crashed the test process with an access +/// violation. Each harness still gets a renderer of its own on the shared device. It is never dropped, so no device is +/// torn down while other threads render. Every kittest harness of the app, in any test module, must render with it. +pub(super) fn shared_wgpu() -> eframe::egui_wgpu::WgpuSetup { + static SHARED: OnceLock = OnceLock::new(); + let shared = SHARED.get_or_init(|| { + let state = + egui_kittest::wgpu::create_render_state(egui_kittest::wgpu::default_wgpu_setup(), eframe::egui_wgpu::RendererOptions::PREDICTABLE); + eframe::egui_wgpu::WgpuSetupExisting { + instance: state.instance.clone(), + adapter: state.adapter.clone(), + device: state.device.clone(), + queue: state.queue.clone(), + } + }); + eframe::egui_wgpu::WgpuSetup::Existing(shared.clone()) +} + /// The harness: the window drawn for the model, its events kept. fn harness(model: Model, theme: egui::Theme) -> Harness<'static, (Model, Vec)> { - Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu().build_ui_state( + Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu_setup(shared_wgpu()).build_ui_state( |ui, (model, events): &mut (Model, Vec)| { events.extend(show(ui, model, NOW)); }, @@ -107,6 +146,9 @@ fn snapshot(model: Model, theme: egui::Theme, name: &str) { let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("snapshots"); let path = dir.join(format!("{name}.png")); if std::env::var_os("UPDATE_SNAPSHOTS").is_some() { + if std::env::var("UPDATE_SNAPSHOT_ONLY").ok().is_some_and(|names| !names.split(',').any(|allowed| allowed == name)) { + return; + } std::fs::create_dir_all(&dir).unwrap(); image.save(&path).unwrap(); return; @@ -155,11 +197,15 @@ fn snapshot_join() { snapshot(model.clone(), egui::Theme::Light, "join"); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX"); snapshot(model.clone(), egui::Theme::Light, "joining"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "folder_has_files".into(), message: "has files".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { + code: "folder_has_files".into(), + message: "has files".into(), + name_taken: None, + }))); snapshot(model.clone(), egui::Theme::Light, "join-folder-has-files"); model.update(Event::SyncExistingFiles(false)); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "invalid_code".into(), message: "x".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "invalid_code".into(), message: "x".into(), name_taken: None }))); snapshot(model, egui::Theme::Dark, "join-invalid-code-dark"); } @@ -167,7 +213,7 @@ fn snapshot_join() { fn snapshot_password_needed() { let mut model = after(&FakeAgent::new(signed_out()), Vec::new()); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX-K7M4-Q9TX"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "password_required".into(), message: "x".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "password_required".into(), message: "x".into(), name_taken: None }))); if let Screen::PasswordNeeded(pw) = &mut model.screen { pw.password = "correct horse".into(); pw.again = "correct house".into(); @@ -182,6 +228,119 @@ fn signed_in_model(status: Status) -> Model { model } +fn usage(stored_bytes: u64, quota_bytes: Option, complete: bool) -> AccountUsage { + AccountUsage { + stored_bytes, + quota_bytes, + trash_bytes: complete.then_some(20), + history_bytes: complete.then_some(30), + estimates: craft_cloud_proto::api::UsageEstimates { + index_complete: complete, + reason: (!complete).then_some(craft_cloud_proto::api::UsageEstimateReason::IndexIncomplete), + }, + } +} + +#[test] +fn snapshot_storage_states() { + let with_quota = signed_in_with_usage(Ok(usage(11, Some(100), true))); + snapshot(with_quota, egui::Theme::Light, "storage-with-quota"); + + let no_quota = signed_in_with_usage(Ok(usage(11, None, true))); + snapshot(no_quota, egui::Theme::Light, "storage-no-quota"); + + let estimates_unavailable = signed_in_with_usage(Ok(usage(11, Some(100), false))); + snapshot(estimates_unavailable, egui::Theme::Light, "storage-estimates-unavailable"); + + let error = signed_in_with_usage(Err(Error::Refused { code: "offline".into(), message: "can't reach the server".into(), name_taken: None })); + snapshot(error, egui::Theme::Light, "storage-error"); +} + +fn signed_in_with_usage(result: Result) -> Model { + let agent = FakeAgent::new(signed_in()); + *agent.usage_result.lock().unwrap() = Some(result); + after(&agent, Vec::new()) +} + +#[test] +fn storage_refresh_requests_a_fresh_server_answer() { + let mut model = signed_in_model(signed_in()); + assert_eq!(model.update(Event::RefreshUsage), vec![Effect::Usage(2)]); + let Screen::SignedIn(overview) = model.screen else { panic!("signed-in screen expected") }; + assert!(overview.usage_loading); + assert_eq!(overview.usage_error, None); +} + +#[test] +fn storage_refresh_click_sends_a_request_and_loading_blocks_duplicates() { + let mut h = harness(signed_in_model(signed_in()), egui::Theme::Light); + h.run_steps(2); + h.query_all_by_label("Refresh").next().unwrap().click(); + h.run_steps(2); + let (model, events) = h.state_mut(); + assert!(events.contains(&Event::RefreshUsage)); + assert_eq!(model.update(Event::RefreshUsage), vec![Effect::Usage(2)]); + assert!(model.update(Event::RefreshUsage).is_empty(), "already loading blocks repeat request"); +} + +#[test] +fn stale_refresh_answer_cannot_replace_a_newer_answer() { + let mut model = Model::new(defaults(), true, String::new()); + let first = model + .update(Event::Found(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("signed-in overview asks for usage"); + model.update(Event::Usage { request_id: first, result: Ok(usage(11, None, true)) }); + let second = model + .update(Event::RefreshUsage) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("refresh asks again"); + model.update(Event::Usage { request_id: second, result: Ok(usage(25, None, true)) }); + model.update(Event::Usage { + request_id: first, + result: Err(Error::Refused { code: "offline".into(), message: "old error".into(), name_taken: None }), + }); + let Screen::SignedIn(overview) = &model.screen else { panic!("signed-in screen expected") }; + assert!(!overview.usage_loading); + assert_eq!(overview.usage_error, None); + assert_eq!(overview.usage.as_ref().map(|value| value.stored_bytes), Some(25)); +} + +#[test] +fn usage_answer_from_previous_sign_in_cannot_replace_new_account_state() { + let mut model = Model::new(defaults(), true, String::new()); + let old_request = model + .update(Event::Found(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("signed-in overview asks for usage"); + model.update(Event::Polled(Ok(signed_out()))); + let new_request = model + .update(Event::Polled(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("new sign-in asks for usage"); + model.update(Event::Usage { request_id: new_request, result: Ok(usage(20, None, true)) }); + model.update(Event::Usage { request_id: old_request, result: Ok(usage(999, None, true)) }); + let Screen::SignedIn(overview) = &model.screen else { panic!("signed-in screen expected") }; + assert!(!overview.usage_loading); + assert_eq!(overview.usage.as_ref().map(|value| value.stored_bytes), Some(20)); +} + #[test] fn snapshot_signed_in() { snapshot(signed_in_model(signed_in()), egui::Theme::Light, "signed-in"); @@ -193,6 +352,147 @@ fn snapshot_signed_in() { snapshot(signed_in_model(paused), egui::Theme::Light, "signed-in-paused"); } +fn versions_model(loaded: bool) -> Model { + let mut model = signed_in_model(signed_in()); + model.update(Event::OpenVersions); + model.update(Event::VersionsPath("Projects/painting.png".into())); + if loaded { + let versions = vec![ + craft_cloud_app::HistoryVersion { + id: craft_cloud_app::VersionId::from_uuid(Uuid::nil()), + parent_version: None, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + seq: 9, + size: 42_000, + mtime_ms: NOW, + device: DeviceId::from_uuid(Uuid::nil()), + created_ms: NOW - 30_000, + save: false, + pruned: false, + }, + craft_cloud_app::HistoryVersion { + id: craft_cloud_app::VersionId::from_uuid(Uuid::from_u128(1)), + parent_version: None, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + seq: 8, + size: 38_000, + mtime_ms: NOW - 60_000, + device: DeviceId::from_uuid(Uuid::from_u128(1)), + created_ms: NOW - 60_000, + save: true, + pruned: false, + }, + ]; + model.update(Event::History { ticket: 1, result: Ok(craft_cloud_app::HistoryResponse { versions, next: Some("next-page".into()) }) }); + } + model +} + +#[test] +fn snapshot_versions() { + snapshot(versions_model(false), egui::Theme::Light, "versions-empty"); + snapshot(versions_model(true), egui::Theme::Light, "versions-loaded"); + let mut model = versions_model(true); + if let Screen::Versions(versions) = &mut model.screen { + versions.confirm = versions.page.as_ref().and_then(|page| page.versions.get(1)).map(|version| version.id); + } + snapshot(model, egui::Theme::Light, "versions-confirm"); +} + +fn trash_model(loaded: bool) -> Model { + let mut model = signed_in_model(signed_in()); + let effects = model.update(Event::OpenTrash); + let ticket = match effects.first() { + Some(Effect::Trash { ticket, .. }) => *ticket, + _ => 0, + }; + let items = if loaded { + vec![ + craft_cloud_app::TrashItem { + id: craft_cloud_app::NodeId::from_uuid(Uuid::nil()), + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(1)), + name: "painting.png".into(), + is_dir: false, + size: Some(42_000), + deleted_ms: NOW - 30_000, + purge_after_ms: NOW + 14 * 86_400_000, + path: Some("Projects/painting.png".into()), + }, + craft_cloud_app::TrashItem { + id: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(2)), + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(3)), + name: "Old exports".into(), + is_dir: true, + size: None, + deleted_ms: NOW - 86_400_000, + purge_after_ms: NOW + 13 * 86_400_000, + path: None, + }, + ] + } else { + Vec::new() + }; + model.update(Event::TrashPage { ticket, result: Ok(craft_cloud_app::TrashResponse { items, next: loaded.then(|| "trash-next".into()) }) }); + model +} + +#[test] +fn snapshot_trash() { + snapshot(trash_model(false), egui::Theme::Light, "trash-empty"); + snapshot(trash_model(true), egui::Theme::Light, "trash-loaded"); + let mut model = trash_model(true); + let id = match &model.screen { + Screen::Trash(t) => t.page.as_ref().and_then(|p| p.items.first()).map(|i| i.id), + _ => None, + }; + if let Some(id) = id { + model.update(Event::ConfirmRemoveTrash(id)); + snapshot(model.clone(), egui::Theme::Light, "trash-confirm"); + model.update(Event::CancelRemoveTrash); + let effect = model.update(Event::RestoreTrash(id)); + let ticket = match effect.first() { + Some(Effect::RestoreTrash { ticket, .. }) => *ticket, + _ => 0, + }; + model.update(Event::TrashRestored { + ticket, + result: Err(craft_cloud_app::Error::Refused { code: "conflict".into(), message: "name taken".into(), name_taken: Some(true) }), + }); + snapshot(model.clone(), egui::Theme::Light, "trash-name-taken"); + let restored = craft_cloud_app::TrashRestoreResponse { + node: id, + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(1)), + name: "painting (1).png".into(), + renamed: true, + version: None, + seq: 2, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + }; + let effect = model.update(Event::RestoreTrash(id)); + let ticket = match effect.first() { + Some(Effect::RestoreTrash { ticket, .. }) => *ticket, + _ => 0, + }; + model.update(Event::TrashRestored { ticket, result: Ok(restored) }); + snapshot(model, egui::Theme::Light, "trash-restored"); + } else { + snapshot(model, egui::Theme::Light, "trash-confirm"); + } +} + +#[test] +fn trash_buttons_send_restore_and_confirmation_events() { + let mut h = harness(trash_model(true), egui::Theme::Light); + h.run_steps(2); + h.query_all_by_label("Restore").next().unwrap().click(); + h.run_steps(2); + h.query_all_by_label("Remove from trash").next().unwrap().click(); + h.run_steps(2); + let events = std::mem::take(&mut h.state_mut().1); + assert!(events.iter().any(|e| matches!(e, Event::RestoreTrash(_))), "{events:?}"); + assert!(events.contains(&Event::ConfirmRemoveTrash(craft_cloud_app::NodeId::from_uuid(Uuid::nil()))), "{events:?}"); +} + #[test] fn snapshot_adding_a_computer() { let mut model = signed_in_model(signed_in()); @@ -266,3 +566,541 @@ fn times_and_addresses_read_as_people_say_them() { let (line, _) = overview::sync_line(&Status { online: false, run_state: RunState::Retrying { message: "x".into() }, ..signed_in() }); assert_eq!(line, "Can't reach cloud.shadows.ch — trying again"); } + +// ---- the admin pages ---- + +use crate::admin::{Action, AdminEvent, CodeKind, RoleChoice, View}; +use crate::agent::fake::sample_user; +use craft_cloud_app::AdminUserInfo; + +/// An agent signed in as `fabio`, an admin, with a few more accounts. +fn admin_agent() -> FakeAgent { + let agent = FakeAgent::new(signed_in()); + agent.admin.store(true, std::sync::atomic::Ordering::SeqCst); + let account = |name: &str, f: &dyn Fn(&mut AdminUserInfo)| { + let mut user = sample_user(name, Uuid::now_v7()); + f(&mut user); + user + }; + *agent.users.lock().unwrap() = vec![ + account("fabio", &|u| { + u.role = "admin".into(); + u.devices_active = 2; + u.stored_bytes = 3_400_000_000; + }), + account("ana", &|u| { + u.devices_active = 1; + u.stored_bytes = 640_000_000; + u.quota_bytes = Some(5 * 1_073_741_824); + }), + account("ben", &|u| { + u.devices_active = 0; + u.disabled_ms = Some(1_789_000_000_000); + }), + account("cleo.m", &|u| { + u.devices_active = 0; + u.invite_pending = true; + }), + ]; + agent +} + +fn on_admin_pages(agent: &FakeAgent, more: Vec) -> Model { + let mut events = vec![Event::Devices(Ok(devices())), Event::Admin(AdminEvent::Open)]; + events.extend(more); + after(agent, events) +} + +fn id_of(model: &Model, name: &str) -> Uuid { + model.admin_screen().unwrap().list.users.iter().find(|u| u.username == name).unwrap().id +} + +fn password_typed(model: &mut Model, password: &str) { + let Screen::Admin(admin) = &mut model.screen else { panic!("{:?}", model.screen) }; + admin.dialog.as_mut().unwrap().confirmation.password = password.to_owned(); +} + +/// The model after the change asked for was sent and the fake answered it. +fn confirm_sent(model: &mut Model, agent: &FakeAgent) { + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let answer = call(agent, &no_start(), effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); +} + +#[test] +fn snapshot_admin_pages() { + let agent = admin_agent(); + // The signed-in view offers the pages to an admin only. + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + assert!(model.is_admin()); + snapshot(model, egui::Theme::Light, "admin-signed-in"); + + let mut model = on_admin_pages(&agent, Vec::new()); + snapshot(model.clone(), egui::Theme::Light, "admin-users"); + snapshot(model.clone(), egui::Theme::Dark, "admin-users-dark"); + snapshot(model.clone(), egui::Theme::Light, "admin-storage"); + + model.update(Event::Admin(AdminEvent::NewUser)); + if let Screen::Admin(admin) = &mut model.screen + && let View::Create(form) = &mut admin.view + { + form.username = "dora".into(); + form.role = RoleChoice::Member; + } + snapshot(model, egui::Theme::Light, "admin-new-account"); + + let ana = id_of(&on_admin_pages(&agent, Vec::new()), "ana"); + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + snapshot(model.clone(), egui::Theme::Light, "admin-account"); + snapshot(model.clone(), egui::Theme::Light, "admin-account-usage"); + if let Screen::Admin(admin) = &mut model.screen + && let View::User(view) = &mut admin.view + { + view.usage = Some(craft_cloud_app::AccountUsage { + stored_bytes: 640_000_000, + quota_bytes: Some(5 * 1_073_741_824), + trash_bytes: None, + history_bytes: None, + estimates: craft_cloud_app::UsageEstimates { index_complete: false, reason: Some(craft_cloud_app::UsageEstimateReason::IndexIncomplete) }, + }); + view.usage_loading = false; + } + snapshot(model.clone(), egui::Theme::Light, "admin-account-usage-unavailable"); + + // A change asks for the password; the wrong one is said in the words of the server's refusal. + model.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + password_typed(&mut model, "not-my-password"); + snapshot(model.clone(), egui::Theme::Light, "admin-confirm"); + let error = craft_cloud_app::Error::Refused { code: "wrong_password".into(), message: "the password is wrong".into(), name_taken: None }; + agent.admin_fails.lock().unwrap().push(error); + confirm_sent(&mut model, &agent); + snapshot(model, egui::Theme::Light, "admin-confirm-wrong-password"); + + // The invite a new account got, shown once. + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::NewUser)]); + if let Screen::Admin(admin) = &mut model.screen + && let View::Create(form) = &mut admin.view + { + form.username = "dora".into(); + } + model.update(Event::Admin(AdminEvent::SubmitCreate)); + password_typed(&mut model, "admin-secret-pw"); + confirm_sent(&mut model, &agent); + let code = model.admin_screen().unwrap().code.clone().unwrap(); + assert_eq!((code.username.as_str(), code.kind), ("dora", CodeKind::Invite)); + snapshot(model, egui::Theme::Light, "admin-invite-code"); +} + +#[test] +fn account_estimates_are_not_added_to_stored_total() { + let agent = admin_agent(); + let ana = id_of(&on_admin_pages(&agent, Vec::new()), "ana"); + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(4); + assert_eq!(h.query_all_by_label("Stored: 610 MiB").count(), 1, "estimates stay separate from stored bytes"); + assert_eq!(h.query_all_by_label("Trash estimate (overlapping): 76.3 MiB").count(), 1); + assert_eq!(h.query_all_by_label("History estimate (overlapping): 229 MiB").count(), 1); +} + +/// A member never sees the way in. +#[test] +fn a_member_has_no_administration_button() { + let agent = admin_agent(); + agent.admin.store(false, std::sync::atomic::Ordering::SeqCst); + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + assert!(!model.is_admin()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Administration").count(), 0); +} + +/// The admin pages' buttons send the events they stand for. +#[test] +fn admin_buttons_send_their_events() { + let agent = admin_agent(); + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Administration").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::Open)]); + + let model = on_admin_pages(&agent, Vec::new()); + let ana = id_of(&model, "ana"); + let fabio = id_of(&model, "fabio"); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("New account").click(); + h.run_steps(2); + h.get_by_label("Search").click(); + h.run_steps(2); + h.get_by_label("Refresh").click(); + h.run_steps(2); + // One Manage per account. + assert_eq!(h.query_all_by_label("Manage").count(), 4); + h.query_all_by_label("Manage").nth(1).unwrap().click(); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + let events = std::mem::take(&mut h.state_mut().1); + assert_eq!( + events, + [ + Event::Admin(AdminEvent::NewUser), + Event::Admin(AdminEvent::Search), + Event::Admin(AdminEvent::Reload), + Event::Admin(AdminEvent::OpenUser(ana)), + Event::Admin(AdminEvent::Close), + ] + ); + + // Ana is enabled and has a computer: the buttons ask for the password of their change. + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + for label in ["Save changes", "Disable", "Remove", "New invite", "Password reset code", "Revoke open code", "Revoke"] { + h.get_by_label(label).click(); + h.run_steps(2); + } + let events = std::mem::take(&mut h.state_mut().1); + let asked: Vec<&Action> = events.iter().filter_map(|e| if let Event::Admin(AdminEvent::Ask(a)) = e { Some(a) } else { None }).collect(); + assert!( + matches!( + asked.as_slice(), + [ + Action::Disable { .. }, + Action::Remove { .. }, + Action::IssueInvite { .. }, + Action::IssueReset { .. }, + Action::RevokeInvite { .. }, + Action::RevokeDevice { .. } + ] + ), + "{events:?}" + ); + assert!(events.contains(&Event::Admin(AdminEvent::SubmitEdit))); + // Your own account has no Disable or Remove. + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(fabio))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Disable").count() + h.query_all_by_label("Remove").count(), 0); + + // The password page: the confirm button and Cancel. + let mut model = on_admin_pages(&agent, Vec::new()); + model.update(Event::Admin(AdminEvent::Ask(Action::Enable { id: ana, username: "ana".into() }))); + password_typed(&mut model, "pw"); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Enable").click(); + h.run_steps(2); + h.get_by_label("Cancel").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::Confirm), Event::Admin(AdminEvent::CancelDialog)]); + + // A code shown: Done forgets it. + let mut model = on_admin_pages(&agent, Vec::new()); + model.update(Event::Admin(AdminEvent::Ask(Action::IssueReset { id: ana, username: "ana".into(), reset_two_factor: false }))); + password_typed(&mut model, "pw"); + confirm_sent(&mut model, &agent); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Done").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::DismissCode)]); +} + +/// egui keeps a text field's undo history, which holds what was typed: for the password field it is dropped once the +/// password was sent, or the page cancelled or left, so Undo can't bring a password back to a field opened later. +#[test] +fn a_typed_password_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + use egui::text_edit::TextEditState; + + let agent = admin_agent(); + let model = on_admin_pages(&agent, Vec::new()); + let ana = id_of(&model, "ana"); + let id = super::security::secret_id(super::admin::PASSWORD_FIELD); + let ask = |h: &mut Harness<'static, (Model, Vec)>| { + h.state_mut().0.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + h.run_steps(3); + }; + // Whether egui could still bring typed text back: its state for the field has an undo history for an empty field. + let remembered = |h: &Harness<'static, (Model, Vec)>| { + TextEditState::load(&h.ctx, id).is_some_and(|state| state.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()) + }; + let password = + |h: &Harness<'static, (Model, Vec)>| h.state().0.admin_screen().unwrap().dialog.as_ref().unwrap().confirmation.password.clone(); + let undo = |h: &mut Harness<'static, (Model, Vec)>| { + h.get_by_role(Role::PasswordInput).focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); + }; + + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert!(!remembered(&h)); + ask(&mut h); + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("hunter2-secret"); + // Long enough for egui to settle the typing into an undo point. + h.run_steps(8); + assert_eq!(password(&h), "hunter2-secret"); + assert!(remembered(&h), "while it is typed, egui has the field's state (the test sees what it guards against)"); + + // Sent, and refused as a wrong password: the dialog is open again, empty, and egui has forgotten the text. + let effects = h.state_mut().0.update(Event::Admin(AdminEvent::Confirm)); + assert!(matches!(effects.as_slice(), [Effect::AdminDo { .. }])); + h.run_steps(2); + assert!(!remembered(&h), "once sent"); + let refused = craft_cloud_app::Error::Refused { code: "wrong_password".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::Admin(AdminEvent::Done(Err(refused)))); + h.run_steps(3); + undo(&mut h); + assert_eq!(password(&h), "", "Undo brings nothing back"); + + // Typed again and cancelled; the same change asked for again: Undo still brings nothing back. + h.get_by_role(Role::PasswordInput).type_text("another-secret"); + h.run_steps(8); + assert!(remembered(&h)); + h.state_mut().0.update(Event::Admin(AdminEvent::CancelDialog)); + h.run_steps(2); + assert!(!remembered(&h), "once cancelled"); + ask(&mut h); + undo(&mut h); + assert_eq!(password(&h), ""); + + // Left with the password typed: the same. + h.get_by_role(Role::PasswordInput).type_text("third-secret"); + h.run_steps(3); + h.state_mut().0.update(Event::Admin(AdminEvent::Close)); + h.run_steps(2); + assert!(!remembered(&h), "once the pages are left"); +} + +/// The same for the confirmation's code field (a code from the app, or a recovery code), beside the password's: both are +/// forgotten once the confirmation was sent, cancelled or left, so Undo can't bring either back into a field opened later. +#[test] +fn a_typed_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + use egui::text_edit::TextEditState; + + let agent = admin_agent(); + agent.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + let model = on_admin_pages(&agent, Vec::new()); + assert!(matches!(model.overview().unwrap().access, crate::admin::Access::Admin { two_factor: true })); + let ana = id_of(&model, "ana"); + let ids = [super::security::secret_id(super::admin::PASSWORD_FIELD), super::security::secret_id(super::admin::CODE_FIELD)]; + let ask = |h: &mut Harness<'static, (Model, Vec)>| { + h.state_mut().0.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + h.run_steps(3); + }; + let remembered = |h: &Harness<'static, (Model, Vec)>, which: usize| { + TextEditState::load(&h.ctx, ids[which]) + .is_some_and(|state| state.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()) + }; + let typed = |h: &Harness<'static, (Model, Vec)>| { + let c = &h.state().0.admin_screen().unwrap().dialog.as_ref().unwrap().confirmation; + (c.password.clone(), c.code.clone()) + }; + let undo = |h: &mut Harness<'static, (Model, Vec)>, which: usize| { + h.get_all_by_role(Role::PasswordInput).nth(which).unwrap().focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); + }; + + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + ask(&mut h); + assert_eq!(h.query_all_by_role(Role::PasswordInput).count(), 2, "a password field and a code field"); + h.get_all_by_role(Role::PasswordInput).next().unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).next().unwrap().type_text("hunter2-secret"); + h.run_steps(2); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().type_text("918273"); + h.run_steps(8); + assert_eq!(typed(&h), ("hunter2-secret".to_owned(), "918273".to_owned())); + assert!(TextEditState::load(&h.ctx, ids[0]).is_some() && remembered(&h, 1), "while they are typed, egui has their state"); + + // Sent and refused (a wrong code): both fields are empty and egui has forgotten both texts; Undo brings nothing back. + let effects = h.state_mut().0.update(Event::Admin(AdminEvent::Confirm)); + assert!(matches!(effects.as_slice(), [Effect::AdminDo { .. }])); + h.run_steps(2); + assert!(!remembered(&h, 0) && !remembered(&h, 1), "once sent"); + let refused = craft_cloud_app::Error::Refused { code: "two_factor_invalid".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::Admin(AdminEvent::Done(Err(refused)))); + h.run_steps(3); + for which in [0, 1] { + undo(&mut h, which); + assert_eq!(typed(&h), (String::new(), String::new()), "Undo of field {which} brings nothing back"); + } + + // Typed again and cancelled: forgotten; the same change asked for again: nothing comes back. + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().type_text("ABCD-EFGH-JKMN-PQRS"); + h.run_steps(8); + assert!(remembered(&h, 1)); + h.state_mut().0.update(Event::Admin(AdminEvent::CancelDialog)); + h.run_steps(2); + assert!(!remembered(&h, 1), "once cancelled"); + ask(&mut h); + undo(&mut h, 1); + assert_eq!(typed(&h).1, ""); +} + +// ---- the audit log, the account's own activity and the second factor's confirmation ---- + +use crate::audit::{AuditMsg, ExportState}; +use craft_cloud_proto::api::{AuditDetails, AuditEvent}; + +/// Events as the server writes them, newest first, with the variety the page tells in words. +fn audit_events() -> Vec { + let event = |n: u64, action: &str, outcome: &str, count: u64, details: AuditDetails| AuditEvent { + outcome: outcome.into(), + count, + details, + at_ms: NOW - (n as i64) * 3_600_000 - 5_000, + actor_user: (!action.starts_with("cli.") && !action.ends_with("login_failed")) + .then(|| Uuid::from_u128(0x0123_4567_89ab_cdef_0000_0000_0000_0000 + u128::from(n))), + target_kind: if action.starts_with("admin.") { "user".into() } else { String::new() }, + target_id: action.starts_with("admin.").then(|| Uuid::from_u128(0xfedc_ba98_7654_3210_0000_0000_0000_0000 + u128::from(n))), + ..FakeAgent::sample_event(n, action) + }; + vec![ + event( + 0, + "admin.role_changed", + "success", + 1, + AuditDetails { role_from: Some("member".into()), role_to: Some("admin".into()), ..Default::default() }, + ), + event(1, "auth.login_failed", "failure", 4, AuditDetails { step: Some("password".into()), ..Default::default() }), + event( + 2, + "auth.login", + "success", + 1, + AuditDetails { client_kind: Some("desktop".into()), platform: Some("windows".into()), ..Default::default() }, + ), + event(3, "device.revoked_others", "success", 1, AuditDetails { revoked: Some(2), ..Default::default() }), + event(4, "admin.quota_changed", "success", 1, AuditDetails { quota_bytes: Some(Some(5_368_709_120)), ..Default::default() }), + event(5, "account.two_factor_enabled", "success", 1, AuditDetails::default()), + ] +} + +fn audit_model(agent: &FakeAgent, more: Vec) -> Model { + *agent.audit_events.lock().unwrap() = audit_events(); + let mut events = vec![Event::Devices(Ok(devices())), Event::Admin(AdminEvent::Open), Event::Audit(AuditMsg::OpenAudit(None))]; + events.extend(more); + after(agent, events) +} + +fn view_of(model: &Model) -> &crate::audit::AuditView { + match &model.admin_screen().unwrap().view { + View::Audit(view) => view, + other => panic!("{other:?}"), + } +} + +#[test] +fn snapshot_audit_and_activity_pages() { + let agent = admin_agent(); + let mut model = audit_model(&agent, Vec::new()); + snapshot(model.clone(), egui::Theme::Light, "audit-log"); + snapshot(model.clone(), egui::Theme::Dark, "audit-log-dark"); + + // Narrowed to one account, with an export that the server cut at its limit, and one that failed. + let ana = id_of(&model, "ana"); + let mut narrowed = audit_model(&agent, vec![Event::Audit(AuditMsg::OpenAudit(Some((ana, "ana".into()))))]); + narrowed.update(Event::Audit(AuditMsg::Export)); + crate::audit::tests::picked(&mut narrowed, Some("C:/Users/fabio/craft-cloud-audit.ndjson")); + snapshot(narrowed.clone(), egui::Theme::Light, "audit-log-exporting"); + crate::audit::tests::finished(&mut narrowed, Ok(craft_cloud_app::ExportOutcome { events: 100_000, truncated: true })); + assert!(matches!(view_of(&narrowed).export, ExportState::Done { .. })); + snapshot(narrowed.clone(), egui::Theme::Light, "audit-log-export-truncated"); + narrowed.update(Event::Audit(AuditMsg::DismissExport)); + narrowed.update(Event::Audit(AuditMsg::Export)); + crate::audit::tests::picked(&mut narrowed, Some("out.ndjson")); + crate::audit::tests::finished(&mut narrowed, Err(craft_cloud_app::Error::Io("the export broke off: connection reset".into()))); + snapshot(narrowed, egui::Theme::Light, "audit-log-export-failed"); + + // The account's own activity, for a member too. + model.update(Event::Admin(AdminEvent::Close)); + *agent.audit_events.lock().unwrap() = audit_events(); + let member = FakeAgent::new(signed_in()); + *member.audit_events.lock().unwrap() = audit_events(); + let activity = after(&member, vec![Event::Devices(Ok(devices())), Event::Audit(AuditMsg::OpenActivity)]); + assert!(activity.activity_screen().is_some()); + snapshot(activity, egui::Theme::Light, "activity"); +} + +/// An account with two-factor sign-in: its page says so and offers to turn it off; confirming asks for a code too. +#[test] +fn snapshot_two_factor_in_the_admin_pages() { + let agent = admin_agent(); + agent.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + agent.users.lock().unwrap()[1].two_factor_enabled = true; + let ana = { + let model = on_admin_pages(&agent, Vec::new()); + assert!(matches!(model.overview().unwrap().access, crate::admin::Access::Admin { two_factor: true })); + id_of(&model, "ana") + }; + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + snapshot(model.clone(), egui::Theme::Light, "admin-account-two-factor"); + model.update(Event::Admin(AdminEvent::Ask(Action::ResetTwoFactor { id: ana, username: "ana".into() }))); + password_typed(&mut model, "not-my-password"); + if let Screen::Admin(admin) = &mut model.screen { + admin.dialog.as_mut().unwrap().confirmation.code = "123456".into(); + } + snapshot(model, egui::Theme::Light, "admin-confirm-two-factor"); +} + +/// The buttons of the audit and activity pages send what they stand for. +#[test] +fn audit_buttons_send_their_events() { + let agent = admin_agent(); + let model = audit_model(&agent, Vec::new()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Refresh").click(); + h.run_steps(2); + h.get_by_label("Export…").click(); + h.run_steps(2); + h.get_by_label("Back to the accounts").click(); + h.run_steps(2); + assert_eq!( + std::mem::take(&mut h.state_mut().1), + [Event::Audit(AuditMsg::Reload), Event::Audit(AuditMsg::Export), Event::Admin(AdminEvent::BackToList)] + ); + // A page of the account list opens the log, and an account's page opens its own part of it. + let model = on_admin_pages(&agent, Vec::new()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Audit log").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenAudit(None))]); + let ana = id_of(&h.state().0, "ana"); + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Activity of this account").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenAudit(Some((ana, "ana".into()))))]); + // The signed-in view has an Activity button for everyone; the activity page goes back. + let member = FakeAgent::new(signed_in()); + let model = after(&member, vec![Event::Devices(Ok(devices()))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Administration").count(), 0); + h.get_by_label("Activity").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenActivity)]); + let model = after(&member, vec![Event::Devices(Ok(devices())), Event::Audit(AuditMsg::OpenActivity)]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::CloseActivity)]); +} diff --git a/apps/craft-cloud-desktop/src/ui/tests_security.rs b/apps/craft-cloud-desktop/src/ui/tests_security.rs new file mode 100644 index 00000000..1cc0a23a --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/tests_security.rs @@ -0,0 +1,630 @@ +//! The Security page and the join's code field: every step against the fake agent (the model, effect by effect), the screens as +//! kittest draws them (snapshots `second-factor*` and `security-*` in `tests/snapshots/`, written with `UPDATE_SNAPSHOTS=1`), +//! and that nothing secret outlives the screen that showed it. + +use std::path::PathBuf; + +use craft_cloud_app::{Error, JoinRequest, RunState, Status}; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; + +use super::*; +use crate::agent::fake::{CODE, FakeAgent, PASSWORD, RECOVERY}; +use crate::agent::{Agent, Starter, call}; +use crate::model::{Effect, JoinDefaults, Model, Screen}; +use crate::security::{SecurityEvent, Step, TwoFactorInfo}; + +const NOW: i64 = 1_790_000_000_000; +const TOLERANCE: u8 = 48; +const MAX_DIFFERENT: usize = 4; + +fn defaults() -> JoinDefaults { + JoinDefaults { folder: "/home/fabio/Craft Cloud".into(), device_name: "lptp1".into() } +} + +fn signed_in() -> Status { + Status { + api: 1, + signed_in: true, + account: Some("fabio".into()), + server: Some("https://cloud.shadows.ch".into()), + folder: Some("/home/fabio/Craft Cloud".into()), + machine: Some("lptp1".into()), + online: true, + run_state: RunState::Idle, + autostart: Some(true), + last_sync_ms: Some(NOW - 120_000), + removed: false, + spaces: Vec::new(), + } +} + +fn signed_out() -> Status { + Status { + signed_in: false, + account: None, + server: None, + folder: None, + machine: None, + online: false, + run_state: RunState::SignedOut, + ..signed_in() + } +} + +fn no_start() -> Starter { + std::sync::Arc::new(|_: &dyn Agent| Err("no agent in tests".to_owned())) +} + +/// Feeds `event` to the model and answers every effect with the fake, as the link does, until nothing is left. +fn feed(model: &mut Model, agent: &FakeAgent, event: Event) { + let start = no_start(); + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Some(answer) = call(agent, &start, effect) { + pending.extend(model.update(answer)); + } + } +} + +fn model_after(agent: &FakeAgent, events: Vec) -> Model { + let mut model = Model::new(defaults(), true, "run.log".into()); + for event in std::iter::once(Event::Start).chain(events) { + feed(&mut model, agent, event); + } + model +} + +fn on_security(agent: &FakeAgent) -> Model { + model_after(agent, vec![Event::OpenSecurity]) +} + +/// A folder that is a full path on every platform (the join form refuses anything else). +fn absolute_folder() -> String { + std::env::temp_dir().to_string_lossy().into_owned() +} + +fn page(model: &mut Model) -> &mut crate::security::SecurityPage { + match &mut model.screen { + Screen::Security(page) => page, + other => panic!("not the Security page: {other:?}"), + } +} + +fn type_into(model: &mut Model, fill: impl FnOnce(&mut Step)) { + fill(&mut page(model).step); +} + +// ---- the model ---- + +#[test] +fn the_page_asks_the_server_and_shows_what_it_says() { + let agent = FakeAgent::new(signed_in()); + let mut model = on_security(&agent); + assert_eq!( + page(&mut model).info, + TwoFactorInfo::Known(craft_cloud_app::TwoFactorStatus { enabled: false, enabled_ms: None, recovery_codes_left: 0, pending: false }) + ); + assert!(agent.asked.lock().unwrap().iter().any(|a| a == "two_factor_status")); + // A server from before two-factor sign-in: said so, the page doesn't break. + let old = FakeAgent::new(signed_in()); + old.factor.lock().unwrap().unsupported = true; + let mut model = on_security(&old); + assert_eq!(page(&mut model).info, TwoFactorInfo::Unsupported); + feed(&mut model, &old, Event::Security(SecurityEvent::StartSetup)); + assert_eq!(page(&mut model).step, Step::Home, "nothing to set up"); + // Back to the signed-in view. + feed(&mut model, &old, Event::Security(SecurityEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); +} + +#[test] +fn setting_it_up_turns_it_on_and_shows_the_recovery_codes_once() { + let agent = FakeAgent::new(signed_in()); + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartSetup)); + // Nothing typed: nothing is asked of the server. + let asked = agent.asked.lock().unwrap().len(); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.is_some()); + assert_eq!(agent.asked.lock().unwrap().len(), asked); + // A wrong password stays on the step, with the password gone from the field. + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = "wrong password!".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("That password isn't right.")); + assert!(matches!(&page(&mut model).step, Step::SetupPassword { password } if password.is_empty())); + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = PASSWORD.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + let Step::SetupCode { secret, uri, .. } = &page(&mut model).step else { panic!("{:?}", page(&mut model).step) }; + assert_eq!(secret, "JBSWY3DPEHPK3PXP"); + assert!(uri.starts_with("otpauth://totp/"), "{uri}"); + // A code that isn't six digits is told so; a wrong one is refused by the server and the secret stays. + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = "12345".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("Type the six digits the app shows.")); + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = "654 321".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.as_deref().is_some_and(|e| e.contains("isn't right")), "{:?}", page(&mut model).error); + assert!(matches!(&page(&mut model).step, Step::SetupCode { .. })); + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = " 123 456 ".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + let Step::Codes { codes, .. } = &page(&mut model).step else { panic!("{:?}", page(&mut model).step) }; + assert_eq!(codes.len(), 10); + assert!(page(&mut model).is_on(), "the status was asked again"); + // Done: the codes are dropped, and so is everything secret of the page. + feed(&mut model, &agent, Event::Security(SecurityEvent::Home)); + assert_eq!(page(&mut model).step, Step::Home); + let shown = format!("{:?}", model.screen); + assert!(!shown.contains("AAAA-BBBB") && !shown.contains("JBSWY"), "{shown}"); +} + +#[test] +fn leaving_the_page_drops_the_recovery_codes() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartRegenerate)); + type_into(&mut model, |step| { + if let Step::Regenerate { password, code } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::Codes { .. })); + feed(&mut model, &agent, Event::Security(SecurityEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + let shown = format!("{:?}", model.screen); + assert!(!shown.contains("AAAA"), "{shown}"); + // Opened again, the page starts at its start with nothing shown. + feed(&mut model, &agent, Event::OpenSecurity); + assert_eq!(page(&mut model).step, Step::Home); +} + +#[test] +fn turning_it_off_takes_the_password_and_a_code_or_a_recovery_code() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartDisable)); + let set = |model: &mut Model, pw: &str, code: &str| { + type_into(model, |step| { + if let Step::Disable { password, code: c, .. } = step { + *password = pw.into(); + *c = code.into(); + } + }); + }; + set(&mut model, PASSWORD, "12"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("A code from the app is six digits.")); + set(&mut model, "wrong password!", CODE); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("That password isn't right.")); + // A lock says how long to wait, and nothing changes. + agent.factor.lock().unwrap().locked = true; + set(&mut model, PASSWORD, CODE); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("Too many wrong codes. Try again in 14 minutes.")); + assert!(agent.factor.lock().unwrap().enabled); + agent.factor.lock().unwrap().locked = false; + // A recovery code, once the field is switched. + feed(&mut model, &agent, Event::Security(SecurityEvent::ToggleRecovery)); + set(&mut model, PASSWORD, RECOVERY); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).step, Step::Home); + assert!(!page(&mut model).is_on()); + assert!(page(&mut model).note.as_deref().is_some_and(|n| n.contains("is off"))); + // Passwords never stay in the page. + assert!(!format!("{:?}", model.screen).contains(PASSWORD)); +} + +#[test] +fn the_password_and_the_other_computers() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().other_computers = 2; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartPassword)); + let set = |model: &mut Model, cur: &str, new: &str, again: &str| { + type_into(model, |step| { + if let Step::Password { current, new: n, again: a } = step { + *current = cur.into(); + *n = new.into(); + *a = again.into(); + } + }); + }; + set(&mut model, PASSWORD, "a brand new password", "another one"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("The two new passwords aren't the same.")); + set(&mut model, PASSWORD, "short", "short"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("passwords are 8 to 1024 bytes")); + set(&mut model, PASSWORD, "a brand new password", "a brand new password"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).note.as_deref().is_some_and(|n| n.starts_with("Password changed"))); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartSignOutOthers)); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).note.as_deref(), Some("Signed 2 other computers out.")); + assert!(!format!("{:?}", model.screen).contains("brand new")); +} + +#[test] +fn a_join_for_an_account_with_two_factor_asks_for_the_code_and_keeps_the_invite() { + let agent = FakeAgent::new(signed_out()); + let required = Error::RefusedWith { + code: "two_factor_required".into(), + message: "needs a code".into(), + two_factor: Some(craft_cloud_app::TwoFactorChallenge { challenge: String::new(), expires_ms: 0 }), + retry_after_secs: None, + }; + agent.joins.lock().unwrap().push(Ok(signed_in())); + agent.joins.lock().unwrap().push(Err(Error::Refused { code: "two_factor_invalid".into(), message: "wrong".into(), name_taken: None })); + agent.joins.lock().unwrap().push(Err(required)); + let mut model = model_after(&agent, vec![]); + let Screen::Join(form) = &mut model.screen else { panic!("{:?}", model.screen) }; + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + let mut sent = Vec::new(); + let start = no_start(); + let run = |model: &mut Model, event: Event, sent: &mut Vec| { + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Effect::Join(request) = &effect { + sent.push(request.clone()); + } + if let Some(answer) = call(&agent, &start, effect) { + pending.extend(model.update(answer)); + } + } + }; + run(&mut model, Event::SubmitJoin, &mut sent); + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert_eq!(second.error, None); + assert_eq!((sent[0].totp_code.clone(), sent[0].recovery_code.clone()), (None, None), "the first join carries no code"); + // Six digits are needed; the same join goes again with them. + second.code = "12".into(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert_eq!(second.error.as_deref(), Some("Type the six digits the app shows.")); + assert_eq!(sent.len(), 1); + second.code = "654 321".into(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + assert_eq!(sent.len(), 2); + assert_eq!( + (sent[1].join.as_str(), sent[1].totp_code.as_deref(), sent[1].recovery_code.as_deref()), + (sent[0].join.as_str(), Some("654321"), None) + ); + // The server said it was wrong: back at the field, with the reason, the code cleared. + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert!(second.error.as_deref().is_some_and(|e| e.contains("isn't right")), "{:?}", second.error); + assert!(second.code.is_empty()); + // A recovery code instead. + second.recovery = true; + second.code = RECOVERY.into(); + *agent.status.lock().unwrap() = signed_in(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + assert_eq!(sent[2].recovery_code.as_deref(), Some(RECOVERY)); + assert!(matches!(model.screen, Screen::SignedIn(_)), "{:?}", model.screen); + // Back from the field goes to the join form, which keeps what was typed. + *agent.status.lock().unwrap() = signed_out(); + let mut again = model_after(&agent, vec![]); + if let Screen::Join(form) = &mut again.screen { + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + } + agent.joins.lock().unwrap().push(Err(Error::RefusedWith { + code: "two_factor_required".into(), + message: "m".into(), + two_factor: None, + retry_after_secs: None, + })); + run(&mut again, Event::SubmitJoin, &mut sent); + run(&mut again, Event::CancelSecondFactor, &mut sent); + assert!(matches!(&again.screen, Screen::Join(form) if form.join == "cloud.shadows.ch/K7M4-Q9TX")); +} + +// ---- the screens ---- + +fn harness(model: Model, theme: egui::Theme) -> Harness<'static, (Model, Vec)> { + Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu_setup(super::tests::shared_wgpu()).build_ui_state( + |ui, (model, events): &mut (Model, Vec)| { + events.extend(show(ui, model, NOW)); + }, + (model, Vec::new()), + ) +} + +fn snapshot(model: Model, name: &str) { + let mut h = harness(model, egui::Theme::Light); + h.run_steps(4); + let image = h.render().unwrap(); + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("snapshots"); + let path = dir.join(format!("{name}.png")); + if std::env::var_os("UPDATE_SNAPSHOTS").is_some() { + std::fs::create_dir_all(&dir).unwrap(); + image.save(&path).unwrap(); + return; + } + let new = dir.join(format!("{name}.new.png")); + let Ok(old) = image::open(&path) else { + image.save(&new).unwrap(); + panic!("no snapshot {} (UPDATE_SNAPSHOTS=1 writes it; this run's is {})", path.display(), new.display()); + }; + let old = old.into_rgba8(); + let same_size = old.dimensions() == image.dimensions(); + let different = old.pixels().zip(image.pixels()).filter(|(a, b)| a.0.iter().zip(b.0.iter()).any(|(x, y)| x.abs_diff(*y) > TOLERANCE)).count(); + let allowed = (image.width() * image.height()) as usize * MAX_DIFFERENT / 1000; + if !same_size || different > allowed { + image.save(&new).unwrap(); + panic!("{name}: {different} pixels differ (at most {allowed}); this run's is {}", new.display()); + } + let _ = std::fs::remove_file(new); +} + +#[test] +fn snapshot_second_factor() { + let agent = FakeAgent::new(signed_out()); + agent.joins.lock().unwrap().push(Err(Error::RefusedWith { + code: "two_factor_required".into(), + message: "m".into(), + two_factor: None, + retry_after_secs: None, + })); + let mut model = model_after(&agent, vec![]); + if let Screen::Join(form) = &mut model.screen { + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + } + feed(&mut model, &agent, Event::SubmitJoin); + assert!(matches!(model.screen, Screen::SecondFactor(_))); + snapshot(model, "second-factor"); +} + +#[test] +fn snapshot_second_factor_with_a_wrong_code() { + let form = crate::security::SecondFactorForm::new( + crate::model::JoinForm { join: "cloud.shadows.ch/K7M4-Q9TX".into(), folder: "/home/fabio/Craft Cloud".into(), ..Default::default() }, + None, + Some("That code isn't right, or it was used already. Wait for the app's next code and try again.".into()), + ); + let mut model = Model::new(defaults(), true, String::new()); + model.screen = Screen::SecondFactor(form); + snapshot(model, "second-factor-wrong-code"); +} + +fn security_model(agent: &FakeAgent, events: Vec) -> Model { + let mut model = on_security(agent); + for event in events { + feed(&mut model, agent, event); + } + model +} + +#[test] +fn snapshot_security_off() { + snapshot(security_model(&FakeAgent::new(signed_in()), vec![]), "security-off"); +} + +#[test] +fn snapshot_security_on() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + snapshot(security_model(&agent, vec![]), "security-on"); +} + +#[test] +fn snapshot_security_unsupported() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().unsupported = true; + snapshot(security_model(&agent, vec![]), "security-unsupported"); +} + +#[test] +fn snapshot_security_setup_with_its_qr_code() { + let agent = FakeAgent::new(signed_in()); + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartSetup)]); + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = PASSWORD.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::SetupCode { .. })); + snapshot(model, "security-setup"); +} + +#[test] +fn snapshot_security_recovery_codes() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartRegenerate)]); + type_into(&mut model, |step| { + if let Step::Regenerate { password, code } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::Codes { .. })); + snapshot(model, "security-recovery-codes"); +} + +#[test] +fn snapshot_security_turning_it_off_when_locked() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartDisable)]); + agent.factor.lock().unwrap().locked = true; + type_into(&mut model, |step| { + if let Step::Disable { password, code, .. } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.is_some()); + snapshot(model, "security-disable-locked"); +} + +// ---- the buttons ---- + +#[test] +fn the_overview_has_a_security_button_and_the_page_its_buttons() { + let model = model_after(&FakeAgent::new(signed_in()), vec![]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Security…").click(); + h.run_steps(2); + assert!(h.state().1.contains(&Event::OpenSecurity), "{:?}", h.state().1); + + let agent = FakeAgent::new(signed_in()); + let mut h = harness(security_model(&agent, vec![]), egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Set up two-factor sign-in…").click(); + h.run_steps(2); + h.get_by_label("Change password…").click(); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + let events = &h.state().1; + assert!(events.contains(&Event::Security(SecurityEvent::StartSetup)), "{events:?}"); + assert!(events.contains(&Event::Security(SecurityEvent::StartPassword)), "{events:?}"); + assert!(events.contains(&Event::Security(SecurityEvent::Close)), "{events:?}"); +} + +// ---- what egui remembers of what was typed ---- + +type H = Harness<'static, (Model, Vec)>; + +/// Whether egui still has state for the field: and whether Undo could bring typed text back into an empty field. +fn remembered(h: &H, name: &str) -> (bool, bool) { + use egui::text_edit::TextEditState; + let state = TextEditState::load(&h.ctx, super::security::secret_id(name)); + let undoable = state.as_ref().is_some_and(|s| s.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()); + (state.is_some(), undoable) +} + +fn undo(h: &mut H, role: egui::accesskit::Role) { + h.get_by_role(role).focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); +} + +#[test] +fn a_typed_password_or_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let model = security_model(&agent, vec![Event::Security(SecurityEvent::StartDisable)]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(3); + let typed = |h: &H| match &h.state().0.screen { + Screen::Security(page) => match &page.step { + Step::Disable { password, code, .. } => (password.clone(), code.clone()), + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + }; + + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("hunter2-secret"); + h.run_steps(8); + assert!(remembered(&h, "disable-password").1, "while typed, egui has the field's history (the test sees what it guards against)"); + h.get_by_role(Role::TextInput).focus(); + h.get_by_role(Role::TextInput).type_text("123456"); + h.run_steps(8); + assert_eq!(typed(&h), ("hunter2-secret".into(), "123456".into())); + assert!(remembered(&h, "disable-code").1, "and the code's"); + + // Sent: the model moved the typed text into the call and cleared the fields. + let effects = h.state_mut().0.update(Event::Security(SecurityEvent::Submit)); + assert!(matches!(effects.as_slice(), [Effect::Security(_)]), "{effects:?}"); + h.run_steps(2); + assert!(!remembered(&h, "disable-password").1, "once sent: no history (a field still drawn has an empty state)"); + assert!(!remembered(&h, "disable-code").1, "once sent: no history (a field still drawn has an empty state)"); + let refused = Error::Refused { code: "wrong_password".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::SecurityAnswer(crate::security::SecurityAnswer::Disable(Err(refused)))); + h.run_steps(3); + undo(&mut h, Role::PasswordInput); + undo(&mut h, Role::TextInput); + assert_eq!(typed(&h), (String::new(), String::new()), "Undo brings nothing back"); + + // Typed again and cancelled, then the step opened again. + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("another-secret"); + h.run_steps(8); + assert!(remembered(&h, "disable-password").1); + h.state_mut().0.update(Event::Security(SecurityEvent::Home)); + h.run_steps(2); + assert_eq!(remembered(&h, "disable-password"), (false, false), "once cancelled"); + h.state_mut().0.update(Event::Security(SecurityEvent::StartDisable)); + h.run_steps(3); + undo(&mut h, Role::PasswordInput); + assert_eq!(typed(&h).0, ""); + + // Left with something typed: the same. + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("third-secret"); + h.run_steps(3); + h.state_mut().0.update(Event::Security(SecurityEvent::Close)); + h.run_steps(2); + assert_eq!(remembered(&h, "disable-password"), (false, false), "once the page is left"); +} + +#[test] +fn a_typed_join_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + let form = crate::security::SecondFactorForm::new( + crate::model::JoinForm { join: "cloud.shadows.ch/K7M4-Q9TX".into(), folder: absolute_folder(), ..Default::default() }, + None, + None, + ); + let mut model = Model::new(defaults(), true, String::new()); + model.screen = Screen::SecondFactor(form); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(3); + h.get_by_role(Role::TextInput).focus(); + h.get_by_role(Role::TextInput).type_text("654321"); + h.run_steps(8); + assert!(remembered(&h, "second-factor-code").1, "while typed, egui has its history"); + // Back to the join form, then the field again. + h.state_mut().0.update(Event::CancelSecondFactor); + h.run_steps(2); + assert_eq!(remembered(&h, "second-factor-code"), (false, false), "once cancelled"); + let form = crate::security::SecondFactorForm::new(crate::model::JoinForm::default(), None, None); + h.state_mut().0.screen = Screen::SecondFactor(form); + h.run_steps(3); + undo(&mut h, Role::TextInput); + match &h.state().0.screen { + Screen::SecondFactor(form) => assert_eq!(form.code, "", "Undo brings nothing back"), + other => panic!("{other:?}"), + } + // A code sent (the model clears or leaves the screen): forgotten as well. + h.get_by_role(Role::TextInput).type_text("111111"); + h.run_steps(8); + h.state_mut().0.screen = Screen::Join(Default::default()); + h.run_steps(2); + assert_eq!(remembered(&h, "second-factor-code"), (false, false), "once the screen is left"); +} diff --git a/apps/craft-cloud-desktop/src/ui/trash.rs b/apps/craft-cloud-desktop/src/ui/trash.rs new file mode 100644 index 00000000..1f2ac02b --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/trash.rs @@ -0,0 +1,102 @@ +//! Browse deleted files and restore or remove them from the trash. + +use egui::RichText; + +use crate::model::{Event, Trash}; + +pub(super) fn show(ui: &mut egui::Ui, trash: &mut Trash, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Trash"); + if ui.button("Back").clicked() { + events.push(Event::BackTrash); + } + if super::wide_button(ui, !trash.loading && trash.action.is_none(), "Refresh") { + events.push(Event::LoadTrash); + } + if trash.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading trash…"); + }); + } + if let Some(error) = &trash.error { + super::error_line(ui, error); + } + if let Some(notice) = &trash.notice { + ui.label(RichText::new(notice).color(super::good(ui))); + } + if let Some(page) = &trash.page { + for item in &page.items { + ui.group(|ui| { + ui.label(RichText::new(&item.name).strong()); + if let Some(path) = &item.path { + ui.label(path); + } + let kind = if item.is_dir { "Folder" } else { "File" }; + ui.label(match item.size { + Some(size) => format!("{kind} · {size} bytes"), + None => format!("{kind} · size unavailable"), + }); + ui.label(format!("Deleted {} · {}", super::ago(now_ms, item.deleted_ms), date(item.deleted_ms))); + ui.label(format!("Removed for good after {}", date(item.purge_after_ms))); + ui.horizontal(|ui| { + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Restore")).clicked() { + events.push(Event::RestoreTrash(item.id)); + } + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Remove from trash")).clicked() { + events.push(Event::ConfirmRemoveTrash(item.id)); + } + }); + }); + ui.add_space(4.0); + } + if page.items.is_empty() { + ui.label("Trash is empty."); + } + if page.next.is_some() && super::wide_button(ui, !trash.loading && trash.action.is_none(), "More") { + events.push(Event::NextTrash); + } + } + if trash.action.is_some() { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Updating trash…"); + }); + } + if trash.confirm.is_some() { + egui::Window::new("Remove from trash?").collapsible(false).resizable(false).show(ui.ctx(), |ui| { + ui.label("This item can no longer be restored from the trash."); + ui.horizontal(|ui| { + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Remove from trash")).clicked() { + events.push(Event::RemoveTrash); + } + if ui.button("Cancel").clicked() { + events.push(Event::CancelRemoveTrash); + } + }); + }); + } +} + +fn date(timestamp_ms: i64) -> String { + let days = timestamp_ms.div_euclid(86_400_000) + 719_468; + let era = if days >= 0 { days } else { days - 146_096 } / 146_097; + let day_of_era = days - era * 146_097; + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_part = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_part + 2) / 5 + 1; + let month = month_part + if month_part < 10 { 3 } else { -9 }; + let year = year + if month <= 2 { 1 } else { 0 }; + format!("{year:04}-{month:02}-{day:02}") +} + +#[cfg(test)] +mod tests { + #[test] + fn dates_use_utc_calendar_days() { + assert_eq!(super::date(0), "1970-01-01"); + assert_eq!(super::date(1_790_000_000_000), "2026-09-21"); + assert_eq!(super::date(-1), "1969-12-31"); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/versions.rs b/apps/craft-cloud-desktop/src/ui/versions.rs new file mode 100644 index 00000000..dce5c7c5 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/versions.rs @@ -0,0 +1,94 @@ +//! Browse and restore versions of a file inside the sync folder. + +use egui::RichText; + +use crate::model::{Event, Versions}; + +pub(super) fn show(ui: &mut egui::Ui, versions: &mut Versions, now_ms: i64, events: &mut Vec) { + super::heading(ui, "File versions"); + if ui.button("Back").clicked() { + events.push(Event::BackVersions); + } + ui.add_space(8.0); + ui.label("Path inside the sync folder"); + let mut path = versions.path.clone(); + let response = ui.add_enabled_ui(!versions.restoring, |ui| ui.add(egui::TextEdit::singleline(&mut path).hint_text("Projects/file.txt"))).inner; + if response.changed() { + events.push(Event::VersionsPath(path)); + } + if super::wide_button(ui, !versions.loading && !versions.restoring, "Load versions") { + events.push(Event::LoadVersions); + } + if versions.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading versions…"); + }); + } + if let Some(error) = &versions.error { + super::error_line(ui, error); + } + if let Some(notice) = &versions.notice { + ui.label(RichText::new(notice).color(super::good(ui))); + } + if let Some(page) = &versions.page { + ui.add_space(8.0); + for version in &page.versions { + let current = Some(version.id) == versions.current_version; + ui.group(|ui| { + ui.label(RichText::new(if current { "Current version" } else { "Saved version" }).strong()); + ui.label(format!("{} · {} · {} bytes", date(version.created_ms), super::ago(now_ms, version.created_ms), version.size)); + ui.label(format!("Device {}", version.device)); + if version.pruned { + ui.label("This version can no longer be restored"); + } else if !current && ui.add_enabled(!versions.loading && !versions.restoring, egui::Button::new("Restore this version")).clicked() { + events.push(Event::ConfirmRestore(version.id)); + } + }); + ui.add_space(4.0); + } + if page.versions.is_empty() { + ui.label("No versions found."); + } + if page.next.is_some() && super::wide_button(ui, !versions.loading && !versions.restoring, "More versions") { + events.push(Event::NextVersions); + } + } + if let Some(version) = versions.confirm { + egui::Window::new("Restore version?").collapsible(false).resizable(false).show(ui.ctx(), |ui| { + ui.label("This makes the selected version the current file. Sync will update this computer."); + ui.horizontal(|ui| { + if ui.add_enabled(!versions.restoring && !versions.loading, egui::Button::new("Restore this version")).clicked() { + events.push(Event::RestoreVersion); + } + if ui.button("Cancel").clicked() { + events.push(Event::CancelRestore); + } + }); + let _ = version; + }); + } +} + +fn date(timestamp_ms: i64) -> String { + let days = timestamp_ms.div_euclid(86_400_000) + 719_468; + let era = if days >= 0 { days } else { days - 146_096 } / 146_097; + let day_of_era = days - era * 146_097; + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_part = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_part + 2) / 5 + 1; + let month = month_part + if month_part < 10 { 3 } else { -9 }; + let year = year + if month <= 2 { 1 } else { 0 }; + format!("{year:04}-{month:02}-{day:02}") +} + +#[cfg(test)] +mod tests { + #[test] + fn dates_use_utc_calendar_days() { + assert_eq!(super::date(0), "1970-01-01"); + assert_eq!(super::date(1_790_000_000_000), "2026-09-21"); + } +} diff --git a/apps/craft-cloud-desktop/tests/snapshots/activity.png b/apps/craft-cloud-desktop/tests/snapshots/activity.png new file mode 100644 index 00000000..7af536ac Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/activity.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png new file mode 100644 index 00000000..d9e5541f Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png new file mode 100644 index 00000000..bb61c09b Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png new file mode 100644 index 00000000..108f0e3d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account.png new file mode 100644 index 00000000..108f0e3d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png new file mode 100644 index 00000000..8b561717 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png new file mode 100644 index 00000000..2910cddc Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png new file mode 100644 index 00000000..914390c9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png b/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png new file mode 100644 index 00000000..a582dbdc Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png b/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png new file mode 100644 index 00000000..9eb8a4d6 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png b/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png new file mode 100644 index 00000000..7031f82c Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png b/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png new file mode 100644 index 00000000..899aedb9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png b/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png new file mode 100644 index 00000000..6cd92ef5 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-users.png b/apps/craft-cloud-desktop/tests/snapshots/admin-users.png new file mode 100644 index 00000000..899aedb9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-users.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png new file mode 100644 index 00000000..51a0258c Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png new file mode 100644 index 00000000..517b829d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png new file mode 100644 index 00000000..0894fd55 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png new file mode 100644 index 00000000..7c256de8 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log.png new file mode 100644 index 00000000..d0d491d1 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png b/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png new file mode 100644 index 00000000..ab5e2747 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/second-factor.png b/apps/craft-cloud-desktop/tests/snapshots/second-factor.png new file mode 100644 index 00000000..1857d806 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/second-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png b/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png new file mode 100644 index 00000000..cce9701e Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-off.png b/apps/craft-cloud-desktop/tests/snapshots/security-off.png new file mode 100644 index 00000000..4283d7dd Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-off.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-on.png b/apps/craft-cloud-desktop/tests/snapshots/security-on.png new file mode 100644 index 00000000..1399264e Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-on.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png b/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png new file mode 100644 index 00000000..a9215832 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-setup.png b/apps/craft-cloud-desktop/tests/snapshots/security-setup.png new file mode 100644 index 00000000..4e4b7082 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-setup.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png b/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png new file mode 100644 index 00000000..ed75428d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png index b443f0bb..371c5428 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png index 8a80cb23..8812a766 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png index b6365788..0fccbc93 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in.png index aef3f674..8fd3c3eb 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-error.png b/apps/craft-cloud-desktop/tests/snapshots/storage-error.png new file mode 100644 index 00000000..fdc3a154 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-error.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png b/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png new file mode 100644 index 00000000..bd504fdf Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png b/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png new file mode 100644 index 00000000..3d395ff9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png b/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png new file mode 100644 index 00000000..92613785 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png new file mode 100644 index 00000000..7057fc91 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png b/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png new file mode 100644 index 00000000..388a63e0 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png b/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png new file mode 100644 index 00000000..e4b47462 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png b/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png new file mode 100644 index 00000000..c78a5fd8 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png b/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png new file mode 100644 index 00000000..35dba9e1 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png new file mode 100644 index 00000000..5ca7a743 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png b/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png new file mode 100644 index 00000000..be3b1e82 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png b/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png new file mode 100644 index 00000000..2a943990 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png differ diff --git a/apps/craft-cloud-server/Cargo.toml b/apps/craft-cloud-server/Cargo.toml index 4c1c3ef6..d472dc3f 100644 --- a/apps/craft-cloud-server/Cargo.toml +++ b/apps/craft-cloud-server/Cargo.toml @@ -12,6 +12,7 @@ publish = false test-support = [] [dependencies] +aes-gcm.workspace = true argon2.workspace = true axum.workspace = true base64.workspace = true @@ -21,20 +22,27 @@ clap.workspace = true craft-cloud-meta.workspace = true craft-cloud-proto.workspace = true craft-cloud-store.workspace = true +# The audit export's body is a stream of pages. +futures-util.workspace = true getrandom.workspace = true +hmac.workspace = true serde.workspace = true serde_json.workspace = true +sha1.workspace = true +subtle.workspace = true thiserror.workspace = true tokio.workspace = true toml.workspace = true tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true +zeroize.workspace = true [dev-dependencies] +# The tests move the server's clock (`AppState::with_clock`): the feature is on for them only. +craft-cloud-server = { path = ".", features = ["test-support"] } reqwest.workspace = true tokio-tungstenite.workspace = true -futures-util.workspace = true serde_json.workspace = true sqlx.workspace = true tempfile.workspace = true diff --git a/apps/craft-cloud-server/src/account.rs b/apps/craft-cloud-server/src/account.rs new file mode 100644 index 00000000..6af4d177 --- /dev/null +++ b/apps/craft-cloud-server/src/account.rs @@ -0,0 +1,170 @@ +//! The signed-in account: `GET /api/v1/account`, `POST /api/v1/account/password`, and the device routes that change +//! a device (`PATCH /api/v1/devices/{id}`, `POST /api/v1/devices/revoke-others`). +//! +//! - **A wrong current password is `403 wrong_password`, never 401**: on a signed-in route the agent reads every 401 +//! as "this device was revoked" and signs out. The same holds for every later route that asks for a password. +//! - **Changing the password** reads the body (at most [`TINY_JSON_BYTES`]), counts the try against the account +//! ([`PASSWORD_CHANGES_PER_USER`] in 15 minutes, wrong passwords included), takes one of the server-wide +//! password-check slots (every slot busy is `429 rate_limited` and the try is given back), checks the current +//! password and the new one's rules, and hashes the new one, all in that one slot. Then one transaction sets the +//! hash, revokes every other device and deletes the account's open invite and pairings +//! ([`craft_cloud_meta::Meta::change_password`]); afterwards each revoked device's live connections end at once. +//! The device that asked stays signed in. +//! - **Nothing secret is logged**: not a password, a hash or a token. The count of revoked devices is logged at +//! `info`, the username at most at `debug`. + +use std::sync::Arc; + +use axum::Json; +use axum::extract::rejection::PathRejection; +use axum::extract::{Extension, Path, Request, State}; +use axum::http::StatusCode; +use craft_cloud_meta::{DeviceAuth, RevokeReason, UserId}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{AccountInfo, ChangePasswordRequest, DeviceInfo, RenameDeviceRequest, RevokedOthers, codes}; + +use crate::api::{PASSWORD_RETRY_SECS, TINY_JSON_BYTES, device_info, read_json}; +use crate::auth::{MAX_PASSWORD_BYTES, check_password_rules, hash_password, verify_password}; +use crate::error::ApiError; +use crate::state::AppState; + +/// Password change tries one account may make in [`AppState::password_changes`]'s window (15 minutes). +pub const PASSWORD_CHANGES_PER_USER: u32 = 5; + +type Shared = Arc; + +/// `403 wrong_password`: the current password was wrong. Never 401, see the module's notes. +pub(crate) fn wrong_password() -> ApiError { + ApiError::new(StatusCode::FORBIDDEN, codes::WRONG_PASSWORD, "the current password is wrong") +} + +/// Checks the signed-in account's own password (`user`, named `username`: a device's account, or a browser session's) for a +/// route that asks for one again (a change of the two-factor settings, an admin's confirmation) and returns the password hash it was verified against, which the change then compares with the +/// stored one under the account's lock. Wrong passwords are counted per account ([`PASSWORD_CHANGES_PER_USER`] in 15 +/// minutes, the window of the password change) and a right one is given back, so only a wrong password stays counted; +/// every check holds one of the server-wide password-check slots, and all busy is `429 rate_limited` that costs no try. A +/// wrong password is `403 wrong_password`, never 401. +pub(crate) async fn confirm_password(state: &AppState, user_id: UserId, username: &str, password: String) -> Result { + if password.len() > MAX_PASSWORD_BYTES { + return Err(ApiError::invalid("passwords are at most 1024 bytes")); + } + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{username}"); + let Some(counted) = state.password_changes.count(&key, PASSWORD_CHANGES_PER_USER, now) else { + return Err(ApiError::rate_limited("too many wrong passwords; wait a few minutes", None)); + }; + // Each check costs 19 MiB and real CPU time: all slots busy is told to come back, not queued, and costs no try. + let Ok(permit) = state.password_checks.clone().try_acquire_owned() else { + state.password_changes.refund(&key, counted); + return Err(ApiError::rate_limited("the server is busy checking other passwords; try again in a moment", Some(PASSWORD_RETRY_SECS))); + }; + let user = match state.meta.user_by_name(username).await { + Ok(Some(user)) if user.id == user_id => user, + other => { + state.password_changes.refund(&key, counted); + other?; + return Err(ApiError::unauthorized()); + } + }; + let hash = user.password_hash.clone(); + let ok = tokio::task::spawn_blocking(move || { + let _permit = permit; + verify_password(&password, &hash) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "password check task failed"); + ApiError::internal() + })?; + if !ok { + return Err(wrong_password()); + } + state.password_changes.refund(&key, counted); + Ok(user.password_hash) +} + +/// `GET /api/v1/account`. +pub async fn get_account(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let account = state.meta.account(auth.user).await?; + Ok(Json(AccountInfo { + user_id: account.id.0, + username: account.username, + role: account.role.as_str().to_owned(), + created_ms: account.created_ms, + password_changed_ms: account.password_changed_ms, + two_factor_enabled: account.two_factor_enabled, + })) +} + +/// `POST /api/v1/account/password`: `204` once the password is changed and the account's other devices are signed out. +pub async fn change_password(State(state): State, Extension(auth): Extension, request: Request) -> Result { + let req: ChangePasswordRequest = read_json(&state, request, TINY_JSON_BYTES).await?; + if req.current_password.len() > MAX_PASSWORD_BYTES || req.new_password.len() > MAX_PASSWORD_BYTES { + return Err(ApiError::invalid("passwords are at most 1024 bytes")); + } + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{}", auth.username); + let Some(counted) = state.password_changes.count(&key, PASSWORD_CHANGES_PER_USER, now) else { + return Err(ApiError::rate_limited("too many password change attempts; wait a few minutes", None)); + }; + // Each check costs 19 MiB and real CPU time (see the sign-in): all slots busy is told to come back, not queued, + // and costs the account no try, since nothing was checked. + let Ok(permit) = state.password_checks.clone().try_acquire_owned() else { + state.password_changes.refund(&key, counted); + return Err(ApiError::rate_limited("the server is busy checking other passwords; try again in a moment", Some(PASSWORD_RETRY_SECS))); + }; + let user = state.meta.user_by_name(&auth.username).await?.ok_or_else(ApiError::unauthorized)?; + let current_hash = user.password_hash.clone(); + let ChangePasswordRequest { current_password, new_password } = req; + let new_hash = tokio::task::spawn_blocking(move || { + let _permit = permit; + if !verify_password(¤t_password, ¤t_hash) { + return Err(wrong_password()); + } + check_password_rules(&new_password).map_err(|e| ApiError::new(StatusCode::BAD_REQUEST, codes::WEAK_PASSWORD, e.message))?; + if new_password == current_password { + return Err(ApiError::new(StatusCode::BAD_REQUEST, codes::WEAK_PASSWORD, "the new password must differ from the current one")); + } + hash_password(&new_password) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "password change task failed"); + ApiError::internal() + })??; + let revoked = state.meta.change_password(auth.user, &user.password_hash, &new_hash, auth.device, state.now_ms()).await?; + for device in &revoked { + state.live.revoke(*device); + } + tracing::debug!(user = %auth.username, "password changed"); + tracing::info!(device = %auth.device, revoked = revoked.len(), "password changed"); + Ok(StatusCode::NO_CONTENT) +} + +/// `PATCH /api/v1/devices/{id}`: renames one of the account's own devices that isn't revoked. Anything else is +/// `404 not_found`, so the answer never says whether another account has the device. +pub async fn rename_device( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let Path(id) = id?; + let req: RenameDeviceRequest = read_json(&state, request, TINY_JSON_BYTES).await?; + crate::api::check_device_name(&req.name)?; + let device = state.meta.rename_device(auth.user, id, &req.name).await?; + Ok(Json(device_info(device, auth.device))) +} + +/// `POST /api/v1/devices/revoke-others`: signs out every other device of the account. +pub async fn revoke_others(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let revoked = state.meta.revoke_other_devices(auth.user, auth.device, RevokeReason::Others, state.now_ms()).await?; + // Their live connections end now, not at their next request. + for device in &revoked { + state.live.revoke(*device); + } + tracing::info!(device = %auth.device, revoked = revoked.len(), "other devices signed out"); + Ok(Json(RevokedOthers { revoked: u32::try_from(revoked.len()).unwrap_or(u32::MAX) })) +} diff --git a/apps/craft-cloud-server/src/admin.rs b/apps/craft-cloud-server/src/admin.rs new file mode 100644 index 00000000..4c4d5db4 --- /dev/null +++ b/apps/craft-cloud-server/src/admin.rs @@ -0,0 +1,453 @@ +//! The admin API, under `/api/v1/admin`: server counters, accounts (list, create, change role and quota, disable, +//! enable, mark for removal), invites, password reset codes, and a user's devices. +//! +//! - **Every route is behind [`require_admin`](crate::auth::require_admin)** (after `require_device`): a member gets `403 +//! forbidden`, a bad token `401`. The role is read from the database for each request, so a demoted admin is out with +//! the next one. +//! - **Every change needs a fresh confirmation in its body**: the acting admin's current `password`. A device token never +//! expires, so a stolen admin token alone must not be enough to make an account an admin's or to sign others out. A +//! missing or mistyped field is `400 invalid`, a wrong password `403 wrong_password` (never 401: on a signed-in route +//! the agent reads 401 as "this device was revoked"). Tries are counted per account in +//! [`AppState::password_changes`], the same window the password change uses, but only a wrong password stays counted; +//! every check holds a server-wide password-check slot. An admin whose account has two-factor sign-in adds `totp_code` +//! (or `recovery_code`): missing or wrong is `403 two_factor_invalid`, too many wrong ones `429 two_factor_locked`. The +//! code is used up by the confirmation (one time step serves one change, as at sign-in), and the check shares its +//! counters and limiter with sign-in. +//! - **Bodies are tiny** ([`TINY_JSON_BYTES`]), the `DELETE` routes carry one too, and are read under the same deadline +//! as every JSON body. +//! - **Nothing secret is logged**: not a password, a code or a join link. Events are logged at `info` with the kind and +//! the target's ID, never a username. +//! - **Live connections** of devices an admin signs out end at once in this process (after the commit); another server +//! process, or the command line, finds out at the next ping (about 30 seconds). + +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::rejection::{PathRejection, QueryRejection}; +use axum::extract::{DefaultBodyLimit, Extension, FromRequest, Path, Query, Request, State}; +use axum::http::StatusCode; +use axum::routing::{get, post}; +use axum::{Router, middleware}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{AdminActor, AdminUser, AdminVia, DeviceAuth, Role, UserId, valid_username}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{ + AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminManifestIndex, AdminPasswordReset, AdminServerInfo, + AdminStorageAccount, AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, UsageEstimateReason, UsageEstimates, +}; +use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::account::confirm_password; +use crate::api::{TINY_JSON_BYTES, body_too_slow, device_info}; +use crate::auth::require_admin; +use crate::error::ApiError; +use crate::state::AppState; + +type Shared = Arc; + +/// Accounts one page of `GET /admin/users` holds when `limit` is not given. +pub(crate) const DEFAULT_PAGE: u32 = 50; +/// The most accounts a page may hold. +pub(crate) const MAX_PAGE: u32 = 200; +/// The longest `query` of the user list, in bytes. +pub(crate) const MAX_QUERY_BYTES: usize = craft_cloud_meta::MAX_SEARCH_BYTES; + +/// The admin routes, all behind `require_admin`. `router` merges them into the signed-in routes, so `require_device` +/// wraps them too and runs first. +pub fn routes() -> Router { + Router::new() + .route("/admin/server", get(server)) + .route("/admin/storage", get(storage)) + .route("/admin/users", get(list_users).post(create_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}", axum::routing::patch(update_user).delete(remove_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/usage", get(user_usage)) + .route("/admin/users/{id}/disable", post(disable_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/enable", post(enable_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/invite", post(issue_invite).delete(revoke_invite).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/password-reset", post(issue_reset).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/reset-two-factor", post(reset_two_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/devices", get(list_devices)) + .route("/admin/users/{id}/devices/{device_id}", axum::routing::delete(revoke_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route_layer(middleware::from_fn(require_admin)) + .route_layer(middleware::from_fn(crate::web::reject_mixed_credentials)) +} + +// ---- helpers ---- + +/// Reads a JSON body of at most [`TINY_JSON_BYTES`] under the usual deadline. Whatever is wrong with its shape (not JSON, +/// a missing or mistyped field, an unknown role) is `400 invalid`; too long is `413 too_large`. +async fn read_body(state: &AppState, request: Request) -> Result { + let bytes = match tokio::time::timeout(state.limits.body_time(TINY_JSON_BYTES), Bytes::from_request(request, &())).await { + Ok(bytes) => bytes?, + Err(_) => return Err(body_too_slow()), + }; + serde_json::from_slice(&bytes) + .map_err(|_| ApiError::invalid("the body must be a JSON object with your current password in `password`, and the fields of this call")) +} + +/// What an admin's body gives for a confirmation: the password, and the second factor when the account has one. +pub(crate) struct Confirmation { + password: Option, + totp_code: Option, + recovery_code: Option, +} + +impl Confirmation { + pub(crate) fn new(password: Option, totp_code: Option, recovery_code: Option) -> Self { + Self { password, totp_code, recovery_code } + } +} + +/// Checks the acting admin's confirmation and returns the actor to hand to the change: the password ([`confirm_password`]: +/// one wrong password stays counted against the account, a right one is given back), then, when the admin's account has +/// two-factor sign-in, the second factor (`totp_code`, or `recovery_code`), which is used up here: the time step or the +/// recovery code can't serve a second change, and wrong ones are counted and can lock ([`crate::two_factor`]). A password +/// that is wrong is told before anything about the second factor. +async fn confirm(state: &AppState, auth: &DeviceAuth, given: Confirmation) -> Result { + confirm_actor(state, auth.user, &auth.username, AdminVia::Device(auth.device), given).await +} + +/// [`confirm`] for any admin who acts: `via` says whether a device or a browser session does (the web pages call this with +/// the same confirmation, the same counters and the same statuses as the JSON routes above). +pub(crate) async fn confirm_actor( + state: &AppState, + user: UserId, + username: &str, + via: AdminVia, + given: Confirmation, +) -> Result { + let password = given.password.ok_or_else(|| ApiError::invalid("this change needs your current password in `password`"))?; + // The shape of the second factor is checked before anything is counted. + let proof = crate::two_factor::parse_proof(given.totp_code, given.recovery_code)?; + let password_hash = confirm_password(state, user, username, password).await?; + let two_factor = crate::two_factor::confirm_admin_second_factor(state, user, username, proof).await?; + Ok(AdminActor { user, via, password_hash, two_factor }) +} + +fn user_id(id: Result, PathRejection>) -> Result { + let Path(id) = id?; + Ok(UserId(id)) +} + +pub(crate) fn role_of(name: &str) -> Result { + match name { + "admin" => Ok(Role::Admin), + "member" => Ok(Role::Member), + _ => Err(ApiError::invalid("role is admin or member")), + } +} + +fn user_info(u: AdminUser) -> AdminUserInfo { + AdminUserInfo { + id: u.id.0, + username: u.username, + role: u.role.as_str().to_owned(), + created_ms: u.created_ms, + disabled_ms: u.disabled_ms, + two_factor_enabled: u.two_factor_enabled, + devices_active: u.devices_active, + stored_bytes: u.stored_bytes, + quota_bytes: u.quota_bytes, + invite_pending: u.invite_pending, + } +} + +/// A fresh invite or reset code, as the admin gets it: shown once, never logged. +pub(crate) fn new_code() -> Result { + Code::generate(INVITE_CHARS).map_err(|e| { + tracing::error!(error = %e, "the system random source failed"); + ApiError::internal() + }) +} + +pub(crate) fn admin_code(state: &AppState, code: Code, expires_ms: i64) -> AdminCode { + let shown = code.as_str().to_owned(); + let join_url = state.public_url.as_ref().map(|server| JoinString::new(server.clone(), code).link()); + AdminCode { code: shown, join_url, expires_ms } +} + +pub(crate) fn end_live_connections(state: &AppState, devices: &[DeviceId]) { + for device in devices { + state.live.revoke(*device); + } +} + +// ---- reads ---- + +/// `GET /admin/server`: counters and kinds only. +async fn server(State(state): State) -> Result, ApiError> { + let counts = state.meta.admin_server_counts().await?; + Ok(Json(AdminServerInfo { + version: env!("CARGO_PKG_VERSION").to_owned(), + users: counts.users, + devices_active: counts.devices_active, + stored_bytes: counts.stored_bytes, + live_bytes: counts.live_bytes, + storage_kind: state.store.kind().to_owned(), + database_kind: state.meta.database_kind().to_owned(), + })) +} + +/// `GET /admin/storage`: bounded account and manifest-index counts, plus the ten largest accounts. +async fn storage(State(state): State) -> Result, ApiError> { + let overview = state.meta.admin_storage_overview().await?; + Ok(Json(AdminStorageOverview { + accounts: overview.accounts, + stored_bytes: overview.stored_bytes, + default_quota_bytes: overview.default_quota_bytes, + accounts_with_override: overview.accounts_with_override, + accounts_near_quota: overview.accounts_near_quota, + accounts_over_quota: overview.accounts_over_quota, + top_accounts: overview + .top_accounts + .into_iter() + .map(|account| AdminStorageAccount { + id: account.id.0, + username: account.username, + stored_bytes: account.stored_bytes, + quota_bytes: account.quota_bytes, + }) + .collect(), + manifest_index: AdminManifestIndex { + complete: overview.manifest_index.complete, + building: overview.manifest_index.building, + not_indexed: overview.manifest_index.not_indexed, + }, + })) +} + +/// `GET /admin/users/{id}/usage`: the account's stored bytes and labeled estimates. +async fn user_usage(State(state): State, id: Result, PathRejection>) -> Result, ApiError> { + let estimate = state.meta.admin_storage_estimate(user_id(id)?).await?; + let index_complete = estimate.reason.is_none(); + Ok(Json(AccountUsage { + stored_bytes: estimate.stored_bytes, + quota_bytes: estimate.quota_bytes, + trash_bytes: estimate.trash_bytes, + history_bytes: estimate.history_bytes, + estimates: UsageEstimates { index_complete, reason: estimate.reason.map(|_| UsageEstimateReason::IndexIncomplete) }, + })) +} + +#[derive(Debug, Deserialize)] +struct ListQuery { + cursor: Option, + limit: Option, + query: Option, +} + +/// The username a cursor stands for. A cursor is the last username of a page, in base64url: opaque to clients, and +/// checked like any username, since it is hostile input. +pub(crate) fn decode_cursor(cursor: &str) -> Result { + let bad = || ApiError::invalid("the cursor is not one this server gave"); + let bytes = URL_SAFE_NO_PAD.decode(cursor).map_err(|_| bad())?; + let name = String::from_utf8(bytes).map_err(|_| bad())?; + if valid_username(&name) { Ok(name) } else { Err(bad()) } +} + +/// `GET /admin/users?cursor&limit&query`: accounts by username, 50 a page (at most 200). `query` finds a username +/// containing it (case does not matter, at most 64 bytes of printable characters). +async fn list_users(State(state): State, q: Result, QueryRejection>) -> Result, ApiError> { + let Query(q) = q?; + let limit = q.limit.unwrap_or(DEFAULT_PAGE); + if !(1..=MAX_PAGE).contains(&limit) { + return Err(ApiError::invalid(format!("limit is 1 to {MAX_PAGE}"))); + } + let after = q.cursor.as_deref().map(decode_cursor).transpose()?; + let search = q.query.as_deref().filter(|s| !s.is_empty()); + if let Some(search) = search + && (search.len() > MAX_QUERY_BYTES || search.chars().any(char::is_control)) + { + return Err(ApiError::invalid(format!("query is at most {MAX_QUERY_BYTES} bytes of printable characters"))); + } + let page = state.meta.admin_list_users(after.as_deref(), search, limit, state.now_ms()).await?; + Ok(Json(AdminUserList { + users: page.users.into_iter().map(user_info).collect(), + next: page.next_after.map(|name| URL_SAFE_NO_PAD.encode(name.as_bytes())), + })) +} + +/// `GET /admin/users/{id}/devices`: every device of the account, revoked ones with when and why. +async fn list_devices( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let devices = state.meta.admin_devices(user_id(id)?).await?; + Ok(Json(AdminDevices { devices: devices.into_iter().map(|d| device_info(d, auth.device)).collect() })) +} + +// ---- changes ---- + +/// `POST /admin/users`: a new account without a password and its invite. +async fn create_user( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: AdminCreateUser = read_body(&state, request).await?; + if !valid_username(&req.username) { + return Err(ApiError::invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); + } + let role = role_of(&req.role)?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let now = state.now_ms(); + let (user, issued) = state.meta.admin_create_user(&actor, &req.username, role, &code.hash(), now, INVITE_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %user.id.0, "admin: account created"); + Ok(Json(AdminCreatedUser { user: user_info(user), invite: admin_code(&state, code, issued.expires_ms) })) +} + +/// `PATCH /admin/users/{id}`: a new role and/or quota. +async fn update_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminUpdateUser = read_body(&state, request).await?; + let role = req.role.as_deref().map(role_of).transpose()?; + if role.is_none() && req.quota_bytes.is_none() { + return Err(ApiError::invalid("give a role or a quota_bytes to change")); + } + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let user = state.meta.admin_update_user(&actor, target, role, req.quota_bytes, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, role_changed = role.is_some(), quota_changed = req.quota_bytes.is_some(), "admin: account changed"); + Ok(Json(user_info(user))) +} + +/// `POST /admin/users/{id}/disable`: 204. +async fn disable_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_disable_user(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + crate::spaces::wake_spaces_of(&state, target).await; + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: account disabled"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /admin/users/{id}/enable`: 204. Devices that were signed out stay signed out. +async fn enable_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_enable_user(&actor, target, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: account enabled"); + Ok(StatusCode::NO_CONTENT) +} + +/// `DELETE /admin/users/{id}`: 202. Marks the account for removal; nothing it stored is deleted. +async fn remove_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_remove_user(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + crate::spaces::wake_spaces_of(&state, target).await; + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: account marked for removal"); + Ok(StatusCode::ACCEPTED) +} + +/// `POST /admin/users/{id}/invite`: a new invite for an account that has no password yet. +async fn issue_invite( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let issued = state.meta.admin_issue_invite(&actor, target, &code.hash(), state.now_ms(), INVITE_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: invite issued"); + Ok(Json(admin_code(&state, code, issued.expires_ms))) +} + +/// `DELETE /admin/users/{id}/invite`: 204. Ends the account's open invite or reset code. +async fn revoke_invite( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_revoke_invite(&actor, target, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: invite revoked"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /admin/users/{id}/password-reset`: a code good for an hour. Redeemed at `POST /join`, it sets a new password, +/// signs every device of the account out, and signs the redeeming computer in. +async fn issue_reset( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminPasswordReset = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let issued = + state.meta.admin_issue_reset(&actor, target, &code.hash(), req.reset_two_factor, state.now_ms(), craft_cloud_meta::RESET_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %target.0, reset_two_factor = req.reset_two_factor, "admin: password reset code issued"); + Ok(Json(admin_code(&state, code, issued.expires_ms))) +} + +/// `POST /admin/users/{id}/reset-two-factor`: 204. Takes the account's two-factor sign-in away and signs every device of +/// it out (for someone who lost their authenticator and their recovery codes). +async fn reset_two_factor( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_reset_two_factor(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: two-factor sign-in reset"); + Ok(StatusCode::NO_CONTENT) +} + +/// `DELETE /admin/users/{id}/devices/{device_id}`: 204. +async fn revoke_device( + State(state): State, + Extension(auth): Extension, + ids: Result, PathRejection>, + request: Request, +) -> Result { + let Path((target, device)) = ids?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_revoke_device(&actor, UserId(target), device, state.now_ms()).await?; + state.live.revoke(device); + tracing::info!(actor = %auth.device, target = %target, %device, "admin: device revoked"); + Ok(StatusCode::NO_CONTENT) +} diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index efc5392f..61db4600 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -1,4 +1,6 @@ -//! The HTTP API, under `/api/v1`. Everything but `info`, `auth/login` and `join` needs a device token. `GET /live` +//! The HTTP API, under `/api/v1`. Everything but `info`, `auth/login`, `auth/login/second-factor` and `join` needs a device +//! token. The account and its devices' routes (`account`, `account/password`, renaming and signing out devices) are in +//! [`crate::account`], two-factor sign-in's in [`crate::two_factor`]. `GET /live` //! becomes the live service's WebSocket ([`crate::live`]); join codes (`pairings`, `join`, and the `/join/{code}` page //! outside the API) are in [`crate::join`]. @@ -11,35 +13,42 @@ use axum::extract::rejection::{PathRejection, QueryRejection}; use axum::extract::{ConnectInfo, DefaultBodyLimit, Extension, FromRequest, Path, Query, Request, State}; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Response}; -use axum::routing::{delete, get, post, put}; +use axum::routing::{MethodRouter, delete, get, post, put}; use axum::{Json, Router, middleware}; -use craft_cloud_meta::{BlobKind, DeviceAuth, valid_username}; +use craft_cloud_meta::{BlobKind, DeviceAuth, DeviceClient, DeviceRecord, NewChallenge, Origin, RevokeReason, SignIn, valid_username}; use craft_cloud_proto::api::{ - CommitRequest, CommitResponse, DeviceInfo, InfoResponse, LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, - PROTOCOL_MAX, PROTOCOL_MIN, SpaceResponse, codes, + AccountUsage, CommitRequest, CommitResponse, DeviceInfo, HISTORY_DEFAULT_LIMIT, HistoryCursor, HistoryResponse, InfoResponse, LoginRequest, + LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, RestoreRequest, RestoreResponse, SpaceResponse, + SpaceRole, TRASH_DEFAULT_LIMIT, TrashCursor, TrashItem, TrashResponse, TrashRestoreResponse, UsageEstimateReason, UsageEstimates, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_JSON_BYTES, MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, Manifest, ManifestId, NodeId, PageCursor}; use craft_cloud_store::{BlobKey, StoreError}; use serde::Deserialize; +use crate::account; +use crate::audit; use crate::auth::{new_token, require_device, verify_password}; use crate::error::ApiError; use crate::join; use crate::ratelimit::{client_ip, ip_key}; +use crate::space::{SpaceAccess, SpaceGate, require_space}; use crate::state::AppState; +use crate::two_factor; /// Chunker parameter sets the server accepts. pub const CHUNKER_PARAMS: &[&str] = &["fastcdc-v1"]; /// Login attempts per account and per client address in five minutes. -const LOGINS_PER_USER: u32 = 10; -const LOGINS_PER_IP: u32 = 30; +pub(crate) const LOGINS_PER_USER: u32 = 10; +pub(crate) const LOGINS_PER_IP: u32 = 30; /// Longest change-feed long-poll. const MAX_WAIT_SECS: u64 = 60; /// `Retry-After` for a refused long-poll. const WAITERS_RETRY_SECS: u32 = 5; /// Small JSON bodies (login, join and missing-chunk queries). pub(crate) const SMALL_JSON_BYTES: usize = 1024 * 1024; +/// Tiny JSON bodies (a password change, a device's new name): a few short strings, so far less than a small body. +pub(crate) const TINY_JSON_BYTES: usize = 4 * 1024; /// `Retry-After` for a sign-in refused because every password check slot is busy. pub(crate) const PASSWORD_RETRY_SECS: u32 = 1; @@ -47,33 +56,77 @@ type Shared = Arc; pub fn router(state: Shared) -> Router { let signed_in = Router::new() + .route("/account", get(account::get_account)) + .route("/account/password", post(account::change_password).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/activity", get(audit::account_activity)) + .route("/account/two-factor", get(two_factor::get_status)) + .route("/account/two-factor/setup", post(two_factor::setup).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/confirm", post(two_factor::confirm).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/disable", post(two_factor::disable).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/recovery-codes", post(two_factor::recovery_codes).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/devices", get(list_devices)) - .route("/devices/{id}", delete(revoke_device)) + .route("/devices/revoke-others", post(account::revoke_others)) + .route("/devices/{id}", delete(revoke_device).patch(account::rename_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/pairings", post(join::create_pairing)) .route("/pairings/{id}", get(join::pairing_status).delete(join::delete_pairing)) - .route("/chunks/missing", post(missing_chunks).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) - .route("/chunks/{id}", put(put_chunk).get(get_chunk).layer(DefaultBodyLimit::max(MAX_CHUNK_BYTES as usize))) - .route("/manifests/{id}", put(put_manifest).get(get_manifest).layer(DefaultBodyLimit::max(MAX_MANIFEST_BYTES))) - .route("/commit", post(commit).layer(DefaultBodyLimit::max(MAX_BATCH_JSON_BYTES))) - .route("/saves", post(attach_save).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) - .route("/space", get(space)) - .route("/tree", get(tree)) - .route("/changes", get(changes)) - .route("/live", get(crate::live::connect)) + .route("/account/usage", get(account_usage)) + .merge(space_routes(&state)) + .merge(crate::spaces::routes()) + // The admin routes carry `require_admin` themselves; `require_device` below wraps them too and runs first. + .merge(crate::admin::routes()) + .merge(audit::admin_routes()) + .merge(crate::spaces::admin_routes()) .route_layer(middleware::from_fn_with_state(state.clone(), require_device)); let public = Router::new() .route("/info", get(info)) .route("/auth/login", post(login).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/auth/login/second-factor", post(two_factor::second_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/join", post(join::join).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))); - Router::new() + let app = Router::new() .nest("/api/v1", public.merge(signed_in)) // Version 1 isn't known yet when a client first asks what the server speaks. .route("/api/info", get(info)) // A join link opened in a browser. .route("/join/{code}", get(join::join_page)) .fallback(no_such_endpoint) - .method_not_allowed_fallback(wrong_method) - .with_state(state) + .method_not_allowed_fallback(wrong_method); + // The admin pages exist only on a server that was set up for them. + let app = + if state.web.is_some() { app.nest("/web", crate::web::routes(state.clone())).merge(crate::web::trailing_slash(state.clone())) } else { app }; + app.with_state(state) +} + +/// A route group's role gate: the layer that refuses a caller whose role in the request's space is below `min` (see +/// [`crate::space`]). Applied per method, so one path can need different roles for different methods. +fn gated(state: &Shared, min: SpaceRole, routes: MethodRouter) -> MethodRouter { + routes.route_layer(middleware::from_fn_with_state(SpaceGate { state: state.clone(), min }, require_space)) +} + +/// The space-scoped routes, each with the least role it needs (`docs/design/collaboration.md`, 3.2). A route not listed here +/// has no space layer and takes no `craft-space` header. A new space-scoped route is added here, with its role. +fn space_routes(state: &Shared) -> Router { + use SpaceRole::{Editor, Owner, Viewer}; + Router::new() + .route("/chunks/missing", gated(state, Editor, post(missing_chunks)).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route( + "/chunks/{id}", + gated(state, Viewer, get(get_chunk)).merge(gated(state, Editor, put(put_chunk))).layer(DefaultBodyLimit::max(MAX_CHUNK_BYTES as usize)), + ) + .route( + "/manifests/{id}", + gated(state, Viewer, get(get_manifest)).merge(gated(state, Editor, put(put_manifest))).layer(DefaultBodyLimit::max(MAX_MANIFEST_BYTES)), + ) + .route("/commit", gated(state, Editor, post(commit)).layer(DefaultBodyLimit::max(MAX_BATCH_JSON_BYTES))) + .route("/saves", gated(state, Editor, post(attach_save)).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/nodes/{id}/history", gated(state, Viewer, get(history))) + .route("/nodes/{id}/restore", gated(state, Editor, post(restore)).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/trash", gated(state, Viewer, get(trash))) + .route("/trash/{id}/restore", gated(state, Editor, post(restore_from_trash))) + .route("/trash/{id}", gated(state, Owner, delete(remove_from_trash))) + .route("/space", gated(state, Viewer, get(space))) + .route("/tree", gated(state, Viewer, get(tree))) + .route("/changes", gated(state, Viewer, get(changes))) + .route("/live", gated(state, Viewer, get(crate::live::connect))) } async fn no_such_endpoint() -> ApiError { @@ -109,6 +162,7 @@ async fn login( return Err(ApiError::invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); } check_device_name(&req.device_name)?; + let client = device_client(req.client_kind.as_deref(), req.platform.as_deref(), req.app_version.as_deref())?; // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. let now = tokio::time::Instant::now().into_std(); let ip = client_ip(peer, &headers, &state.trusted_proxies); @@ -131,7 +185,10 @@ async fn login( return Err(ApiError::invalid("passwords are at most 1024 bytes")); } let user = state.meta.user_by_name(&username).await?; - let hash = user.as_ref().map_or(state.dummy_hash.as_str(), |u| u.password_hash.as_str()).to_owned(); + let known = user.as_ref().map(|u| u.id); + // An account without a password (an open invite) costs the same Argon2id check as a name nobody has, and never signs in. + let usable = user.as_ref().is_some_and(|u| crate::auth::usable_hash(&u.password_hash)); + let hash = user.as_ref().filter(|_| usable).map_or(state.dummy_hash.as_str(), |u| u.password_hash.as_str()).to_owned(); let password = req.password.clone(); let ok = tokio::task::spawn_blocking(move || { let _permit = permit; @@ -142,11 +199,29 @@ async fn login( tracing::error!(error = %e, "password check task failed"); ApiError::internal() })?; - let (Some(user), true) = (user, ok) else { + let (Some(user), true) = (user, ok && usable) else { + // Written after the answer is decided and in a transaction of its own: a log that cannot be written changes + // neither the answer nor the limits above. A name no account has is never put in the log. + audit::login_failed(&state, known).await; return Err(ApiError::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "wrong username or password")); }; let (token, token_hash) = new_token()?; - let device = state.meta.create_device(user.id, &req.device_name, &token_hash, state.now_ms()).await?; + // A challenge is made only if the account needs a second factor; the token the client gets for it is a secret of its own. + let (challenge_token, challenge_hash) = new_token()?; + let challenge = NewChallenge { id_hash: challenge_hash, client_key: ip_key(ip) }; + // Created only if the password just checked is still the account's: one changed meanwhile (and the devices + // signed out with it) must not be followed by a device that the old password let in. Answered as a wrong password. The + // account's two-factor state is read under the same lock: with it on, there is a challenge and no device. + let signed_in = + state.meta.sign_in(user.id, (&user.password_hash, user.auth_gen), &req.device_name, &client, &token_hash, &challenge, state.now_ms()).await?; + let device = match signed_in { + None => return Err(ApiError::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "wrong username or password")), + Some(SignIn::Device(device)) => device, + Some(SignIn::Challenge { expires_ms }) => { + tracing::info!("sign-in needs a second factor"); + return Err(ApiError::two_factor_required(challenge_token, expires_ms)); + } + }; // The username is the only personal identifier the server could log here: kept to debug. tracing::debug!(user = %user.username, "sign-in"); tracing::info!(%device, "device signed in"); @@ -162,26 +237,63 @@ pub(crate) fn check_device_name(name: &str) -> Result<(), ApiError> { Ok(()) } -async fn space(State(state): State, Extension(auth): Extension) -> Result, ApiError> { - let info = state.meta.space_info(auth.space).await?; - Ok(Json(SpaceResponse { space: auth.space, root: info.root, epoch: info.epoch, seq: info.seq, username: auth.username })) +/// The client fields of a sign-in or join, checked before anything is counted: an unknown `client_kind`, a field over +/// 32 bytes or a control character is `400 invalid`. The fields are only shown in the device list. +pub(crate) fn device_client(kind: Option<&str>, platform: Option<&str>, app_version: Option<&str>) -> Result { + Ok(DeviceClient::new(kind, platform, app_version)?) +} + +/// A device as the API shows it; `current` is the device making the request. +pub(crate) fn device_info(d: DeviceRecord, current: DeviceId) -> DeviceInfo { + DeviceInfo { + current: d.id == current, + id: d.id, + name: d.name, + created_ms: d.created_ms, + last_used_ms: d.last_used_ms, + revoked: d.revoked, + client_kind: d.client.kind, + platform: d.client.platform, + app_version: d.client.app_version, + revoked_ms: d.revoked_ms, + revoke_reason: d.revoke_reason.map(|r| r.as_str().to_owned()), + } +} + +async fn space( + State(state): State, + Extension(auth): Extension, + Extension(access): Extension, +) -> Result, ApiError> { + let info = state.meta.space_info(access.space).await?; + let name = if access.personal { None } else { state.meta.team_space_name(access.space).await? }; + Ok(Json(SpaceResponse { + space: access.space, + root: info.root, + epoch: info.epoch, + seq: info.seq, + username: auth.username, + kind: Some(if access.personal { "personal" } else { "team" }.to_owned()), + name, + role: Some(access.role), + })) +} + +async fn account_usage(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let estimate = state.meta.storage_estimate(auth.personal_space()).await?; + let index_complete = estimate.reason.is_none(); + Ok(Json(AccountUsage { + stored_bytes: estimate.stored_bytes, + quota_bytes: estimate.quota_bytes, + trash_bytes: estimate.trash_bytes, + history_bytes: estimate.history_bytes, + estimates: UsageEstimates { index_complete, reason: estimate.reason.map(|_| UsageEstimateReason::IndexIncomplete) }, + })) } async fn list_devices(State(state): State, Extension(auth): Extension) -> Result>, ApiError> { let devices = state.meta.list_devices(auth.user).await?; - Ok(Json( - devices - .into_iter() - .map(|d| DeviceInfo { - current: d.id == auth.device, - id: d.id, - name: d.name, - created_ms: d.created_ms, - last_used_ms: d.last_used_ms, - revoked: d.revoked, - }) - .collect(), - )) + Ok(Json(devices.into_iter().map(|d| device_info(d, auth.device)).collect())) } async fn revoke_device( @@ -190,7 +302,7 @@ async fn revoke_device( id: Result, PathRejection>, ) -> Result { let Path(id) = id?; - state.meta.revoke_device(auth.user, id).await?; + state.meta.revoke_device_by(auth.user, id, RevokeReason::User, Origin::Device(auth.device), state.now_ms()).await?; // Its live connections end now, not at their next request. state.live.revoke(id); Ok(StatusCode::NO_CONTENT) @@ -198,14 +310,14 @@ async fn revoke_device( async fn missing_chunks( State(state): State, - Extension(auth): Extension, + Extension(access): Extension, request: Request, ) -> Result, ApiError> { let req: MissingRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; if req.ids.len() > MAX_MISSING_IDS { return Err(ApiError::invalid(format!("ask about at most {MAX_MISSING_IDS} chunks at once"))); } - Ok(Json(MissingResponse { missing: state.meta.missing_chunks(auth.space, &req.ids).await? })) + Ok(Json(MissingResponse { missing: state.meta.missing_chunks(access.space, &req.ids).await? })) } fn parse_chunk_id(id: &str) -> Result { @@ -252,13 +364,14 @@ pub(crate) async fn read_json(state: &AppState, } } -fn body_too_slow() -> ApiError { +pub(crate) fn body_too_slow() -> ApiError { ApiError::new(StatusCode::REQUEST_TIMEOUT, codes::TIMEOUT, "the request body took too long to arrive") } async fn put_chunk( State(state): State, Extension(auth): Extension, + Extension(access): Extension, id: Result, PathRejection>, request: Request, ) -> Result { @@ -271,22 +384,22 @@ async fn put_chunk( return Err(ApiError::invalid("a chunk can't be empty")); } // A space at its cap doesn't get to store bytes it would then be refused for. - state.meta.check_room(auth.space, BlobKind::Chunk, id.as_bytes(), body.len() as u64 + 1).await?; + state.meta.check_room(access.space, BlobKind::Chunk, id.as_bytes(), body.len() as u64 + 1).await?; // Bytes first, record second: the database never points at a blob that isn't stored. state.store.put(BlobKey::chunk(&id), &body).await?; - state.meta.add_chunk(auth.space, &id, body.len() as u64 + 1, state.now_ms()).await?; + access.meta(&state, auth.user).add_chunk(access.space, &id, body.len() as u64 + 1, state.now_ms()).await?; Ok(StatusCode::NO_CONTENT) } async fn get_chunk( State(state): State, - Extension(auth): Extension, + Extension(access): Extension, id: Result, PathRejection>, ) -> Result { let Path(id) = id?; let id = parse_chunk_id(&id)?; // Only chunks this space uploaded; another space's chunk is "not found", not "forbidden". - if !state.meta.missing_chunks(auth.space, &[id]).await?.is_empty() { + if !state.meta.missing_chunks(access.space, &[id]).await?.is_empty() { return Err(ApiError::not_found("chunk")); } Ok(octets(stored_blob(&state, BlobKey::chunk(&id)).await?)) @@ -295,6 +408,7 @@ async fn get_chunk( async fn put_manifest( State(state): State, Extension(auth): Extension, + Extension(access): Extension, id: Result, PathRejection>, request: Request, ) -> Result { @@ -314,7 +428,7 @@ async fn put_manifest( return Err(ApiError::invalid(format!("unknown chunker parameter set {}", manifest.chunker().as_str()))); } let ids: Vec = manifest.chunks().iter().map(|c| c.id).collect(); - let mut missing = state.meta.missing_chunks(auth.space, &ids).await?; + let mut missing = state.meta.missing_chunks(access.space, &ids).await?; if !missing.is_empty() { let total = missing.len(); missing.truncate(MAX_MISSING_IDS); @@ -325,34 +439,43 @@ async fn put_manifest( } return Err(err); } - state.meta.check_room(auth.space, BlobKind::Manifest, id.as_bytes(), body.len() as u64 + 1).await?; + state.meta.check_room(access.space, BlobKind::Manifest, id.as_bytes(), body.len() as u64 + 1).await?; state.store.put(BlobKey::manifest(&id), &body).await?; - state.meta.add_manifest(auth.space, &manifest, body.len() as u64 + 1, state.now_ms()).await?; + access.meta(&state, auth.user).add_manifest(access.space, &manifest, body.len() as u64 + 1, state.now_ms()).await?; + // The index of its chunks is complete before the upload is acknowledged: a manifest of up to the index budget was indexed + // in the transaction above, a longer one is finished here in batches (and by `index build` if this is cut short). + state.meta.index_complete(&manifest).await?; Ok(StatusCode::NO_CONTENT) } async fn get_manifest( State(state): State, - Extension(auth): Extension, + Extension(access): Extension, id: Result, PathRejection>, ) -> Result { let Path(id) = id?; let id = parse_manifest_id(&id)?; // No permit: one indexed query and one blob read. (Uploads' slots are per user and never block this.) - if !state.meta.has_manifest(auth.space, &id).await? { + if !state.meta.has_manifest(access.space, &id).await? { return Err(ApiError::not_found("manifest")); } Ok(octets(stored_blob(&state, BlobKey::manifest(&id)).await?)) } -async fn commit(State(state): State, Extension(auth): Extension, request: Request) -> Result, ApiError> { +async fn commit( + State(state): State, + Extension(auth): Extension, + Extension(access): Extension, + request: Request, +) -> Result, ApiError> { let req: CommitRequest = read_json(&state, request, MAX_BATCH_JSON_BYTES).await?; // Commit and publish run in their own task: a client that disconnects mid-request drops this handler, // and the publish must not be skipped for a commit that went through. let task_state = state.clone(); + let meta = access.meta(&state, auth.user); let outcome = tokio::spawn(async move { - let outcome = task_state.meta.commit(auth.space, auth.device, &req.ops, task_state.now_ms()).await?; - task_state.notifier.publish(auth.space, outcome.seq); + let outcome = meta.commit(access.space, auth.device, &req.ops, task_state.now_ms()).await?; + task_state.notifier.publish(access.space, outcome.seq); Ok::<_, ApiError>(outcome) }) .await @@ -367,14 +490,16 @@ async fn commit(State(state): State, Extension(auth): Extension, Extension(auth): Extension, + Extension(access): Extension, request: Request, ) -> Result, ApiError> { let req: craft_cloud_proto::api::SaveAttach = read_json(&state, request, SMALL_JSON_BYTES).await?; let task_state = state.clone(); + let meta = access.meta(&state, auth.user); // As a commit: the publish is never skipped for a change that went through. let outcome = tokio::spawn(async move { - let outcome = task_state.meta.attach_save(auth.space, req.node, req.version, &req.save).await?; - task_state.notifier.publish(auth.space, outcome.seq); + let outcome = meta.attach_save(access.space, req.node, req.version, &req.save).await?; + task_state.notifier.publish(access.space, outcome.seq); Ok::<_, ApiError>(outcome) }) .await @@ -385,6 +510,134 @@ async fn attach_save( Ok(Json(CommitResponse { seq: outcome.seq, epoch: outcome.epoch })) } +#[derive(Debug, Deserialize)] +struct HistoryQuery { + cursor: Option, + limit: Option, +} + +/// `GET /nodes/{id}/history`: the versions of a file of the caller's own space, newest first. The node is looked +/// up in the space of the device token, so another account's node is `not_found`. +async fn history( + State(state): State, + Extension(access): Extension, + id: Result, PathRejection>, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad history cursor"))?; + let page = state.meta.history(access.space, node, after, q.limit.unwrap_or(HISTORY_DEFAULT_LIMIT)).await?; + Ok(Json(HistoryResponse { versions: page.versions, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /nodes/{id}/restore`: a new version with the content of an old one, as an ordinary update of the file. +async fn restore( + State(state): State, + Extension(auth): Extension, + Extension(access): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let Path(node) = id?; + let req: RestoreRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let meta = access.meta(&state, auth.user); + let outcome = tokio::spawn(async move { + let outcome = meta.restore_version(access.space, auth.device, node, req.version, req.expected_version, task_state.now_ms()).await?; + task_state.notifier.publish(access.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "restore task failed"); + ApiError::internal() + })??; + Ok(Json(RestoreResponse { version: outcome.version, parent_version: outcome.parent_version, seq: outcome.seq, epoch: outcome.epoch })) +} + +const MS_PER_DAY: i64 = 86_400_000; + +#[derive(Debug, Deserialize)] +struct TrashQuery { + cursor: Option, + limit: Option, +} + +/// `GET /trash`: the deleted nodes of the caller's own space, newest deletion first. +async fn trash( + State(state): State, + Extension(access): Extension, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad trash cursor"))?; + let page = state.meta.trash(access.space, after, q.limit.unwrap_or(TRASH_DEFAULT_LIMIT), state.now_ms()).await?; + if page.repaired > 0 { + tracing::warn!(count = page.repaired, "deleted nodes without a deletion time were stamped with the current time"); + } + let keep = i64::from(state.trash_days).saturating_mul(MS_PER_DAY); + let items = page + .entries + .into_iter() + .map(|e| TrashItem { + id: e.id, + parent: e.parent, + name: e.name, + is_dir: e.is_dir, + size: e.size, + deleted_ms: e.deleted_ms, + purge_after_ms: e.deleted_ms.saturating_add(keep), + }) + .collect(); + Ok(Json(TrashResponse { items, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /trash/{id}/restore`: a deleted node of the caller's space comes back, in one change. +async fn restore_from_trash( + State(state): State, + Extension(auth): Extension, + Extension(access): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let meta = access.meta(&state, auth.user); + let outcome = tokio::spawn(async move { + let outcome = meta.restore_from_trash(access.space, auth.device, node, task_state.now_ms()).await?; + task_state.notifier.publish(access.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "trash restore task failed"); + ApiError::internal() + })??; + Ok(Json(TrashRestoreResponse { + node: outcome.node, + parent: outcome.parent, + name: outcome.name, + renamed: outcome.renamed, + version: outcome.version, + seq: outcome.seq, + epoch: outcome.epoch, + })) +} + +/// `DELETE /trash/{id}`: takes a deleted node out of the trash. Only marks it: nothing is freed. +async fn remove_from_trash( + State(state): State, + Extension(auth): Extension, + Extension(access): Extension, + id: Result, PathRejection>, +) -> Result { + let Path(node) = id?; + access.meta(&state, auth.user).request_purge(access.space, node, state.now_ms()).await?; + Ok(StatusCode::NO_CONTENT) +} + fn cursor(after_seq: Option, after_node: Option) -> Result, ApiError> { match (after_seq, after_node) { (Some(seq), Some(node)) => Ok(Some(PageCursor { seq, node })), @@ -402,11 +655,24 @@ struct PageQuery { async fn tree( State(state): State, - Extension(auth): Extension, + Extension(access): Extension, q: Result, QueryRejection>, ) -> Result, ApiError> { let Query(q) = q?; - Ok(Json(state.meta.tree(auth.space, cursor(q.after_seq, q.after_node)?, q.limit.unwrap_or(MAX_PAGE_NODES)).await?)) + Ok(Json(state.meta.tree(access.space, cursor(q.after_seq, q.after_node)?, q.limit.unwrap_or(MAX_PAGE_NODES)).await?)) +} + +/// Resolves when `device` is revoked (or some revocations were missed, so that it must be looked up). +async fn revoked(revocations: &mut tokio::sync::broadcast::Receiver, device: DeviceId) { + use tokio::sync::broadcast::error::RecvError; + loop { + match revocations.recv().await { + Ok(revoked) if revoked == device => return, + Ok(_) => {} + Err(RecvError::Lagged(_)) => return, + Err(RecvError::Closed) => std::future::pending::<()>().await, + } + } } // The paging fields are written out here rather than flattened from `PageQuery`: serde's `flatten` buffers @@ -427,28 +693,56 @@ struct ChangesQuery { async fn changes( State(state): State, Extension(auth): Extension, + Extension(access): Extension, q: Result, QueryRejection>, ) -> Result, ApiError> { let Query(q) = q?; let cursor = cursor(q.after_seq, q.after_node)?; let slot = if q.wait > 0 && cursor.is_none() { - let slot = state.notifier.begin_wait(auth.space); + let slot = state.notifier.begin_wait(access.space); Some(slot.ok_or_else(|| ApiError::rate_limited("too many waiting requests for this space", Some(WAITERS_RETRY_SECS)))?) } else { None }; if slot.is_some() { // Subscribe before reading the sequence number, so a commit in between still wakes us. - let mut updates = state.notifier.subscribe(auth.space); + let mut updates = state.notifier.subscribe(access.space); + let mut members = state.notifier.subscribe_members(access.space); let mut stopping = state.notifier.shutdown_signal(); - if state.meta.space_info(auth.space).await?.seq == q.since { + // Listening for revocations first, then looking the device up: a revoke before the lookup is seen by it, one + // after it ends the wait, so none is missed between the sign-in check and the wait. + let mut revocations = state.live.revocations(); + if !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } + if state.meta.space_info(access.space).await?.seq == q.since { let deadline = Duration::from_secs(q.wait.min(MAX_WAIT_SECS)); - // Whichever comes first: a commit, the deadline, or the server shutting down. + // Whichever comes first: a commit, the deadline, the server shutting down, or this device signed out. tokio::select! { _ = tokio::time::timeout(deadline, updates.wait_for(|seq| *seq > q.since)) => {} _ = stopping.wait_for(|stop| *stop) => {} + () = revoked(&mut revocations, auth.device) => {} + // A change to the space's members: the access is looked at again below, at once. + _ = members.changed() => {} } } } - Ok(Json(state.meta.changes(auth.space, q.since, cursor, q.limit.unwrap_or(MAX_PAGE_NODES)).await?)) + // A team space's access can end while the request waits (a member removed, the role lowered, an account disabled): the + // wait is over by now, and the answer is the one a request from outside the space gets. Nothing is read for the + // caller's own space on a request without the header. + if slot.is_some() && !access.personal { + match state.meta.space_access(auth.user, access.space).await? { + Some(role) if role >= SpaceRole::Viewer => {} + _ => return Err(ApiError::not_found("space")), + } + } + let changes = state.meta.changes(access.space, q.since, cursor, q.limit.unwrap_or(MAX_PAGE_NODES)).await?; + // A long-poll looks the device up once more after the fetch, the last thing before the answer: a device signed + // out while it waited or while the change set was read (by the API, or by the command line in another process, + // which wakes nothing) gets no change feed. A revoke that commits after this lookup is a sign-out that happened + // after the answer was decided. + if slot.is_some() && !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } + Ok(Json(changes)) } diff --git a/apps/craft-cloud-server/src/audit.rs b/apps/craft-cloud-server/src/audit.rs new file mode 100644 index 00000000..5571b9a3 --- /dev/null +++ b/apps/craft-cloud-server/src/audit.rs @@ -0,0 +1,486 @@ +//! The security audit log over HTTP: `GET /api/v1/account/activity` (the signed-in account's own events), and for +//! administrators `GET /api/v1/admin/audit` (the whole log, filtered) and `GET /api/v1/admin/audit/export` (the same as +//! NDJSON, streamed). +//! +//! - **Events are written by the changes they record**, in the transaction of the change (`craft_cloud_meta::audit`). +//! This module only reads, and writes the one event that is not part of a change: a failed sign-in +//! ([`login_failed`]), after the answer is decided, whose failure to be written never changes the answer. +//! - **Pages** are newest first in the order `(at_ms, id)` descending. `cursor` is opaque (base64url of that pair) and +//! exclusive: the next page starts strictly behind it, so events with one timestamp are neither repeated nor lost. +//! A cursor this server did not make is `400 invalid`. `cursor` wins over `before_ms`, which only limits a listing's +//! first call. `limit` over the route's most is `400 invalid` rather than quietly smaller. +//! - **The account's own activity** holds the events the account acted in or was the target of, and shows the device that +//! acted only when it is the account's own. +//! - **The export** is bounded in every way: pages of [`AuditExportLimits::page`] rows per query, a channel of +//! [`AuditExportLimits::channel`] pages between the database and the connection (so a slow client slows the reading +//! and does not fill memory), the end of the log fixed when it starts, at most [`AuditExportLimits::max_rows`] event +//! lines. The last line says how it ended: `{"complete":true,...}`, or `{"truncated":true,...}` when there were more +//! events than it holds. A database error in the middle ends the connection with an error and no last line, and a +//! client that goes away stops the reading at once. A client counts an export as whole only with the `complete` line. +//! - **Nothing is logged of the events.** A failure to write or read one is logged without its content. + +use std::future::Future; +use std::sync::Arc; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::rejection::QueryRejection; +use axum::extract::{Extension, Query, State}; +use axum::http::{StatusCode, header}; +use axum::response::Response; +use axum::routing::get; +use axum::{Json, middleware}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{ActionFilter, AuditCursor, AuditEvent, AuditFailure, AuditQuery, DeviceAuth, MetaError, Step, UserId}; +use craft_cloud_proto::api::{AuditDetails, AuditExportEnd, AuditPage}; +use serde::Deserialize; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::auth::require_admin; +use crate::error::ApiError; +use crate::state::AppState; + +type Shared = Arc; + +/// Events one page of the account's activity holds when `limit` is not given, and the most it may hold. +const DEFAULT_PAGE: u32 = 50; +const ACCOUNT_MAX_PAGE: u32 = 100; +/// The same for the admin's list. +const ADMIN_MAX_PAGE: u32 = 200; +/// How an export is offered: always under this name, whatever the filters. +const EXPORT_DISPOSITION: &str = "attachment; filename=\"craft-cloud-audit.ndjson\""; + +/// How an export is cut. The defaults are the contract in `docs/design/sync.md`; tests make them smaller. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditExportLimits { + /// Rows one database query reads. + pub page: u32, + /// Pages the channel to the connection holds. + pub channel: usize, + /// Event lines in an export; more events than that make it `truncated`. + pub max_rows: u64, +} + +impl Default for AuditExportLimits { + fn default() -> Self { + Self { page: 500, channel: 2, max_rows: 100_000 } + } +} + +/// The admin routes, behind `require_admin`; `crate::api::router` merges them into the signed-in routes, so +/// `require_device` runs first. +pub fn admin_routes() -> Router { + Router::new() + .route("/admin/audit", get(admin_list)) + .route("/admin/audit/export", get(admin_export)) + .route_layer(middleware::from_fn(require_admin)) + .route_layer(middleware::from_fn(crate::web::reject_mixed_credentials)) +} + +/// Writes the event of a sign-in that failed at its password, in a transaction of its own. `target` is the account the +/// guess was about, `None` for a name no account has (the log never holds a name that was typed). A failure is logged and +/// otherwise ignored: the answer to the client and the limits around it do not depend on the log. +pub(crate) async fn login_failed(state: &AppState, target: Option) { + record_login_failure(state, target, false).await; +} + +pub(crate) async fn web_login_failed(state: &AppState, target: Option) { + record_login_failure(state, target, true).await; +} + +async fn record_login_failure(state: &AppState, target: Option, web: bool) { + let failure = AuditFailure { action: craft_cloud_meta::AuditAction::AuthLoginFailed, target, step: Step::Password }; + let result = if web { + state.meta.audit_record_web_failure(failure, state.now_ms()).await + } else { + state.meta.audit_record_failure(failure, state.now_ms()).await + }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } +} + +/// Writes the event of a sign-in whose challenge was not good (unknown, expired, used, or out of tries), as +/// [`login_failed`] does. `target` is the account when the challenge was found, `None` when it was not (nothing a client sent +/// is stored). The wrong second factors themselves are written by the store, where the account is known. +pub(crate) async fn challenge_failed(state: &AppState, target: Option) { + record_challenge_failure(state, target, false).await; +} + +pub(crate) async fn web_challenge_failed(state: &AppState, target: Option) { + record_challenge_failure(state, target, true).await; +} + +async fn record_challenge_failure(state: &AppState, target: Option, web: bool) { + let failure = AuditFailure { action: craft_cloud_meta::AuditAction::AuthSecondFactorFailed, target, step: Step::ChallengeExpired }; + let result = if web { + state.meta.audit_record_web_failure(failure, state.now_ms()).await + } else { + state.meta.audit_record_failure(failure, state.now_ms()).await + }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } +} + +// ---- reading parameters ---- + +/// The query of the two lists and the export. Unknown parameters are ignored, as everywhere in the API. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct Params { + pub(crate) cursor: Option, + before_ms: Option, + limit: Option, + pub(crate) user: Option, + pub(crate) action: Option, +} + +impl Params { + /// The same filters with the empty ones taken out: a form that was sent with a field left blank asks for no filter. + pub(crate) fn nonempty(&self) -> Params { + let keep = |field: &Option| field.clone().filter(|text| !text.is_empty()); + Params { cursor: keep(&self.cursor), before_ms: self.before_ms, limit: self.limit, user: keep(&self.user), action: keep(&self.action) } + } +} + +fn bad_cursor() -> ApiError { + ApiError::invalid("the cursor is not one this server gave") +} + +pub(crate) fn encode_cursor(cursor: AuditCursor) -> String { + URL_SAFE_NO_PAD.encode(cursor.to_bytes()) +} + +pub(crate) fn decode_cursor(text: &str) -> Result { + let bytes = URL_SAFE_NO_PAD.decode(text).map_err(|_| bad_cursor())?; + AuditCursor::from_bytes(&bytes).ok_or_else(bad_cursor) +} + +/// The limit of a list: 1 to `max`, `DEFAULT_PAGE` when not given. +fn page_limit(limit: Option, max: u32) -> Result { + let limit = limit.unwrap_or(DEFAULT_PAGE); + if (1..=max).contains(&limit) { Ok(limit) } else { Err(ApiError::invalid(format!("limit is 1 to {max}"))) } +} + +/// The `user` and `action` filters. A `user` is an account ID, an `action` one of the names or a prefix of them ending +/// in a dot like `auth.`; anything else is `400 invalid`, so nothing typed reaches a query. +pub(crate) fn filters(params: &Params) -> Result<(Option, Option), ApiError> { + let user = + params.user.as_deref().map(|text| Uuid::parse_str(text).map(UserId).map_err(|_| ApiError::invalid("user is an account ID"))).transpose()?; + let action = params + .action + .as_deref() + .map(|text| ActionFilter::parse(text).ok_or_else(|| ApiError::invalid("action is an event name like auth.login, or a prefix like auth."))) + .transpose()?; + Ok((user, action)) +} + +/// The event as the API shows it. `own`: an account whose own activity this is, which sees the device only of its own acts, and +/// the details of an event about another account (a former administrator's past act on someone else) not at all. +fn wire(event: AuditEvent, own: Option) -> craft_cloud_proto::api::AuditEvent { + let device_visible = own.is_none_or(|own| event.actor_user == Some(own)); + let about_another = own.is_some_and(|own| event.target_kind == "user" && event.target_id.is_some_and(|t| t != own.0)); + craft_cloud_proto::api::AuditEvent { + id: event.id, + at_ms: event.at_ms, + actor_user: event.actor_user.map(|u| u.0), + actor_device: if device_visible { event.actor_device } else { None }, + action: event.action.as_str().to_owned(), + target_kind: event.target_kind, + target_id: event.target_id, + outcome: event.outcome.as_str().to_owned(), + // Written by this code from closed lists; a row that does not read as such shows no details rather than failing the page. + details: if about_another { AuditDetails::default() } else { serde_json::from_str::(&event.details).unwrap_or_default() }, + count: event.count, + } +} + +/// One page: asks for one more event than `limit` to know whether there is a next one. +async fn page(state: &AppState, params: &Params, max: u32, restrict_to: Option) -> Result, ApiError> { + let limit = page_limit(params.limit, max)?; + let (user, action) = filters(params)?; + if params.before_ms.is_some_and(|before| before < 0) { + return Err(ApiError::invalid("before_ms is a time in milliseconds")); + } + let after = params.cursor.as_deref().map(decode_cursor).transpose()?; + let query = AuditQuery { + // An account sees nobody's events but its own, whatever the request names. + user: restrict_to.or(user), + action, + // The cursor is a position and wins over the time. + before_ms: if after.is_some() { None } else { params.before_ms }, + until_ms: None, + after, + limit: limit.saturating_add(1), + }; + let mut events = state.meta.audit_list(&query).await?; + let more = events.len() > limit as usize; + events.truncate(limit as usize); + let next = if more { events.last().map(|event| encode_cursor(event.cursor())) } else { None }; + Ok(Json(AuditPage { events: events.into_iter().map(|event| wire(event, restrict_to)).collect(), next })) +} + +/// `GET /api/v1/account/activity?cursor&before_ms&limit`: the signed-in account's own security events. +pub(crate) async fn account_activity( + State(state): State, + Extension(auth): Extension, + params: Result, QueryRejection>, +) -> Result, ApiError> { + let Query(params) = params?; + // The account sees events about itself; `user` and `action` are the admin's filters and are not read here. + let own = Params { user: None, action: None, ..params }; + page(&state, &own, ACCOUNT_MAX_PAGE, Some(auth.user)).await +} + +/// `GET /api/v1/admin/audit?cursor&before_ms&limit&user&action`. +async fn admin_list(State(state): State, params: Result, QueryRejection>) -> Result, ApiError> { + let Query(params) = params?; + page(&state, ¶ms, ADMIN_MAX_PAGE, None).await +} + +// ---- the export ---- + +/// One page of the export's reading: the events strictly behind `after`, at most `rows`. +type Fetch<'a> = + Box, u32) -> std::pin::Pin, MetaError>> + Send + 'a>> + Send + 'a>; + +/// Reads the log in pages with `fetch` and sends NDJSON to `tx`, a page at a time, ending with the line that says how it +/// ended. Returns without a last line when `fetch` fails (the error is sent, so the connection ends in error) or when the +/// receiver is gone (the reading stops there and then: it also stops a query that is under way). +async fn export_pages(mut fetch: Fetch<'_>, limits: AuditExportLimits, tx: mpsc::Sender>) { + let page = limits.page.max(1); + let mut after = None; + let mut exported = 0u64; + loop { + let rows = tokio::select! { + () = tx.closed() => return, + rows = fetch(after, page) => rows, + }; + let rows = match rows { + Ok(rows) => rows, + Err(e) => { + // The content of the error is the database's; the client only learns that the stream broke. + tracing::warn!(error = %e, "the audit export broke off"); + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + } + }; + let more_in_log = rows.len() >= page as usize; + let room = limits.max_rows.saturating_sub(exported); + let take = usize::try_from(room).unwrap_or(usize::MAX).min(rows.len()); + let mut chunk = Vec::new(); + for event in rows.iter().take(take) { + // Serializing these plain structs cannot fail; if it did, the stream breaks like any other error. + let Ok(line) = serde_json::to_vec(&wire(event.clone(), None)) else { + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + }; + chunk.extend_from_slice(&line); + chunk.push(b'\n'); + } + exported = exported.saturating_add(take as u64); + let cut = take < rows.len(); + if let Some(last) = take.checked_sub(1).and_then(|i| rows.get(i)) { + after = Some(last.cursor()); + } + // At the limit with a full page behind it there may be more, which one more row tells. + let at_limit = !cut && exported >= limits.max_rows && more_in_log; + let ended = if cut { + Some(false) + } else if !more_in_log { + Some(true) + } else if at_limit { + let probe = tokio::select! { + () = tx.closed() => return, + probe = fetch(after, 1) => probe, + }; + match probe { + Ok(rows) => Some(rows.is_empty()), + Err(e) => { + tracing::warn!(error = %e, "the audit export broke off"); + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + } + } + } else { + None + }; + if let Some(complete) = ended { + let end = AuditExportEnd { complete: complete.then_some(true), truncated: (!complete).then_some(true), exported }; + if let Ok(line) = serde_json::to_vec(&end) { + chunk.extend_from_slice(&line); + chunk.push(b'\n'); + } + } + if !chunk.is_empty() && tx.send(Ok(Bytes::from(chunk))).await.is_err() { + return; + } + if ended.is_some() { + return; + } + } +} + +/// `GET /api/v1/admin/audit/export?user&action&before_ms`: the log as NDJSON (`before_ms` also limits it, besides the fixed end). +async fn admin_export(State(state): State, params: Result, QueryRejection>) -> Result { + let Query(params) = params?; + export_response(&state, params).await +} + +/// The export as a response, for the JSON route and the web pages' download alike: the same bounds, the same last line. +pub(crate) async fn export_response(state: &Shared, params: Params) -> Result { + let (user, action) = filters(¶ms)?; + if params.before_ms.is_some_and(|before| before < 0) { + return Err(ApiError::invalid("before_ms is a time in milliseconds")); + } + let before_ms = params.before_ms; + let limits = state.audit_export; + // The end of the log is fixed now: events that arrive while the export runs are not in it. + let until_ms = state.now_ms(); + let (tx, rx) = mpsc::channel(limits.channel.max(1)); + let reader = state.clone(); + let fetch: Fetch<'static> = Box::new(move |after, rows| { + let meta = reader.meta.clone(); + Box::pin(async move { meta.audit_list(&AuditQuery { user, action, before_ms, until_ms: Some(until_ms), after, limit: rows }).await }) + }); + tokio::spawn(export_pages(fetch, limits, tx)); + let body = Body::from_stream(futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|item| (item, rx)) })); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::OK; + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, header::HeaderValue::from_static("application/x-ndjson")); + headers.insert(header::CONTENT_DISPOSITION, header::HeaderValue::from_static(EXPORT_DISPOSITION)); + headers.insert(header::CACHE_CONTROL, header::HeaderValue::from_static("no-store")); + Ok(response) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use craft_cloud_meta::{AuditAction, AuditOutcome}; + + use super::*; + + fn event(n: i64) -> AuditEvent { + AuditEvent { + id: Uuid::from_u128(n as u128), + at_ms: 1000 - n, + actor_user: None, + actor_device: None, + action: AuditAction::AuthLogin, + target_kind: String::new(), + target_id: None, + outcome: AuditOutcome::Success, + details: "{}".to_owned(), + count: 1, + } + } + + /// A log of `total` events, newest first, read `rows` at a time; `fail_at` makes that call (counting from 0) fail. + fn fake_log(total: i64, calls: Arc, fail_at: Option) -> Fetch<'static> { + Box::new(move |after, rows| { + let call = calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + if fail_at == Some(call) { + return Err(MetaError::Corrupt("a test failure")); + } + let start = after.map_or(0, |c| 1000 - c.at_ms); + Ok((start + 1..=total).take(rows as usize).map(event).collect()) + }) + }) + } + + async fn run(fetch: Fetch<'static>, limits: AuditExportLimits) -> (Vec, bool) { + let (tx, mut rx) = mpsc::channel(limits.channel); + tokio::spawn(export_pages(fetch, limits, tx)); + let mut lines = Vec::new(); + let mut broke = false; + while let Some(item) = rx.recv().await { + match item { + Ok(bytes) => lines.extend(String::from_utf8(bytes.to_vec()).unwrap().lines().map(str::to_owned)), + Err(_) => broke = true, + } + } + (lines, broke) + } + + fn limits(page: u32, max_rows: u64) -> AuditExportLimits { + AuditExportLimits { page, channel: 1, max_rows } + } + + #[tokio::test] + async fn an_export_ends_with_complete_when_it_holds_every_event() { + for total in [0, 1, 4, 5, 6, 12] { + let (lines, broke) = run(fake_log(total, Arc::default(), None), limits(5, 100)).await; + assert!(!broke); + assert_eq!(lines.len() as i64, total + 1, "total {total}"); + assert_eq!(lines.last().unwrap(), &format!("{{\"complete\":true,\"exported\":{total}}}")); + for line in &lines[..lines.len() - 1] { + serde_json::from_str::(line).unwrap(); + } + } + } + + #[tokio::test] + async fn an_export_over_the_limit_ends_with_truncated_and_counts_only_events() { + // 12 events, at most 7 lines of events: truncated, in the middle of a page. + let (lines, broke) = run(fake_log(12, Arc::default(), None), limits(5, 7)).await; + assert!(!broke); + assert_eq!(lines.len(), 8); + assert_eq!(lines.last().unwrap(), r#"{"truncated":true,"exported":7}"#); + // Exactly at the limit with events behind it: truncated, found by the probe after a full page. + let (lines, _) = run(fake_log(12, Arc::default(), None), limits(5, 10)).await; + assert_eq!(lines.len(), 11); + assert_eq!(lines.last().unwrap(), r#"{"truncated":true,"exported":10}"#); + // Exactly the limit and nothing behind it: complete, not truncated (a full page, then the probe finds nothing). + let (lines, _) = run(fake_log(10, Arc::default(), None), limits(5, 10)).await; + assert_eq!(lines.last().unwrap(), r#"{"complete":true,"exported":10}"#); + // The limit is a multiple of nothing: a page ending short of it. + let (lines, _) = run(fake_log(7, Arc::default(), None), limits(5, 7)).await; + assert_eq!(lines.last().unwrap(), r#"{"complete":true,"exported":7}"#); + } + + #[tokio::test] + async fn a_database_error_in_the_middle_ends_the_stream_without_a_last_line() { + let (lines, broke) = run(fake_log(20, Arc::default(), Some(2)), limits(5, 100)).await; + assert!(broke); + assert_eq!(lines.len(), 10, "the two pages before the error arrived"); + assert!(lines.iter().all(|line| !line.contains("complete") && !line.contains("truncated"))); + } + + #[tokio::test] + async fn a_client_that_goes_away_stops_the_reading() { + let calls = Arc::new(AtomicUsize::new(0)); + let (tx, mut rx) = mpsc::channel(1); + let task = tokio::spawn(export_pages(fake_log(1_000_000, calls.clone(), None), limits(5, u64::MAX), tx)); + // The first page is read, and the client leaves. + assert!(rx.recv().await.is_some()); + drop(rx); + tokio::time::timeout(std::time::Duration::from_secs(10), task).await.unwrap().unwrap(); + let after_leaving = calls.load(Ordering::SeqCst); + // Reads in flight when it left may finish, the channel's depth at most; nothing goes on after that. + assert!(after_leaving <= 4, "{after_leaving}"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!(calls.load(Ordering::SeqCst), after_leaving); + } + + #[test] + fn cursors_are_opaque_and_checked() { + let cursor = AuditCursor { at_ms: 12, id: Uuid::now_v7() }; + assert_eq!(decode_cursor(&encode_cursor(cursor)).unwrap(), cursor); + for bad in ["", "!", "AAAA", &URL_SAFE_NO_PAD.encode([0u8; 23]), &URL_SAFE_NO_PAD.encode([0xffu8; 24]), &URL_SAFE_NO_PAD.encode([0u8; 25])] { + assert!(decode_cursor(bad).is_err(), "{bad}"); + } + } + + #[test] + fn a_limit_is_one_to_the_most_of_the_route() { + assert_eq!(page_limit(None, 100).unwrap(), 50); + assert_eq!(page_limit(Some(100), 100).unwrap(), 100); + assert!(page_limit(Some(101), 100).is_err()); + assert!(page_limit(Some(0), 100).is_err()); + } +} diff --git a/apps/craft-cloud-server/src/auth.rs b/apps/craft-cloud-server/src/auth.rs index 719e9cb3..2eeedaf4 100644 --- a/apps/craft-cloud-server/src/auth.rs +++ b/apps/craft-cloud-server/src/auth.rs @@ -15,6 +15,7 @@ use axum::middleware::Next; use axum::response::Response; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{DeviceAuth, Role}; use craft_cloud_proto::api::EPOCH_HEADER; use crate::error::ApiError; @@ -44,6 +45,13 @@ pub fn verify_password(password: &str, hash: &str) -> bool { } } +/// Whether a stored password hash can be verified at all. An account made by an invite has no password yet (an empty hash): a +/// sign-in must still spend a full Argon2id check on it (against the dummy hash, see [`AppState::dummy_hash`]) and then fail +/// whatever was typed, or the answer's timing tells a pending account from an unknown name. +pub fn usable_hash(hash: &str) -> bool { + PasswordHash::new(hash).is_ok() +} + /// A new device token and the hash to store for it. pub fn new_token() -> Result<(String, [u8; 32]), ApiError> { let mut raw = [0u8; 32]; @@ -67,25 +75,49 @@ fn bearer(request: &Request) -> Option<&str> { } /// Signs the request in from its device token, puts the device in the request extensions, and adds the -/// space's epoch to the answer. The epoch is read before the handler runs, so a failed read stops the +/// personal space's epoch to the answer (the space layer, [`crate::space::require_space`], sets the epoch of another space). The epoch is read before the handler runs, so a failed read stops the /// request before anything has happened. pub async fn require_device(State(state): State>, mut request: Request, next: Next) -> Result { let token = bearer(&request).ok_or_else(ApiError::unauthorized)?; let device = state.meta.device_by_token(&token_hash(token)).await?.ok_or_else(ApiError::unauthorized)?; - // Off the request's path: the write may wait for the SQLite write lock, and the request needn't wait with it. - let touching = state.clone(); - let (id, now) = (device.device, state.now_ms()); - tokio::spawn(async move { touching.touch(id, now).await }); - let space = device.space; + // Usage reads and the storage overview are fingerprinted as read-only; do not update last-used timestamps for them. + let path = request.uri().path(); + let admin_usage = ["/api/v1/admin/users/", "/admin/users/"] + .iter() + .any(|prefix| path.strip_prefix(prefix).and_then(|id| id.strip_suffix("/usage")).is_some_and(|id| !id.is_empty() && !id.contains('/'))); + if !matches!(path, "/api/v1/account/usage" | "/account/usage" | "/api/v1/admin/storage" | "/admin/storage") && !admin_usage { + // Off the request's path: the write may wait for the SQLite write lock, and the request needn't wait with it. + let touching = state.clone(); + let (id, now) = (device.device, state.now_ms()); + tokio::spawn(async move { touching.touch(id, now).await }); + } + let space = device.personal_space(); let epoch = state.meta.space_info(space).await?.epoch; request.extensions_mut().insert(device); let mut response = next.run(request).await; - if let Ok(value) = HeaderValue::from_str(&epoch.to_string()) { + // A space-scoped route that worked on another space has put that space's epoch there already. + if !response.headers().contains_key(EPOCH_HEADER) + && response.extensions().get::().is_none() + && let Ok(value) = HeaderValue::from_str(&epoch.to_string()) + { response.headers_mut().insert(EPOCH_HEADER, value); } Ok(response) } +/// Lets only an admin through, after [`require_device`] (it reads the device that layer put in the request extensions). +/// The role is the one `device_by_token` read from the database for this request, never cached, so a demoted admin +/// loses access with the next request. It is only the first check: every admin change checks the role again inside +/// its transaction. +pub async fn require_admin(request: Request, next: Next) -> Result { + match request.extensions().get::() { + Some(device) if device.role == Role::Admin => Ok(next.run(request).await), + Some(_) => Err(ApiError::forbidden("this needs an administrator account")), + // Not reachable behind `require_device`; a request without a device is not signed in. + None => Err(ApiError::unauthorized()), + } +} + impl AppState { /// Updates the device's `last used` time, at most once a minute (every write takes the SQLite write lock). async fn touch(&self, device: craft_cloud_proto::DeviceId, now: i64) { @@ -118,6 +150,13 @@ mod tests { assert!(!verify_password("correct horse", "not a hash")); } + #[test] + fn only_a_parseable_hash_is_usable() { + assert!(usable_hash(&hash_password("correct horse").unwrap())); + assert!(!usable_hash(""), "an invited account has no password yet"); + assert!(!usable_hash("not a hash")); + } + #[test] fn password_rules() { assert!(check_password_rules("short").is_err()); diff --git a/apps/craft-cloud-server/src/cli.rs b/apps/craft-cloud-server/src/cli.rs index a9b4908e..26936c61 100644 --- a/apps/craft-cloud-server/src/cli.rs +++ b/apps/craft-cloud-server/src/cli.rs @@ -2,19 +2,22 @@ //! appear in the process list or shell history: `printf '%s\n' "$PASSWORD" | craft-cloud-server user add …`. use std::io::{BufRead, Read, Write}; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; use clap::{Parser, Subcommand}; -use craft_cloud_meta::{Meta, Role, valid_username}; -use craft_cloud_proto::DeviceId; +use craft_cloud_meta::{ActionFilter, AuditQuery, Meta, Origin, RevokeReason, Role, SpaceActor, valid_username}; +use craft_cloud_proto::api::SpaceRole; use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; +use craft_cloud_proto::{DeviceId, SpaceId, SpaceName}; use craft_cloud_store::Store; use craft_cloud_store::disk::DiskBackend; use craft_cloud_store::s3::S3Backend; use crate::auth::{check_password_rules, hash_password}; use crate::config::{Config, StorageConfig}; +use crate::secretbox::TwoFactorKey; use crate::state::now_ms; #[derive(Debug, Parser)] @@ -65,8 +68,17 @@ pub enum Command { #[command(subcommand)] action: DeviceAction, }, - /// Start a new epoch for a user's space (or every space with --all). Run after restoring the server from a backup. - #[command(group(clap::ArgGroup::new("target").required(true).multiple(false).args(["username", "all"])))] + /// Manage team spaces: create one with its first owner, list them, see and add members, archive. + Space { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: SpaceAction, + }, + /// Start a new epoch for a user's space, a team space (--space) or every space with --all. Run after restoring the server + /// from a backup. + #[command(group(clap::ArgGroup::new("target").required(true).multiple(false).args(["username", "all", "space"])))] ResetEpoch { /// Path to config.toml. May come before or after the action. #[arg(long, global = true)] @@ -76,6 +88,64 @@ pub enum Command { /// Reset every space on the server. #[arg(long)] all: bool, + /// Reset this space, by ID (a team space's, or any user's). + #[arg(long)] + space: Option, + }, + /// Read and prune the security audit log. + Audit { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: AuditCommand, + }, + /// Check the database and the blob store against each other and report: the reference index of every manifest, the + /// manifests and chunks the database names against the objects in the store, and what names what. Changes nothing; + /// exits with an error when it finds a problem. Reads every chunk object (`--skip-chunk-reads` leaves that out). + Verify { + /// Path to config.toml. + #[arg(long)] + config: Option, + /// Print the report as JSON. + #[arg(long)] + json: bool, + /// Check chunks against the database only; do not read their objects from the store. + #[arg(long)] + skip_chunk_reads: bool, + }, + /// Inventory the store and report possible collection candidates without changing anything. + Gc { + #[arg(long)] + config: Option, + #[arg(long)] + dry_run: bool, + #[arg(long)] + json: bool, + }, + /// Clean up the admin pages' browser sessions. + WebSessions { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: WebSessionsCommand, + }, + /// Manage the key for two-factor secrets. + TwoFactor { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: TwoFactorAction, + }, + /// The reference index of what each manifest names. + Index { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: IndexAction, }, /// Manage live documents. Live { @@ -85,6 +155,102 @@ pub enum Command { #[command(subcommand)] action: LiveAction, }, + /// Ask a running server whether it answers (`GET /api/info`). Prints nothing and exits 0 if it does; otherwise + /// exits 1 with one line on standard error. For a container's health check, which has no shell and no `curl`. + Healthcheck { + /// The address the server listens on, as seen from here. + #[arg(long, default_value = "127.0.0.1:8080")] + addr: String, + }, +} + +#[derive(Debug, Subcommand)] +pub enum SpaceAction { + /// Create a team space. Its first owner is a member from the start, with no invitation to accept. + Create { + /// The space's display name (1 to 64 characters, unique on the server in any case). + name: String, + /// The first owner's username. + #[arg(long)] + owner: String, + }, + /// List the team spaces: ID, members, owners, stored bytes and name. + List, + /// List a team space's members and invitations. The space is its ID or its name. + Members { space: String }, + /// Add an account to a team space, accepted at once (whoever runs the command line has the files anyway). + Add { + /// The space's ID or name. + space: String, + /// The account's username. + user: String, + /// viewer, editor or owner. + #[arg(long, default_value = "editor")] + role: String, + }, + /// Archive a team space: every member becomes read-only, nothing is deleted. + Archive { + /// The space's ID or name. + space: String, + }, +} + +#[derive(Debug, Subcommand)] +pub enum AuditCommand { + /// Print the newest security events, newest first, as text. An event names accounts by ID, never by name. + List { + /// Only events this account acted in or was the target of. + #[arg(long)] + user: Option, + /// Only this event (like auth.login) or a group of them (like auth.). + #[arg(long)] + action: Option, + /// How many events at most (1 to 200). + #[arg(long, default_value_t = 50)] + limit: u32, + }, + /// Delete the events older than `[audit] retention_days` and say how many. Run it now and then (daily, from cron or + /// the task scheduler): the server also prunes when it writes an event, but not while no one signs in or changes anything. + Prune { + /// Say how many events would go and delete none. + #[arg(long)] + dry_run: bool, + }, +} + +#[derive(Debug, Subcommand)] +pub enum WebSessionsCommand { + /// Delete the browser sessions that are over (logged out, past `[web_admin] absolute_hours`, or unused for longer than + /// `idle_minutes`) and say how many. The server also deletes a few at each sign-in, but not while no one signs in; run this + /// now and then (daily, from cron or the task scheduler) to empty the table. A session that is still valid is never touched. + Prune { + /// Say how many sessions would go and delete none. + #[arg(long)] + dry_run: bool, + }, +} + +#[derive(Debug, Subcommand)] +pub enum TwoFactorAction { + /// Rewrite every two-factor secret that is sealed with the older key (`key_file_old`) under the current one + /// (`key_file`). To rotate the key: put the new key in `key_file` and the old one in `key_file_old`, run this, then + /// remove `key_file_old`. It can be run again, changes nothing when it is done, and refuses (changing nothing) when a + /// secret is sealed with a key that is neither. Prints counts, never a secret or a key. Restart every server instance on + /// the database with the new `key_file` and `key_file_old` first (a running server holds the keys it started with), and + /// remove `key_file_old` only after this reports that every secret is under the current key. + Rekey, +} + +#[derive(Debug, Subcommand)] +pub enum IndexAction { + /// Complete the index of every manifest whose index is not complete: reads each manifest from the store, checks it + /// against its ID, and writes its edges in batches. Writes only that derived index. Safe to stop and run again, and to + /// run beside the server. + Build { + /// Edges written per database transaction (default 5000). + #[arg(long)] + batch_rows: Option, + }, } #[derive(Debug, Subcommand)] @@ -103,11 +269,26 @@ pub enum UserAction { #[arg(long)] admin: bool, }, - /// Set a new password, read from standard input. - ResetPassword { username: String }, + /// Set a new password, read from standard input. Every device of the account is signed out and its open invite + /// and add-a-computer codes are deleted, so a stolen device or code does not outlive the old password; with + /// `--keep-devices` the devices stay signed in. A device's live connection ends at its next ping (about 30 + /// seconds) and a change-feed long-poll within 60 seconds, since the server runs in another process. + ResetPassword { + username: String, + /// Leave the account's devices signed in. + #[arg(long)] + keep_devices: bool, + }, + /// Take away an account's two-factor sign-in, for someone who lost their authenticator and their recovery codes: its + /// secret, recovery codes and open sign-in challenges are deleted, the lock after wrong codes goes, **every device of the + /// account is signed out** and its open invite and add-a-computer codes are deleted. Prints how many devices were signed + /// out. The person signs in with their password again and can set it up anew. A device's live connection ends at its next + /// ping (about 30 seconds), since the server runs in another process. + #[command(name = "reset-2fa")] + Reset2fa { username: String }, /// Print an invite code for an account, good once for 7 days, to join a computer with in Craft Cloud. Creates the /// account (without a password, chosen when the invite is redeemed) if there is none, and replaces its earlier - /// invite. + /// invite or password reset code. Invite { username: String, /// Revoke the account's invite instead. @@ -118,8 +299,18 @@ pub enum UserAction { #[derive(Debug, Subcommand)] pub enum DeviceAction { - List { username: String }, - Revoke { username: String, device: DeviceId }, + List { + username: String, + }, + Revoke { + username: String, + device: DeviceId, + }, + /// Sign out every device of an account. Live connections end at their next ping (about 30 seconds), change-feed + /// long-polls within 60 seconds. + RevokeAll { + username: String, + }, } #[derive(Debug, thiserror::Error)] @@ -130,16 +321,24 @@ pub enum CliError { Meta(#[from] craft_cloud_meta::MetaError), #[error(transparent)] Store(#[from] craft_cloud_store::StoreError), + #[error(transparent)] + Gc(#[from] crate::gc::GcError), #[error("{0}")] Usage(String), + /// A check found something wrong (its report was written first). + #[error("{0}")] + Inconsistent(String), #[error("I/O error: {0}")] Io(#[from] std::io::Error), } /// Opens the database and blob storage a config names. pub async fn open(config: &Config) -> Result<(Meta, Store), CliError> { - let meta = - Meta::open(&config.database.url).await?.with_stored_bytes_cap(config.stored_bytes_per_user).with_live_bytes_cap(config.live_bytes_per_user); + let meta = Meta::open(&config.database.url) + .await? + .with_stored_bytes_cap(config.stored_bytes_per_user) + .with_live_bytes_cap(config.live_bytes_per_user) + .with_audit_retention_days(config.audit.retention_days); let store = match (&config.storage, config.storage.s3()?) { (_, Some(s3)) => Store::new(S3Backend::new(&s3)?), (StorageConfig::Disk { path }, None) => Store::new(DiskBackend::new(path)?), @@ -148,6 +347,23 @@ pub async fn open(config: &Config) -> Result<(Meta, Store), CliError> { Ok((meta, store)) } +/// Like [`open`], for a command that only reads: the database and the storage must exist already, and nothing is +/// created or upgraded. A config that points at the wrong place is an error, not a new empty server that has no problems. +pub async fn open_existing(config: &Config) -> Result<(Meta, Store), CliError> { + let meta = Meta::open_existing(&config.database.url).await?; + let store = match (&config.storage, config.storage.s3()?) { + (_, Some(s3)) => Store::new(S3Backend::new(&s3)?), + (StorageConfig::Disk { path }, None) => { + if !path.is_dir() { + return Err(CliError::Usage(format!("the storage directory {} does not exist (is the config's path right?)", path.display()))); + } + Store::new(DiskBackend::new(path)?) + } + (StorageConfig::S3 { .. }, None) => return Err(CliError::Usage("S3 storage without settings".into())), + }; + Ok((meta, store)) +} + /// `--config` is global, which clap can't also make required, so it is checked here. fn config_path(config: Option) -> Result { config.ok_or_else(|| CliError::Usage("--config is required".into())) @@ -172,6 +388,76 @@ async fn user_id(meta: &Meta, username: &str) -> Result Result { + if let Ok(id) = text.parse::() { + return Ok(id); + } + let name = SpaceName::new(text).map_err(|e| CliError::Usage(format!("{text:?} is neither a space ID nor a space name: {e}")))?; + meta.team_space_by_name(&name).await?.ok_or_else(|| CliError::Usage(format!("no team space {text:?}"))) +} + +async fn space_command(meta: &Meta, action: SpaceAction, out: &mut impl Write) -> Result<(), CliError> { + match action { + SpaceAction::Create { name, owner } => { + let name = SpaceName::new(&name).map_err(|e| CliError::Usage(format!("a space's name: {e}")))?; + let owner = user_id(meta, &owner).await?; + let space = meta.create_team_space(&SpaceActor::Cli, &name, owner.id, None, now_ms()).await?; + writeln!(out, "created the team space {space}, owned by {}", owner.username)?; + } + SpaceAction::List => { + let mut after = None; + loop { + let page = meta.admin_list_spaces(after.as_deref(), None, 200).await?; + for space in &page.spaces { + let archived = if space.archived_ms.is_some() { " (archived)" } else { "" }; + writeln!( + out, + "{} {} member(s), {} owner(s), {} bytes {}{archived}", + space.id, space.members, space.owners, space.stored_bytes, space.name + )?; + } + match page.next_after { + Some(next) => after = Some(next), + None => break, + } + } + } + SpaceAction::Members { space } => { + let space = resolve_space(meta, &space).await?; + for member in meta.list_members(space).await? { + writeln!(out, "{} {} {}", member.username, member.role.as_str(), if member.accepted() { "member" } else { "invited" })?; + } + } + SpaceAction::Add { space, user, role } => { + let space = resolve_space(meta, &space).await?; + let role = SpaceRole::parse(&role).ok_or_else(|| CliError::Usage("the role is viewer, editor or owner".into()))?; + let user = user_id(meta, &user).await?; + let current = meta.list_members(space).await?.into_iter().find(|m| m.user == user.id); + match current { + None => { + meta.invite_member(&SpaceActor::Cli, space, user.id, role, now_ms()).await?; + } + // Already in the space: the command sets the role instead, and accepts a pending invitation (whoever runs the + // command line has the files). + Some(member) => { + meta.change_member_role(&SpaceActor::Cli, space, user.id, role, now_ms()).await?; + if !member.accepted() { + writeln!(out, "{} had only been invited so far; the invitation is accepted now", user.username)?; + } + } + } + writeln!(out, "{} is {} of the space", user.username, role.as_str())?; + } + SpaceAction::Archive { space } => { + let space = resolve_space(meta, &space).await?; + meta.set_space_archived(&SpaceActor::Cli, space, true, now_ms()).await?; + writeln!(out, "archived the space {space}; nothing was deleted")?; + } + } + Ok(()) +} + /// Runs every command except `serve` (which `main` runs, since it never returns). pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Write) -> Result<(), CliError> { match command { @@ -179,6 +465,11 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ init(&dir, &admin, &listen, (behind_proxy, &trusted_proxies), input, out).await } Command::Serve { .. } => Err(CliError::Usage("serve is run by main".into())), + Command::Healthcheck { addr } => { + let addr: SocketAddr = addr.parse().map_err(|_| CliError::Usage(format!("bad address {addr}")))?; + // Blocking for at most `HEALTH_TIMEOUT`: this process does nothing else. + check_health(addr, HEALTH_TIMEOUT) + } Command::User { config, action } => { let config = Config::load(&config_path(config)?)?; let (meta, _) = open(&config).await?; @@ -190,6 +481,12 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ writeln!(out, "revoked the invite for {}", username.to_ascii_lowercase())?; } UserAction::Invite { username, revoke: false } => invite(&meta, &config, &username, out).await?, + UserAction::Reset2fa { username } => { + let user = user_id(&meta, &username).await?; + let revoked = meta.reset_two_factor(user.id, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "two-factor sign-in reset"); + writeln!(out, "two-factor sign-in of {} is off; signed out {} device(s)", user.username, revoked.len())?; + } UserAction::Add { username, admin } => { let password = read_password(input)?; let role = if admin { Role::Admin } else { Role::Member }; @@ -197,12 +494,17 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ let user = meta.create_user(&username, &hash, role, now_ms()).await?; writeln!(out, "added {} ({})", user.username, if admin { "admin" } else { "member" })?; } - UserAction::ResetPassword { username } => { + UserAction::ResetPassword { username, keep_devices } => { let user = user_id(&meta, &username).await?; let password = read_password(input)?; let hash = hash_password(&password).map_err(|e| CliError::Usage(e.message))?; - meta.set_password_hash(user.id, &hash).await?; - writeln!(out, "password changed for {}", user.username)?; + let revoked = meta.reset_password(user.id, &hash, keep_devices, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "password reset"); + if keep_devices { + writeln!(out, "password changed for {}; the devices stay signed in", user.username)?; + } else { + writeln!(out, "password changed for {}; signed out {} device(s)", user.username, revoked.len())?; + } } } Ok(()) @@ -218,15 +520,38 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } DeviceAction::Revoke { username, device } => { let user = user_id(&meta, &username).await?; - meta.revoke_device(user.id, device).await?; + meta.revoke_device_by(user.id, device, RevokeReason::Admin, Origin::Cli, now_ms()).await?; writeln!(out, "revoked {device}")?; } + DeviceAction::RevokeAll { username } => { + let user = user_id(&meta, &username).await?; + let revoked = meta.revoke_all_devices(user.id, RevokeReason::Admin, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "all devices revoked"); + writeln!(out, "revoked {} device(s) of {}", revoked.len(), user.username)?; + } } Ok(()) } - Command::ResetEpoch { config, username, all } => { + Command::Audit { config, action } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + audit(&meta, &config, action, out).await + } + Command::WebSessions { config, action } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + web_sessions(&meta, &config, action, out).await + } + Command::Space { config, action } => { + let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; + space_command(&meta, action, out).await + } + Command::ResetEpoch { config, username, all, space } => { let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; - if all { + if let Some(space) = space { + let epoch = meta.reset_epoch(space).await?; + writeln!(out, "new epoch {epoch} for space {space}")?; + } else if all { let spaces = meta.all_spaces().await?; for space in &spaces { let epoch = meta.reset_epoch(*space).await?; @@ -241,6 +566,70 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } Ok(()) } + Command::TwoFactor { config, action: TwoFactorAction::Rekey } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + let keys = crate::two_factor::load_keys(&config).map_err(|e| CliError::Usage(e.to_string()))?; + let crate::two_factor::KeyState::Loaded(secret_box) = keys else { + return Err(CliError::Usage( + "there is no key: set [two_factor] key_file (and key_file_old for the key to rotate away from) in the config".into(), + )); + }; + let done = crate::two_factor::rekey(&meta, &secret_box).await.map_err(CliError::Usage)?; + writeln!(out, "rewrote {} secret(s) under the current key; {} were under it already", done.rewritten, done.already_current)?; + if secret_box.old_id().is_some() { + writeln!( + out, + "every secret is under the current key now. key_file_old can be removed once every server instance on this database has been \ + restarted with key_file (the new key) and key_file_old, as it must have been before this command: an instance still running with \ + only the old key would seal new secrets with it" + )?; + } + Ok(()) + } + Command::Verify { config, json, skip_chunk_reads } => { + let (meta, store) = open_existing(&Config::load(&config_path(config)?)?).await?; + let report = crate::verify::verify(&meta, &store, crate::verify::VerifyOptions { read_chunks: !skip_chunk_reads }).await?; + if json { + writeln!(out, "{}", report.to_json())?; + } else { + write!(out, "{}", report.to_text())?; + } + match report.problem_count() { + 0 => Ok(()), + n => Err(CliError::Inconsistent(format!("verify found {n} problem(s)"))), + } + } + Command::Gc { dry_run: false, .. } => Err(CliError::Usage("deleting is not implemented yet; run with --dry-run".into())), + Command::Gc { config, dry_run: true, json } => { + let config = Config::load(&config_path(config)?)?; + let (meta, store) = open_existing(&config).await?; + let report = crate::gc::dry_run(&meta, &store).await?; + if json { + writeln!(out, "{}", report.to_json())?; + } else { + write!(out, "{}", report.to_text())?; + } + if report.has_problems() { Err(CliError::Inconsistent("gc dry-run found problems".into())) } else { Ok(()) } + } + Command::Index { config, action: IndexAction::Build { batch_rows } } => { + let (meta, store) = open(&Config::load(&config_path(config)?)?).await?; + let meta = match batch_rows { + Some(0) => return Err(CliError::Usage("--batch-rows must be at least 1".into())), + Some(rows) => meta.with_index_batch_rows(rows), + None => meta, + }; + let report = crate::verify::build_index(&meta, &store).await?; + writeln!(out, "indexed {} manifest(s)", report.indexed)?; + for (id, why) in &report.unreadable { + writeln!(out, "could not index manifest {id}: {why}")?; + } + if report.unreadable.is_empty() { + Ok(()) + } else { + Err(CliError::Inconsistent(format!("{} manifest(s) could not be indexed", report.unreadable.len()))) + } + } Command::Live { config, action: LiveAction::ResetDoc { username, app } } => { let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; let user = user_id(&meta, &username).await?; @@ -259,6 +648,125 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } } +/// How long `healthcheck` waits for the whole exchange: connecting, sending and reading the status line. +pub const HEALTH_TIMEOUT: Duration = Duration::from_secs(3); + +/// Sends `GET /api/info` to `addr` and succeeds only if the status line says 200. The whole exchange must finish +/// within `timeout`. The body is never read, and nothing the server sends is printed. +pub fn check_health(addr: SocketAddr, timeout: Duration) -> Result<(), CliError> { + let fail = |what: &str| CliError::Usage(format!("{addr} is not healthy: {what}")); + let deadline = Instant::now() + timeout; + let left = || deadline.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()); + let mut stream = TcpStream::connect_timeout(&addr, timeout).map_err(|e| fail(&format!("can't connect: {e}")))?; + let request = format!("GET /api/info HTTP/1.0\r\nHost: {addr}\r\nConnection: close\r\n\r\n"); + stream.set_write_timeout(left()).map_err(|e| fail(&e.to_string()))?; + stream.write_all(request.as_bytes()).map_err(|e| fail(&format!("can't send the request: {e}")))?; + // The status line fits in 64 bytes. It counts only when it is complete (ends in CRLF): an answer that stops inside + // it, or goes on for 64 bytes without a line end, is not a healthy server. + let mut head = [0u8; 64]; + let mut len = 0; + let line_end = loop { + if let Some(end) = head.get(..len).and_then(|read| read.windows(2).position(|pair| pair == b"\r\n")) { + break end; + } + let wait = left().ok_or_else(|| fail("no answer in time"))?; + stream.set_read_timeout(Some(wait)).map_err(|e| fail(&e.to_string()))?; + let Some(room) = head.get_mut(len..).filter(|room| !room.is_empty()) else { return Err(fail("not an HTTP answer")) }; + match stream.read(room) { + Ok(0) => return Err(fail("the answer ended inside the status line")), + Ok(n) => len += n, + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => return Err(fail("no answer in time")), + Err(e) => return Err(fail(&format!("can't read the answer: {e}"))), + } + }; + let line = String::from_utf8_lossy(head.get(..line_end).unwrap_or_default()); + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some(version), Some("200")) if version.starts_with("HTTP/1.") => Ok(()), + (Some(version), Some(status)) if version.starts_with("HTTP/") => Err(fail(&format!("status {}", status.chars().take(3).collect::()))), + _ => Err(fail("not an HTTP answer")), + } +} + +/// Sessions one batch of `web-sessions prune` deletes. +const WEB_SESSION_PRUNE_BATCH: u32 = 1000; + +/// `web-sessions prune`: the batches go on until none is left, each in a transaction of its own. +async fn web_sessions(meta: &Meta, config: &Config, action: WebSessionsCommand, out: &mut impl Write) -> Result<(), CliError> { + let idle_ms = config.web_admin.idle_ms(); + let WebSessionsCommand::Prune { dry_run } = action; + if dry_run { + let due = meta.web_sessions_count_over(now_ms(), idle_ms).await?; + writeln!(out, "{due} web session(s) are over; dry run, nothing deleted")?; + return Ok(()); + } + let mut deleted = 0u64; + loop { + let batch = meta.web_sessions_prune(now_ms(), idle_ms, WEB_SESSION_PRUNE_BATCH).await?; + deleted = deleted.saturating_add(batch); + if batch < u64::from(WEB_SESSION_PRUNE_BATCH) { + break; + } + } + tracing::info!(deleted, "web sessions pruned"); + writeln!(out, "deleted {deleted} web session(s) that were over")?; + Ok(()) +} + +/// `audit list` and `audit prune`. +async fn audit(meta: &Meta, config: &Config, action: AuditCommand, out: &mut impl Write) -> Result<(), CliError> { + match action { + AuditCommand::List { user, action, limit } => { + if !(1..=200).contains(&limit) { + return Err(CliError::Usage("--limit is 1 to 200".into())); + } + let user = match user { + Some(name) => Some(user_id(meta, &name).await?.id), + None => None, + }; + let action = match action { + Some(text) => Some( + ActionFilter::parse(&text) + .ok_or_else(|| CliError::Usage("--action is an event name like auth.login, or a group like auth.".into()))?, + ), + None => None, + }; + let events = meta.audit_list(&AuditQuery { user, action, limit, ..AuditQuery::default() }).await?; + let id = |id: Option| id.map_or_else(|| "-".to_owned(), |id| id.to_string()); + for e in &events { + // Only closed words, IDs and numbers: nothing here is a secret or a name anyone typed. + writeln!( + out, + "{} {} {} actor={} device={} target={}:{} count={} {}", + e.at_ms, + e.action.as_str(), + e.outcome.as_str(), + id(e.actor_user.map(|u| u.0)), + id(e.actor_device.map(|d| *d.as_uuid())), + if e.target_kind.is_empty() { "-" } else { e.target_kind.as_str() }, + id(e.target_id), + e.count, + e.details, + )?; + } + writeln!(out, "{} event(s)", events.len())?; + } + AuditCommand::Prune { dry_run } => { + let days = config.audit.retention_days; + let before = now_ms().saturating_sub(i64::from(days).saturating_mul(24 * 60 * 60 * 1000)); + if dry_run { + let due = meta.audit_count_before(before).await?; + writeln!(out, "{due} audit event(s) are older than {days} days; dry run, nothing deleted")?; + } else { + let deleted = meta.audit_prune(before).await?; + tracing::info!(deleted, "audit events pruned"); + writeln!(out, "deleted {deleted} audit event(s) older than {days} days")?; + } + } + } + Ok(()) +} + /// `user invite`: makes the invite and prints how to use it. The code is printed only here, never logged. async fn invite(meta: &Meta, config: &Config, username: &str, out: &mut impl Write) -> Result<(), CliError> { let code = Code::generate(INVITE_CHARS).map_err(|e| CliError::Usage(format!("the system random source failed: {e}")))?; @@ -320,7 +828,8 @@ async fn init( let blobs = without_verbatim_prefix(&dir.join("blobs").display().to_string()).to_owned(); let trusted: Vec = trusted_proxies.iter().map(|ip| format!("\"{ip}\"")).collect(); let config_text = format!( - "listen = \"{listen}\"\nbehind_proxy = {behind_proxy}\ntrusted_proxies = [{}]\n\n[database]\nurl = {}\n\n[storage]\nkind = \"disk\"\npath = {}\n", + "listen = \"{listen}\"\nbehind_proxy = {behind_proxy}\ntrusted_proxies = [{}]\n\n[database]\nurl = {}\n\n[storage]\nkind = \"disk\"\npath = {}\n\n\ + [two_factor]\nkey_file = \"{KEY_FILE_NAME}\"\n", trusted.join(", "), toml_string(&format!("sqlite:{db}")), toml_string(&blobs), @@ -335,12 +844,18 @@ async fn init( return Err(e); } writeln!(out, "created {}\nrun: craft-cloud-server serve --config {}", dir.display(), dir.join("config.toml").display())?; + writeln!( + out, + "two-factor key: {}. Keep a copy apart from your database backups: without it the two-factor secrets in a backup can't be opened.", + dir.join(KEY_FILE_NAME).display() + )?; Ok(()) } async fn create_files(dir: &Path, config_text: &str, config: &Config, admin: &str, hash: &str) -> Result<(), CliError> { create_data_dir(dir)?; write_config(&dir.join("config.toml"), config_text)?; + write_key_file(&dir.join(KEY_FILE_NAME))?; let (meta, _) = open(config).await?; meta.create_user(admin, hash, Role::Admin, now_ms()).await?; for name in ["meta.db", "meta.db-wal", "meta.db-shm"] { @@ -406,10 +921,28 @@ fn write_config(path: &Path, text: &str) -> Result<(), CliError> { Ok(()) } +/// The key file `init` makes, next to `config.toml`. +const KEY_FILE_NAME: &str = "two-factor.key"; + +/// Writes a new random key for two-factor secrets: 32 bytes, owner-only on Unix, and never over a file that is there. +fn write_key_file(path: &Path) -> Result<(), CliError> { + let key = TwoFactorKey::generate().map_err(|e| CliError::Usage(format!("the system random source failed: {e}")))?; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(key.to_file_bytes().as_slice())?; + Ok(()) +} + /// Removes what a failed `init` made. The directory was empty (or absent) when `init` started, so /// everything in it is `init`'s. Best effort: the original error is what the admin needs to see. fn remove_made(dir: &Path, created_dir: bool) { - for name in ["config.toml", "meta.db", "meta.db-wal", "meta.db-shm"] { + for name in ["config.toml", KEY_FILE_NAME, "meta.db", "meta.db-wal", "meta.db-shm"] { let _ = std::fs::remove_file(dir.join(name)); } let _ = std::fs::remove_dir_all(dir.join("blobs")); @@ -464,7 +997,7 @@ mod tests { let parent = tempfile::tempdir().unwrap(); let dir = parent.path().join("data"); std::fs::create_dir_all(dir.join("blobs")).unwrap(); - for name in ["config.toml", "meta.db", "meta.db-wal", "blobs/chunk"] { + for name in ["config.toml", "two-factor.key", "meta.db", "meta.db-wal", "blobs/chunk"] { std::fs::write(dir.join(name), b"x").unwrap(); } remove_made(&dir, true); @@ -513,6 +1046,14 @@ mod tests { } } + #[test] + fn healthcheck_parses_with_and_without_an_address() { + let cli = Cli::try_parse_from(["craft-cloud-server", "healthcheck"]).unwrap(); + assert!(matches!(&cli.command, Command::Healthcheck { addr } if addr == "127.0.0.1:8080")); + let cli = Cli::try_parse_from(["craft-cloud-server", "healthcheck", "--addr", "127.0.0.1:9000"]).unwrap(); + assert!(matches!(&cli.command, Command::Healthcheck { addr } if addr == "127.0.0.1:9000")); + } + #[test] fn reset_epoch_takes_a_username_or_all_but_not_both_or_neither() { assert!(Cli::try_parse_from(["craft-cloud-server", "reset-epoch", "--config", "c.toml", "--all"]).is_ok()); diff --git a/apps/craft-cloud-server/src/config.rs b/apps/craft-cloud-server/src/config.rs index 725e9a48..e3e066fa 100644 --- a/apps/craft-cloud-server/src/config.rs +++ b/apps/craft-cloud-server/src/config.rs @@ -9,6 +9,16 @@ //! # live_bytes_per_user = 1_073_741_824 # cap on what one account's live documents hold, in bytes (1 GiB) //! # public_url = "https://cloud.example.com" # where computers reach the server; `user invite` prints it //! +//! # [audit] +//! # retention_days = 365 # how long security events are kept (1 to 3650); `audit prune` deletes older ones +//! +//! # [web_admin] # the admin pages under /web, off unless enabled; needs behind_proxy and an https public_url +//! # enabled = true +//! # idle_minutes = 480 # a browser session ends after this long unused (5 to 1440) +//! # absolute_hours = 168 # and at the latest after this long (1 to 720) +//! # [retention] +//! # trash_days = 30 # how long a deleted file stays in the trash before the server may remove it (1 to 3650) +//! //! [database] //! url = "sqlite:/var/lib/craft-cloud/meta.db" # or postgres://user:pass@host/db //! @@ -16,6 +26,10 @@ //! kind = "disk" //! path = "/var/lib/craft-cloud/blobs" //! # kind = "s3", bucket, prefix, region, endpoint, access_key_id, secret_access_key, allow_http +//! +//! [two_factor] # optional: without it, two-factor sign-in can't be set up +//! key_file = "two-factor.key" # 32 random bytes, made by `init`; relative to this file +//! # key_file_old = "old.key" # while rotating the key, see `two-factor rekey` //! ``` //! //! The server speaks plain HTTP. It refuses to listen anywhere but loopback unless `behind_proxy = true`, @@ -55,10 +69,51 @@ pub enum ConfigError { agent could not tell a full account from a file that is too big. Raise it, or remove it for no cap" )] StoredBytesCapTooSmall(u64, u32), + #[error("retention.trash_days = {0} is not between {MIN_TRASH_DAYS} and {MAX_TRASH_DAYS}")] + TrashDays(u32), #[error("public_url = {0:?} is not a server address like https://cloud.example.com (a host, an optional port and path, no query)")] PublicUrl(String), #[error("storage.{0} is missing (set it in the config or the {1} environment variable)")] MissingS3Credential(&'static str, &'static str), + #[error("audit.retention_days = {0} is out of range: keep security events for {1} to {2} days")] + AuditRetention(u32, u32, u32), + #[error( + "web_admin.enabled = true needs public_url = \"https://host[:port]\" with no path: the browser's cookie is only sent over \ + HTTPS, and the pages check the Origin of every form against that address" + )] + WebAdminPublicUrl, + #[error( + "web_admin.enabled = true needs behind_proxy = true: the server speaks plain HTTP, and the session cookie is marked Secure, \ + so a TLS reverse proxy must stand in front of it" + )] + WebAdminNeedsProxy, + #[error("web_admin.{0} = {1} is out of range: {2} to {3}")] + WebAdminRange(&'static str, u32, u32, u32), +} + +/// Fewest and most days a deleted node waits in the trash. +pub const MIN_TRASH_DAYS: u32 = 1; +pub const MAX_TRASH_DAYS: u32 = 3650; +/// Days in the trash when `retention.trash_days` is not set. +pub const DEFAULT_TRASH_DAYS: u32 = 30; + +/// `[retention]`: how long things are kept. Only the trash today; nothing is removed by it yet. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RetentionConfig { + /// Days a deleted node stays in the trash before the server may remove it for good (shown as `purge_after_ms`). + #[serde(default = "default_trash_days")] + pub trash_days: u32, +} + +impl Default for RetentionConfig { + fn default() -> Self { + Self { trash_days: DEFAULT_TRASH_DAYS } + } +} + +fn default_trash_days() -> u32 { + DEFAULT_TRASH_DAYS } #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -90,14 +145,107 @@ pub struct Config { /// trailing slash. Unset: `user invite` prints the bare code and says to put the address in front. #[serde(default)] pub public_url: Option, + /// The security audit log. The section is optional and its defaults are safe. + #[serde(default)] + pub audit: AuditConfig, + /// The admin pages in the browser (`/web`). Off unless the section says `enabled = true`. + #[serde(default)] + pub web_admin: WebAdminConfig, + #[serde(default)] + pub retention: RetentionConfig, pub database: DatabaseConfig, pub storage: StorageConfig, + /// The key for the secrets of two-factor sign-in. Without it two-factor sign-in can't be set up (`503 + /// two_factor_unavailable`), and a server whose database already holds secrets refuses to start. `init` makes the + /// key file and writes this section. The server never makes the key itself: a second server on the same database, + /// or a restore without the key, would otherwise seal secrets nobody can open. + #[serde(default)] + pub two_factor: Option, + /// The directory the config file is in, which `key_file` paths are relative to. Set by [`Config::load`], never read + /// from the file. + #[serde(skip)] + pub dir: Option, +} + +/// `[two_factor]`: where the key for two-factor secrets is. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TwoFactorConfig { + /// A file of 32 random bytes (mode 0600; Windows has no equivalent here, keep it out of everyone's reach). A relative + /// path is relative to the directory of `config.toml`. + pub key_file: PathBuf, + /// The key the secrets were sealed with before a rotation: new secrets are sealed with `key_file`, old ones still + /// open with this one until `two-factor rekey` has rewritten them. Remove it after that. + #[serde(default)] + pub key_file_old: Option, } fn default_live_bytes() -> u64 { craft_cloud_meta::DEFAULT_LIVE_BYTES_PER_SPACE } +/// `[audit]`: how the security audit log is kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuditConfig { + /// Days a security event is kept: `audit prune` deletes older ones, and a write does so now and then (at most once an + /// hour). 1 to 3650, default 365. A server that no one writes to keeps expired events until `audit prune` runs. + #[serde(default = "default_retention_days")] + pub retention_days: u32, +} + +impl Default for AuditConfig { + fn default() -> Self { + Self { retention_days: default_retention_days() } + } +} + +fn default_retention_days() -> u32 { + 365 +} + +/// `[web_admin]`: the admin pages in the browser and the lifetimes of their sessions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WebAdminConfig { + /// Serve the admin pages under `/web`. Default off: a server that was not told to speak to browsers does not. Needs + /// `behind_proxy = true` and an `https://` `public_url` without a path. + #[serde(default)] + pub enabled: bool, + /// A session ends when it was not used for this many minutes: 5 to 1440, default 480 (8 hours). + #[serde(default = "default_idle_minutes")] + pub idle_minutes: u32, + /// A session ends this many hours after sign-in, however busy it is: 1 to 720, default 168 (7 days). + #[serde(default = "default_absolute_hours")] + pub absolute_hours: u32, +} + +impl Default for WebAdminConfig { + fn default() -> Self { + Self { enabled: false, idle_minutes: default_idle_minutes(), absolute_hours: default_absolute_hours() } + } +} + +fn default_idle_minutes() -> u32 { + 8 * 60 +} + +fn default_absolute_hours() -> u32 { + 7 * 24 +} + +impl WebAdminConfig { + /// The idle lifetime of a session in milliseconds. + pub fn idle_ms(&self) -> i64 { + i64::from(self.idle_minutes) * 60_000 + } + + /// The absolute lifetime of a session in milliseconds. + pub fn absolute_ms(&self) -> i64 { + i64::from(self.absolute_hours) * 3_600_000 + } +} + #[derive(Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct DatabaseConfig { @@ -164,7 +312,19 @@ impl std::fmt::Debug for Secret { impl Config { pub fn load(path: &Path) -> Result { let text = std::fs::read_to_string(path).map_err(|source| ConfigError::Read { path: path.to_owned(), source })?; - Self::parse(&text) + let mut config = Self::parse(&text)?; + // A relative path to the config file has a parent too, possibly empty: that is the working directory. + config.dir = Some(path.parent().map(Path::to_owned).unwrap_or_default()); + Ok(config) + } + + /// `path` of a `[two_factor]` key file as the server opens it: relative paths are relative to the config file's + /// directory (when the config was loaded from a file). + pub fn key_path(&self, path: &Path) -> PathBuf { + match (&self.dir, path.is_absolute()) { + (Some(dir), false) => dir.join(path), + _ => path.to_owned(), + } } pub fn parse(text: &str) -> Result { @@ -186,10 +346,36 @@ impl Config { { return Err(ConfigError::StoredBytesCapTooSmall(cap, MAX_CHUNK_BYTES)); } + if !(craft_cloud_meta::MIN_RETENTION_DAYS..=craft_cloud_meta::MAX_RETENTION_DAYS).contains(&config.audit.retention_days) { + return Err(ConfigError::AuditRetention( + config.audit.retention_days, + craft_cloud_meta::MIN_RETENTION_DAYS, + craft_cloud_meta::MAX_RETENTION_DAYS, + )); + } + let web = &config.web_admin; + for (name, value, low, high) in [("idle_minutes", web.idle_minutes, 5, 1440), ("absolute_hours", web.absolute_hours, 1, 720)] { + if !(low..=high).contains(&value) { + return Err(ConfigError::WebAdminRange(name, value, low, high)); + } + } + if !(MIN_TRASH_DAYS..=MAX_TRASH_DAYS).contains(&config.retention.trash_days) { + return Err(ConfigError::TrashDays(config.retention.trash_days)); + } let mut config = config; if let Some(url) = &config.public_url { config.public_url = Some(public_url(url)?); } + if config.web_admin.enabled { + // The pages' address is an origin: https, a host and maybe a port, and no path to put in front of `/web`. + let origin_only = config.public_url.as_deref().and_then(|url| url.strip_prefix("https://")).is_some_and(|rest| !rest.contains('/')); + if !origin_only { + return Err(ConfigError::WebAdminPublicUrl); + } + if !config.behind_proxy { + return Err(ConfigError::WebAdminNeedsProxy); + } + } Ok(config) } } @@ -323,11 +509,96 @@ path = "/tmp/blobs" assert!(matches!(Config::parse(&no_proxy), Err(ConfigError::IgnoreForwardedForMisplaced))); } + #[test] + fn the_audit_section_is_optional_and_checked() { + assert_eq!(Config::parse(DISK).unwrap().audit.retention_days, 365, "a year by default"); + let with = |section: &str| Config::parse(&format!("{DISK}\n{section}\n")); + assert_eq!(with("[audit]").unwrap().audit.retention_days, 365); + assert_eq!(with("[audit]\nretention_days = 30").unwrap().audit.retention_days, 30); + for days in [1, 3650] { + assert_eq!(with(&format!("[audit]\nretention_days = {days}")).unwrap().audit.retention_days, days); + } + for days in ["0", "3651", "-1", "\"30\"", "1.5"] { + assert!(with(&format!("[audit]\nretention_days = {days}")).is_err(), "{days}"); + } + assert!(matches!(with("[audit]\nretention_days = 0"), Err(ConfigError::AuditRetention(0, 1, 3650)))); + assert!(matches!(with("[audit]\nkeep = 5"), Err(ConfigError::Parse(_))), "unknown keys are refused"); + } + + #[test] + fn the_web_admin_is_off_by_default_and_checked_when_on() { + let c = Config::parse(DISK).unwrap(); + assert_eq!(c.web_admin, WebAdminConfig { enabled: false, idle_minutes: 480, absolute_hours: 168 }, "off, 8 hours idle, 7 days at most"); + assert_eq!((c.web_admin.idle_ms(), c.web_admin.absolute_ms()), (8 * 3_600_000, 7 * 24 * 3_600_000)); + let proxied = DISK.replace("127.0.0.1:8080", "0.0.0.0:8080"); + let with = |top: &str, section: &str| Config::parse(&format!("{top}\n{proxied}\n[web_admin]\n{section}\n")); + let proxy = "behind_proxy = true\ntrusted_proxies = [\"10.0.0.1\"]"; + let good = format!("{proxy}\npublic_url = \"https://cloud.example.com\""); + let c = with(&good, "enabled = true\nidle_minutes = 30\nabsolute_hours = 12").unwrap(); + assert_eq!((c.web_admin.enabled, c.web_admin.idle_minutes, c.web_admin.absolute_hours), (true, 30, 12)); + // Each missing piece of the setup is refused with its own message. + assert!(matches!(with(proxy, "enabled = true"), Err(ConfigError::WebAdminPublicUrl)), "no public_url"); + for url in ["http://cloud.example.com", "https://cloud.example.com/cloud"] { + let bad = format!("{proxy}\npublic_url = \"{url}\""); + assert!(matches!(with(&bad, "enabled = true"), Err(ConfigError::WebAdminPublicUrl)), "{url}"); + } + let no_proxy = format!("public_url = \"https://cloud.example.com\"\n{DISK}\n[web_admin]\nenabled = true\n"); + assert!(matches!(Config::parse(&no_proxy), Err(ConfigError::WebAdminNeedsProxy))); + // Off, none of it matters; the lifetimes are always in range; unknown keys are errors. + assert!(!Config::parse(&format!("{DISK}\n[web_admin]\nenabled = false\n")).unwrap().web_admin.enabled); + for section in ["idle_minutes = 4", "idle_minutes = 1441", "absolute_hours = 0", "absolute_hours = 721"] { + assert!(matches!(Config::parse(&format!("{DISK}\n[web_admin]\n{section}\n")), Err(ConfigError::WebAdminRange(..))), "{section}"); + } + assert!(matches!(Config::parse(&format!("{DISK}\n[web_admin]\ncookie = 1\n")), Err(ConfigError::Parse(_)))); + } + + #[test] + fn the_trash_period_has_a_default_and_bounds() { + assert_eq!(Config::parse(DISK).unwrap().retention, RetentionConfig { trash_days: 30 }); + // A table has to come after the top-level keys, so the section goes at the end. + let at_end = |days: &str| Config::parse(&format!("{DISK}\n[retention]\ntrash_days = {days}\n")); + assert_eq!(at_end("1").unwrap().retention.trash_days, 1); + assert_eq!(at_end("3650").unwrap().retention.trash_days, 3650); + for bad in ["0", "3651", "-1", "\"30\"", "1.5"] { + assert!(at_end(bad).is_err(), "{bad}"); + } + assert!(matches!(at_end("0"), Err(ConfigError::TrashDays(0)))); + assert!(matches!(at_end("3651"), Err(ConfigError::TrashDays(3651)))); + // Nothing but `trash_days` is accepted in the section, and the section may be empty. + assert!(matches!(Config::parse(&format!("{DISK}\n[retention]\ngc_wait_hours = 24\n")), Err(ConfigError::Parse(_)))); + assert_eq!(Config::parse(&format!("{DISK}\n[retention]\n")).unwrap().retention.trash_days, 30); + } + #[test] fn unknown_keys_are_errors() { assert!(matches!(Config::parse(&format!("lisen = 1\n{DISK}")), Err(ConfigError::Parse(_)))); } + #[test] + fn the_two_factor_section_is_optional_and_strict() { + assert_eq!(Config::parse(DISK).unwrap().two_factor, None, "a config from before two-factor sign-in still loads"); + let with = |section: &str| Config::parse(&format!("{DISK}\n[two_factor]\n{section}\n")); + let c = with("key_file = \"two-factor.key\"").unwrap(); + assert_eq!(c.two_factor, Some(TwoFactorConfig { key_file: "two-factor.key".into(), key_file_old: None })); + let c = with("key_file = \"new.key\"\nkey_file_old = \"old.key\"").unwrap(); + assert_eq!(c.two_factor.unwrap().key_file_old, Some("old.key".into())); + assert!(matches!(with("key_file = \"k\"\nkey = \"x\""), Err(ConfigError::Parse(_))), "an unknown key in the section"); + assert!(matches!(with("key_file_old = \"old.key\""), Err(ConfigError::Parse(_))), "key_file is required"); + } + + #[test] + fn key_files_are_relative_to_the_config_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, format!("{DISK}\n[two_factor]\nkey_file = \"two-factor.key\"\n")).unwrap(); + let c = Config::load(&path).unwrap(); + assert_eq!(c.key_path(Path::new("two-factor.key")), dir.path().join("two-factor.key")); + let absolute = dir.path().join("elsewhere.key"); + assert_eq!(c.key_path(&absolute), absolute); + // A config that was parsed from text has no directory: the path stays as it is. + assert_eq!(Config::parse(DISK).unwrap().key_path(Path::new("k")), PathBuf::from("k")); + } + #[test] fn s3_config_hides_its_secret() { let text = r#" diff --git a/apps/craft-cloud-server/src/error.rs b/apps/craft-cloud-server/src/error.rs index 73d5a18f..21d2dc71 100644 --- a/apps/craft-cloud-server/src/error.rs +++ b/apps/craft-cloud-server/src/error.rs @@ -6,7 +6,7 @@ use axum::extract::rejection::{BytesRejection, JsonRejection, PathRejection, Que use axum::http::{HeaderValue, StatusCode, header}; use axum::response::{IntoResponse, Response}; use craft_cloud_meta::MetaError; -use craft_cloud_proto::api::{ErrorBody, QuotaUse, codes}; +use craft_cloud_proto::api::{ErrorBody, QuotaUse, TwoFactorChallenge, codes}; use craft_cloud_proto::{ChunkId, Conflict}; use craft_cloud_store::StoreError; @@ -21,11 +21,13 @@ pub struct ApiError { pub quota: Option>, /// Seconds for the `Retry-After` header. pub retry_after: Option, + /// The challenge of `two_factor_required`. Boxed for the same reason as `quota`. + pub two_factor: Option>, } impl ApiError { pub fn new(status: StatusCode, code: &'static str, message: impl Into) -> Self { - Self { status, code, message: message.into(), conflicts: None, missing: None, quota: None, retry_after: None } + Self { status, code, message: message.into(), conflicts: None, missing: None, quota: None, retry_after: None, two_factor: None } } /// `507 quota_exceeded`: the account would store more than the server lets it. @@ -36,6 +38,11 @@ impl ApiError { e } + /// `403 forbidden`: signed in, but not allowed to do this. + pub fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, codes::FORBIDDEN, message) + } + pub fn invalid(message: impl Into) -> Self { Self::new(StatusCode::BAD_REQUEST, codes::INVALID, message) } @@ -69,6 +76,49 @@ impl ApiError { Self::new(StatusCode::BAD_REQUEST, codes::PASSWORD_REQUIRED, "this invite is for a new account: choose a password for it") } + /// `401 two_factor_required`: the password (or code) was right and the account needs its second factor. No token. + /// `challenge` is the secret to send back to `POST /auth/login/second-factor` (empty for `/join`, which has none). + pub fn two_factor_required(challenge: String, expires_ms: i64) -> Self { + let mut e = Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, "this account needs a code from its authenticator app"); + e.two_factor = Some(Box::new(TwoFactorChallenge { challenge, expires_ms })); + e + } + + /// `403 two_factor_invalid`: a wrong code or recovery code on a signed-in route (never 401: on a signed-in route the + /// agent reads 401 as "this device was revoked"). [`ApiError::on_public_route`] makes it 401 for sign-in and join. + pub fn two_factor_invalid() -> Self { + Self::new(StatusCode::FORBIDDEN, codes::TWO_FACTOR_INVALID, "the code is wrong, or was used already") + } + + /// `429 two_factor_locked` with `Retry-After`. + pub fn two_factor_locked(retry_after_secs: u32) -> Self { + let mut e = Self::new( + StatusCode::TOO_MANY_REQUESTS, + codes::TWO_FACTOR_LOCKED, + "too many wrong codes: this account's second factor is locked for a while; try again later, or ask the server's admin", + ); + e.retry_after = Some(retry_after_secs); + e + } + + /// `503 two_factor_unavailable`: the server has no key for two-factor secrets. + pub fn two_factor_unavailable() -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + codes::TWO_FACTOR_UNAVAILABLE, + "this server has no key for two-factor secrets; the admin sets [two_factor] key_file in its config", + ) + } + + /// The answer on a route that is not signed in (sign-in's second step, join): a wrong second factor is `401` there, + /// where it is `403` on every signed-in route. + pub fn on_public_route(mut self) -> Self { + if self.code == codes::TWO_FACTOR_INVALID { + self.status = StatusCode::UNAUTHORIZED; + } + self + } + pub fn internal() -> Self { Self::new(StatusCode::INTERNAL_SERVER_ERROR, codes::INTERNAL, "the server hit an internal error; the admin can find details in its log") } @@ -95,12 +145,31 @@ impl From for ApiError { MetaError::NotFound(what) => Self::not_found(what), MetaError::InvalidCode => Self::invalid_code(), MetaError::PasswordRequired => Self::password_required(), + MetaError::DeviceRevoked => Self::unauthorized(), + MetaError::Forbidden(message) => Self::forbidden(message), + MetaError::LastAdmin => Self::new(StatusCode::CONFLICT, codes::LAST_ADMIN, e.to_string()), + MetaError::ReadOnly => crate::space::read_only(), + MetaError::LastOwner => Self::new(StatusCode::CONFLICT, codes::LAST_OWNER, e.to_string()), + MetaError::AccountDisabled => Self::new(StatusCode::FORBIDDEN, codes::ACCOUNT_DISABLED, "this account is disabled; ask an administrator"), + // The status of a wrong second factor depends on the route: 403 here (every signed-in route), made 401 by + // `on_public_route` for sign-in and join. + MetaError::TwoFactorInvalid => Self::two_factor_invalid(), + MetaError::TwoFactorLocked { retry_after_ms } => Self::two_factor_locked(craft_cloud_meta::retry_after_secs(retry_after_ms)), + MetaError::TwoFactorPending => { + Self::new(StatusCode::CONFLICT, codes::TWO_FACTOR_PENDING, "a two-factor set-up is open on another device: finish it there, or wait") + } + // Only the join route can see this and it builds its own answer; a signed-in route never asks a second factor. + MetaError::TwoFactorRequired => Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, "this account needs its second factor"), + MetaError::ChallengeGone => { + Self::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "this sign-in step doesn't work any more: sign in again") + } + MetaError::Conflict(message) => Self::new(StatusCode::CONFLICT, codes::CONFLICT, message), MetaError::UsernameTaken => Self::new(StatusCode::CONFLICT, codes::USERNAME_TAKEN, "the username is taken"), MetaError::QuotaExceeded { used, limit } => Self::quota_exceeded(used, limit), MetaError::DocFull { .. } | MetaError::LiveFull { .. } | MetaError::TooManyDocs { .. } | MetaError::TooManyDocRows { .. } => { Self::new(StatusCode::INSUFFICIENT_STORAGE, codes::QUOTA_EXCEEDED, e.to_string()) } - MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::UnsupportedUrl => { + MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::Missing(_) | MetaError::UnsupportedUrl => { tracing::error!(error = %e, "metadata error"); Self::internal() } @@ -160,6 +229,7 @@ impl IntoResponse for ApiError { conflicts: self.conflicts, missing: self.missing, quota: self.quota.map(|q| *q), + two_factor: self.two_factor.map(|t| *t), }; let mut response = (self.status, Json(body)).into_response(); if let Some(secs) = self.retry_after { @@ -168,3 +238,42 @@ impl IntoResponse for ApiError { response } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_change_that_lost_a_race_is_a_conflict_and_a_signed_out_caller_is_unauthorized() { + let conflict = ApiError::from(MetaError::Conflict("changed meanwhile")); + assert_eq!((conflict.status, conflict.code), (StatusCode::CONFLICT, codes::CONFLICT)); + let revoked = ApiError::from(MetaError::DeviceRevoked); + assert_eq!((revoked.status, revoked.code), (StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED)); + let forbidden = ApiError::from(MetaError::Forbidden("no")); + assert_eq!((forbidden.status, forbidden.code), (StatusCode::FORBIDDEN, codes::FORBIDDEN)); + let last = ApiError::from(MetaError::LastAdmin); + assert_eq!((last.status, last.code), (StatusCode::CONFLICT, codes::LAST_ADMIN)); + let disabled = ApiError::from(MetaError::AccountDisabled); + assert_eq!((disabled.status, disabled.code), (StatusCode::FORBIDDEN, codes::ACCOUNT_DISABLED)); + } + + #[test] + fn second_factor_errors_are_403_on_signed_in_routes_and_401_on_public_ones() { + let invalid = ApiError::from(MetaError::TwoFactorInvalid); + assert_eq!((invalid.status, invalid.code), (StatusCode::FORBIDDEN, codes::TWO_FACTOR_INVALID)); + let public = invalid.on_public_route(); + assert_eq!((public.status, public.code), (StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_INVALID)); + let locked = ApiError::from(MetaError::TwoFactorLocked { retry_after_ms: 1500 }); + assert_eq!((locked.status, locked.code, locked.retry_after), (StatusCode::TOO_MANY_REQUESTS, codes::TWO_FACTOR_LOCKED, Some(2))); + let locked = locked.on_public_route(); + assert_eq!(locked.status, StatusCode::TOO_MANY_REQUESTS); + let pending = ApiError::from(MetaError::TwoFactorPending); + assert_eq!((pending.status, pending.code), (StatusCode::CONFLICT, codes::TWO_FACTOR_PENDING)); + assert_eq!(ApiError::two_factor_unavailable().status, StatusCode::SERVICE_UNAVAILABLE); + let gone = ApiError::from(MetaError::ChallengeGone); + assert_eq!((gone.status, gone.code), (StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED)); + let required = ApiError::two_factor_required("challenge-secret".into(), 5); + assert_eq!((required.status, required.code), (StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED)); + assert!(!format!("{required:?}").contains("challenge-secret"), "the challenge is hidden in Debug"); + } +} diff --git a/apps/craft-cloud-server/src/gc.rs b/apps/craft-cloud-server/src/gc.rs new file mode 100644 index 00000000..5e5b2c49 --- /dev/null +++ b/apps/craft-cloud-server/src/gc.rs @@ -0,0 +1,197 @@ +//! Read-only store inventory and garbage-collection preview. + +use craft_cloud_meta::{BlobKind, Meta, MetaError}; +use craft_cloud_store::{ListCursor, MAX_LIST_PAGE_SIZE, Store}; +use serde_json::{Value, json}; + +use crate::verify::{self, VerifyOptions}; + +const EXAMPLES: usize = 20; + +#[derive(Debug, thiserror::Error)] +pub enum GcError { + #[error(transparent)] + Meta(#[from] MetaError), +} + +#[derive(Default)] +struct Class { + count: u64, + bytes: u64, + examples: Vec, +} + +/// Objects the database records: how many and their stored bytes as `blobs.stored_size` has them (no object is read). +#[derive(Default)] +struct Recorded { + count: u64, + bytes: u64, +} + +pub struct Report { + chunks_recorded: Recorded, + manifests_recorded: Recorded, + chunks_orphans: Class, + manifests_orphans: Class, + foreign: Class, + inventory_error: Option, + db_check: verify::Report, + incomplete: bool, + chunk_count: u64, + chunk_candidates: u64, + chunk_candidate_bytes: u64, + chunk_protected: u64, + chunk_examples: Vec, +} + +fn add(class: &mut Class, id: String, bytes: u64) { + class.count = class.count.saturating_add(1); + class.bytes = class.bytes.saturating_add(bytes); + if class.examples.len() < EXAMPLES { + class.examples.push(id); + } +} + +pub async fn dry_run(meta: &Meta, store: &Store) -> Result { + let audit = meta.index_audit(EXAMPLES).await?; + let incomplete = audit.absent > 0 || audit.building > 0; + let mut chunks_recorded = Recorded::default(); + let mut manifests_recorded = Recorded::default(); + let mut chunks_orphans = Class::default(); + let mut manifests_orphans = Class::default(); + let mut foreign = Class::default(); + let mut inventory_error = None; + for (kind, label) in [(BlobKind::Chunk, "chunks"), (BlobKind::Manifest, "manifests")] { + let mut cursor: Option = None; + let mut page_no = 1u64; + loop { + let page = match store.list(kind, cursor.clone(), MAX_LIST_PAGE_SIZE).await { + Ok(p) => p, + Err(source) => { + inventory_error = Some(format!("store listing failed for {label} page {page_no}: {source}")); + break; + } + }; + let ids: Vec<[u8; 32]> = page.blobs.iter().map(|b| b.id).collect(); + let recorded = meta.recorded_blob_sizes(kind, &ids).await?; + let (recorded_total, orphans) = + if kind == BlobKind::Chunk { (&mut chunks_recorded, &mut chunks_orphans) } else { (&mut manifests_recorded, &mut manifests_orphans) }; + for blob in &page.blobs { + if let Some(size) = recorded.get(&blob.id) { + recorded_total.count = recorded_total.count.saturating_add(1); + recorded_total.bytes = recorded_total.bytes.saturating_add(*size); + continue; + } + match store.stored_size(*blob).await { + Ok(size) => add(orphans, hex(&blob.id), size), + Err(error) => { + add(orphans, hex(&blob.id), 0); + inventory_error = Some(format!("could not read stored size for {} object {}: {error}", label, hex(&blob.id))); + break; + } + } + } + if inventory_error.is_some() { + // A size that cannot be read (a backend without metadata-only sizes, or an I/O error) is reported once and + // ends the inventory; it is not retried for every remaining object. + break; + } + for key in &page.foreign { + let id = blake3::hash(key.as_bytes()).to_hex().to_string(); + add(&mut foreign, id, 0); + } + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + page_no = page_no.saturating_add(1); + } + if inventory_error.is_some() { + break; + } + } + let db_check = verify::verify(meta, store, VerifyOptions { read_chunks: true }).await?; + let chunk_count = audit.chunks; + let incomplete = incomplete || db_check.audit.absent > 0 || db_check.audit.building > 0; + let fail_closed = incomplete || db_check.problem_count() > 0; + // `index_audit` reads all reference roots in one read snapshot. The inventory is diagnostic; a later collector must + // recheck roots before claiming anything because store listing and metadata snapshots are not atomic together. + let chunk_candidates = if fail_closed { 0 } else { audit.unreferenced_chunks.count }; + let chunk_candidate_bytes = if fail_closed { 0 } else { audit.unreferenced_chunk_bytes }; + let chunk_protected = if fail_closed { chunk_count } else { chunk_count.saturating_sub(chunk_candidates) }; + let chunk_examples = if fail_closed { Vec::new() } else { audit.unreferenced_chunks.examples }; + Ok(Report { + chunks_recorded, + manifests_recorded, + chunks_orphans, + manifests_orphans, + foreign, + inventory_error, + db_check, + incomplete, + chunk_count, + chunk_candidates, + chunk_candidate_bytes, + chunk_protected, + chunk_examples, + }) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +impl Report { + pub fn has_problems(&self) -> bool { + self.inventory_error.is_some() || self.db_check.problem_count() > 0 + } + pub fn to_json(&self) -> Value { + json!({"schema_version":1,"store":{"chunk_recorded":{"count":self.chunks_recorded.count,"stored_bytes":self.chunks_recorded.bytes,"bytes_source":"database blobs.stored_size"},"manifest_recorded":{"count":self.manifests_recorded.count,"stored_bytes":self.manifests_recorded.bytes,"bytes_source":"database blobs.stored_size"},"chunk_orphans":{"count":self.chunks_orphans.count,"stored_bytes":self.chunks_orphans.bytes,"examples":self.chunks_orphans.examples},"manifest_orphans":{"count":self.manifests_orphans.count,"stored_bytes":self.manifests_orphans.bytes,"examples":self.manifests_orphans.examples},"foreign":{"count":self.foreign.count,"stored_bytes":null,"examples":self.foreign.examples},"listing_error":self.inventory_error},"database":{"problems":self.db_check.problem_count(),"verify":self.db_check.to_json()},"candidates":{"orphans":{"count":0,"potential_count":self.chunks_orphans.count.saturating_add(self.manifests_orphans.count),"reason":"eligible only after first-sight grace; would need two runs apart","protection":"first sight is not recorded"},"chunks":{"count":self.chunk_candidates,"stored_bytes":self.chunk_candidate_bytes,"examples":self.chunk_examples,"reason":"no complete-indexed manifest names chunk","protected_count":self.chunk_protected,"protection":if self.incomplete {"index incomplete: dependencies unknown"} else if self.db_check.problem_count()>0 {"protected: database verification reported problems"} else {"versions, saved versions, trash items, live snapshots and import snapshots in the database read snapshot; future collection must recheck"}},"manifests":{"count":0,"stored_bytes":0,"protected_count":self.db_check.audit.unrooted_manifests.count,"protection":"protected: may be a live document's asset"}},"index_incomplete":self.incomplete,"nothing_was_changed":true}) + } + pub fn to_text(&self) -> String { + let mut out = format!( + "store chunks: {} recorded, {} stored bytes (as the database records them)\nstore manifests: {} recorded, {} stored bytes (as the database records them)\nstore chunks: {} orphan(s), {} stored bytes\nstore manifests: {} orphan(s), {} stored bytes\nforeign objects: {}\n", + self.chunks_recorded.count, + self.chunks_recorded.bytes, + self.manifests_recorded.count, + self.manifests_recorded.bytes, + self.chunks_orphans.count, + self.chunks_orphans.bytes, + self.manifests_orphans.count, + self.manifests_orphans.bytes, + self.foreign.count + ); + out.push_str(&format!( + "orphan candidates: {}; potential: {}; protection: first sight is not recorded; would need two runs apart\nchunk candidates: {}; {} stored bytes; protected: {}\nforeign stored bytes: not reported\n", + 0, self.chunks_orphans.count.saturating_add(self.manifests_orphans.count), self.chunk_candidates, self.chunk_candidate_bytes, self.chunk_protected + )); + for (label, examples) in [ + ("chunk orphan IDs", &self.chunks_orphans.examples), + ("manifest orphan IDs", &self.manifests_orphans.examples), + ("foreign object pseudonyms", &self.foreign.examples), + ] { + if !examples.is_empty() { + out.push_str(&format!("{label}: {}\n", examples.join(", "))); + } + } + if self.incomplete { + out.push_str(&format!("chunk protection: index incomplete: dependencies unknown; protected {} recorded chunk(s)\n", self.chunk_count)); + } + out.push_str("chunk roots: current and saved file versions (including trash history), live snapshots and import snapshots\n"); + if !self.chunk_examples.is_empty() { + out.push_str(&format!("chunk candidate IDs from database snapshot: {}\n", self.chunk_examples.join(", "))); + } + out.push_str(&format!( + "manifest protection: protected: may be a live document's asset ({} unrooted recorded manifest(s))\n", + self.db_check.audit.unrooted_manifests.count + )); + if let Some(error) = &self.inventory_error { + out.push_str(&format!("listing error: {error}\n")); + } + if self.db_check.problem_count() > 0 { + out.push_str(&self.db_check.to_text()); + } + out.push_str(&format!("database problems: {}\nnothing was changed\n", self.db_check.problem_count())); + out + } +} diff --git a/apps/craft-cloud-server/src/join.rs b/apps/craft-cloud-server/src/join.rs index 41d57bdc..0bf3266b 100644 --- a/apps/craft-cloud-server/src/join.rs +++ b/apps/craft-cloud-server/src/join.rs @@ -4,11 +4,18 @@ //! - **One answer for every code that doesn't work**: wrong, expired, used, malformed, or a pairing whose maker has //! signed out are all `404 invalid_code` with the same message, so an answer never says which. //! - **Failed redeems are limited per client address** (behind trusted proxies, as sign-ins are): [`JOINS_PER_IP`] -//! in a minute, then `429 rate_limited`, even for a code that works. Only `invalid_code` counts. A successful -//! redeem doesn't, and neither does `password_required` or `weak_password`: they are only answered for a real, -//! unused invite, so they tell a guesser nothing a success wouldn't. Every try is counted before the code is -//! looked at, and given back once it turns out not to be `invalid_code`, so tries sent at the same moment are held -//! to the limit as well. +//! in a minute, then `429 rate_limited`, even for a code that works. `invalid_code` and `two_factor_invalid` count. A +//! successful redeem doesn't, and neither does `password_required`, `weak_password` or `two_factor_required`: they are +//! only answered for a real, unused invite, so they tell a guesser nothing a success wouldn't. Every try is counted +//! before the code is looked at, and given back once it turns out not to be one of those two, so tries sent at the same +//! moment are held to the limit as well. +//! - **An account with two-factor sign-in** needs its second factor (`totp_code`, or `recovery_code`) to redeem an invite or +//! a password reset code, unless the reset code was made to turn it off. Without it the answer is `401 +//! two_factor_required` and the code is **not** used up: the same body is sent again with the factor (there is no +//! challenge for `/join`, so the answer's `challenge` is empty). A wrong factor is `401 two_factor_invalid`, leaves the +//! code as it was and counts against the account (and its lock) and, with the client address, against the +//! second-factor limiter. A pairing code needs no second factor: only a signed-in device can make one, and turning +//! two-factor sign-in on signs the account's other devices out. //! - **An invite to a new account** must carry a password (the server's password rules), hashed with Argon2id under //! the same server-wide slots as sign-ins; it becomes the account's in the transaction that creates the device. //! - **Codes are never logged**, and neither are tokens or passwords; the database keeps only codes' hashes. @@ -21,16 +28,17 @@ use axum::extract::rejection::PathRejection; use axum::extract::{ConnectInfo, Extension, Path, Request, State}; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::IntoResponse; -use craft_cloud_meta::DeviceAuth; +use craft_cloud_meta::{CHALLENGE_TTL_MS, DeviceAuth, DeviceClient, MetaError}; use craft_cloud_proto::api::{JoinRequest, LoginResponse, PairingCreated, PairingStatus, codes}; use craft_cloud_proto::join::{Code, PAIRING_CHARS, PAIRING_TTL_MS}; use uuid::Uuid; -use crate::api::{PASSWORD_RETRY_SECS, SMALL_JSON_BYTES, check_device_name, read_json}; +use crate::api::{PASSWORD_RETRY_SECS, SMALL_JSON_BYTES, check_device_name, device_client, read_json}; use crate::auth::{check_password_rules, hash_password, new_token}; use crate::error::ApiError; use crate::ratelimit::{client_ip, ip_key}; use crate::state::AppState; +use crate::two_factor::{Prepared, ProofInput, SECOND_FACTORS_PER_IP, parse_proof}; /// Failed redeems one client address may make in a minute. pub const JOINS_PER_IP: u32 = 10; @@ -82,22 +90,50 @@ pub async fn join( ) -> Result, ApiError> { let req: JoinRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; check_device_name(&req.device_name)?; + // Checked before the try is counted, as the device name is: a body that is refused for it costs nothing. + let client = device_client(req.client_kind.as_deref(), req.platform.as_deref(), req.app_version.as_deref())?; + let proof = parse_proof(req.totp_code.clone(), req.recovery_code.clone())?; // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. let now = tokio::time::Instant::now().into_std(); let key = ip_key(client_ip(peer, &headers, &state.trusted_proxies)); // Counted before the redeem, so tries made at the same moment are held to the limit too; given back unless the // code didn't work. let Some(counted) = state.joins.count(&key, JOINS_PER_IP, now) else { - return Err(ApiError::rate_limited("too many tries with codes that don't work; wait a minute", None)); + return Err(limited("too many tries with codes that don't work; wait a minute", state.joins.retry_after(&key, now))); }; - let result = redeem(&state, &req).await; - if !result.as_ref().is_err_and(|e| e.code == codes::INVALID_CODE) { + // A second factor is also counted against the address's second-factor limiter: wrong ones stay counted. + let second = match proof { + Some(_) => match state.second_factors.count(&key, SECOND_FACTORS_PER_IP, now) { + Some(counted) => Some(counted), + None => { + state.joins.refund(&key, counted); + return Err(limited("too many wrong codes from this address; wait a few minutes", state.second_factors.retry_after(&key, now))); + } + }, + None => None, + }; + let result = redeem(&state, &req, &client, proof).await; + // Wrong codes (of the invite or of the second factor) stay counted; everything else is given back. + if !result.as_ref().is_err_and(|e| e.code == codes::INVALID_CODE || e.code == codes::TWO_FACTOR_INVALID) { state.joins.refund(&key, counted); } + if let Some(second) = second + && !result.as_ref().is_err_and(|e| e.code == codes::TWO_FACTOR_INVALID) + { + state.second_factors.refund(&key, second); + } result.map(Json) } -async fn redeem(state: &AppState, req: &JoinRequest) -> Result { +/// The answer for a join a limiter refuses: the same for every code (a real invite, a locked account's, a made-up one), with the +/// time until the limiter's window ends in `Retry-After`, so that the refusal tells a guesser nothing about codes or locks. +/// A lock is only told to a request the limiters let through (and counted), as at sign-in. +fn limited(message: &'static str, window: std::time::Duration) -> ApiError { + let secs = u32::try_from(window.as_secs().saturating_add(1)).unwrap_or(u32::MAX).max(1); + ApiError::rate_limited(message, Some(secs)) +} + +async fn redeem(state: &AppState, req: &JoinRequest, client: &DeviceClient, proof: Option) -> Result { let code = Code::parse_any(&req.code).ok_or_else(ApiError::invalid_code)?; let code_hash = code.hash(); let now_ms = state.now_ms(); @@ -110,8 +146,22 @@ async fn redeem(state: &AppState, req: &JoinRequest) -> Result None, }; let (token, token_hash) = new_token()?; - let redeemed = state.meta.redeem(&code_hash, &req.device_name, &token_hash, new_password_hash.as_deref(), now_ms).await?; - tracing::info!(device = %redeemed.device, "device joined with a code"); + let prepared = proof.map(|proof| proof.prepare(state, now_ms)); + let second_factor = prepared.as_ref().map(Prepared::proof); + let redeemed = state + .meta + .redeem_with_second_factor(&code_hash, &req.device_name, client, &token_hash, new_password_hash.as_deref(), second_factor.as_ref(), now_ms) + .await + .map_err(|e| match e { + // No challenge row for a join: the answer only says that the same code goes again with the second factor. + MetaError::TwoFactorRequired => ApiError::two_factor_required(String::new(), now_ms.saturating_add(CHALLENGE_TTL_MS)), + e => ApiError::from(e).on_public_route(), + })?; + // A reset code signed the account's other devices out: their live connections end now, not at their next request. + for device in &redeemed.revoked { + state.live.revoke(*device); + } + tracing::info!(device = %redeemed.device, revoked = redeemed.revoked.len(), "device joined with a code"); Ok(LoginResponse { device: redeemed.device, token }) } diff --git a/apps/craft-cloud-server/src/lib.rs b/apps/craft-cloud-server/src/lib.rs index 97ffd436..c609c531 100644 --- a/apps/craft-cloud-server/src/lib.rs +++ b/apps/craft-cloud-server/src/lib.rs @@ -2,16 +2,27 @@ //! (`craft-cloud-store`), the live service on a WebSocket ([`live`]), plus the admin command line. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod account; +pub mod admin; pub mod api; +pub mod audit; pub mod auth; pub mod cli; pub mod config; pub mod error; +pub mod gc; pub mod join; pub mod live; pub mod notify; pub mod ratelimit; +pub mod secretbox; +pub mod space; +pub mod spaces; pub mod state; +pub mod totp; +pub mod two_factor; +pub mod verify; +pub mod web; use std::net::SocketAddr; use std::sync::Arc; diff --git a/apps/craft-cloud-server/src/live.rs b/apps/craft-cloud-server/src/live.rs index 859d5c88..e8bd8f33 100644 --- a/apps/craft-cloud-server/src/live.rs +++ b/apps/craft-cloud-server/src/live.rs @@ -65,6 +65,7 @@ use tokio::sync::broadcast; use tokio::time::Instant; use crate::error::ApiError; +use crate::space::SpaceAccess; use crate::state::{AppState, Limits}; /// Live connections one device may hold at once (a reconnect may overlap the old connection for a moment). @@ -369,11 +370,16 @@ impl Session { /// A session for the signed-in device `auth`. The caller listens to the space ([`LiveHub::listen`]) before the /// session reads anything, and hands it what it hears ([`Session::note`], [`Session::lagged`]). pub fn new(state: Arc, auth: &DeviceAuth) -> Self { + Self::in_space(state, auth.device, auth.personal_space()) + } + + /// A session of `device` on `space`, which the caller has checked the device's account may use. + pub fn in_space(state: Arc, device: DeviceId, space: SpaceId) -> Self { let budget = Budget::new(&state.limits); Self { state, - space: auth.space, - device: auth.device, + space, + device, welcomed: false, protocol: 0, id: NEXT_SESSION.fetch_add(1, Ordering::Relaxed), @@ -761,20 +767,34 @@ impl Session { pub async fn connect( State(state): State>, Extension(auth): Extension, + Extension(access): Extension, upgrade: Result, ) -> Result { + // Live documents of a team space need live protocol 3 (the role per frame, the space in the welcome), which a later + // change brings. Until then the upgrade answers a team space as it does a space the caller has no access to, so a viewer + // can never write through a protocol that has no role in it. An agent does not ask: it opens no live connection for a + // team space on live protocol 2. + if !access.personal { + return Err(ApiError::not_found("space")); + } let upgrade = upgrade.map_err(|e| ApiError::new(e.status(), codes::INVALID, e.body_text()))?; if !take(&state.live.connections, auth.device, MAX_CONNECTIONS_PER_DEVICE) { return Err(ApiError::rate_limited(format!("a device may hold at most {MAX_CONNECTIONS_PER_DEVICE} live connections"), Some(5))); } let slot = ConnectionSlot { hub: state.clone(), device: auth.device }; + // Listening for revocations before the device is looked up once more, and before the upgrade: a revoke between + // the sign-in check and here is seen by the lookup, one after it by the receiver, so none is missed. + let revocations = state.live.revocations(); + if !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } // No frame is over an update and its envelope (snapshots travel through the chunk API). Ok(upgrade.max_message_size(MAX_FRAME_BYTES).max_frame_size(MAX_FRAME_BYTES).on_upgrade(move |socket| async move { let _slot = slot; // Listening before the session reads anything, so no update falls between a read and the notification. - let notes = state.live.listen(auth.space); - let presence = state.live.listen_presence(auth.space); - serve(socket, Session::new(state, &auth), notes, presence).await; + let notes = state.live.listen(access.space); + let presence = state.live.listen_presence(access.space); + serve(socket, Session::in_space(state, auth.device, access.space), notes, presence, revocations).await; })) } @@ -801,9 +821,14 @@ async fn send_all(socket: &mut WebSocket, frames: &[ServerFrame]) -> bool { /// The connection loop: frames in, frames out, the log pumped while a document has news, pings, and an end on /// shutdown, silence or a broken connection. -async fn serve(mut socket: WebSocket, mut session: Session, mut notes: broadcast::Receiver, mut presence: broadcast::Receiver) { +async fn serve( + mut socket: WebSocket, + mut session: Session, + mut notes: broadcast::Receiver, + mut presence: broadcast::Receiver, + mut revocations: broadcast::Receiver, +) { let mut stopping = session.state.notifier.shutdown_signal(); - let mut revocations = session.state.live.revocations(); let mut last_heard = Instant::now(); let mut ping = tokio::time::interval_at(Instant::now() + PING_EVERY, PING_EVERY); let hello_by = Instant::now() + HELLO_TIMEOUT; diff --git a/apps/craft-cloud-server/src/main.rs b/apps/craft-cloud-server/src/main.rs index 99384030..0c46a890 100644 --- a/apps/craft-cloud-server/src/main.rs +++ b/apps/craft-cloud-server/src/main.rs @@ -5,6 +5,7 @@ use std::process::ExitCode; use clap::Parser; use craft_cloud_server::cli::{Cli, Command, open}; use craft_cloud_server::config::Config; +use craft_cloud_server::web::WebSettings; use craft_cloud_server::{AppState, serve}; #[tokio::main] @@ -35,9 +36,22 @@ async fn run_server(config_path: &std::path::Path) -> Result<(), String> { tracing::warn!("ignore_forwarded_for is set: every client shares one address for sign-in rate limits"); } let (meta, store) = open(&config).await.map_err(|e| e.to_string())?; - let state = AppState::new(meta, store, config.trusted_proxies.clone()).map_err(|e| e.message)?; + // Fail closed: a database that holds two-factor secrets the key can't open is not served. The server never makes the key. + let keys = craft_cloud_server::two_factor::load_keys(&config).map_err(|e| e.to_string())?; + let secrets = craft_cloud_server::two_factor::check_startup(&meta, keys, &config, craft_cloud_server::state::now_ms()).await?; + // `Config::load` has checked the setup the pages need (an https public_url, a proxy in front). + let web = match (config.web_admin.enabled, &config.public_url) { + (true, Some(public_url)) => Some(WebSettings::new(public_url, config.web_admin.idle_ms(), config.web_admin.absolute_ms())?), + _ => None, + }; + let state = AppState::new(meta, store, config.trusted_proxies.clone()) + .map_err(|e| e.message)? + .with_public_url(config.public_url.clone()) + .with_web_admin(web) + .with_two_factor(secrets) + .with_trash_days(config.retention.trash_days); let listener = tokio::net::TcpListener::bind(config.listen).await.map_err(|e| format!("can't listen on {}: {e}", config.listen))?; - tracing::info!(listen = %config.listen, behind_proxy = config.behind_proxy, "Craft Cloud server running"); + tracing::info!(listen = %config.listen, behind_proxy = config.behind_proxy, web_admin = config.web_admin.enabled, "Craft Cloud server running"); let shutdown = async { wait_for_stop_signal().await; tracing::info!("shutting down"); diff --git a/apps/craft-cloud-server/src/notify.rs b/apps/craft-cloud-server/src/notify.rs index 8952e835..02e9aa45 100644 --- a/apps/craft-cloud-server/src/notify.rs +++ b/apps/craft-cloud-server/src/notify.rs @@ -11,13 +11,15 @@ pub const MAX_WAITERS_PER_SPACE: usize = 16; pub struct Notifier { spaces: Mutex>>, + /// Counts the changes to a space's members, roles and life cycle, for the waiters to re-check their access. + members: Mutex>>, waiters: Mutex>, shutdown: watch::Sender, } impl Default for Notifier { fn default() -> Self { - Self { spaces: Mutex::default(), waiters: Mutex::default(), shutdown: watch::channel(false).0 } + Self { spaces: Mutex::default(), members: Mutex::default(), waiters: Mutex::default(), shutdown: watch::channel(false).0 } } } @@ -68,6 +70,22 @@ impl Notifier { spaces.entry(space).or_insert_with(|| watch::channel(0).0).subscribe() } + /// A receiver that sees every later [`Notifier::members_changed`] for the space. + pub fn subscribe_members(&self, space: SpaceId) -> watch::Receiver { + let mut members = self.members.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + members.entry(space).or_insert_with(|| watch::channel(0).0).subscribe() + } + + /// Tells the space's waiting long-polls that its members, a role or its life cycle (archived) changed, so each looks + /// at its own access again at once. A space nobody watches is a no-op. Another server process on the same database is + /// not told: its waiters notice at the end of their wait (at most a minute). + pub fn members_changed(&self, space: SpaceId) { + let members = self.members.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(sender) = members.get(&space) { + sender.send_modify(|count| *count = count.wrapping_add(1)); + } + } + /// Raises the space's sequence to `seq` and wakes its subscribers. Lower values are ignored, and a /// space nobody watches is a no-op. pub fn publish(&self, space: SpaceId, seq: u64) { diff --git a/apps/craft-cloud-server/src/ratelimit.rs b/apps/craft-cloud-server/src/ratelimit.rs index 27e5b4ad..1a9f33c9 100644 --- a/apps/craft-cloud-server/src/ratelimit.rs +++ b/apps/craft-cloud-server/src/ratelimit.rs @@ -32,14 +32,16 @@ impl Limiter { Self { windows: Mutex::new(Windows { entries: HashMap::new(), last_sweep: None }), period, capacity } } - /// Counts one attempt for `key`; false once `max` attempts were made in the current window, or when + /// Counts one attempt for `key`; false once `max` attempts were counted in the current window, or when /// the key is new and the limiter is full. pub fn allow(&self, key: &str, max: u32, now: Instant) -> bool { self.count(key, max, now).is_some() } /// [`Limiter::allow`] for a caller that may give the attempt back: the window it was counted in, for - /// [`Limiter::refund`], or `None` when it is refused. + /// [`Limiter::refund`], or `None` when it is refused. A refused attempt is not counted: the count never goes past + /// `max`, so refused tries made at the same moment as a counted one cannot make that one's refund (or the window's + /// end) leave the key with more counted attempts than were accepted. pub fn count(&self, key: &str, max: u32, now: Instant) -> Option { let mut windows = self.windows.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let sweep_due = windows.last_sweep.is_none_or(|last| now.saturating_duration_since(last) >= self.period); @@ -55,8 +57,18 @@ impl Limiter { if now.saturating_duration_since(entry.0) >= self.period { *entry = (now, 0); } - entry.1 = entry.1.saturating_add(1); - (entry.1 <= max).then_some(Counted(entry.0)) + if entry.1 >= max { + return None; + } + entry.1 += 1; + Some(Counted(entry.0)) + } + + /// How long until `key`'s current window ends (when its count starts again from nothing): what a refused caller is + /// told in `Retry-After`. A key that has no window running is `Duration::ZERO`. + pub fn retry_after(&self, key: &str, now: Instant) -> Duration { + let windows = self.windows.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + windows.entries.get(key).map_or(Duration::ZERO, |(start, _)| (*start + self.period).saturating_duration_since(now)) } /// Gives back one attempt [`Limiter::count`] counted for `key`, if the window it was counted in is still the @@ -145,6 +157,21 @@ mod tests { /// An attempt counted near the end of a window whose redeem finishes in the next one: its refund would give the /// new window one try more than the limit. + #[test] + fn a_refused_attempt_is_not_counted_so_a_refund_frees_exactly_one_place() { + let limiter = Limiter::new(Duration::from_secs(60)); + let t0 = Instant::now(); + let first = limiter.count("user:a", 2, t0).unwrap(); + let _second = limiter.count("user:a", 2, t0).unwrap(); + // Tries made while the key is full are refused, however many there are. + for _ in 0..10 { + assert!(limiter.count("user:a", 2, t0).is_none()); + } + limiter.refund("user:a", first); + assert!(limiter.count("user:a", 2, t0).is_some(), "the refund freed one place"); + assert!(limiter.count("user:a", 2, t0).is_none(), "and only one"); + } + #[test] fn a_refund_never_crosses_into_a_later_window() { let limiter = Limiter::new(Duration::from_secs(60)); diff --git a/apps/craft-cloud-server/src/secretbox.rs b/apps/craft-cloud-server/src/secretbox.rs new file mode 100644 index 00000000..e2685f1b --- /dev/null +++ b/apps/craft-cloud-server/src/secretbox.rs @@ -0,0 +1,279 @@ +//! Sealing the TOTP secrets that the database keeps. +//! +//! - **The key** is 32 random bytes in a file the server's config names (`[two_factor] key_file`, see +//! [`crate::two_factor`]). It is never used as it is: the AES-256 key is [`blake3::derive_key`] of it under a fixed +//! context. A key shows in no log, no `Debug` output and no error message, and its bytes are wiped when it is dropped. +//! - **A sealed secret** is one byte naming the key (see below), a 12-byte random nonce and the AES-256-GCM ciphertext +//! with its tag. The associated data is `craft-cloud:totp:v1:` and the 16 bytes of the account's ID, so a ciphertext +//! copied to another account does not open. +//! - **The key's name** is the first byte of the BLAKE3 hash of the key file's bytes: one of 256 values, only there to tell +//! the current key from the older one, so two keys that share one are refused when the server starts. It does not +//! identify a key, and the authentication tag refuses a wrong key whatever the name says. With a second, older key (`key_file_old`) configured, the +//! name picks the key to open a secret with; sealing always uses the current key. The `two-factor rekey` command +//! rewrites the secrets under the current key, after which the old key can go. +//! - **Errors say nothing about their input**: not the key, not the plaintext, not the ciphertext. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use aes_gcm::aead::{Aead, KeyInit, Payload}; +use aes_gcm::{Aes256Gcm, Nonce}; +use craft_cloud_meta::UserId; +use zeroize::Zeroizing; + +/// The bytes a key file holds. +pub const KEY_BYTES: usize = 32; +const NONCE_BYTES: usize = 12; +const TAG_BYTES: usize = 16; +const KDF_CONTEXT: &str = "craft-cloud 2026 totp secret v1"; +const AAD_PREFIX: &[u8] = b"craft-cloud:totp:v1:"; + +/// A secret could not be sealed or opened. Carries nothing of what was sealed or opened. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("a two-factor secret could not be sealed or opened")] +pub struct SecretBoxError; + +/// A key file that cannot be used. The message names the file and what is wrong with it, never its content. +#[derive(Debug, thiserror::Error)] +pub enum KeyError { + #[error("the two-factor key file {path} does not exist")] + Missing { path: PathBuf }, + #[error("can't read the two-factor key file {path}: {source}")] + Read { path: PathBuf, source: std::io::Error }, + #[error("the two-factor key file {path} must hold exactly 32 bytes")] + Length { path: PathBuf }, + #[error( + "key_file and key_file_old are the same key, or two keys that share a name: make a new key for key_file (the first byte of the keys' BLAKE3 \ + hashes must differ)" + )] + SameName, +} + +/// A key as read from a key file. `Debug` shows nothing of it. +pub struct TwoFactorKey { + bytes: Zeroizing<[u8; KEY_BYTES]>, + id: u8, +} + +impl std::fmt::Debug for TwoFactorKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("TwoFactorKey()") + } +} + +impl TwoFactorKey { + /// A key from its 32 bytes. + pub fn from_bytes(bytes: [u8; KEY_BYTES]) -> Self { + let id = blake3::hash(&bytes).as_bytes().first().copied().unwrap_or(0); + Self { bytes: Zeroizing::new(bytes), id } + } + + /// A new random key (for `init`). + pub fn generate() -> Result { + let mut bytes = [0u8; KEY_BYTES]; + getrandom::fill(&mut bytes)?; + Ok(Self::from_bytes(bytes)) + } + + /// The key file's bytes: only for writing a new key file. + pub fn to_file_bytes(&self) -> Zeroizing<[u8; KEY_BYTES]> { + self.bytes.clone() + } + + /// Reads a key file: exactly [`KEY_BYTES`] bytes (a longer file is not read further). + pub fn from_file(path: &Path) -> Result { + let mut file = std::fs::File::open(path).map_err(|source| match source.kind() { + std::io::ErrorKind::NotFound => KeyError::Missing { path: path.to_owned() }, + _ => KeyError::Read { path: path.to_owned(), source }, + })?; + let mut bytes = Zeroizing::new(Vec::with_capacity(KEY_BYTES + 1)); + (&mut file).take(KEY_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|source| KeyError::Read { path: path.to_owned(), source })?; + let bytes: [u8; KEY_BYTES] = bytes.as_slice().try_into().map_err(|_| KeyError::Length { path: path.to_owned() })?; + Ok(Self::from_bytes(bytes)) + } + + /// The name sealed secrets carry for this key. + pub fn id(&self) -> u8 { + self.id + } +} + +struct KeyEntry { + id: u8, + cipher: Aes256Gcm, +} + +impl KeyEntry { + fn new(key: &TwoFactorKey) -> Result { + let derived = Zeroizing::new(blake3::derive_key(KDF_CONTEXT, key.bytes.as_slice())); + let cipher = Aes256Gcm::new_from_slice(derived.as_slice()).map_err(|_| SecretBoxError)?; + Ok(Self { id: key.id, cipher }) + } +} + +/// Seals and opens TOTP secrets with the current key and, for reading, an older one. +pub struct SecretBox { + current: KeyEntry, + old: Option, +} + +impl std::fmt::Debug for SecretBox { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SecretBox()") + } +} + +fn aad(user: UserId) -> Vec { + let mut aad = Vec::with_capacity(AAD_PREFIX.len() + 16); + aad.extend_from_slice(AAD_PREFIX); + aad.extend_from_slice(user.0.as_bytes()); + aad +} + +impl SecretBox { + /// A box for `current` and, optionally, the `old` key secrets are still sealed with. Two keys with one name are + /// refused. + pub fn new(current: &TwoFactorKey, old: Option<&TwoFactorKey>) -> Result { + if old.is_some_and(|old| old.id == current.id) { + return Err(KeyError::SameName); + } + let entry = |key: &TwoFactorKey| KeyEntry::new(key).map_err(|_| KeyError::SameName); + Ok(Self { current: entry(current)?, old: old.map(entry).transpose()? }) + } + + /// The name of the current key. + pub fn current_id(&self) -> u8 { + self.current.id + } + + /// The name of the older key, if one is configured. + pub fn old_id(&self) -> Option { + self.old.as_ref().map(|old| old.id) + } + + /// The name of the key a sealed secret was sealed with (`None` for something too short to be one). + pub fn key_id(sealed: &[u8]) -> Option { + (sealed.len() >= 1 + NONCE_BYTES + TAG_BYTES).then(|| sealed.first().copied()).flatten() + } + + /// Seals `plaintext` for `user` with the current key and a fresh random nonce. + pub fn seal(&self, user: UserId, plaintext: &[u8]) -> Result, SecretBoxError> { + let mut nonce = [0u8; NONCE_BYTES]; + getrandom::fill(&mut nonce).map_err(|_| SecretBoxError)?; + let ciphertext = + self.current.cipher.encrypt(Nonce::from_slice(&nonce), Payload { msg: plaintext, aad: &aad(user) }).map_err(|_| SecretBoxError)?; + let mut sealed = Vec::with_capacity(1 + NONCE_BYTES + ciphertext.len()); + sealed.push(self.current.id); + sealed.extend_from_slice(&nonce); + sealed.extend_from_slice(&ciphertext); + Ok(sealed) + } + + /// Opens a secret sealed for `user`. A secret of another account, a damaged or truncated one, one sealed with a key this + /// box doesn't hold: all the same error. + pub fn open(&self, user: UserId, sealed: &[u8]) -> Result>, SecretBoxError> { + let id = Self::key_id(sealed).ok_or(SecretBoxError)?; + let entry = if id == self.current.id { &self.current } else { self.old.as_ref().filter(|old| old.id == id).ok_or(SecretBoxError)? }; + let (nonce, ciphertext) = sealed.get(1..).and_then(|rest| rest.split_at_checked(NONCE_BYTES)).ok_or(SecretBoxError)?; + let plaintext = entry.cipher.decrypt(Nonce::from_slice(nonce), Payload { msg: ciphertext, aad: &aad(user) }).map_err(|_| SecretBoxError)?; + Ok(Zeroizing::new(plaintext)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn user() -> UserId { + UserId(uuid::Uuid::now_v7()) + } + + fn key(byte: u8) -> TwoFactorKey { + TwoFactorKey::from_bytes([byte; KEY_BYTES]) + } + + #[test] + fn a_sealed_secret_opens_again() { + let sb = SecretBox::new(&key(1), None).unwrap(); + let ana = user(); + let sealed = sb.seal(ana, b"12345678901234567890").unwrap(); + assert_eq!(sb.open(ana, &sealed).unwrap().as_slice(), b"12345678901234567890"); + assert_eq!(sealed.first().copied(), Some(key(1).id())); + assert_eq!(sealed.len(), 1 + NONCE_BYTES + 20 + TAG_BYTES); + assert_ne!(sealed, sb.seal(ana, b"12345678901234567890").unwrap(), "a fresh nonce each time"); + assert!(!sealed.windows(20).any(|w| w == b"12345678901234567890"), "no plaintext in the ciphertext"); + } + + #[test] + fn a_wrong_key_another_account_or_a_damaged_ciphertext_fails() { + let sb = SecretBox::new(&key(1), None).unwrap(); + let (ana, bob) = (user(), user()); + let sealed = sb.seal(ana, b"secret").unwrap(); + assert_eq!(sb.open(bob, &sealed), Err(SecretBoxError), "the account is bound in"); + let other = SecretBox::new(&key(2), None).unwrap(); + assert_eq!(other.open(ana, &sealed), Err(SecretBoxError), "wrong key"); + // A key with the same name but other bytes fails on the tag. + let mut bytes = [1u8; KEY_BYTES]; + let mut same_name = None; + for i in 0..=255u8 { + bytes[0] = i; + if TwoFactorKey::from_bytes(bytes).id() == key(1).id() && bytes != [1u8; KEY_BYTES] { + same_name = Some(TwoFactorKey::from_bytes(bytes)); + break; + } + } + if let Some(same_name) = same_name { + assert_eq!(SecretBox::new(&same_name, None).unwrap().open(ana, &sealed), Err(SecretBoxError)); + } + for cut in [0, 1, 12, 13, sealed.len() - 1] { + assert_eq!(sb.open(ana, &sealed[..cut]), Err(SecretBoxError), "cut at {cut}"); + } + let mut flipped = sealed.clone(); + if let Some(last) = flipped.last_mut() { + *last ^= 1; + } + assert_eq!(sb.open(ana, &flipped), Err(SecretBoxError)); + } + + #[test] + fn the_older_key_opens_what_it_sealed_and_the_new_one_seals() { + let ana = user(); + let old_only = SecretBox::new(&key(1), None).unwrap(); + let sealed_old = old_only.seal(ana, b"secret").unwrap(); + let rotated = SecretBox::new(&key(2), Some(&key(1))).unwrap(); + assert_eq!(rotated.open(ana, &sealed_old).unwrap().as_slice(), b"secret"); + assert_eq!(rotated.old_id(), Some(key(1).id())); + let sealed_new = rotated.seal(ana, b"secret").unwrap(); + assert_eq!(SecretBox::key_id(&sealed_new), Some(key(2).id())); + assert_eq!(old_only.open(ana, &sealed_new), Err(SecretBoxError), "the old key alone does not open the new"); + // Without the old key configured, what it sealed is a stranger's. + assert_eq!(SecretBox::new(&key(2), None).unwrap().open(ana, &sealed_old), Err(SecretBoxError)); + } + + #[test] + fn two_keys_with_one_name_are_refused() { + assert!(matches!(SecretBox::new(&key(1), Some(&key(1))), Err(KeyError::SameName))); + } + + #[test] + fn a_key_is_read_from_a_file_of_exactly_32_bytes() { + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.key"); + std::fs::write(&good, [7u8; KEY_BYTES]).unwrap(); + assert_eq!(TwoFactorKey::from_file(&good).unwrap().id(), key(7).id()); + for (name, len) in [("short.key", 31), ("long.key", 33), ("empty.key", 0), ("huge.key", 100_000)] { + let path = dir.path().join(name); + std::fs::write(&path, vec![7u8; len]).unwrap(); + assert!(matches!(TwoFactorKey::from_file(&path), Err(KeyError::Length { .. })), "{name}"); + } + assert!(matches!(TwoFactorKey::from_file(&dir.path().join("nope.key")), Err(KeyError::Missing { .. }))); + } + + #[test] + fn keys_and_boxes_show_nothing_in_debug_and_errors_say_nothing() { + let k = key(9); + assert_eq!(format!("{k:?}"), "TwoFactorKey()"); + assert_eq!(format!("{:?}", SecretBox::new(&k, None).unwrap()), "SecretBox()"); + assert_eq!(SecretBoxError.to_string(), "a two-factor secret could not be sealed or opened"); + } +} diff --git a/apps/craft-cloud-server/src/space.rs b/apps/craft-cloud-server/src/space.rs new file mode 100644 index 00000000..ff353094 --- /dev/null +++ b/apps/craft-cloud-server/src/space.rs @@ -0,0 +1,134 @@ +//! The space layer: which space a request works on, and whether the signed-in account may do what the route does there. +//! +//! [`require_space`] runs after [`crate::auth::require_device`] on every space-scoped route and puts a [`SpaceAccess`] into the +//! request. A request names its space with the `craft-space` header ([`SPACE_HEADER`]); without it the space is the +//! device's personal space and nothing else is looked up, so an agent or app that never heard of team spaces behaves exactly +//! as before. Every answer echoes the space the server worked on, so a client talking to an older server (which ignores the +//! header and answers about the personal space) can tell. +//! +//! - **No access is `404 not_found`**, the same answer as for a space that does not exist. A member whose role is too low is +//! `403 read_only`, never 401 (the agent reads every 401 on a signed-in route as "this device was signed out"). +//! - **Handlers take their space from [`SpaceAccess`]**, never from the device token's own space. The device token's space +//! stays the account's personal space for the account routes ([`DeviceAuth::personal_space`]). +//! - **The role is checked twice.** The layer reads it from the database on every request (nothing is cached) and refuses +//! before the handler runs. A write then asks for [`SpaceAccess::meta`], a store handle whose write transaction reads the +//! role again after locking the space's row, so a role lowered between the two is respected. +//! - **A malformed header** is `400 invalid` and has no echo (there is no space to echo). The layer's own refusals carry no +//! `craft-epoch` either; the echo of the requested ID is all they say about the space. +//! - **The epoch header** of an answer is the working space's epoch, as the personal space's is for a request without the +//! header. + +use std::sync::Arc; + +use axum::extract::{Request, State}; +use axum::http::HeaderValue; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use craft_cloud_meta::{DeviceAuth, Meta, UserId, WriteGuard}; +use craft_cloud_proto::SpaceId; +use craft_cloud_proto::api::{EPOCH_HEADER, SPACE_HEADER, SpaceRole}; + +use crate::error::ApiError; +use crate::state::AppState; + +/// What the layer decided about the request's space; in the request's extensions for the handler. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpaceAccess { + /// The space the request works on. + pub space: SpaceId, + /// The caller's role in it (`Owner` in the caller's own personal space). + pub role: SpaceRole, + /// Whether it is the caller's personal space. + pub personal: bool, + /// The least role the route needs, which `role` has reached. + pub required: SpaceRole, +} + +impl SpaceAccess { + /// The store handle for a write of this route. In a team space it re-checks the caller's role inside the write + /// transaction ([`Meta::guarded`]); in the caller's own personal space there is nothing to re-check (its owner is its + /// only member), and the handle is the plain one. + pub fn meta(&self, state: &AppState, user: UserId) -> Meta { + if self.personal { state.meta.clone() } else { state.meta.guarded(WriteGuard { user, min: self.required }) } + } +} + +/// Marks a response the space layer refused itself, so that [`crate::auth::require_device`] adds no `craft-epoch` to it. +#[derive(Debug, Clone, Copy)] +pub struct NoEpoch; + +/// The layer's state: the server and the role its route needs. +#[derive(Clone)] +pub struct SpaceGate { + pub state: Arc, + pub min: SpaceRole, +} + +/// `403 read_only`: a member whose role does not allow this. +pub fn read_only() -> ApiError { + ApiError::new(axum::http::StatusCode::FORBIDDEN, craft_cloud_proto::api::codes::READ_ONLY, "your role in this space does not allow changes") +} + +/// The space a request names: `Ok(None)` without the header, `400 invalid` for a header that is not exactly one space ID. +pub fn named_space(request: &Request) -> Result, ApiError> { + let mut values = request.headers().get_all(SPACE_HEADER).iter(); + let Some(value) = values.next() else { return Ok(None) }; + let bad = || ApiError::invalid(format!("{SPACE_HEADER} is one space ID")); + if values.next().is_some() { + return Err(bad()); + } + let text = value.to_str().map_err(|_| bad())?; + // The hyphenated form only: the form the server echoes. + if text.len() != 36 { + return Err(bad()); + } + text.parse::().map(Some).map_err(|_| bad()) +} + +/// Resolves the request's space and the caller's role in it, refuses what the route's role does not allow, and echoes +/// the space. After [`crate::auth::require_device`]; a request that is not signed in is `401`. +pub async fn require_space(State(gate): State, mut request: Request, next: Next) -> Result { + let auth = request.extensions().get::().cloned().ok_or_else(ApiError::unauthorized)?; + let named = named_space(&request)?; + let space = named.unwrap_or_else(|| auth.personal_space()); + let echo = HeaderValue::from_str(&space.to_string()).ok(); + let work = async { + let (role, personal) = match named { + // The device's own space, which no header names: its owner has every right (the device was just looked up with + // an account that can sign in), so nothing more is read and the epoch is the one `require_device` adds. + None => (SpaceRole::Owner, true), + Some(space) => { + let role = gate.state.meta.space_access(auth.user, space).await?.ok_or_else(|| ApiError::not_found("space"))?; + (role, space == auth.personal_space()) + } + }; + if role < gate.min { + return Err(read_only()); + } + let epoch = match named { + Some(space) => Some(gate.state.meta.space_info(space).await?.epoch), + None => None, + }; + request.extensions_mut().insert(SpaceAccess { space, role, personal, required: gate.min }); + let mut response = next.run(request).await; + if let Some(epoch) = epoch + && let Ok(value) = HeaderValue::from_str(&epoch.to_string()) + { + response.headers_mut().insert(EPOCH_HEADER, value); + } + Ok::<_, ApiError>(response) + }; + let mut response = match work.await { + Ok(response) => response, + Err(e) => { + // The layer's own refusal says nothing about the space's epoch (a 404 must not): the device layer leaves it out. + let mut refusal = e.into_response(); + refusal.extensions_mut().insert(NoEpoch); + refusal + } + }; + if let Some(echo) = echo { + response.headers_mut().insert(SPACE_HEADER, echo); + } + Ok(response) +} diff --git a/apps/craft-cloud-server/src/spaces.rs b/apps/craft-cloud-server/src/spaces.rs new file mode 100644 index 00000000..9b54e1d6 --- /dev/null +++ b/apps/craft-cloud-server/src/spaces.rs @@ -0,0 +1,537 @@ +//! The team-space routes (`docs/design/collaboration.md`, 4.2): the account's spaces and invitations, a space's members, +//! usage, name and life cycle, and the administrator's list, creation, archive and owner routes. +//! +//! - **The space is in the path here**, not in the `craft-space` header: these routes are about the spaces themselves, so +//! they sit outside the space layer ([`crate::space`]) and decide access with [`Meta::space_access`] or the member row. +//! No access is `404 not_found`, the same answer as for a space that does not exist; a member whose role does not allow the +//! change is `403 read_only`. +//! - **Who acts.** A change to a space's members, name or life cycle is made by one of its owners, by their own role and +//! without a password, or by a server administrator, who carries the administrator confirmation of every admin change (the +//! password, and the second factor when on; see [`crate::admin`]) and has no access to the space's content. An owner +//! archiving their space gives their password. Changing a space's limit is an administrator's change always. +//! - **Invitations** are rate limited per account ([`INVITES_PER_MINUTE`]). Unknown, disabled and removed accounts are one +//! answer (`404 not_found`). +//! - **Wake-ups.** Every change that can end someone's access or lower their role calls +//! [`Notifier::members_changed`](crate::notify::Notifier::members_changed) after it committed, so the change-feed +//! long-polls of that space look at their access again at once. Disabling or removing an account does the same for every +//! space it was a member of ([`wake_spaces_of`]). +//! - Bodies are tiny ([`TINY_JSON_BYTES`]); the `DELETE` and archive routes may carry one (the confirmation) or none. + +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::rejection::{PathRejection, QueryRejection}; +use axum::extract::{DefaultBodyLimit, Extension, FromRequest, Path, Query, Request, State}; +use axum::http::StatusCode; +use axum::routing::{get, post}; +use axum::{Router, middleware}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{AdminVia, DeviceAuth, Member, Role, SpaceActor, SpaceSummary, UserId, UserSpace, valid_username}; +use craft_cloud_proto::api::{ + AddSpaceMember, AdminCreateSpace, AdminSetSpaceOwner, AdminSpaceInfo, AdminSpaceList, ChangeSpaceMember, SpaceConfirm, SpaceList, SpaceListItem, + SpaceMember, SpaceMembers, SpaceRole, SpaceUsage, UpdateSpace, UsageEstimateReason, UsageEstimates, +}; +use craft_cloud_proto::{SpaceId, SpaceName}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::account::confirm_password; +use crate::admin::{Confirmation, confirm_actor}; +use crate::api::{TINY_JSON_BYTES, body_too_slow}; +use crate::auth::require_admin; +use crate::error::ApiError; +use crate::space::read_only; +use crate::state::AppState; + +type Shared = Arc; + +/// Invitations one account may make per minute (the window of [`AppState::space_invites`]). +pub const INVITES_PER_MINUTE: u32 = 30; +/// Spaces one page of `GET /admin/spaces` holds when `limit` is not given, and the most it may hold. +const DEFAULT_PAGE: u32 = 50; +const MAX_PAGE: u32 = 200; + +/// The routes about the spaces an account is in, behind `require_device` only (the space is in the path). +pub fn routes() -> Router { + let tiny = || DefaultBodyLimit::max(TINY_JSON_BYTES); + Router::new() + .route("/spaces", get(list_spaces)) + .route("/spaces/{id}", axum::routing::patch(update_space).layer(tiny())) + .route("/spaces/{id}/members", get(list_members).post(add_member).layer(tiny())) + .route("/spaces/{id}/members/{user}", axum::routing::patch(change_member).delete(remove_member).layer(tiny())) + .route("/spaces/{id}/accept", post(accept)) + .route("/spaces/{id}/decline", post(decline)) + .route("/spaces/{id}/usage", get(usage)) + .route("/spaces/{id}/archive", post(archive).layer(tiny())) + .route("/spaces/{id}/unarchive", post(unarchive).layer(tiny())) +} + +/// The administrator's routes, behind `require_admin` like the rest of `/admin`. +pub fn admin_routes() -> Router { + let tiny = || DefaultBodyLimit::max(TINY_JSON_BYTES); + Router::new() + .route("/admin/spaces", get(admin_list).post(admin_create).layer(tiny())) + .route("/admin/spaces/{id}/archive", post(admin_archive).layer(tiny())) + .route("/admin/spaces/{id}/unarchive", post(admin_unarchive).layer(tiny())) + .route("/admin/spaces/{id}/owner", axum::routing::put(admin_set_owner).layer(tiny())) + .route_layer(middleware::from_fn(require_admin)) + .route_layer(middleware::from_fn(crate::web::reject_mixed_credentials)) +} + +// ---- helpers ---- + +/// The palette index an account's colour is drawn from: BLAKE3 of a fixed label and the account's ID, read as a big-endian +/// number, modulo the 12 colours. Derived, not stored, so every machine computes the same one. +pub fn member_colour(user: Uuid) -> u8 { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"craft-cloud member colour v1"); + hasher.update(user.as_bytes()); + let digest = hasher.finalize(); + let remainder = digest.as_bytes().iter().fold(0u32, |acc, byte| (acc * 256 + u32::from(*byte)) % 12); + // At most 11. + u8::try_from(remainder).unwrap_or(0) +} + +fn space_id(id: Result, PathRejection>) -> Result { + Ok(id?.0) +} + +/// A JSON body of at most [`TINY_JSON_BYTES`] under the usual deadline; none at all is the type's default (for the routes whose +/// body only carries a confirmation an owner does not need). +async fn optional_body(state: &AppState, request: Request) -> Result { + let bytes = match tokio::time::timeout(state.limits.body_time(TINY_JSON_BYTES), Bytes::from_request(request, &())).await { + Ok(bytes) => bytes?, + Err(_) => return Err(body_too_slow()), + }; + if bytes.iter().all(u8::is_ascii_whitespace) { + return Ok(T::default()); + } + serde_json::from_slice(&bytes).map_err(|_| ApiError::invalid("the body must be a JSON object with the fields of this call")) +} + +fn confirmation(confirm: SpaceConfirm) -> Confirmation { + Confirmation::new(confirm.password, confirm.totp_code, confirm.recovery_code) +} + +fn user_actor(auth: &DeviceAuth) -> SpaceActor { + SpaceActor::User { user: auth.user, device: Some(auth.device) } +} + +/// Who makes a change to the space's members, name or life cycle: an owner, by their role; otherwise an administrator, with +/// the confirmation; a member who is neither is `read_only`, and anyone else gets the answer for a space that is not there. +async fn actor_for(state: &AppState, auth: &DeviceAuth, space: SpaceId, confirm: SpaceConfirm) -> Result { + let role = state.meta.space_member_role(auth.user, space).await?; + // An administrator who sends the confirmation acts as an administrator, also in a space they own: the change is then + // audited and confirmed as an administrator's (an owner's own change carries no password). + let confirmed = confirm.password.is_some() || confirm.totp_code.is_some() || confirm.recovery_code.is_some(); + if auth.role == Role::Admin && confirmed { + return admin_actor(state, auth, confirm).await; + } + if role == Some(SpaceRole::Owner) { + return Ok(user_actor(auth)); + } + if auth.role == Role::Admin { + return admin_actor(state, auth, confirm).await; + } + Err(if role.is_some() { read_only() } else { ApiError::not_found("space") }) +} + +async fn admin_actor(state: &AppState, auth: &DeviceAuth, confirm: SpaceConfirm) -> Result { + let admin = confirm_actor(state, auth.user, &auth.username, AdminVia::Device(auth.device), confirmation(confirm)).await?; + Ok(SpaceActor::Admin(admin)) +} + +fn item(space: UserSpace) -> SpaceListItem { + SpaceListItem { + id: space.id, + kind: if space.personal { "personal" } else { "team" }.to_owned(), + name: space.name, + role: space.role, + state: if space.invited { "invited" } else { "active" }.to_owned(), + archived: space.archived, + created_ms: space.created_ms, + members: space.members, + root: space.root, + epoch: space.epoch, + joined_ms: space.joined_ms, + } +} + +fn member_info(member: Member) -> SpaceMember { + let state = if member.accepted() { "active" } else { "invited" }.to_owned(); + SpaceMember { user: member.user.0, username: member.username, role: member.role, state, colour: member_colour(member.user.0) } +} + +fn summary(space: SpaceSummary) -> AdminSpaceInfo { + AdminSpaceInfo { + id: space.id, + name: space.name, + created_ms: space.created_ms, + archived_ms: space.archived_ms, + members: space.members, + owners: space.owners, + stored_bytes: space.stored_bytes, + quota_bytes: space.quota_bytes, + } +} + +fn space_name(text: &str) -> Result { + SpaceName::new(text).map_err(|e| ApiError::invalid(format!("a space's name: {e}"))) +} + +/// After a committed change: the space's long-polls look at their access again. +pub(crate) fn changed(state: &AppState, space: SpaceId) { + state.notifier.members_changed(space); +} + +/// After an account was disabled or marked for removal: the long-polls of every team space it was a member of look again. +pub(crate) async fn wake_spaces_of(state: &AppState, user: UserId) { + match state.meta.member_space_ids(user).await { + Ok(spaces) => spaces.into_iter().for_each(|space| changed(state, space)), + Err(e) => tracing::warn!(error = %e, "couldn't list the spaces to wake"), + } +} + +/// Counts one invitation against the account's minute. +fn count_invite(state: &AppState, user: UserId) -> Result<(), ApiError> { + let key = format!("user:{}", user.0); + let now = tokio::time::Instant::now().into_std(); + if state.space_invites.allow(&key, INVITES_PER_MINUTE, now) { + Ok(()) + } else { + let wait = state.space_invites.retry_after(&key, now); + // Rounded up: told to wait 1 s when 1.4 s are left, a client would be refused again. + let secs = wait.as_secs().saturating_add(u64::from(wait.subsec_nanos() > 0)).max(1); + Err(ApiError::rate_limited("too many invitations; wait a minute", Some(u32::try_from(secs).unwrap_or(60)))) + } +} + +// ---- the account's spaces ---- + +/// `GET /spaces`. +async fn list_spaces(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let spaces = state.meta.user_spaces(auth.user).await?; + Ok(Json(SpaceList { spaces: spaces.into_iter().map(item).collect() })) +} + +/// `POST /spaces/{id}/accept`: the invited account joins; the answer is the space as `GET /spaces` lists it. +async fn accept( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let space = space_id(id)?; + state.meta.accept_invitation(auth.user, Some(auth.device), space, state.now_ms()).await?; + changed(&state, space); + let listed = state.meta.user_spaces(auth.user).await?.into_iter().find(|s| s.id == space).ok_or_else(|| ApiError::not_found("space"))?; + Ok(Json(item(listed))) +} + +/// `POST /spaces/{id}/decline`: 204. +async fn decline( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result { + let space = space_id(id)?; + state.meta.decline_invitation(auth.user, Some(auth.device), space, state.now_ms()).await?; + changed(&state, space); + Ok(StatusCode::NO_CONTENT) +} + +/// `GET /spaces/{id}/usage`: any member. +async fn usage( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let space = space_id(id)?; + state.meta.space_access(auth.user, space).await?.ok_or_else(|| ApiError::not_found("space"))?; + let estimate = state.meta.storage_estimate(space).await?; + let index_complete = estimate.reason.is_none(); + Ok(Json(SpaceUsage { + stored_bytes: estimate.stored_bytes, + quota_bytes: estimate.quota_bytes, + trash_bytes: estimate.trash_bytes, + history_bytes: estimate.history_bytes, + live_bytes: state.meta.space_live_bytes(space).await?, + estimates: UsageEstimates { index_complete, reason: estimate.reason.map(|_| UsageEstimateReason::IndexIncomplete) }, + })) +} + +// ---- members ---- + +/// `GET /spaces/{id}/members`: any member, invitations included. +async fn list_members( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let space = space_id(id)?; + state.meta.space_access(auth.user, space).await?.ok_or_else(|| ApiError::not_found("space"))?; + let members = state.meta.list_members(space).await?; + Ok(Json(SpaceMembers { members: members.into_iter().map(member_info).collect() })) +} + +/// `POST /spaces/{id}/members`: an owner (or an administrator with the confirmation) invites an account. The answer is the +/// member row, `invited`. +async fn add_member( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let space = space_id(id)?; + let req: AddSpaceMember = crate::api::read_json(&state, request, TINY_JSON_BYTES).await?; + if !valid_username(&req.username) { + return Err(ApiError::invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); + } + let actor = actor_for(&state, &auth, space, req.confirm).await?; + count_invite(&state, auth.user)?; + // Unknown, disabled and removed accounts are one answer: the store says `NotFound("user")` for the last two as well. + let target = state.meta.user_by_name(&req.username).await?.ok_or_else(|| ApiError::not_found("user"))?; + let member = state.meta.invite_member(&actor, space, target.id, req.role, state.now_ms()).await?; + changed(&state, space); + Ok(Json(member_info(member))) +} + +/// `PATCH /spaces/{id}/members/{user}`: a new role for a member or an invitation; lowering the last owner is `409 last_owner`. +async fn change_member( + State(state): State, + Extension(auth): Extension, + ids: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let (space, target) = ids?.0; + let req: ChangeSpaceMember = crate::api::read_json(&state, request, TINY_JSON_BYTES).await?; + let actor = actor_for(&state, &auth, space, req.confirm).await?; + let member = state.meta.change_member_role(&actor, space, UserId(target), req.role, state.now_ms()).await?; + changed(&state, space); + Ok(Json(member_info(member))) +} + +/// `DELETE /spaces/{id}/members/{user}`: 204. An owner (or an administrator with the confirmation) removes a member or +/// withdraws an invitation; a member's own row is the member leaving (an invited account's own row is declined) and needs no +/// role. The last owner leaving or being removed is `409 last_owner`. +async fn remove_member( + State(state): State, + Extension(auth): Extension, + ids: Result, PathRejection>, + request: Request, +) -> Result { + let (space, target) = ids?.0; + let confirm: SpaceConfirm = optional_body(&state, request).await?; + let now = state.now_ms(); + if target == auth.user.0 { + match state.meta.leave_space(auth.user, Some(auth.device), space, now).await { + Err(craft_cloud_meta::MetaError::NotFound("member")) => state.meta.decline_invitation(auth.user, Some(auth.device), space, now).await?, + other => other?, + } + } else { + let actor = actor_for(&state, &auth, space, confirm).await?; + state.meta.remove_member(&actor, space, UserId(target), now).await?; + } + changed(&state, space); + Ok(StatusCode::NO_CONTENT) +} + +// ---- name, limit, life cycle ---- + +/// `PATCH /spaces/{id}`: a new name (an owner, or an administrator with the confirmation) and/or a new limit (an +/// administrator with the confirmation). The answer is the space as the administrator's list shows it. +async fn update_space( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let space = space_id(id)?; + let req: UpdateSpace = crate::api::read_json(&state, request, TINY_JSON_BYTES).await?; + if req.name.is_none() && req.quota_bytes.is_none() { + return Err(ApiError::invalid("give a name or a quota_bytes to change")); + } + let name = req.name.as_deref().map(space_name).transpose()?; + let now = state.now_ms(); + // Checked before anything is written, so a bad limit cannot follow a rename that already committed. + if let Some(Some(quota)) = req.quota_bytes + && !(craft_cloud_meta::MIN_QUOTA_BYTES..=i64::MAX as u64).contains("a) + { + return Err(ApiError::invalid("quota_bytes is at least one chunk (4194304) and at most 9223372036854775807")); + } + if let Some(quota) = req.quota_bytes { + if auth.role != Role::Admin { + // A member learns that this is not theirs to change; anyone else learns nothing about the space. + return Err(if state.meta.space_member_role(auth.user, space).await?.is_some() { + ApiError::forbidden("only an administrator changes a space's limit") + } else { + ApiError::not_found("space") + }); + } + let SpaceActor::Admin(admin) = admin_actor(&state, &auth, req.confirm).await? else { return Err(ApiError::internal()) }; + if let Some(name) = &name { + state.meta.rename_team_space(&SpaceActor::Admin(admin.clone()), space, name, now).await?; + } + state.meta.set_space_quota(&admin, space, quota, now).await?; + } else if let Some(name) = &name { + let actor = actor_for(&state, &auth, space, req.confirm).await?; + state.meta.rename_team_space(&actor, space, name, now).await?; + } + changed(&state, space); + Ok(Json(summary(state.meta.admin_space(space).await?))) +} + +/// An owner archives or unarchives their space with their own password. +async fn owner_archive( + state: &AppState, + auth: &DeviceAuth, + id: Result, PathRejection>, + request: Request, + archived: bool, +) -> Result { + let space = space_id(id)?; + let confirm: SpaceConfirm = optional_body(state, request).await?; + match state.meta.space_member_role(auth.user, space).await? { + Some(SpaceRole::Owner) => {} + Some(_) => return Err(read_only()), + None => return Err(ApiError::not_found("space")), + } + let password = confirm.password.ok_or_else(|| ApiError::invalid("this change needs your current password in `password`"))?; + confirm_password(state, auth.user, &auth.username, password).await?; + state.meta.set_space_archived(&user_actor(auth), space, archived, state.now_ms()).await?; + changed(state, space); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /spaces/{id}/archive`: 204. Every member becomes read-only and the space leaves the lists of all but its owners. +async fn archive( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + owner_archive(&state, &auth, id, request, true).await +} + +/// `POST /spaces/{id}/unarchive`: 204. +async fn unarchive( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + owner_archive(&state, &auth, id, request, false).await +} + +// ---- the administrator ---- + +#[derive(Debug, Deserialize)] +struct ListQuery { + cursor: Option, + limit: Option, + query: Option, +} + +/// `GET /admin/spaces?cursor&limit&query`: the team spaces by name, with counts and sizes. A cursor is opaque. +async fn admin_list(State(state): State, q: Result, QueryRejection>) -> Result, ApiError> { + let Query(q) = q?; + let limit = q.limit.unwrap_or(DEFAULT_PAGE); + if !(1..=MAX_PAGE).contains(&limit) { + return Err(ApiError::invalid(format!("limit is 1 to {MAX_PAGE}"))); + } + let bad = || ApiError::invalid("the cursor is not one this server gave"); + let after = match q.cursor.as_deref() { + Some(cursor) => Some(String::from_utf8(URL_SAFE_NO_PAD.decode(cursor).map_err(|_| bad())?).map_err(|_| bad())?), + None => None, + }; + let search = q.query.as_deref().filter(|s| !s.is_empty()); + if let Some(search) = search + && (search.len() > crate::admin::MAX_QUERY_BYTES || search.chars().any(char::is_control)) + { + return Err(ApiError::invalid(format!("query is at most {} bytes of printable characters", crate::admin::MAX_QUERY_BYTES))); + } + let page = state.meta.admin_list_spaces(after.as_deref(), search, limit).await?; + Ok(Json(AdminSpaceList { + spaces: page.spaces.into_iter().map(summary).collect(), + next: page.next_after.map(|key| URL_SAFE_NO_PAD.encode(key.as_bytes())), + })) +} + +/// `POST /admin/spaces`: a team space with its first owner, who is an accepted member from the start. +async fn admin_create( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: AdminCreateSpace = crate::api::read_json(&state, request, TINY_JSON_BYTES).await?; + let name = space_name(&req.name)?; + let actor = admin_actor(&state, &auth, req.confirm).await?; + let space = state.meta.create_team_space(&actor, &name, UserId(req.owner), req.quota_bytes, state.now_ms()).await?; + tracing::info!(actor = %auth.device, %space, "admin: team space created"); + Ok(Json(summary(state.meta.admin_space(space).await?))) +} + +async fn admin_archive_to( + state: &AppState, + auth: &DeviceAuth, + id: Result, PathRejection>, + request: Request, + archived: bool, +) -> Result { + let space = space_id(id)?; + let confirm: SpaceConfirm = crate::api::read_json(state, request, TINY_JSON_BYTES).await?; + let actor = admin_actor(state, auth, confirm).await?; + state.meta.set_space_archived(&actor, space, archived, state.now_ms()).await?; + changed(state, space); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /admin/spaces/{id}/archive`: 204. +async fn admin_archive( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + admin_archive_to(&state, &auth, id, request, true).await +} + +/// `POST /admin/spaces/{id}/unarchive`: 204. +async fn admin_unarchive( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + admin_archive_to(&state, &auth, id, request, false).await +} + +/// `PUT /admin/spaces/{id}/owner`: 204. Makes an account an accepted owner (for a space whose owners are all gone). +async fn admin_set_owner( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let space = space_id(id)?; + let req: AdminSetSpaceOwner = crate::api::read_json(&state, request, TINY_JSON_BYTES).await?; + let SpaceActor::Admin(admin) = admin_actor(&state, &auth, req.confirm).await? else { return Err(ApiError::internal()) }; + state.meta.set_owner_accepted(&admin, space, UserId(req.user), state.now_ms()).await?; + changed(&state, space); + Ok(StatusCode::NO_CONTENT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_colour_is_one_of_twelve_and_the_same_every_time() { + let user = Uuid::now_v7(); + assert!(member_colour(user) < 12); + assert_eq!(member_colour(user), member_colour(user)); + let seen: std::collections::BTreeSet = (0..200).map(|_| member_colour(Uuid::now_v7())).collect(); + assert!(seen.len() > 6, "the colours are spread: {seen:?}"); + } +} diff --git a/apps/craft-cloud-server/src/state.rs b/apps/craft-cloud-server/src/state.rs index 10d7edc8..edcdb01d 100644 --- a/apps/craft-cloud-server/src/state.rs +++ b/apps/craft-cloud-server/src/state.rs @@ -10,11 +10,13 @@ use craft_cloud_proto::DeviceId; use craft_cloud_store::Store; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use crate::audit::AuditExportLimits; use crate::auth::hash_password; use crate::error::ApiError; use crate::live::LiveHub; use crate::notify::Notifier; use crate::ratelimit::Limiter; +use crate::secretbox::SecretBox; /// How much one user, or one request, may hold on the server at once. Each user's limits are their own, so a /// stalled or greedy client only ever slows its own account. @@ -109,10 +111,26 @@ pub struct AppState { pub logins: Limiter, /// Failed join-code redeems per client address, in one-minute windows ([`crate::join`]). pub joins: Limiter, + /// Password change tries per account, in 15-minute windows ([`crate::account`]). + pub password_changes: Limiter, + /// Invitations to team spaces per account, in one-minute windows ([`crate::spaces`]). + pub space_invites: Limiter, + /// Wrong second factors per account and per client address, in five-minute windows ([`crate::two_factor`]). + pub second_factors: Limiter, + /// Sign-in attempts to the admin pages per client address and per account, in five-minute windows ([`crate::web`]). + pub web_logins: Limiter, + /// The admin pages' settings; `None`: they are off and `/web` does not exist. + pub web: Option, + /// Seals and opens the secrets of two-factor sign-in; `None`: no key is configured, so it can't be set up. + pub two_factor: Option>, pub notifier: Notifier, /// The live service's connections and subscriptions ([`crate::live`]). pub live: LiveHub, pub limits: Limits, + /// How the audit export is cut into pages and bounded ([`crate::audit`]). + pub audit_export: AuditExportLimits, + /// The address computers reach the server at (`public_url` in the config), for the join links of the admin API. + pub public_url: Option, /// Chunk and manifest bodies being received, per user ([`Limits::bodies_per_user`]). pub bodies: PerUser, /// Manifests being decoded and checked, per user ([`Limits::manifests_per_user`]). @@ -122,6 +140,8 @@ pub struct AppState { pub(crate) touches: Mutex>, /// Verified against when the username doesn't exist, so both cases cost the same. pub dummy_hash: String, + /// Days a deleted node stays in the trash before the server may remove it (`retention.trash_days`). + pub trash_days: u32, /// The time of day for everything a request records (a device's last use, a stored blob, a commit). clock: Clock, } @@ -135,14 +155,23 @@ impl AppState { trusted_proxies, logins: Limiter::new(Duration::from_secs(300)), joins: Limiter::new(Duration::from_secs(60)), + password_changes: Limiter::new(Duration::from_secs(900)), + space_invites: Limiter::new(Duration::from_secs(60)), + second_factors: Limiter::new(Duration::from_secs(300)), + web_logins: Limiter::new(Duration::from_secs(300)), + web: None, + two_factor: None, notifier: Notifier::default(), live: LiveHub::default(), bodies: PerUser::new(limits.bodies_per_user), manifests: PerUser::new(limits.manifests_per_user), password_checks: Arc::new(Semaphore::new(limits.password_checks)), limits, + audit_export: AuditExportLimits::default(), + public_url: None, touches: Mutex::new(HashMap::new()), dummy_hash: hash_password("craft-cloud dummy password")?, + trash_days: 30, clock: Arc::new(now_ms), }) } @@ -156,10 +185,42 @@ impl AppState { self } + /// The state with other bounds for the audit export (tests use small ones). + pub fn with_audit_export(mut self, limits: AuditExportLimits) -> Self { + self.audit_export = limits; + self + } + + /// The state with the key for two-factor secrets (`None`: none is configured and two-factor sign-in can't be set up). + pub fn with_two_factor(mut self, secrets: Option) -> Self { + self.two_factor = secrets.map(Arc::new); + self + } + + /// The state with the admin pages on (`None`: off, the default): their address, session lifetimes and the like. + pub fn with_web_admin(mut self, web: Option) -> Self { + self.web = web; + self + } + + /// The state with the server's public address (`public_url`), which the admin API puts in the join links it makes. + pub fn with_public_url(mut self, public_url: Option) -> Self { + self.public_url = public_url; + self + } + + /// The state with another trash period in days (`retention.trash_days`; the default is 30). + pub fn with_trash_days(mut self, days: u32) -> Self { + self.trash_days = days; + self + } + /// The state with another clock: a simulation's, so that what requests record and when a device's last use is /// written again follow simulated time, never the real one. A test seam (the `test-support` feature). #[cfg(feature = "test-support")] pub fn with_clock(mut self, clock: Clock) -> Self { + // The store judges a browser session's expiry inside an admin change by the same simulated time. + self.meta = self.meta.clone().with_clock(clock.clone()); self.clock = clock; self } diff --git a/apps/craft-cloud-server/src/totp.rs b/apps/craft-cloud-server/src/totp.rs new file mode 100644 index 00000000..aa7dd107 --- /dev/null +++ b/apps/craft-cloud-server/src/totp.rs @@ -0,0 +1,221 @@ +//! TOTP (RFC 6238) as authenticator apps speak it: HMAC-SHA-1, six digits, 30-second steps. +//! +//! - A code is accepted for the step the clock is in and the one before and after it (a clock a little off); the step +//! it matched is handed back, and the database lets an account accept a step only once, and only a later one than the +//! last (see `craft_cloud_meta`'s two-factor module), so a code can't be used twice. +//! - A code is exactly six ASCII digits. Anything else is refused before it is compared. +//! - Codes are compared in constant time. The secret is 20 random bytes, shown to the person as Base32 without padding. + +use hmac::{Hmac, KeyInit, Mac}; +use sha1::Sha1; +use subtle::ConstantTimeEq; +use zeroize::Zeroizing; + +/// Bytes of a new secret: 160 bits. +pub const SECRET_BYTES: usize = 20; +/// Digits of a code. +pub const DIGITS: usize = 6; +/// Seconds a code is good for. +pub const STEP_SECS: i64 = 30; +/// What the authenticator app shows as the service's name. +pub const ISSUER: &str = "Craft Cloud"; + +/// A new random secret. +pub fn generate_secret() -> Result, getrandom::Error> { + let mut secret = Zeroizing::new([0u8; SECRET_BYTES]); + getrandom::fill(secret.as_mut_slice())?; + Ok(secret) +} + +/// The time step of `now_ms` (milliseconds since the Unix epoch). +pub fn step_at(now_ms: i64) -> i64 { + now_ms.div_euclid(1000).div_euclid(STEP_SECS) +} + +/// Whether `code` has the shape of a code: exactly six ASCII digits. +pub fn is_code(code: &str) -> bool { + code.len() == DIGITS && code.bytes().all(|b| b.is_ascii_digit()) +} + +/// The code for time step `step` (HOTP's counter), with `secret`. `None` for a step before the epoch. +pub fn code_for(secret: &[u8], step: i64) -> Option { + let counter = u64::try_from(step).ok()?; + let mut mac = as KeyInit>::new_from_slice(secret).ok()?; + mac.update(&counter.to_be_bytes()); + let digest = mac.finalize().into_bytes(); + let offset = usize::from(digest.last().copied()? & 0x0f); + let word = digest.get(offset..offset.checked_add(4)?)?; + let value = u32::from_be_bytes([*word.first()?, *word.get(1)?, *word.get(2)?, *word.get(3)?]) & 0x7fff_ffff; + Some(format!("{:0width$}", value % 10u32.pow(DIGITS as u32), width = DIGITS)) +} + +/// The time step `code` is right for at `now_ms`, if it is right for the step of `now_ms` or the one before or after it +/// (the latest if several match, which is all but impossible); `None` for a code that is wrong or is not six digits. All +/// three steps are always computed and compared in constant time. +pub fn verify(secret: &[u8], code: &str, now_ms: i64) -> Option { + if !is_code(code) { + return None; + } + let now = step_at(now_ms); + let mut matched = None; + for step in [now.saturating_sub(1), now, now.saturating_add(1)] { + let expected = code_for(secret, step).unwrap_or_default(); + // Equal lengths (six) for a real code; `ct_eq` on slices of different lengths is false. + if bool::from(expected.as_bytes().ct_eq(code.as_bytes())) && !expected.is_empty() { + matched = Some(step); + } + } + matched +} + +const BASE32: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +/// RFC 4648 Base32 without padding. +pub fn base32_encode(bytes: &[u8]) -> String { + let mut out = String::with_capacity((bytes.len() * 8).div_ceil(5)); + let (mut buffer, mut bits) = (0u32, 0u32); + for &byte in bytes { + buffer = (buffer << 8) | u32::from(byte); + bits += 8; + while bits >= 5 { + bits -= 5; + if let Some(&c) = BASE32.get(((buffer >> bits) & 31) as usize) { + out.push(char::from(c)); + } + } + buffer &= (1 << bits) - 1; + } + if bits > 0 + && let Some(&c) = BASE32.get(((buffer << (5 - bits)) & 31) as usize) + { + out.push(char::from(c)); + } + out +} + +/// The inverse of [`base32_encode`] (upper or lower case, no padding, no spaces); `None` for anything else. +pub fn base32_decode(text: &str) -> Option> { + let mut out = Vec::with_capacity(text.len() * 5 / 8); + let (mut buffer, mut bits) = (0u32, 0u32); + for c in text.bytes() { + let value = BASE32.iter().position(|&b| b == c.to_ascii_uppercase())?; + buffer = (buffer << 5) | u32::try_from(value).ok()?; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push(u8::try_from((buffer >> bits) & 0xff).ok()?); + buffer &= (1 << bits) - 1; + } + } + // Leftover bits must be the zeros the encoder pads with. + (buffer == 0).then_some(out) +} + +/// `text` with everything but `A-Z a-z 0-9 - _ . ~` percent-encoded (UTF-8 bytes), for a URI's path and query. +pub fn percent_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// The `otpauth://` URI an authenticator app reads (from a QR code or typed in): the issuer and the account's username +/// as the label, both percent-encoded. +pub fn otpauth_uri(username: &str, secret_base32: &str) -> String { + let issuer = percent_encode(ISSUER); + format!( + "otpauth://totp/{issuer}:{}?secret={secret_base32}&issuer={issuer}&algorithm=SHA1&digits={DIGITS}&period={STEP_SECS}", + percent_encode(username) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// RFC 6238's SHA-1 key; its eight-digit test codes, cut to the last six digits. + const RFC_KEY: &[u8] = b"12345678901234567890"; + + #[test] + fn the_rfc_6238_vectors_hold_for_six_digits() { + for (time, code) in [ + (59i64, "287082"), + (1_111_111_109, "081804"), + (1_111_111_111, "050471"), + (1_234_567_890, "005924"), + (2_000_000_000, "279037"), + (20_000_000_000, "353130"), + ] { + assert_eq!(code_for(RFC_KEY, step_at(time * 1000)).as_deref(), Some(code), "t = {time}"); + } + // RFC 4226's own HOTP vectors, counter 0 and 1. + assert_eq!(code_for(RFC_KEY, 0).as_deref(), Some("755224")); + assert_eq!(code_for(RFC_KEY, 1).as_deref(), Some("287082")); + assert_eq!(code_for(RFC_KEY, -1), None); + } + + #[test] + fn the_window_is_one_step_either_way_and_reports_the_matched_step() { + let now_ms = 1_111_111_109_000; + let step = step_at(now_ms); + for (offset, accepted) in [(-2, false), (-1, true), (0, true), (1, true), (2, false)] { + let code = code_for(RFC_KEY, step + offset).unwrap(); + assert_eq!(verify(RFC_KEY, &code, now_ms), accepted.then_some(step + offset), "offset {offset}"); + } + // Just inside and just outside the step's edge. + let edge = step_at(1_111_111_109_000) * STEP_SECS * 1000; + assert_eq!(step_at(edge), step); + assert_eq!(step_at(edge - 1), step - 1); + } + + #[test] + fn only_six_ascii_digits_are_codes() { + for bad in ["", "12345", "1234567", "12 456", "١٢٣٤٥٦", "12345a", " 123456", "123456 ", "+12345"] { + assert!(!is_code(bad), "{bad:?}"); + assert_eq!(verify(RFC_KEY, bad, 59_000), None, "{bad:?}"); + } + assert!(is_code("000000")); + assert_eq!(verify(RFC_KEY, "287082", 59_000), Some(1)); + assert_eq!(verify(RFC_KEY, "287083", 59_000), None); + } + + #[test] + fn base32_round_trips_and_matches_the_rfc() { + assert_eq!(base32_encode(b""), ""); + assert_eq!(base32_encode(b"f"), "MY"); + assert_eq!(base32_encode(b"fo"), "MZXQ"); + assert_eq!(base32_encode(b"foo"), "MZXW6"); + assert_eq!(base32_encode(b"foob"), "MZXW6YQ"); + assert_eq!(base32_encode(b"fooba"), "MZXW6YTB"); + assert_eq!(base32_encode(b"foobar"), "MZXW6YTBOI"); + assert_eq!(base32_encode(RFC_KEY), "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ"); + for text in ["", "MY", "MZXQ", "MZXW6", "MZXW6YQ", "MZXW6YTB", "MZXW6YTBOI"] { + assert_eq!(base32_encode(&base32_decode(text).unwrap()), text); + } + assert_eq!(base32_decode("mzxw6ytboi").as_deref(), Some(b"foobar".as_slice())); + assert_eq!(base32_decode("MZXW6YTB1"), None, "1 is not in the alphabet"); + assert_eq!(base32_decode("MZXW6Y=="), None, "no padding"); + assert_eq!(base32_decode("MZ"), None, "the leftover bits must be zero"); + let secret = generate_secret().unwrap(); + assert_eq!(base32_decode(&base32_encode(secret.as_slice())).unwrap(), secret.to_vec()); + assert_eq!(base32_encode(secret.as_slice()).len(), 32); + } + + #[test] + fn the_otpauth_uri_encodes_the_label() { + let uri = otpauth_uri("ana", "GEZDGNBVGY3TQOJQ"); + assert_eq!(uri, "otpauth://totp/Craft%20Cloud:ana?secret=GEZDGNBVGY3TQOJQ&issuer=Craft%20Cloud&algorithm=SHA1&digits=6&period=30"); + let odd = otpauth_uri("a.b_c-d~e", "AAAA"); + assert!(odd.contains(":a.b_c-d~e?"), "{odd}"); + // Not a real username today, but the encoder must hold for whatever one comes to be. + let wild = otpauth_uri("a b&c=d/é?#:", "AAAA"); + assert!(wild.contains(":a%20b%26c%3Dd%2F%C3%A9%3F%23%3A?"), "{wild}"); + assert_eq!(wild.matches('?').count(), 1); + assert_eq!(wild.matches('#').count(), 0); + } +} diff --git a/apps/craft-cloud-server/src/two_factor.rs b/apps/craft-cloud-server/src/two_factor.rs new file mode 100644 index 00000000..98149aad --- /dev/null +++ b/apps/craft-cloud-server/src/two_factor.rs @@ -0,0 +1,617 @@ +//! Two-factor sign-in over HTTP: `GET /api/v1/account/two-factor`, `POST …/setup`, `…/confirm`, `…/disable`, +//! `…/recovery-codes`, and the second step of a sign-in, `POST /api/v1/auth/login/second-factor`. The admin API's +//! confirmation uses [`confirm_admin_second_factor`], `POST /join` the same parsing and proofs. The key for the secrets, +//! its checks at start and `two-factor rekey` are here too. +//! +//! - **TOTP and recovery codes** (see [`crate::totp`]): the secret is 160 random bits, shown once as Base32 and as an +//! `otpauth://` URI, and kept only sealed with the key from `[two_factor] key_file` ([`crate::secretbox`]). Ten recovery +//! codes of 16 characters (80 bits, the alphabet of the join codes) are shown once; the database keeps their BLAKE3 +//! hashes, so they work whatever happens to the key, and 80 bits can't be searched even from a leaked database. +//! - **Statuses**: a wrong password is `403 wrong_password`; a wrong code `403 two_factor_invalid` on a signed-in route and +//! `401` at sign-in and join; a body that is not what it should be, a code that is not six digits and a recovery code +//! that is not 16 letters and digits `400 invalid` (nothing is counted); a lock `429 two_factor_locked` with +//! `Retry-After`. 401 is otherwise the answer for a token that is revoked, and for a challenge that doesn't work. +//! - **Every change takes the account's lock in `craft_cloud_meta`** and checks again there what the handler checked +//! before (the device still signed in, the password still the verified one, the account's current state). Handlers end +//! the live connections of the devices a change signed out only after it committed. +//! - **Counting**: a wrong password counts against `password_changes` (as a password change does), a wrong second factor +//! against the account (the lock) and, at sign-in's second step, join and the admin's confirmation, against +//! [`AppState::second_factors`] ([`SECOND_FACTORS_PER_USER`] per account and [`SECOND_FACTORS_PER_IP`] per client +//! address in five minutes; a right proof is given back). Wrong codes at the second step also count against the +//! sign-in limiter. +//! - **Nothing secret is logged**: not a secret, a code, a recovery code, a challenge, the key or a password. +//! - **The key**: [`load_keys`] reads the key files, [`check_startup`] refuses to start a server that has secrets it can't +//! open (all of them are opened, not a sample), and [`rekey`] rewrites the secrets under the current key. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::{ConnectInfo, Extension, FromRequest, Request, State}; +use axum::http::{HeaderMap, StatusCode}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{DeviceActor, DeviceAuth, Meta, MetaError, Proof, RECOVERY_CODES, TotpVerifier, UnreadableSecret, UserId}; +use craft_cloud_proto::api::{ + LoginResponse, RecoveryCodes, SecondFactorRequest, TwoFactorConfirmRequest, TwoFactorProofRequest, TwoFactorSetupRequest, TwoFactorSetupResponse, + TwoFactorStatus, +}; +use craft_cloud_proto::join::{Code, INVITE_CHARS}; + +use crate::account::confirm_password; +use crate::api::{LOGINS_PER_IP, LOGINS_PER_USER, TINY_JSON_BYTES, body_too_slow}; +use crate::auth::{new_token, token_hash}; +use crate::config::Config; +use crate::error::ApiError; +use crate::ratelimit::{client_ip, ip_key}; +use crate::secretbox::{KeyError, SecretBox, TwoFactorKey}; +use crate::state::AppState; +use crate::totp; + +/// Wrong second factors one account may give in [`AppState::second_factors`]'s window (five minutes). +pub const SECOND_FACTORS_PER_USER: u32 = 10; +/// Wrong second factors one client address may give in the same window. +pub const SECOND_FACTORS_PER_IP: u32 = 30; + +type Shared = Arc; + +// ---- reading and checking what a client sends ---- + +/// Reads a tiny JSON body (at most [`TINY_JSON_BYTES`]) under the usual deadline. Whatever is wrong with its shape (not +/// JSON, a missing or mistyped field) is `400 invalid`, one too long `413 too_large`, one too slow `408`. +pub(crate) async fn read_tiny(state: &AppState, request: Request) -> Result { + let bytes = match tokio::time::timeout(state.limits.body_time(TINY_JSON_BYTES), Bytes::from_request(request, &())).await { + Ok(bytes) => bytes?, + Err(_) => return Err(body_too_slow()), + }; + serde_json::from_slice(&bytes).map_err(|_| ApiError::invalid("the body must be a JSON object with the fields of this call")) +} + +/// A second factor as a client sent it, checked for its shape only. +pub(crate) enum ProofInput { + /// Six digits. + Totp(String), + /// The BLAKE3 hash of the normalised recovery code. + Recovery([u8; 32]), +} + +/// The second factor of a body: `totp_code` (six ASCII digits) or `recovery_code` (16 letters and digits, case, dashes and +/// spaces forgiven, `O` for `0` and `I`, `L` for `1`), never both. `None` when the body has neither. Anything else is `400 +/// invalid`, before anything is counted. +pub(crate) fn parse_proof(totp_code: Option, recovery_code: Option) -> Result, ApiError> { + match (totp_code, recovery_code) { + (None, None) => Ok(None), + (Some(_), Some(_)) => Err(ApiError::invalid("send either totp_code or recovery_code, not both")), + (Some(code), None) => { + if !totp::is_code(&code) { + return Err(ApiError::invalid("totp_code is the six digits from the authenticator app")); + } + Ok(Some(ProofInput::Totp(code))) + } + (None, Some(code)) => match Code::parse(&code, INVITE_CHARS) { + Some(code) => Ok(Some(ProofInput::Recovery(code.hash()))), + None => Err(ApiError::invalid("recovery_code is 16 letters and digits, like ABCD-EFGH-JKMN-PQRS")), + }, + } +} + +/// A second factor ready to hand to `craft_cloud_meta`: a recovery code as its hash, a TOTP code as the check that opens +/// the account's stored secret (under the account's lock, in the transaction) and compares. +pub(crate) enum Prepared { + Totp(Box>), + Recovery([u8; 32]), +} + +impl Prepared { + pub(crate) fn proof(&self) -> Proof<'_> { + match self { + Prepared::Totp(verify) => Proof::Totp(&**verify), + Prepared::Recovery(hash) => Proof::Recovery(*hash), + } + } +} + +impl ProofInput { + pub(crate) fn prepare(self, state: &AppState, now_ms: i64) -> Prepared { + match self { + ProofInput::Recovery(hash) => Prepared::Recovery(hash), + ProofInput::Totp(code) => Prepared::Totp(totp_check(state.two_factor.clone(), code, now_ms)), + } + } +} + +/// The TOTP check for `code` at `now_ms`: opens the stored secret (sealed for the account in question), compares, and +/// hands back the time step. No key, or a secret the key doesn't open, is [`UnreadableSecret`]. +fn totp_check(secrets: Option>, code: String, now_ms: i64) -> Box> { + Box::new(move |user: UserId, sealed: &[u8]| { + let secrets = secrets.as_ref().ok_or(UnreadableSecret)?; + let secret = secrets.open(user, sealed).map_err(|_| UnreadableSecret)?; + Ok(totp::verify(&secret, &code, now_ms)) + }) +} + +fn random_failed(e: getrandom::Error) -> ApiError { + tracing::error!(error = %e, "the system random source failed"); + ApiError::internal() +} + +fn sealing_failed(_: crate::secretbox::SecretBoxError) -> ApiError { + tracing::error!("a two-factor secret could not be sealed"); + ApiError::internal() +} + +/// Ten new, distinct recovery codes. +fn new_recovery_codes() -> Result, ApiError> { + let mut codes: Vec = Vec::with_capacity(RECOVERY_CODES); + while codes.len() < RECOVERY_CODES { + let code = Code::generate(INVITE_CHARS).map_err(random_failed)?; + // A repeated code is all but impossible (80 bits), and is drawn again if it comes. + if !codes.iter().any(|known| known.hash() == code.hash()) { + codes.push(code); + } + } + Ok(codes) +} + +/// The codes as shown once, and their hashes as stored. +fn shown(codes: &[Code]) -> (RecoveryCodes, Vec<[u8; 32]>) { + (RecoveryCodes { recovery_codes: codes.iter().map(Code::display).collect() }, codes.iter().map(Code::hash).collect()) +} + +fn no_second_factor() -> ApiError { + ApiError::invalid("send a code in totp_code or a recovery code in recovery_code") +} + +// ---- the account's own routes ---- + +/// `GET /api/v1/account/two-factor`. +pub async fn get_status(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let info = state.meta.two_factor_info(auth.user, state.now_ms()).await?; + Ok(Json(TwoFactorStatus { + enabled: info.enabled_ms.is_some(), + enabled_ms: info.enabled_ms, + recovery_codes_left: info.recovery_codes_left, + pending: info.pending, + })) +} + +/// `POST /api/v1/account/two-factor/setup`: the password starts a set-up and gets the new secret, shown once. Nothing is on +/// until `confirm`. +pub async fn setup( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: TwoFactorSetupRequest = read_tiny(&state, request).await?; + let secrets = state.two_factor.clone().ok_or_else(ApiError::two_factor_unavailable)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let secret = totp::generate_secret().map_err(random_failed)?; + let sealed = secrets.seal(auth.user, secret.as_slice()).map_err(sealing_failed)?; + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + let expires_ms = state.meta.two_factor_begin_setup(&actor, &sealed, state.now_ms()).await?; + let encoded = totp::base32_encode(secret.as_slice()); + tracing::info!(device = %auth.device, "two-factor set-up started"); + Ok(Json(TwoFactorSetupResponse { otpauth_uri: totp::otpauth_uri(&auth.username, &encoded), secret: encoded, expires_ms })) +} + +/// `POST /api/v1/account/two-factor/confirm`: the first code turns two-factor sign-in on and shows the recovery codes, +/// once. Every other device is signed out. +pub async fn confirm(State(state): State, Extension(auth): Extension, request: Request) -> Result, ApiError> { + let req: TwoFactorConfirmRequest = read_tiny(&state, request).await?; + let Some(ProofInput::Totp(code)) = parse_proof(Some(req.totp_code), None)? else { return Err(no_second_factor()) }; + let now_ms = state.now_ms(); + let verify = totp_check(state.two_factor.clone(), code, now_ms); + let (answer, hashes) = shown(&new_recovery_codes()?); + let revoked = state.meta.two_factor_confirm(auth.user, auth.device, &*verify, &hashes, now_ms).await?; + // Only after the commit: their live connections end now, not at their next request. + end_live_connections(&state, &revoked); + tracing::info!(device = %auth.device, revoked = revoked.len(), "two-factor sign-in turned on"); + Ok(Json(answer)) +} + +/// `POST /api/v1/account/two-factor/disable`: the password and a code or recovery code turn it off. Every other device is +/// signed out. +pub async fn disable(State(state): State, Extension(auth): Extension, request: Request) -> Result { + let req: TwoFactorProofRequest = read_tiny(&state, request).await?; + let proof = parse_proof(req.totp_code, req.recovery_code)?.ok_or_else(no_second_factor)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let now_ms = state.now_ms(); + let prepared = proof.prepare(&state, now_ms); + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + let revoked = state.meta.two_factor_disable(&actor, &prepared.proof(), now_ms).await?; + end_live_connections(&state, &revoked); + tracing::info!(device = %auth.device, revoked = revoked.len(), "two-factor sign-in turned off"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /api/v1/account/two-factor/recovery-codes`: the password and a code make ten new recovery codes, shown once; the +/// old ones stop working. +pub async fn recovery_codes( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: TwoFactorProofRequest = read_tiny(&state, request).await?; + if req.recovery_code.is_some() { + return Err(ApiError::invalid("new recovery codes are made with a code from the authenticator app (totp_code)")); + } + let proof = parse_proof(req.totp_code, None)?.ok_or_else(no_second_factor)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let now_ms = state.now_ms(); + let prepared = proof.prepare(&state, now_ms); + let (answer, hashes) = shown(&new_recovery_codes()?); + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + state.meta.two_factor_new_recovery_codes(&actor, &prepared.proof(), &hashes, now_ms).await?; + tracing::info!(device = %auth.device, "new recovery codes made"); + Ok(Json(answer)) +} + +fn end_live_connections(state: &AppState, devices: &[craft_cloud_proto::DeviceId]) { + for device in devices { + state.live.revoke(*device); + } +} + +// ---- the second step of a sign-in ---- + +/// The hash of a challenge token as the client sent it; `None` for text that is no token (it is answered like an unknown +/// challenge). +pub(crate) fn challenge_hash(challenge: &str) -> Option<[u8; 32]> { + let is_token = challenge.len() == 43 && URL_SAFE_NO_PAD.decode(challenge).is_ok(); + is_token.then(|| token_hash(challenge)) +} + +/// `POST /api/v1/auth/login/second-factor` (no sign-in): the challenge `POST /auth/login` gave and a code or recovery +/// code make the device, answered as a sign-in is. A challenge that is unknown, expired, used, from before a password +/// change or a reset, or fit no more is `401 unauthorized`, the same for each. A wrong code is `401 two_factor_invalid` +/// (five of them delete the challenge), a lock `429 two_factor_locked`. +pub async fn second_factor( + State(state): State, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + request: Request, +) -> Result, ApiError> { + let req: SecondFactorRequest = read_tiny(&state, request).await?; + let proof = parse_proof(req.totp_code, req.recovery_code)?.ok_or_else(no_second_factor)?; + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let now_ms = state.now_ms(); + let ip = ip_key(client_ip(peer, &headers, &state.trusted_proxies)); + // Counted before anything is looked at, so tries at the same moment are held to the limits too, and given back for a + // right proof or a lock (which checked nothing). + let Some(ip_counted) = state.second_factors.count(&ip, SECOND_FACTORS_PER_IP, now) else { + return Err(ApiError::rate_limited("too many wrong codes from this address; wait a few minutes", None)); + }; + let gone = || ApiError::from(MetaError::ChallengeGone); + let Some(hash) = challenge_hash(&req.challenge) else { + crate::audit::challenge_failed(&state, None).await; + return Err(gone()); + }; + let Some((user, username)) = state.meta.challenge_owner(&hash, now_ms).await? else { + crate::audit::challenge_failed(&state, None).await; + return Err(gone()); + }; + let user_key = format!("user:{username}"); + let Some(user_counted) = state.second_factors.count(&user_key, SECOND_FACTORS_PER_USER, now) else { + state.second_factors.refund(&ip, ip_counted); + // Ten wrong codes both fill this window and lock the account: say the lock, which tells how long. + if let Some(locked_ms) = state.meta.two_factor_locked_for(user, now_ms).await? { + return Err(ApiError::two_factor_locked(craft_cloud_meta::retry_after_secs(locked_ms))); + } + return Err(ApiError::rate_limited("too many wrong codes for this account; wait a few minutes", None)); + }; + let (token, new_token_hash) = new_token()?; + let prepared = proof.prepare(&state, now_ms); + let result = state.meta.complete_second_factor(&hash, &prepared.proof(), &new_token_hash, now_ms).await; + match result { + Ok(done) => { + state.second_factors.refund(&ip, ip_counted); + state.second_factors.refund(&user_key, user_counted); + // The username is the only personal identifier the server could log here: kept to debug. + tracing::debug!(%username, "sign-in with a second factor"); + tracing::info!(device = %done.device, "device signed in with a second factor"); + Ok(Json(LoginResponse { device: done.device, token })) + } + Err(MetaError::TwoFactorInvalid) => { + // A wrong second factor counts against sign-in too. + let _ = state.logins.allow(&ip, LOGINS_PER_IP, now); + let _ = state.logins.allow(&user_key, LOGINS_PER_USER, now); + Err(ApiError::from(MetaError::TwoFactorInvalid).on_public_route()) + } + Err(MetaError::TwoFactorLocked { retry_after_ms }) => { + state.second_factors.refund(&ip, ip_counted); + state.second_factors.refund(&user_key, user_counted); + Err(ApiError::from(MetaError::TwoFactorLocked { retry_after_ms })) + } + Err(MetaError::ChallengeGone) => { + // The challenge was found a moment ago and has gone since (used, out of tries, expired, or the account changed). + crate::audit::challenge_failed(&state, Some(user)).await; + Err(gone()) + } + Err(e) => Err(e.into()), + } +} + +// ---- the admin's confirmation ---- + +/// The second factor of an admin's confirmation (see [`crate::admin`], the account is a device's or a browser session's): when the admin's account has two-factor sign-in, +/// `proof` is checked and used up (`403 two_factor_invalid` when it is missing, wrong or a time step that was used, `429 +/// two_factor_locked` while locked), counted against [`AppState::second_factors`] for the account. Whether the account +/// has it is read again under its lock when the proof is checked. The answer says whether it had: the change compares +/// that with the account's state in its own transaction. +pub(crate) async fn confirm_admin_second_factor(state: &AppState, user: UserId, username: &str, proof: Option) -> Result { + let enabled = state.meta.two_factor_enabled(user).await?; + if !enabled { + // A code given to an account that has none is not looked at: the change compares the account's state again anyway. + return Ok(false); + } + let Some(proof) = proof else { + let mut e = ApiError::two_factor_invalid(); + e.message = "this account has two-factor sign-in: add a code in totp_code (or a recovery code in recovery_code)".to_owned(); + return Err(e); + }; + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{username}"); + let now_ms = state.now_ms(); + let Some(counted) = state.second_factors.count(&key, SECOND_FACTORS_PER_USER, now) else { + if let Some(locked_ms) = state.meta.two_factor_locked_for(user, now_ms).await? { + return Err(ApiError::two_factor_locked(craft_cloud_meta::retry_after_secs(locked_ms))); + } + return Err(ApiError::rate_limited("too many wrong codes; wait a few minutes", None)); + }; + let prepared = proof.prepare(state, now_ms); + match state.meta.confirm_second_factor(user, Some(&prepared.proof()), now_ms).await { + Ok(had) => { + state.second_factors.refund(&key, counted); + Ok(had) + } + Err(MetaError::TwoFactorInvalid) => Err(MetaError::TwoFactorInvalid.into()), + Err(e) => { + state.second_factors.refund(&key, counted); + Err(e.into()) + } + } +} + +// ---- the key ---- + +/// Where the server's key for two-factor secrets stands, from its config. +#[derive(Debug)] +pub enum KeyState { + /// The config has no `[two_factor]` section. + NotConfigured, + /// The config names a key file that is not there. + Missing(PathBuf), + /// The key (and the older one, if named) is read. + Loaded(Box), +} + +/// Reads the key files the config names. A key file that is not there is [`KeyState::Missing`] (whether that is fatal +/// depends on whether the database holds secrets, see [`check_startup`]); one that can't be read, has the wrong length, or +/// shares its name with the older key is an error, naming the file. +pub fn load_keys(config: &Config) -> Result { + let Some(section) = &config.two_factor else { return Ok(KeyState::NotConfigured) }; + let path = config.key_path(§ion.key_file); + let current = match TwoFactorKey::from_file(&path) { + Ok(key) => key, + Err(KeyError::Missing { path }) => return Ok(KeyState::Missing(path)), + Err(e) => return Err(e), + }; + warn_if_readable_by_others(&path); + let old = match §ion.key_file_old { + Some(old) => { + let old = config.key_path(old); + warn_if_readable_by_others(&old); + Some(TwoFactorKey::from_file(&old)?) + } + None => None, + }; + Ok(KeyState::Loaded(Box::new(SecretBox::new(¤t, old.as_ref())?))) +} + +#[cfg(unix)] +fn warn_if_readable_by_others(path: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(path) + && meta.permissions().mode() & 0o077 != 0 + { + tracing::warn!(path = %path.display(), "the two-factor key file is readable by other users: make it owner-only (chmod 600)"); + } +} + +#[cfg(not(unix))] +fn warn_if_readable_by_others(_path: &std::path::Path) {} + +/// Refuses to start a server that holds two-factor secrets it can't open: with secrets in the database and no key (not +/// configured, or the file is missing), or a key that doesn't open **every** one of them. The message names the key file +/// the config expects. Without secrets there is nothing to open, and a server without a key runs, with set-up answering `503 +/// two_factor_unavailable`. Returns the box to give [`AppState::with_two_factor`]. +pub async fn check_startup(meta: &Meta, keys: KeyState, config: &Config, now_ms: i64) -> Result, String> { + let secrets = meta.two_factor_secrets(now_ms).await.map_err(|e| e.to_string())?; + let active: Vec<_> = secrets.iter().filter_map(|row| row.active.as_ref().map(|sealed| (row.user, sealed))).collect(); + let expected = config.two_factor.as_ref().map(|section| config.key_path(§ion.key_file)); + match keys { + KeyState::Loaded(secret_box) => { + let unreadable = active.iter().filter(|(user, sealed)| secret_box.open(*user, sealed).is_err()).count(); + if unreadable > 0 { + let path = expected.map(|p| p.display().to_string()).unwrap_or_default(); + return Err(format!( + "{unreadable} of {} two-factor secrets in the database can't be opened with the key in {path}{}: this is not the key they were sealed \ + with. Put the right key there (and a previous one in key_file_old while rotating); the server won't start without it", + active.len(), + if config.two_factor.as_ref().is_some_and(|section| section.key_file_old.is_some()) { " or key_file_old" } else { "" }, + )); + } + Ok(Some(*secret_box)) + } + KeyState::Missing(path) => { + if active.is_empty() { + tracing::warn!(path = %path.display(), "the two-factor key file is missing: two-factor sign-in can't be set up"); + Ok(None) + } else { + Err(format!( + "{} accounts use two-factor sign-in but the key file {} does not exist: restore it (it is not part of the database), or the server won't start", + active.len(), + path.display() + )) + } + } + KeyState::NotConfigured => { + if active.is_empty() { + Ok(None) + } else { + Err(format!( + "{} accounts use two-factor sign-in but the config has no [two_factor] key_file: set it to the key file the server had (it is not part of the \ + database), or the server won't start", + active.len() + )) + } + } + } +} + +/// What [`rekey`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Rekeyed { + /// Secrets rewritten under the current key. + pub rewritten: usize, + /// Secrets that were under it already. + pub already_current: usize, +} + +/// Rewrites every secret (the active ones and the set-ups', expired ones too) that is sealed with the older key under the +/// current one, one account at a time, each in a transaction that reads the account's ciphertexts fresh under its lock and +/// rewrites both, so a confirmation that races it can't leave one behind. It refuses, before changing anything, when a secret +/// is sealed with a key that is neither (a key that was lost) or can't be opened. It then scans again and goes round again +/// (at most five times) until no ciphertext is under any key but the current one, and fails if some still is: only a +/// successful return means `key_file_old` can be removed **and** that no server still running holds only the old key, which +/// is the operator's to see to (every instance must run with both keys before this, see `docs/server.md`). Neither the +/// secrets nor the keys are printed. It can be run again. +pub async fn rekey(meta: &Meta, secret_box: &SecretBox) -> Result { + rekey_hooked(meta, secret_box, &|| {}).await +} + +/// [`rekey`] with a test seam: `after_scan` is called once, after the first scan of what is stored and before anything is +/// rewritten, so a test can let other changes (a confirmation moving a set-up's secret) land exactly there. It blocks the +/// thread it runs on, so a test runs the rest on others. +pub async fn rekey_hooked(meta: &Meta, secret_box: &SecretBox, after_scan: &(dyn Fn() + Sync)) -> Result { + let neither = || { + "some secrets are sealed with a key that is neither key_file nor key_file_old: nothing was changed; put that key in key_file_old first" + .to_owned() + }; + let mut already_current = 0usize; + for row in meta.two_factor_all_secrets().await.map_err(|e| e.to_string())? { + for sealed in [row.active, row.pending].into_iter().flatten() { + let id = SecretBox::key_id(&sealed); + if id == Some(secret_box.current_id()) { + // Under the current key: it must open, or the rest of the work would hide a lost key. + secret_box.open(row.user, &sealed).map_err(|_| "a secret that carries the current key's name does not open with it".to_owned())?; + already_current += 1; + } else if id.is_some() && id == secret_box.old_id() { + secret_box.open(row.user, &sealed).map_err(|_| "a secret does not open with key_file_old: nothing was changed".to_owned())?; + } else { + return Err(neither()); + } + } + } + after_scan(); + let current = secret_box.current_id(); + let rewrite = |user: UserId, sealed: &[u8]| -> Result>, UnreadableSecret> { + if SecretBox::key_id(sealed) == Some(current) { + return Ok(None); + } + let secret = secret_box.open(user, sealed).map_err(|_| UnreadableSecret)?; + secret_box.seal(user, &secret).map(Some).map_err(|_| UnreadableSecret) + }; + let mut rewritten = 0usize; + for _round in 0..5 { + let behind: Vec = meta + .two_factor_all_secrets() + .await + .map_err(|e| e.to_string())? + .into_iter() + .filter(|row| [&row.active, &row.pending].into_iter().flatten().any(|sealed| SecretBox::key_id(sealed) != Some(current))) + .map(|row| row.user) + .collect(); + if behind.is_empty() { + return Ok(Rekeyed { rewritten, already_current }); + } + for user in behind { + let (done, _) = meta.two_factor_rewrite(user, &rewrite).await.map_err(|e| e.to_string())?; + rewritten += done; + } + } + // One last look: a secret still under another key means the migration is not complete. + let left = meta + .two_factor_all_secrets() + .await + .map_err(|e| e.to_string())? + .iter() + .flat_map(|row| [&row.active, &row.pending]) + .flatten() + .filter(|sealed| SecretBox::key_id(sealed) != Some(current)) + .count(); + if left == 0 { + Ok(Rekeyed { rewritten, already_current }) + } else { + Err(format!( + "{left} secret(s) are still sealed with another key after rewriting: a server may still be sealing new ones with the old key. \ + Restart every instance with key_file and key_file_old, and run this again; do not remove key_file_old yet" + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_proof_has_the_shape_of_one_code_only() { + assert!(matches!(parse_proof(None, None), Ok(None))); + assert!(matches!(parse_proof(Some("123456".into()), None), Ok(Some(ProofInput::Totp(c))) if c == "123456")); + for bad in ["12345", "1234567", "abcdef", " 12345", "١٢٣٤٥٦", ""] { + assert!(parse_proof(Some(bad.into()), None).is_err(), "{bad:?}"); + } + assert!(parse_proof(Some("123456".into()), Some("ABCD-EFGH-JKMN-PQRS".into())).is_err(), "both"); + let a = parse_proof(None, Some("abcd-efgh-jkmn-pqrs".into())).unwrap(); + let b = parse_proof(None, Some("ABCDEFGHJKMNPQRS".into())).unwrap(); + let (Some(ProofInput::Recovery(a)), Some(ProofInput::Recovery(b))) = (a, b) else { panic!("recovery codes") }; + assert_eq!(a, b, "case and dashes don't matter"); + // O is 0, and I and L are 1, as in the join codes. + let zero = parse_proof(None, Some("0000-0000-0000-0000".into())).unwrap(); + let letter = parse_proof(None, Some("OOOO-OOOO-OOOO-OOOO".into())).unwrap(); + let (Some(ProofInput::Recovery(zero)), Some(ProofInput::Recovery(letter))) = (zero, letter) else { panic!("recovery codes") }; + assert_eq!(zero, letter); + for bad in ["ABCD-EFGH-JKMN-PQR", "ABCD-EFGH-JKMN-PQRSA", "ABCD-EFGH-JKMN-PQRU", ""] { + assert!(parse_proof(None, Some(bad.into())).is_err(), "{bad:?}"); + } + } + + #[test] + fn ten_distinct_recovery_codes_of_16_characters_are_shown_in_groups() { + let codes = new_recovery_codes().unwrap(); + assert_eq!(codes.len(), 10); + let (answer, hashes) = shown(&codes); + assert_eq!(answer.recovery_codes.len(), 10); + let unique: std::collections::BTreeSet<_> = hashes.iter().collect(); + assert_eq!(unique.len(), 10); + for (shown, code) in answer.recovery_codes.iter().zip(&codes) { + assert_eq!(shown.len(), 19, "16 characters in four groups"); + assert_eq!(shown.replace('-', ""), code.as_str()); + assert_eq!(Code::parse(shown, INVITE_CHARS).map(|c| c.hash()), Some(code.hash())); + } + // The hash is BLAKE3 of the normalised code, as the join codes' are, and reproducible. + assert_eq!(hashes.first().copied(), codes.first().map(|c| *blake3::hash(c.as_str().as_bytes()).as_bytes())); + } + + #[test] + fn a_challenge_is_a_43_character_token_or_nothing() { + let (token, hash) = new_token().unwrap(); + assert_eq!(challenge_hash(&token), Some(hash)); + for bad in ["", "short", &"x".repeat(44), &format!("{}=", &token[..42]), &format!("{}!", &token[..42])] { + assert_eq!(challenge_hash(bad), None, "{bad:?}"); + } + } +} diff --git a/apps/craft-cloud-server/src/verify.rs b/apps/craft-cloud-server/src/verify.rs new file mode 100644 index 00000000..a40edcb7 --- /dev/null +++ b/apps/craft-cloud-server/src/verify.rs @@ -0,0 +1,370 @@ +//! The invariant checker behind `craft-cloud-server verify`, and the index builder behind `craft-cloud-server index +//! build`. +//! +//! `verify` reads and reports; it changes nothing: the database and the storage must exist already (a wrong path is an +//! error, not a new empty server with no problems), and a database whose schema is older is refused, not upgraded. It +//! checks what a later garbage collector would have to trust: that every manifest's list of +//! chunks in the database is complete and equal to the manifest object in the store, that every chunk a manifest names +//! is recorded and its object is there and intact, and that nothing in the database names a manifest or a chunk that does +//! not exist. Anything it cannot establish is reported, not guessed: a manifest it cannot read or whose index is not +//! complete has unknown dependencies, and unknown means kept. +//! +//! Two kinds of finding. *Problems* are inconsistencies and make `verify` fail. *Notices* are facts a reader needs but +//! that are not wrong by themselves: manifests whose index is not complete yet (`index build` completes it), manifests +//! no version or snapshot names (a live document's assets are named only inside the document, where the server cannot +//! read them, so these are not garbage and must not be treated as garbage), and chunks no complete manifest names. + +use std::collections::BTreeMap; + +use craft_cloud_meta::{EdgeCheck, Finding, IndexAudit, IndexState, Meta, MetaError}; +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use craft_cloud_store::{BlobKey, Store, StoreError}; +use serde_json::{Value, json}; + +/// Problems listed in full in a report; further ones are only counted. +const MAX_LISTED_PROBLEMS: usize = 1000; +/// Examples of each audit finding. +const EXAMPLES: usize = 20; +/// IDs one metadata query covers. +const PAGE: usize = 200; + +/// What `verify` should look at. +#[derive(Debug, Clone, Copy)] +pub struct VerifyOptions { + /// Read every chunk object from the store, which checks its hash. Without it only manifests are read and chunks are + /// checked against the database. + pub read_chunks: bool, +} + +impl Default for VerifyOptions { + fn default() -> Self { + Self { read_chunks: true } + } +} + +/// One inconsistency. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Problem { + pub kind: &'static str, + /// The IDs involved in hex (`manifest` or `manifest/chunk`). + pub subject: String, + pub detail: Option, +} + +/// The result of a `verify` run. +#[derive(Debug, Clone)] +pub struct Report { + pub audit: IndexAudit, + /// The first [`MAX_LISTED_PROBLEMS`] problems. + pub problems: Vec, + /// Every problem, by kind. + pub counts: BTreeMap<&'static str, u64>, + pub manifests_checked: u64, + pub chunks_checked: u64, + pub chunk_objects_read: bool, +} + +impl Report { + /// How many problems there are in all. + pub fn problem_count(&self) -> u64 { + self.counts.values().sum() + } + + /// Whether the index covers every recorded manifest, completely. + pub fn index_complete(&self) -> bool { + self.audit.manifests == self.audit.complete + } + + fn add(&mut self, kind: &'static str, subject: String, detail: Option) { + *self.counts.entry(kind).or_insert(0) += 1; + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject, detail }); + } + } + + /// An audit finding that is a problem: counted in full, its examples listed. + fn add_finding(&mut self, kind: &'static str, finding: &Finding) { + if finding.count == 0 { + return; + } + *self.counts.entry(kind).or_insert(0) += finding.count; + for example in &finding.examples { + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject: example.clone(), detail: None }); + } + } + } + + /// The report as JSON. + pub fn to_json(&self) -> Value { + let notice = |f: &Finding| json!({ "count": f.count, "examples": f.examples }); + json!({ + "ok": self.problem_count() == 0, + "index": { + "manifests": self.audit.manifests, + "complete": self.audit.complete, + "building": self.audit.building, + "absent": self.audit.absent, + "all_complete": self.index_complete(), + }, + "problems": { + "total": self.problem_count(), + "by_kind": self.counts, + "listed": self.problems.iter().map(|p| json!({ "kind": p.kind, "id": p.subject, "detail": p.detail })).collect::>(), + "truncated": self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX), + }, + "notices": { + "manifests_without_complete_index": self.audit.absent.saturating_add(self.audit.building), + "roots_not_indexed": notice(&self.audit.roots_not_indexed), + "unrooted_manifests": notice(&self.audit.unrooted_manifests), + "unreferenced_chunks": notice(&self.audit.unreferenced_chunks), + }, + "checked": { + "manifests": self.manifests_checked, + "chunks": self.chunks_checked, + "chunk_objects_read": self.chunk_objects_read, + }, + }) + } + + /// The report for people. + pub fn to_text(&self) -> String { + let a = &self.audit; + let mut out = format!( + "manifests: {} recorded, index complete for {}, being built for {}, not indexed for {}\nchecked {} manifest(s) and {} chunk(s){}\n", + a.manifests, + a.complete, + a.building, + a.absent, + self.manifests_checked, + self.chunks_checked, + if self.chunk_objects_read { " (chunk objects read and hashed)" } else { " (chunk objects not read)" }, + ); + if self.problem_count() == 0 { + out.push_str("no problems found\n"); + } else { + out.push_str(&format!("{} problem(s) found:\n", self.problem_count())); + for (kind, count) in &self.counts { + out.push_str(&format!(" {kind}: {count}\n")); + } + for p in &self.problems { + match &p.detail { + Some(detail) => out.push_str(&format!(" {} {} ({detail})\n", p.kind, p.subject)), + None => out.push_str(&format!(" {} {}\n", p.kind, p.subject)), + } + } + if self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX) { + out.push_str(" (more problems than are listed; use --json for counts by kind)\n"); + } + } + if !self.index_complete() { + out.push_str("notice: the index is not complete; `craft-cloud-server index build` completes it. Manifests without a complete index have unknown dependencies.\n"); + } + for (what, finding) in [ + ( + "manifests named by no version or snapshot (live-document assets are named only inside their documents, so these are not garbage)", + &a.unrooted_manifests, + ), + ("recorded chunks no complete manifest names", &a.unreferenced_chunks), + ] { + if finding.count > 0 { + out.push_str(&format!("notice: {} {what}\n", finding.count)); + } + } + out + } +} + +/// Why a manifest or chunk object could not be read. +fn store_problem(error: &StoreError) -> (&'static str, Option) { + match error { + StoreError::NotFound(_) => ("object_missing", None), + StoreError::HashMismatch(_) | StoreError::Corrupt { .. } | StoreError::TooLarge { .. } => ("object_corrupt", Some(error.to_string())), + StoreError::Io(_) | StoreError::Backend(_) => ("store_error", Some(error.to_string())), + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Reads a manifest object and decodes it; `Err` says what is wrong with it. +pub async fn read_manifest(store: &Store, id: &ManifestId) -> Result)> { + read_manifest_sized(store, id).await.map(|(manifest, _)| manifest) +} + +/// Like [`read_manifest`], with the bytes the object is stored as (the format byte included). +async fn read_manifest_sized(store: &Store, id: &ManifestId) -> Result<(Manifest, u64), (&'static str, Option)> { + let (bytes, stored) = store.get_sized(BlobKey::manifest(id)).await.map_err(|e| { + let (kind, detail) = store_problem(&e); + ( + if kind == "object_missing" { + "manifest_object_missing" + } else if kind == "object_corrupt" { + "manifest_object_corrupt" + } else { + "store_error" + }, + detail, + ) + })?; + let manifest = Manifest::decode(&bytes).map_err(|e| ("manifest_not_decodable", Some(e.to_string())))?; + if manifest.id() != *id { + return Err(("manifest_id_mismatch", None)); + } + Ok((manifest, stored)) +} + +/// Checks the database and the store against each other and reports. Changes nothing. +pub async fn verify(meta: &Meta, store: &Store, options: VerifyOptions) -> Result { + let audit = meta.index_audit(EXAMPLES).await?; + let mut report = Report { + audit: audit.clone(), + problems: Vec::new(), + counts: BTreeMap::new(), + manifests_checked: 0, + chunks_checked: 0, + chunk_objects_read: options.read_chunks, + }; + report.add_finding("index_without_manifest", &audit.index_without_manifest); + report.add_finding("orphan_edges", &audit.edges_without_index); + report.add_finding("index_wrong_count", &audit.complete_with_wrong_count); + report.add_finding("unknown_dependency", &audit.unknown_dependencies); + report.add_finding("root_manifest_unknown", &audit.roots_without_manifest); + report.add_finding("space_blob_without_blob", &audit.space_blobs_without_blob); + + // Every recorded manifest: its object, its index, the chunks it names. + let mut after: Option<[u8; 32]> = None; + loop { + let records = meta.blob_records_after(BlobKind::Manifest, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; + after = Some(last); + for record in records { + let raw = record.id; + report.manifests_checked += 1; + let id = ManifestId::from_bytes(raw); + let (manifest, stored) = match read_manifest_sized(store, &id).await { + Ok(read) => read, + Err((kind, detail)) => { + report.add(kind, hex(&raw), detail); + continue; + } + }; + // What the database records of the manifest is what the object is: the bytes it is stored as, and the size + // of the file it describes. + if record.stored_size != stored { + report.add( + "manifest_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + if record.file_size != Some(manifest.size()) { + let recorded = record.file_size.map_or_else(|| "none".to_owned(), |n| n.to_string()); + report.add( + "manifest_file_size_mismatch", + hex(&raw), + Some(format!("the database records a file of {recorded} bytes, the manifest says {}", manifest.size())), + ); + } + if let IndexState::Complete { .. } = meta.index_state(&id).await? + && let EdgeCheck::Mismatch(problem) = meta.index_check(&manifest).await? + { + report.add("index_mismatch", hex(&raw), Some(format!("ordinal {}: {}", problem.ordinal, problem.kind.as_str()))); + } + for refs in manifest.chunks().chunks(500) { + let part: Vec = refs.iter().map(|c| c.id).collect(); + let recorded = meta.recorded_chunk_sizes(&part).await?; + for chunk in &part { + if !recorded.contains_key(chunk) { + report.add("chunk_not_recorded", format!("{}/{}", hex(&raw), hex(chunk.as_bytes())), None); + } + } + // The manifest says how long each chunk is; the database records the bytes it is stored as, one more + // than that (the format byte). With the chunk objects read below, this ties the length the manifest + // names to the bytes in the store. + for chunk in refs { + if let Some(stored) = recorded.get(&chunk.id) + && *stored != u64::from(chunk.len).saturating_add(1) + { + report.add( + "chunk_length_mismatch", + format!("{}/{}", hex(&raw), hex(chunk.id.as_bytes())), + Some(format!("the manifest says {} bytes, the database records {stored} stored", chunk.len)), + ); + } + } + } + } + } + + // Every recorded chunk's object (it also finds chunks no manifest names, which a manifest walk would not). + let mut after: Option<[u8; 32]> = None; + loop { + let records = meta.blob_records_after(BlobKind::Chunk, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; + after = Some(last); + for record in records { + let raw = record.id; + report.chunks_checked += 1; + if !options.read_chunks { + continue; + } + match store.get_sized(BlobKey::chunk(&ChunkId::from_bytes(raw))).await { + // Read, and its hash is right: its length is what the database records. + Ok((_, stored)) => { + if stored != record.stored_size { + report.add( + "chunk_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + } + Err(e) => { + let (kind, detail) = store_problem(&e); + let kind = match kind { + "object_missing" => "chunk_object_missing", + "object_corrupt" => "chunk_object_corrupt", + other => other, + }; + report.add(kind, hex(&raw), detail); + } + } + } + } + Ok(report) +} + +/// What an `index build` run did. +#[derive(Debug, Clone, Default)] +pub struct BuildReport { + /// Manifests whose index is complete after the run that were not before. + pub indexed: u64, + /// Manifests that could not be indexed: `(manifest, why)`. Their index stays incomplete. + pub unreadable: Vec<(String, &'static str)>, +} + +/// Completes the index of every recorded manifest whose index is not complete, in batches of at most the index budget +/// per transaction, so it can run beside the server and be stopped and run again. A manifest is read from the store and +/// checked against its ID first; one that cannot be read is reported and left unindexed (its chunks stay unknown). Only +/// the derived index is written. +pub async fn build_index(meta: &Meta, store: &Store) -> Result { + let mut report = BuildReport::default(); + let mut after: Option = None; + loop { + let ids = meta.manifests_to_index(after, PAGE).await?; + let Some(last) = ids.last().copied() else { break }; + after = Some(last); + for id in ids { + match read_manifest(store, &id).await { + Ok(manifest) => { + meta.index_complete(&manifest).await?; + report.indexed += 1; + } + Err((kind, _)) => report.unreadable.push((hex(id.as_bytes()), kind)), + } + } + } + Ok(report) +} diff --git a/apps/craft-cloud-server/src/web/html.rs b/apps/craft-cloud-server/src/web/html.rs new file mode 100644 index 00000000..21f6fb36 --- /dev/null +++ b/apps/craft-cloud-server/src/web/html.rs @@ -0,0 +1,255 @@ +//! The pages' building blocks: escaping, the layout, and the form fields. +//! +//! Pages are plain `format!`ed HTML. The rule that keeps them safe is small: **every value that is not a literal in this +//! code goes through [`esc`]** before it is put into markup, and the helpers here escape their own arguments, so a page +//! built from them has no place to forget it. There is no script, no inline style and no external file anywhere (the +//! CSP, see [`super::secure_headers`], forbids all three), so a value that did slip through could still not run code. + +use std::fmt::Write; + +/// `text` escaped for HTML text and for attribute values in double or single quotes. +pub fn esc(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + // Control characters other than the usual whitespace have no place in a page. + c if c.is_control() && !matches!(c, '\n' | '\t') => out.push('\u{fffd}'), + c => out.push(c), + } + } + out +} + +/// A time in milliseconds since the Unix epoch as `2026-11-04 13:05:09 UTC`, or `-` for one that is out of range. +pub fn fmt_time(ms: i64) -> String { + let secs = ms.div_euclid(1000); + let (days, rest) = (secs.div_euclid(86_400), secs.rem_euclid(86_400)); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm), in checked arithmetic. + let date = || -> Option<(i64, i64, i64)> { + let z = days.checked_add(719_468)?; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe.checked_add(era.checked_mul(400)?)?.checked_add(i64::from(month <= 2))?; + Some((year, month, day)) + }; + match date() { + Some((y, m, d)) if (0..=9999).contains(&y) => format!("{y:04}-{m:02}-{d:02} {:02}:{:02}:{:02} UTC", rest / 3600, rest % 3600 / 60, rest % 60), + _ => "-".to_owned(), + } +} + +/// A size in bytes as `1.5 GiB`, exact below one KiB. +pub fn fmt_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["KiB", "MiB", "GiB", "TiB", "PiB"]; + if bytes < 1024 { + return format!("{bytes} B"); + } + // Tenths of the unit, in 128 bits so that nothing overflows. + let mut tenths = u128::from(bytes) * 10 / 1024; + let mut unit = 0usize; + while tenths >= 10_240 && unit + 1 < UNITS.len() { + tenths /= 1024; + unit += 1; + } + format!("{}.{} {}", tenths / 10, tenths % 10, UNITS.get(unit).copied().unwrap_or("PiB")) +} + +/// What a page shows about the signed-in admin. +pub struct Ctx { + pub username: String, + /// The CSRF token the forms of this session carry. + pub csrf: String, +} + +/// The whole page: `body` is markup built with this module's helpers, `title` plain text. +pub fn page(ctx: Option<&Ctx>, title: &str, body: &str) -> String { + let nav = match ctx { + Some(ctx) => format!( + "", + esc(&ctx.csrf), + esc(&ctx.username) + ), + None => String::new(), + }; + format!( + "\n\ + {} - Craft Cloud admin\ + Skip to the content\ +

Craft Cloud admin

{nav}
{body}
\n", + esc(title) + ) +} + +/// A page heading and its first paragraph. +pub fn heading(text: &str) -> String { + format!("

{}

", esc(text)) +} + +/// A notice that something worked (`role=status`) or failed (`role=alert`). +pub fn notice(text: &str, error: bool) -> String { + let (class, role) = if error { ("notice error", "alert") } else { ("notice", "status") }; + format!("

{}

", esc(text)) +} + +/// A link to a path made by this code (never to a value from the outside as it is). +pub fn link(href: &str, text: &str) -> String { + format!("{}", esc(href), esc(text)) +} + +/// Percent-encoding for a value that goes into a query string. +pub fn url_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + out.push(char::from(byte)); + } else { + let _ = write!(out, "%{byte:02X}"); + } + } + out +} + +/// A labelled text field. `kind` is an input type from this code (`text`, `password`, `search`, `number`). +pub fn field(form: &str, name: &str, label: &str, kind: &str, value: &str, extra: &str) -> String { + let id = esc(&format!("{form}-{name}")); + format!( + "

", + esc(label), + esc(name), + esc(value) + ) +} + +/// A labelled select; `options` are `(value, text)` pairs and `selected` the value that is chosen. +pub fn select(form: &str, name: &str, label: &str, options: &[(&str, &str)], selected: &str) -> String { + let id = esc(&format!("{form}-{name}")); + let mut out = format!("

"); + out +} + +/// A labelled checkbox. +pub fn checkbox(form: &str, name: &str, label: &str) -> String { + let id = esc(&format!("{form}-{name}")); + format!( + "

", + esc(name), + esc(label) + ) +} + +/// The confirmation every change asks for: the admin's password and, when their account has two-factor sign-in, a code or a +/// recovery code. The server asks again whatever the page showed. +pub fn confirm_fields(form: &str, two_factor: bool) -> String { + let mut out = + field(form, "password", "Your password, to confirm", "password", "", "required autocomplete=\"current-password\" maxlength=\"1024\""); + if two_factor { + out.push_str(&field( + form, + "totp_code", + "Code from your authenticator app", + "text", + "", + "inputmode=\"numeric\" autocomplete=\"one-time-code\" pattern=\"[0-9]{6}\" maxlength=\"6\"", + )); + out.push_str(&field(form, "recovery_code", "Or a recovery code", "text", "", "autocomplete=\"off\" maxlength=\"32\" spellcheck=\"false\"")); + } + out +} + +/// A form that posts to `action` with the session's CSRF token, `inner` being markup built with the helpers above. +pub fn post_form(ctx: &Ctx, action: &str, class: &str, inner: &str) -> String { + format!( + "
{inner}
", + esc(action), + esc(class), + esc(&ctx.csrf) + ) +} + +/// A table with a caption and column headers. Every cell is markup already built with [`esc`]. +pub fn table(caption: &str, headers: &[&str], rows: &[Vec]) -> String { + let mut out = format!("", esc(caption)); + for header in headers { + let _ = write!(out, "", esc(header)); + } + out.push_str(""); + for row in rows { + out.push_str(""); + for (n, cell) in row.iter().enumerate() { + if n == 0 { + let _ = write!(out, ""); + } else { + let _ = write!(out, ""); + } + } + out.push_str(""); + } + out.push_str("
{}
{}
{cell}{cell}
"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn markup_characters_are_escaped_everywhere_a_value_goes() { + let nasty = "'&"; + assert_eq!(esc(nasty), "<script>alert("x")</script>'&"); + assert_eq!(esc("a\u{0}b\u{7}c\nd"), "a\u{fffd}b\u{fffd}c\nd"); + for html in [ + heading(nasty), + notice(nasty, true), + link(nasty, nasty), + field("f", nasty, nasty, "text", nasty, ""), + select("f", nasty, nasty, &[(nasty, nasty)], nasty), + checkbox("f", nasty, nasty), + page(Some(&Ctx { username: nasty.to_owned(), csrf: nasty.to_owned() }), nasty, ""), + post_form(&Ctx { username: String::new(), csrf: nasty.to_owned() }, nasty, nasty, ""), + table(nasty, &[nasty], &[]), + ] { + assert!(!html.contains("".into(), + stored_bytes: 12, + quota_bytes: Some(100), + }], + manifest_index: craft_cloud_meta::StorageIndexCounts { complete: 1, building: 0, not_indexed: 0 }, + }; + let html = storage_section(&storage); + assert!(html.contains("<script>alert('x')</script>"), "{html}"); + assert!(!html.contains(""; + let platform = "&\""; + let res = s + .api( + None, + Method::POST, + "/auth/login", + Some(json!({ "username": "ana", "password": PASSWORD, "device_name": name, "platform": platform, "app_version": "" })), + ) + .await; + assert_eq!(res.status(), StatusCode::OK); + // A username can't hold markup, but a row in a damaged or hand-edited database could: the pages must hold anyway. + use sqlx::Connection; + sqlx::any::install_default_drivers(); + let mut conn = sqlx::AnyConnection::connect(&s.url).await.unwrap(); + sqlx::query("UPDATE users SET username = $1 WHERE username = 'ana'").bind("").execute(&mut conn).await.unwrap(); + drop(conn); + let pages = [ + s.get("/web", Some(&sess.token)).await, + s.get("/web/users", Some(&sess.token)).await, + s.get(&format!("/web/users/{}", ana.id.0), Some(&sess.token)).await, + s.get("/web/audit", Some(&sess.token)).await, + s.get("/web/users?query=%3Cscript%3Ealert(2)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/audit?action=%22%3E%3Cscript%3Ealert(3)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/audit?user=%3Cscript%3Ealert(4)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/%3Cscript%3Ealert(5)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get(&format!("/web/users/{}?notice=%3Cscript%3E", ana.id.0), Some(&sess.token)).await, + ]; + let mut shown = String::new(); + for res in pages { + let body = res.text().await.unwrap(); + assert!(!body.contains(""), "{body}"); + shown.push_str(&body); + } + assert!(shown.contains("<script>alert(1)</script>"), "the user name is shown, escaped"); + assert!(shown.contains("<script>alert('d')</script>"), "the device name is shown, escaped"); + assert!(shown.contains("<b onmouseover=x>&""), "the platform is shown, escaped"); + assert!(shown.contains("<script>alert(2)</script>"), "the search is shown back, escaped"); + assert!(shown.contains("Storage") && shown.contains("Top accounts by stored bytes"), "the storage overview is shown"); + assert!(shown.contains("<script>alert(1)</script>"), "the top-account name is escaped"); + assert!( + shown.contains("Effective quota") && shown.contains("Trash estimate (overlapping)") && shown.contains("History estimate (overlapping)"), + "the account usage is shown" + ); +} + +async fn old_sessions_are_cleaned_up_at_sign_in(s: &Web) { + let root = s.add_admin("root").await; + s.clock.store(craft_cloud_server::state::now_ms(), Ordering::SeqCst); + for _ in 0..5 { + s.sign_in("root").await; + } + let out = s.sign_in("root").await; + s.post("/web/logout", Some(&out.token), &[("csrf", &out.csrf)]).await; + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 1, "the logged-out one"); + s.advance(IDLE_MS + 1); + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 6, "five idle ones and the logged-out one"); + // The next sign-in deletes them on its way and is itself unharmed. + let fresh = s.sign_in("root").await; + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 0); + assert!(signed_in(s, &fresh).await); + assert_eq!(s.meta.web_sessions_open(root.id, s.at(), IDLE_MS).await.unwrap(), 1); +} + +async fn form_bodies_are_small_and_odd_ones_refused(s: &Web) { + s.add_admin("root").await; + let sess = s.sign_in("root").await; + // More than 4 KiB is 413, and no form at all is not a sign-in. + let big = "x".repeat(5000); + let res = s.post("/web/login", None, &[("username", "root"), ("password", &big)]).await; + assert_eq!(res.status(), StatusCode::PAYLOAD_TOO_LARGE); + let res = s + .client + .post(s.url("/web/login")) + .header("origin", ORIGIN) + .header("content-type", "application/json") + .body("{\"username\":\"root\",\"password\":\"x\"}") + .send() + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST, "a body that is not a form is a malformed request"); + let res = s + .client + .post(s.url("/web/logout")) + .header("origin", ORIGIN) + .header("cookie", format!("{COOKIE_NAME}={}", sess.token)) + .header("content-type", "application/x-www-form-urlencoded") + .body(vec![0xff, 0xfe, 0x25, 0x00]) + .send() + .await + .unwrap(); + assert!(res.status().is_client_error()); + assert!(signed_in(s, &sess).await); + // Parameters of the wrong kind in a query are a 400 page. + assert_eq!(s.get("/web/users?query=a&query=b&cursor=%FF", Some(&sess.token)).await.status().as_u16() / 100, 4); +} + +// ---- off by default, and its own tests ---- + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_pages_do_not_exist_unless_the_server_was_set_up_for_them() { + let s = sqlite_with(vec![], false).await; + for path in ["/web", "/web/", "/web/login", "/web/style.css", "/web/users"] { + let res = s.get(path, None).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND, "{path}"); + assert_eq!(header(&res, "content-type"), "application/json", "{path}: the API's own not-found answer"); + } + let res = s.post("/web/login", None, &[("username", "root"), ("password", PASSWORD)]).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND); + // The API is unharmed. + s.add_admin("root").await; + assert!(!s.device("root").await.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_client_address_behind_a_trusted_proxy_is_what_the_limits_count() { + let s = sqlite_with(vec!["127.0.0.1".parse().unwrap()], true).await; + s.add_admin("root").await; + // Thirty wrong tries from one forwarded address use it up, and another forwarded address still signs in. + for n in 0..30 { + let name = format!("guess{n}"); + let res = s + .client + .post(s.url("/web/login")) + .header("origin", ORIGIN) + .header("x-forwarded-for", "203.0.113.7") + .header("content-type", "application/x-www-form-urlencoded") + .body(encode(&[("username", &name), ("password", "x")])) + .send() + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); + } + let post = |from: &'static str| { + let (client, url) = (s.client.clone(), s.url("/web/login")); + async move { + client + .post(url) + .header("origin", ORIGIN) + .header("x-forwarded-for", from) + .header("content-type", "application/x-www-form-urlencoded") + .body(encode(&[("username", "root"), ("password", PASSWORD)])) + .send() + .await + .unwrap() + } + }; + assert_eq!(post("203.0.113.7").await.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(post("203.0.113.8").await.status(), StatusCode::SEE_OTHER); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn wrong_confirmation_passwords_are_limited_per_account_on_sqlite() { + wrong_confirmation_passwords_are_limited_per_account(&sqlite_web().await).await; +} + +// ---- review fixes ---- + +async fn an_account_without_a_password_never_signs_in(s: &Web) { + // What an invite makes: an account whose password hash is empty. It is answered like a name nobody has, whatever is typed, + // the server's own dummy password included (which is what an unknown name is checked against). + s.meta.create_user("pending", "", Role::Admin, 0).await.unwrap(); + let unknown = refused(s.post("/web/login", None, &[("username", "nobody"), ("password", PASSWORD)]).await).await; + for password in [PASSWORD, "", "craft-cloud dummy password", "$argon2id$"] { + let answer = refused(s.post("/web/login", None, &[("username", "pending"), ("password", password)]).await).await; + assert_eq!(answer, unknown, "{password:?}"); + } + for password in [PASSWORD, "craft-cloud dummy password"] { + let res = s.api(None, Method::POST, "/auth/login", Some(json!({ "username": "pending", "password": password, "device_name": "pc" }))).await; + assert_eq!(res.status(), StatusCode::UNAUTHORIZED, "{password:?}: the JSON sign-in too"); + } +} + +async fn the_slash_path_has_the_same_headers_and_policy(s: &Web) { + let get = s.get("/web/", None).await; + assert_eq!((get.status(), location(&get).as_str()), (StatusCode::SEE_OTHER, "/web")); + let head = s.client.head(s.url("/web/")).send().await.unwrap(); + assert_eq!((head.status(), location(&head).as_str()), (StatusCode::SEE_OTHER, "/web")); + let wrong_method = s.post("/web/", None, &[]).await; + assert_eq!(wrong_method.status(), StatusCode::METHOD_NOT_ALLOWED); + let no_origin = s.post_from("/web/", None, None, None, &[]).await; + assert_eq!(no_origin.status(), StatusCode::FORBIDDEN, "a post from nowhere is refused here too"); + for res in [get, head, wrong_method, no_origin] { + let csp = header(&res, "content-security-policy"); + assert!(csp.contains("script-src 'none'") && csp.contains("frame-ancestors 'none'"), "{}: {csp}", res.status()); + assert_eq!(header(&res, "x-frame-options"), "DENY"); + assert_eq!(header(&res, "x-content-type-options"), "nosniff"); + assert_eq!(header(&res, "referrer-policy"), "no-referrer"); + assert_eq!(header(&res, "cache-control"), "no-store"); + } +} + +/// A change whose body is still arriving when its session ends is refused, though the session was valid when the request began. +async fn a_change_that_outlives_its_session_is_refused(s: &Web) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + s.add_admin("root").await; + let vic = s.add_user("vic", Role::Member).await; + s.clock.store(craft_cloud_server::state::now_ms(), Ordering::SeqCst); + let sess = s.sign_in("root").await; + let body = encode(&[("csrf", &sess.csrf), ("password", PASSWORD)]); + let (first, rest) = body.split_at(8); + let host = s.base.trim_start_matches("http://").to_owned(); + let mut stream = tokio::net::TcpStream::connect(&host).await.unwrap(); + let head = format!( + "POST /web/users/{}/disable HTTP/1.1\r\nHost: {host}\r\nOrigin: {ORIGIN}\r\nCookie: {COOKIE_NAME}={}\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{first}", + vic.id.0, + sess.token, + body.len() + ); + stream.write_all(head.as_bytes()).await.unwrap(); + // The server has looked at the session by now (it does so before it reads the form); the session ends; the body goes on. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + s.advance(IDLE_MS + 1); + stream.write_all(rest.as_bytes()).await.unwrap(); + let mut answer = String::new(); + stream.read_to_string(&mut answer).await.unwrap(); + assert!(answer.starts_with("HTTP/1.1 401"), "{answer}"); + assert!(s.meta.admin_user(vic.id, s.at()).await.unwrap().unwrap().disabled_ms.is_none(), "nothing changed"); +} + +/// Disabling or removing an account over the pages ends the long-polls it has on its team spaces at once, as the JSON routes do. +async fn disabling_or_removing_over_the_pages_wakes_the_spaces_long_polls(s: &Web) { + use craft_cloud_meta::SpaceActor; + use craft_cloud_proto::SpaceName; + use craft_cloud_proto::api::SpaceRole; + s.add_admin("root").await; + let owner = s.add_user("own", Role::Member).await; + let (away, gone) = (s.add_user("away", Role::Member).await, s.add_user("gone", Role::Member).await); + let space = s.meta.create_team_space(&SpaceActor::Cli, &SpaceName::new("Studio").unwrap(), owner.id, None, 0).await.unwrap(); + for member in [&away, &gone] { + s.meta.invite_member(&SpaceActor::Cli, space, member.id, SpaceRole::Viewer, 0).await.unwrap(); + } + let boss = s.sign_in("root").await; + for (name, verb, id) in [("away", "disable", away.id.0), ("gone", "remove", gone.id.0)] { + let device = s.device(name).await; + let poll = { + let (client, url) = (s.client.clone(), s.url("/api/v1/changes?since=0&wait=30")); + tokio::spawn(async move { client.get(url).bearer_auth(device).header("craft-space", space.to_string()).send().await.unwrap() }) + }; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert!(!poll.is_finished(), "{name}: the long-poll waits"); + assert_eq!(s.act(&boss, &format!("/web/users/{id}/{verb}"), &[]).await.status(), StatusCode::SEE_OTHER); + let res = tokio::time::timeout(std::time::Duration::from_secs(5), poll) + .await + .unwrap_or_else(|_| panic!("{name}: the waiter was not woken")) + .unwrap(); + assert!(matches!(res.status(), StatusCode::NOT_FOUND | StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN), "{name}: {}", res.status()); + } +} + +// ---- the Team spaces pages ---- + +/// A team space made by the command line (so a test starts from a space without going through the page under test). +async fn cli_space(s: &Web, name: &str, owner: &craft_cloud_meta::UserRecord) -> craft_cloud_proto::SpaceId { + use craft_cloud_meta::SpaceActor; + use craft_cloud_proto::SpaceName; + s.meta.create_team_space(&SpaceActor::Cli, &SpaceName::new(name).unwrap(), owner.id, None, 0).await.unwrap() +} + +/// Every write of the Team spaces page, for the space `id`. +fn space_changes(id: &str) -> Vec<(String, Vec<(&'static str, &'static str)>)> { + vec![ + ("/web/spaces".to_owned(), vec![("name", "Fresh"), ("owner", "own")]), + (format!("/web/spaces/{id}/quota"), vec![("quota_mib", "8")]), + (format!("/web/spaces/{id}/archive"), vec![]), + (format!("/web/spaces/{id}/unarchive"), vec![]), + (format!("/web/spaces/{id}/owner"), vec![("owner", "newcomer")]), + ] +} + +async fn the_space_is_untouched(s: &Web, space: craft_cloud_proto::SpaceId) { + let found = s.meta.admin_space(space).await.unwrap(); + assert_eq!((found.name.as_str(), found.archived_ms, found.quota_bytes, found.owners, found.members), ("Studio", None, None, 1, 1)); + assert_eq!(s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.len(), 1, "no space was made"); +} + +async fn the_team_spaces_pages_are_for_administrators_and_need_csrf(s: &Web) { + s.add_admin("root").await; + s.add_admin("demoted").await; + let owner = s.add_user("own", Role::Member).await; + s.add_user("newcomer", Role::Member).await; + let space = cli_space(s, "Studio", &owner).await; + let id = space.as_uuid().to_string(); + let (boss, demoted) = (s.sign_in("root").await, s.sign_in("demoted").await); + let res = s.act(&boss, &format!("/web/users/{}/update", s.id_of("demoted").await), &[("role", "member")]).await; + assert_eq!(res.status(), StatusCode::SEE_OTHER); + + // No session: the sign-in page for a page, 401 for a change. A demoted administrator's session: 403 for both. + for path in ["/web/spaces".to_owned(), format!("/web/spaces/{id}")] { + let res = s.get(&path, None).await; + assert_eq!((res.status(), location(&res).as_str()), (StatusCode::SEE_OTHER, "/web/login"), "{path}"); + let res = s.get(&path, Some(&demoted.token)).await; + assert_eq!(res.status(), StatusCode::FORBIDDEN, "{path}"); + assert!(!res.text().await.unwrap().contains("Studio"), "{path}"); + } + for (path, extra) in space_changes(&id) { + let mut fields = extra.clone(); + fields.push(("csrf", &boss.csrf)); + fields.push(("password", PASSWORD)); + assert_eq!(s.post(&path, None, &fields).await.status(), StatusCode::UNAUTHORIZED, "{path}: no session"); + let res = s.act(&demoted, &path, &extra).await; + assert_eq!(res.status(), StatusCode::FORBIDDEN, "{path}: a demoted administrator"); + // The CSRF token: missing, wrong, and another session's. + let mut fields = extra.clone(); + fields.push(("password", PASSWORD)); + assert_eq!(s.post(&path, Some(&boss.token), &fields).await.status(), StatusCode::FORBIDDEN, "{path}: no token"); + for csrf in ["", "wrong", demoted.csrf.as_str()] { + let mut fields = fields.clone(); + fields.push(("csrf", csrf)); + let res = s.post(&path, Some(&boss.token), &fields).await; + assert_eq!(res.status(), StatusCode::FORBIDDEN, "{path}: {csrf:?}"); + assert!(res.text().await.unwrap().contains("out of date, or did not come from this site")); + } + // The origin: another site's form is refused before anything is read. + let mut fields = extra.clone(); + fields.push(("csrf", &boss.csrf)); + fields.push(("password", PASSWORD)); + let res = s.post_from(&path, Some(&boss.token), Some("https://evil.example"), None, &fields).await; + assert_eq!(res.status(), StatusCode::FORBIDDEN, "{path}: another origin"); + // A GET changes nothing. + let get = s.get(&path, Some(&boss.token)).await.status(); + assert_eq!(get, if path == "/web/spaces" { StatusCode::OK } else { StatusCode::METHOD_NOT_ALLOWED }, "{path}"); + } + the_space_is_untouched(s, space).await; +} + +async fn every_team_space_change_needs_the_confirmation(s: &Web) { + let owner = s.add_user("own", Role::Member).await; + s.add_user("newcomer", Role::Member).await; + let space = cli_space(s, "Studio", &owner).await; + for (n, (path, extra)) in space_changes(&space.as_uuid().to_string()).iter().enumerate() { + // A fresh administrator per route: wrong passwords are counted per account. + let name = format!("adm{n}"); + s.add_admin(&name).await; + let sess = s.sign_in(&name).await; + let mut fields: Vec<(&str, &str)> = extra.clone(); + fields.push(("csrf", &sess.csrf)); + let res = s.post(path, Some(&sess.token), &fields).await; + assert_eq!(res.status(), StatusCode::BAD_REQUEST, "{path}: no password"); + let mut wrong = fields.clone(); + wrong.push(("password", "a wrong password")); + let res = s.post(path, Some(&sess.token), &wrong).await; + assert_eq!(res.status(), StatusCode::FORBIDDEN, "{path}: a wrong password"); + assert!(res.text().await.unwrap().contains("the current password is wrong"), "{path}"); + let mut empty = fields.clone(); + empty.push(("password", "")); + assert_eq!(s.post(path, Some(&sess.token), &empty).await.status(), StatusCode::BAD_REQUEST, "{path}: an empty password"); + let mut long = fields.clone(); + let too_long = "x".repeat(1025); + long.push(("password", &too_long)); + assert_eq!(s.post(path, Some(&sess.token), &long).await.status(), StatusCode::BAD_REQUEST, "{path}: a password that is too long"); + the_space_is_untouched(s, space).await; + } +} + +async fn team_spaces_are_made_listed_and_changed(s: &Web) { + s.add_admin("root").await; + let owner = s.add_user("own", Role::Member).await; + let heir = s.add_user("heir", Role::Member).await; + let gone = s.add_user("gone", Role::Member).await; + let boss = s.sign_in("root").await; + + // The page: an empty list and the form, with the confirmation's password field and the navigation entry. + let html = s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("No team spaces match.") && html.contains("action=\"/web/spaces\"") && html.contains("name=\"password\""), "{html}"); + assert!(html.contains("Team spaces"), "the navigation names the page"); + assert!(!html.contains("name=\"totp_code\""), "no code field without two-factor sign-in"); + + // Create: the errors first, each with its status, and nothing made. + let create = |fields: Vec<(&'static str, &'static str)>| { + let boss = &boss; + async move { s.act(boss, "/web/spaces", &fields).await } + }; + assert_eq!(create(vec![("name", ""), ("owner", "own")]).await.status(), StatusCode::BAD_REQUEST, "an empty name"); + assert_eq!(create(vec![("name", " padded "), ("owner", "own")]).await.status(), StatusCode::BAD_REQUEST, "a padded name"); + assert_eq!(create(vec![("name", "Studio"), ("owner", "no such")]).await.status(), StatusCode::BAD_REQUEST, "an owner that is no username"); + assert_eq!(create(vec![("name", "Studio"), ("owner", "nobody")]).await.status(), StatusCode::NOT_FOUND, "an unknown owner"); + assert_eq!(create(vec![("name", "Studio"), ("owner", "own"), ("quota_mib", "3")]).await.status(), StatusCode::BAD_REQUEST, "below one chunk"); + assert_eq!(create(vec![("name", "Studio"), ("owner", "own"), ("quota_mib", "lots")]).await.status(), StatusCode::BAD_REQUEST, "not a number"); + assert_eq!( + create(vec![("name", "Studio"), ("owner", "own"), ("remove_quota", "on")]).await.status(), + StatusCode::BAD_REQUEST, + "nothing to remove" + ); + assert!(s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.is_empty()); + s.act(&boss, &format!("/web/users/{}/disable", gone.id.0), &[]).await; + assert_eq!(create(vec![("name", "Studio"), ("owner", "gone")]).await.status(), StatusCode::NOT_FOUND, "a disabled owner is the answer for none"); + assert!(s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.is_empty()); + + // Create: success goes to the space's page, which says so; the owner is a member from the start. + let res = create(vec![("name", "Studio"), ("owner", "Own"), ("quota_mib", "16")]).await; + assert_eq!(res.status(), StatusCode::SEE_OTHER); + let page_path = location(&res); + assert!(page_path.starts_with("/web/spaces/") && page_path.ends_with("?notice=created"), "{page_path}"); + let space = s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.remove(0); + assert_eq!((space.name.as_str(), space.owners, space.members, space.quota_bytes), ("Studio", 1, 1, Some(16 * 1024 * 1024))); + assert_eq!(create(vec![("name", "STUDIO"), ("owner", "heir")]).await.status(), StatusCode::CONFLICT, "a name is taken in any case"); + assert_eq!(s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.len(), 1); + + let html = s.get(&page_path, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("The team space was created.") && html.contains("Team space Studio"), "{html}"); + assert!(html.contains("16.0 MiB") && html.contains(&format!("/web/users/{}", owner.id.0)) && html.contains("owner"), "{html}"); + let html = s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("Studio") && html.contains("0 B of 16.0 MiB") && html.contains("active"), "{html}"); + let base = format!("/web/spaces/{}", space.id.as_uuid()); + + // The limit: set, refused, removed. + let post = |what: &str, fields: Vec<(&'static str, &'static str)>| { + let (boss, path) = (&boss, format!("{base}/{what}")); + async move { s.act(boss, &path, &fields).await } + }; + let res = post("quota", vec![("quota_mib", "64")]).await; + assert_eq!((res.status(), location(&res)), (StatusCode::SEE_OTHER, format!("{base}?notice=quota"))); + assert_eq!(s.meta.admin_space(space.id).await.unwrap().quota_bytes, Some(64 * 1024 * 1024)); + for (fields, status) in [ + (vec![], StatusCode::BAD_REQUEST), + (vec![("quota_mib", "1")], StatusCode::BAD_REQUEST), + (vec![("quota_mib", "-5")], StatusCode::BAD_REQUEST), + (vec![("quota_mib", "8"), ("remove_quota", "on")], StatusCode::BAD_REQUEST), + ] { + assert_eq!(post("quota", fields.clone()).await.status(), status, "{fields:?}"); + } + assert_eq!(s.meta.admin_space(space.id).await.unwrap().quota_bytes, Some(64 * 1024 * 1024), "a refused limit changes nothing"); + assert_eq!(post("quota", vec![("remove_quota", "on")]).await.status(), StatusCode::SEE_OTHER); + assert_eq!(s.meta.admin_space(space.id).await.unwrap().quota_bytes, None); + let html = s.get(&format!("{base}?notice=quota"), Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("The space's limit was changed.") && html.contains("
Limit
none
"), "{html}"); + + // Archive and unarchive: repeating either is no error. + for _ in 0..2 { + let res = post("archive", vec![]).await; + assert_eq!((res.status(), location(&res)), (StatusCode::SEE_OTHER, format!("{base}?notice=archived"))); + assert!(s.meta.admin_space(space.id).await.unwrap().archived_ms.is_some()); + } + let html = s.get(&base, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("archived 20") && html.contains("Make the space active again") && !html.contains("Archive the space"), "{html}"); + assert!(s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap().contains("archived 20"), "the list says so"); + for _ in 0..2 { + let res = post("unarchive", vec![]).await; + assert_eq!((res.status(), location(&res)), (StatusCode::SEE_OTHER, format!("{base}?notice=unarchived"))); + assert!(s.meta.admin_space(space.id).await.unwrap().archived_ms.is_none()); + } + + // The owner: an account named becomes an accepted owner at once; unknown, disabled and no username are refused. + assert_eq!(post("owner", vec![("owner", "nobody")]).await.status(), StatusCode::NOT_FOUND); + assert_eq!(post("owner", vec![("owner", "gone")]).await.status(), StatusCode::NOT_FOUND); + assert_eq!(post("owner", vec![("owner", "")]).await.status(), StatusCode::BAD_REQUEST); + assert_eq!(post("owner", vec![("owner", "a b")]).await.status(), StatusCode::BAD_REQUEST); + assert_eq!(s.meta.admin_space(space.id).await.unwrap().owners, 1); + let res = post("owner", vec![("owner", "heir")]).await; + assert_eq!((res.status(), location(&res)), (StatusCode::SEE_OTHER, format!("{base}?notice=owner"))); + assert_eq!(post("owner", vec![("owner", "heir")]).await.status(), StatusCode::SEE_OTHER, "naming an owner twice is no error"); + let members = s.meta.list_members(space.id).await.unwrap(); + assert_eq!(members.iter().filter(|m| m.accepted() && m.role == craft_cloud_proto::api::SpaceRole::Owner).count(), 2); + assert!(members.iter().any(|m| m.user == heir.id)); + let html = s.get(&base, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("heir"), "{html}"); + + // A space that is not there, or an ID that is none: 404 for the page and for every change. + let missing = uuid::Uuid::now_v7(); + for path in [format!("/web/spaces/{missing}"), "/web/spaces/not-an-id".to_owned()] { + assert_eq!(s.get(&path, Some(&boss.token)).await.status(), StatusCode::NOT_FOUND, "{path}"); + for what in ["quota", "archive", "unarchive", "owner"] { + let res = s.act(&boss, &format!("{path}/{what}"), &[("quota_mib", "8"), ("owner", "heir")]).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND, "{path}/{what}"); + } + } + // A personal space is no team space for these pages either. + let personal = s.meta.user_by_name("own").await.unwrap().unwrap().space; + assert_eq!(s.get(&format!("/web/spaces/{}", personal.as_uuid()), Some(&boss.token)).await.status(), StatusCode::NOT_FOUND); + assert_eq!(s.act(&boss, &format!("/web/spaces/{}/archive", personal.as_uuid()), &[]).await.status(), StatusCode::NOT_FOUND); + // An unknown notice has no words. + let html = s.get(&format!("{base}?notice=%3Cb%3E"), Some(&boss.token)).await.text().await.unwrap(); + assert!(!html.contains("") && !html.contains("class=\"notice\""), "{html}"); +} + +async fn team_spaces_are_listed_in_pages_and_searched(s: &Web) { + use craft_cloud_meta::SpaceActor; + use craft_cloud_proto::SpaceName; + s.add_admin("root").await; + let owner = s.add_user("own", Role::Member).await; + for n in 0..52 { + let name = SpaceName::new(&format!("Team {n:02}")).unwrap(); + s.meta.create_team_space(&SpaceActor::Cli, &name, owner.id, None, 0).await.unwrap(); + } + cli_space(s, "Düsseldorf Büro", &owner).await; + let boss = s.sign_in("root").await; + let first = s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap(); + assert!(first.contains("Team 00") && !first.contains("Team 51") && !first.contains("First page"), "the first page has 50"); + let marker = "href=\"/web/spaces?query=&cursor="; + let next = first.split(marker).nth(1).expect("a next page").split('"').next().unwrap().to_owned(); + let second = s.get(&format!("/web/spaces?query=&cursor={next}"), Some(&boss.token)).await.text().await.unwrap(); + assert!(second.contains("Team 51") && !second.contains("Team 00") && second.contains("First page") && !second.contains("Next page"), "{second}"); + // A search is a part of a name, in any case, also with letters that fold; an empty result says so; it is literal text, not a pattern. + let found = s.get("/web/spaces?query=B%C3%9CRO", Some(&boss.token)).await.text().await.unwrap(); + assert!(found.contains("Düsseldorf Büro") && !found.contains("Team 00"), "{found}"); + let found = s.get("/web/spaces?query=team%2051", Some(&boss.token)).await.text().await.unwrap(); + assert!(found.contains("Team 51") && !found.contains("Team 50"), "{found}"); + assert!(s.get("/web/spaces?query=zzz", Some(&boss.token)).await.text().await.unwrap().contains("No team spaces match.")); + let found = s.get("/web/spaces?query=100%25_", Some(&boss.token)).await.text().await.unwrap(); + assert!(found.contains("No team spaces match."), "a search is literal text, not a pattern"); + // Bad queries: too long, a control character, a cursor that is none. + for bad in [ + format!("/web/spaces?query={}", "a".repeat(65)), + "/web/spaces?query=a%00b".to_owned(), + "/web/spaces?cursor=***".to_owned(), + "/web/spaces?cursor=_-8".to_owned(), + ] { + assert_eq!(s.get(&bad, Some(&boss.token)).await.status(), StatusCode::BAD_REQUEST, "{bad}"); + } +} + +async fn team_space_changes_are_audited_with_the_web_session_and_wake_the_long_polls(s: &Web) { + use craft_cloud_meta::AuditAction; + s.add_admin("root").await; + let root = s.id_of("root").await; + let owner = s.add_user("own", Role::Member).await; + let heir = s.add_user("heir", Role::Member).await; + let boss = s.sign_in("root").await; + let res = s.act(&boss, "/web/spaces", &[("name", "Studio"), ("owner", "own"), ("quota_mib", "8")]).await; + assert_eq!(res.status(), StatusCode::SEE_OTHER); + let space = s.meta.admin_list_spaces(None, None, 10).await.unwrap().spaces.remove(0).id; + let base = format!("/web/spaces/{}", space.as_uuid()); + + // The owner's long-poll on the space is woken by every change of the space the page makes. + let device = s.device("own").await; + for (what, extra) in [("archive", vec![]), ("quota", vec![("quota_mib", "16")]), ("unarchive", vec![]), ("owner", vec![("owner", "heir")])] { + let poll = { + let (client, url, device) = (s.client.clone(), s.url("/api/v1/changes?since=0&wait=30"), device.clone()); + tokio::spawn(async move { client.get(url).bearer_auth(device).header("craft-space", space.to_string()).send().await.unwrap() }) + }; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert!(!poll.is_finished(), "{what}: the long-poll waits"); + assert_eq!(s.act(&boss, &format!("{base}/{what}"), &extra).await.status(), StatusCode::SEE_OTHER, "{what}"); + tokio::time::timeout(std::time::Duration::from_secs(5), poll).await.unwrap_or_else(|_| panic!("{what}: the waiter was not woken")).unwrap(); + } + // Repeating a change that changes nothing writes no second event. + assert_eq!(s.act(&boss, &format!("{base}/unarchive"), &[]).await.status(), StatusCode::SEE_OTHER); + assert_eq!(s.act(&boss, &format!("{base}/owner"), &[("owner", "heir")]).await.status(), StatusCode::SEE_OTHER); + + let events = s.meta.audit_list(&craft_cloud_meta::AuditQuery { limit: 200, ..Default::default() }).await.unwrap(); + let on_space: Vec<_> = events.iter().filter(|e| e.target_id == Some(*space.as_uuid())).collect(); + let count = |action: AuditAction| on_space.iter().filter(|e| e.action == action).count(); + assert_eq!( + [ + count(AuditAction::SpaceCreated), + count(AuditAction::SpaceQuotaChanged), + count(AuditAction::SpaceArchived), + count(AuditAction::SpaceUnarchived), + count(AuditAction::SpaceMemberRoleChanged) + ], + [1, 1, 1, 1, 1], + "{on_space:?}" + ); + assert_eq!(on_space.len(), 5); + for event in &on_space { + // The administrator acted over a browser session: the account is the actor and there is no device, as for the other pages. + assert_eq!((event.actor_user.map(|u| u.0), event.actor_device, event.target_kind.as_str()), (Some(root), None, "space"), "{event:?}"); + assert!(!event.details.contains("Studio"), "{}", event.details); + } + let owner_event = on_space.iter().find(|e| e.action == AuditAction::SpaceMemberRoleChanged).unwrap(); + assert!(owner_event.details.contains(&heir.id.0.to_string()) && owner_event.details.contains("\"owner\""), "{}", owner_event.details); + let created = on_space.iter().find(|e| e.action == AuditAction::SpaceCreated).unwrap(); + assert!(created.details.contains(&owner.id.0.to_string()) && created.details.contains("8388608"), "{}", created.details); + // The audit log's page shows them. + let html = s.get("/web/audit?action=space.", Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("space.created") && html.contains("space.quota_changed") && html.contains("space.archived"), "{html}"); +} + +async fn an_admin_with_two_factor_confirms_a_team_space_with_a_code(s: &Web) { + s.add_admin("root").await; + let owner = s.add_user("own", Role::Member).await; + let space = cli_space(s, "Studio", &owner).await; + let tf = turn_on_two_factor(s, "root").await; + let sess = sign_in_with_factor(s, "root", &tf).await; + let base = format!("/web/spaces/{}", space.as_uuid()); + for path in ["/web/spaces".to_owned(), base.clone()] { + let html = s.get(&path, Some(&sess.token)).await.text().await.unwrap(); + assert!(html.contains("name=\"totp_code\"") && html.contains("name=\"recovery_code\""), "{path}"); + } + let archive = format!("{base}/archive"); + assert_eq!(s.act(&sess, &archive, &[]).await.status(), StatusCode::FORBIDDEN, "no code"); + assert_eq!(s.act(&sess, &archive, &[("totp_code", "000000")]).await.status(), StatusCode::FORBIDDEN, "a wrong code"); + assert_eq!(s.act(&sess, &archive, &[("totp_code", "12")]).await.status(), StatusCode::BAD_REQUEST, "a code of the wrong shape"); + assert!(s.meta.admin_space(space).await.unwrap().archived_ms.is_none()); + let code = tf.fresh_code(s); + assert_eq!(s.act(&sess, &archive, &[("totp_code", &code)]).await.status(), StatusCode::SEE_OTHER); + assert!(s.meta.admin_space(space).await.unwrap().archived_ms.is_some()); + assert_eq!(s.act(&sess, &format!("{base}/unarchive"), &[("totp_code", &code)]).await.status(), StatusCode::FORBIDDEN, "the code was used up"); + let recovery = tf.recovery.first().unwrap().clone(); + assert_eq!(s.act(&sess, &format!("{base}/unarchive"), &[("recovery_code", &recovery)]).await.status(), StatusCode::SEE_OTHER); + assert_eq!(s.act(&sess, &archive, &[("recovery_code", &recovery)]).await.status(), StatusCode::FORBIDDEN, "a recovery code serves once"); + + // A mistyped owner is refused before the confirmation is looked at: the recovery code is not used up by it. + s.add_user("heir", Role::Member).await; + let spare = tf.recovery.get(1).unwrap().clone(); + for (path, fields) in [ + ("/web/spaces".to_owned(), vec![("name", "Fresh"), ("owner", "her")]), + ("/web/spaces".to_owned(), vec![("name", "Fresh"), ("owner", "a b")]), + (format!("{base}/owner"), vec![("owner", "her")]), + (format!("{base}/owner"), vec![("owner", "")]), + ] { + let mut fields = fields.clone(); + fields.push(("recovery_code", &spare)); + let status = s.act(&sess, &path, &fields).await.status(); + assert!(matches!(status, StatusCode::NOT_FOUND | StatusCode::BAD_REQUEST), "{path}: {status}"); + } + assert_eq!(s.act(&sess, &format!("{base}/owner"), &[("owner", "heir"), ("recovery_code", &spare)]).await.status(), StatusCode::SEE_OTHER); +} + +/// An owner whose account is disabled or marked for removal has no access: the pages say so, count no owner, and warn. +async fn a_space_whose_owners_cannot_sign_in_is_shown_as_having_none(s: &Web) { + s.add_admin("root").await; + let (own, gone) = (s.add_user("own", Role::Member).await, s.add_user("gone", Role::Member).await); + s.add_user("heir", Role::Member).await; + let (one, two) = (cli_space(s, "Studio", &own).await, cli_space(s, "Press", &gone).await); + let boss = s.sign_in("root").await; + let (one_page, two_page) = (format!("/web/spaces/{}", one.as_uuid()), format!("/web/spaces/{}", two.as_uuid())); + let html = s.get(&one_page, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("
Owners who can sign in
1
") && !html.contains("no owner who can sign in"), "{html}"); + assert!(!s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap().contains("(disabled)")); + + assert_eq!(s.act(&boss, &format!("/web/users/{}/disable", own.id.0), &[]).await.status(), StatusCode::SEE_OTHER); + assert_eq!(s.act(&boss, &format!("/web/users/{}/remove", gone.id.0), &[]).await.status(), StatusCode::SEE_OTHER); + let list = s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap(); + assert!(list.contains(">own (disabled)") && list.contains(">gone (marked for removal)"), "{list}"); + assert_eq!(list.matches("no active owner").count(), 2, "{list}"); + for page in [&one_page, &two_page] { + let html = s.get(page, Some(&boss.token)).await.text().await.unwrap(); + assert!( + html.contains("
Owners who can sign in
0
") && html.contains("no owner who can sign in, so nobody can manage it"), + "{html}" + ); + assert!(html.contains("account disabled") || html.contains("account marked for removal"), "{html}"); + } + + // Naming an owner brings the space back: the warning goes and the new owner is listed. + assert_eq!(s.act(&boss, &format!("{one_page}/owner"), &[("owner", "heir")]).await.status(), StatusCode::SEE_OTHER); + let html = s.get(&one_page, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("
Owners who can sign in
1
") && !html.contains("no owner who can sign in"), "{html}"); + assert!(html.contains("account disabled"), "the old owner is still shown, with its state"); + let list = s.get("/web/spaces", Some(&boss.token)).await.text().await.unwrap(); + assert_eq!(list.matches("no active owner").count(), 1, "{list}"); +} + +async fn team_space_names_are_escaped(s: &Web) { + s.add_admin("root").await; + s.add_user("own", Role::Member).await; + let boss = s.sign_in("root").await; + let nasty = ""; + let res = s.act(&boss, "/web/spaces", &[("name", nasty), ("owner", "own")]).await; + assert_eq!(res.status(), StatusCode::SEE_OTHER, "a name may hold these characters"); + let page = location(&res); + for path in ["/web/spaces", page.as_str()] { + let html = s.get(path, Some(&boss.token)).await.text().await.unwrap(); + assert!(html.contains("<script>alert(1)</script>") && !html.to_ascii_lowercase().contains(">>); + +impl std::io::Write for Capture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Capture { + type Writer = Capture; + + fn make_writer(&'a self) -> Capture { + self.clone() + } +} + +const PASSWORD: &str = "correct horse battery"; +const ORIGIN: &str = "https://cloud.example.org"; + +fn form(fields: &[(&str, &str)]) -> String { + fields + .iter() + .map(|(k, v)| { + let esc: String = v + .bytes() + .map(|b| { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~') { char::from(b).to_string() } else { format!("%{b:02X}") } + }) + .collect(); + format!("{k}={esc}") + }) + .collect::>() + .join("&") +} + +async fn post_form(client: &Client, base: &str, path: &str, token: Option<&str>, fields: Vec<(&'static str, String)>) -> reqwest::Response { + let borrowed: Vec<(&str, &str)> = fields.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let mut req = client + .post(format!("{base}{path}")) + .header("origin", ORIGIN) + .header("content-type", "application/x-www-form-urlencoded") + .body(form(&borrowed)); + if let Some(token) = token { + req = req.header("cookie", format!("{COOKIE_NAME}={token}")); + } + req.send().await.unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_admin_pages_log_no_secret() { + let log = Capture::default(); + tracing::subscriber::set_global_default( + tracing_subscriber::fmt().with_writer(log.clone()).with_max_level(tracing::Level::DEBUG).with_ansi(false).finish(), + ) + .unwrap(); + + let db = tempfile::tempdir().unwrap(); + let blobs = tempfile::tempdir().unwrap(); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]) + .unwrap() + .with_public_url(Some(ORIGIN.to_owned())) + .with_web_admin(Some(WebSettings::new(ORIGIN, 30 * 60_000, 4 * 3_600_000).unwrap())) + .with_two_factor(Some(SecretBox::new(&TwoFactorKey::from_bytes([42; 32]), None).unwrap())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending::<()>())); + let client = Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap(); + meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + let ana_id = meta.user_by_name("ana").await.unwrap().unwrap().id.0; + + // Two-factor sign-in for root, from a device; its token and codes are secrets too. + let device = client + .post(format!("{base}/api/v1/auth/login")) + .json(&json!({ "username": "root", "password": PASSWORD, "device_name": "pc" })) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .token; + let setup: TwoFactorSetupResponse = client + .post(format!("{base}/api/v1/account/two-factor/setup")) + .bearer_auth(&device) + .json(&json!({ "password": PASSWORD })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let first_code = totp::code_for(&secret, totp::step_at(craft_cloud_server::state::now_ms())).unwrap(); + let recovery: RecoveryCodes = client + .post(format!("{base}/api/v1/account/two-factor/confirm")) + .bearer_auth(&device) + .json(&json!({ "totp_code": first_code })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let recovery_code = recovery.recovery_codes.first().unwrap().clone(); + + // A wrong password, the password step with a challenge, a wrong code, a recovery code that signs in. + let wrong = post_form(&client, &base, "/web/login", None, vec![("username", "root".into()), ("password", "a wrong password".into())]).await; + assert_eq!(wrong.status(), StatusCode::UNAUTHORIZED); + let step = + post_form(&client, &base, "/web/login", None, vec![("username", "root".into()), ("password", PASSWORD.into())]).await.text().await.unwrap(); + let marker = "name=\"challenge\" value=\""; + let challenge = step[step.find(marker).unwrap() + marker.len()..].split('"').next().unwrap().to_owned(); + let bad = + post_form(&client, &base, "/web/login/second-factor", None, vec![("challenge", challenge.clone()), ("totp_code", "000000".into())]).await; + assert_eq!(bad.status(), StatusCode::FORBIDDEN); + let signed = + post_form(&client, &base, "/web/login/second-factor", None, vec![("challenge", challenge.clone()), ("recovery_code", recovery_code.clone())]) + .await; + assert_eq!(signed.status(), StatusCode::SEE_OTHER); + let line = signed.headers().get("set-cookie").unwrap().to_str().unwrap().to_owned(); + let token = line.strip_prefix(&format!("{COOKIE_NAME}=")).unwrap().split(';').next().unwrap().to_owned(); + let activity: serde_json::Value = + client.get(format!("{base}/api/v1/account/activity?limit=100")).bearer_auth(&device).send().await.unwrap().json().await.unwrap(); + assert!(activity["events"].as_array().unwrap().iter().any(|event| event["action"] == "auth.login" && event["details"]["client_kind"] == "web")); + let audit: serde_json::Value = + client.get(format!("{base}/api/v1/admin/audit?limit=200")).bearer_auth(&device).send().await.unwrap().json().await.unwrap(); + let events = audit["events"].as_array().unwrap(); + assert!(events.iter().any(|event| event["action"] == "auth.login" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.login_failed" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.second_factor_failed" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.recovery_code_used" && event["details"]["client_kind"] == "web")); + let export = client.get(format!("{base}/api/v1/admin/audit/export")).bearer_auth(&device).send().await.unwrap().text().await.unwrap(); + assert!(export.lines().any(|line| line.contains("\"action\":\"auth.login\"") && line.contains("\"client_kind\":\"web\"")), "{export}"); + assert!(export.lines().any(|line| line.contains("\"action\":\"auth.login_failed\"") && line.contains("\"client_kind\":\"web\"")), "{export}"); + assert!( + export.lines().any(|line| line.contains("\"action\":\"auth.second_factor_failed\"") && line.contains("\"client_kind\":\"web\"")), + "{export}" + ); + assert!( + export.lines().any(|line| line.contains("\"action\":\"auth.recovery_code_used\"") && line.contains("\"client_kind\":\"web\"")), + "{export}" + ); + let audit_page = + client.get(format!("{base}/web/audit")).header("cookie", format!("{COOKIE_NAME}={token}")).send().await.unwrap().text().await.unwrap(); + assert!(audit_page.contains(""client_kind":"web""), "{audit_page}"); + let page = client.get(format!("{base}/web")).header("cookie", format!("{COOKIE_NAME}={token}")).send().await.unwrap().text().await.unwrap(); + let marker = "name=\"csrf\" value=\""; + let csrf = page[page.find(marker).unwrap() + marker.len()..].split('"').next().unwrap().to_owned(); + + // Changes: a new account with its invite, a reset code, a wrong confirmation, a disable, and the sign-out. + let code_of = |html: &str| html.split("").nth(1).unwrap().split("").next().unwrap().to_owned(); + // Every change is confirmed with a recovery code of its own (each works once). + let mut spare = recovery.recovery_codes.iter().skip(1).cloned(); + let mut fields = |extra: Vec<(&'static str, String)>| { + let mut all = extra; + all.push(("csrf", csrf.clone())); + all.push(("password", PASSWORD.to_owned())); + all.push(("recovery_code", spare.next().unwrap())); + all + }; + let created = post_form(&client, &base, "/web/users", Some(&token), fields(vec![("username", "newbie".into()), ("role", "member".into())])).await; + assert_eq!(created.status(), StatusCode::OK); + let invite = code_of(&created.text().await.unwrap()); + let reset = post_form(&client, &base, &format!("/web/users/{ana_id}/password-reset"), Some(&token), fields(vec![])).await; + assert_eq!(reset.status(), StatusCode::OK); + let reset_code = code_of(&reset.text().await.unwrap()); + let refused = post_form( + &client, + &base, + &format!("/web/users/{ana_id}/disable"), + Some(&token), + vec![("csrf", csrf.clone()), ("password", "another wrong password".into()), ("totp_code", "000000".into())], + ) + .await; + assert_eq!(refused.status(), StatusCode::FORBIDDEN); + let disabled = post_form(&client, &base, &format!("/web/users/{ana_id}/disable"), Some(&token), fields(vec![])).await; + assert_eq!(disabled.status(), StatusCode::SEE_OTHER); + let out = post_form(&client, &base, "/web/logout", Some(&token), vec![("csrf", csrf.clone())]).await; + assert_eq!(out.status(), StatusCode::SEE_OTHER); + + let text = String::from_utf8(log.0.lock().unwrap().clone()).unwrap(); + // The events are there, with the target's ID. + assert!( + text.contains("web admin signed in") && text.contains("admin (web): account created") && text.contains("admin (web): account disabled"), + "{text}" + ); + assert!(text.contains(&ana_id.to_string()), "the target's ID is logged"); + // Nothing secret is. + let join_url = format!("{ORIGIN}/join/{invite}"); + let all_codes = recovery.recovery_codes.clone(); + let secrets = [ + PASSWORD, + "a wrong password", + "another wrong password", + token.as_str(), + csrf.as_str(), + challenge.as_str(), + invite.as_str(), + reset_code.as_str(), + recovery_code.as_str(), + device.as_str(), + setup.secret.as_str(), + first_code.as_str(), + ]; + for secret in secrets { + assert!(!text.contains(secret), "{secret} in the log"); + } + for code in &all_codes { + assert!( + !text.contains(code.as_str()) && !text.to_ascii_lowercase().contains(&code.replace('-', "").to_ascii_lowercase()), + "a recovery code in the log" + ); + } + let lower = text.to_ascii_lowercase(); + for code in [&invite, &reset_code, &recovery_code.replace('-', "")] { + assert!(!lower.contains(&code.to_ascii_lowercase()), "a code in the log"); + } + assert!(!text.contains(&join_url) && !text.contains("/join/")); + // The usernames are not logged at info or above. + assert!(!text.lines().any(|l| l.contains(" INFO ") && (l.contains("newbie") || l.contains("root") || l.contains("ana"))), "{text}"); +} diff --git a/crates/api/src/api.rs b/crates/api/src/api.rs index 699dfae0..c8639b63 100644 --- a/crates/api/src/api.rs +++ b/crates/api/src/api.rs @@ -1,11 +1,13 @@ //! The parts of the HTTP API's bodies (`/api/v1/…`) that apps see through the agent's local API: the error codes, the -//! error answer's code and message, the account's devices and add-a-computer codes. The rest of the HTTP API (commit +//! error answer's code and message, the account's devices and add-a-computer codes, and the admin API's bodies (which the +//! agent passes through for the Craft Cloud app's admin pages). The rest of the HTTP API (commit //! batches, manifests, the full error body) is the server's and the agent's, in `craft_cloud_proto::api`, which //! re-exports these. use serde::{Deserialize, Serialize}; +use uuid::Uuid; -use crate::ids::DeviceId; +use crate::ids::{DeviceId, Epoch, NodeId, SpaceId}; /// Stable error codes, the `error` field of an error answer ([`ErrorAnswer`]). pub mod codes { @@ -32,15 +34,116 @@ pub mod codes { pub const PASSWORD_REQUIRED: &str = "password_required"; /// `join`: the new password breaks the server's rules (the message says which). pub const WEAK_PASSWORD: &str = "weak_password"; + /// A signed-in route that asks for the account's password (changing it) got a wrong one (HTTP 403, never 401: a + /// 401 on a signed-in route means the device was revoked, and the agent signs out on it). The local API's + /// `login` uses the same code when the server refuses the username or password. + pub const WRONG_PASSWORD: &str = "wrong_password"; + /// The device is signed in but may not do this (HTTP 403): a member calling an `/admin` route, or an admin doing + /// what their account may not. The local API refuses with it a request that didn't come from a local program. + pub const FORBIDDEN: &str = "forbidden"; + /// A sign-in or a join with the right password or code, for an account an admin disabled or marked for removal + /// (HTTP 403). Said only after the password or code was right, never to a guesser. + pub const ACCOUNT_DISABLED: &str = "account_disabled"; + /// An admin change would leave the server without an enabled administrator (HTTP 409). + pub const LAST_ADMIN: &str = "last_admin"; + /// A change to a team space's members would leave it without an owner (HTTP 409). + pub const LAST_OWNER: &str = "last_owner"; + /// The account is a member of the team space but its role does not allow this (HTTP 403): a viewer writing, an editor + /// removing from the trash. The agent switches that space to read-only on it. Never 401, which reads as "signed out". + pub const READ_ONLY: &str = "read_only"; + /// A password sign-in (or the redeeming of an invite or reset code for an account with two-factor sign-in) was + /// right, but needs a second factor (HTTP 401, no token). The answer's `two_factor` carries the challenge. + pub const TWO_FACTOR_REQUIRED: &str = "two_factor_required"; + /// A code or recovery code was wrong, or a code was used before (HTTP 401 at sign-in and join, which are not + /// signed in, and 403 on a signed-in route, where a 401 would read as "this device was revoked"). + pub const TWO_FACTOR_INVALID: &str = "two_factor_invalid"; + /// The server has no key for two-factor secrets (`[two_factor] key_file`), so it can't set it up (HTTP 503). + pub const TWO_FACTOR_UNAVAILABLE: &str = "two_factor_unavailable"; + /// A set-up is running on another device (HTTP 409). + pub const TWO_FACTOR_PENDING: &str = "two_factor_pending"; + /// Too many wrong codes: every check of this account's second factor is refused until `Retry-After` is over + /// (HTTP 429). A lock never lasts for good. + pub const TWO_FACTOR_LOCKED: &str = "two_factor_locked"; pub const INTERNAL: &str = "internal"; } /// What a client reads of every error answer: `error`, a stable code (one of [`codes`], or the local API's), and -/// `message`, for people. Every other field is ignored (the full body is `craft_cloud_proto::api::ErrorBody`). +/// `message`, for people. The local trash restore call also keeps `name_taken` when a server `conflict` includes that +/// cause; any other server-only fields are ignored (the full body is `craft_cloud_proto::api::ErrorBody`). #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub struct ErrorAnswer { pub error: String, pub message: String, + /// Whether the server's conflict details say that no available restored name remains. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name_taken: Option, +} + +/// The second step of a sign-in: what `two_factor_required` carries. `challenge` is a secret, good once and until +/// `expires_ms` (milliseconds since the Unix epoch, by the server's clock); the code or recovery code goes with it to +/// `POST /auth/login/second-factor`. Its `Debug` output hides the challenge. For `POST /join` there is no challenge +/// (the code is simply sent again with the second factor): `challenge` is empty there. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorChallenge { + pub challenge: String, + pub expires_ms: i64, +} + +impl std::fmt::Debug for TwoFactorChallenge { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorChallenge").field("challenge", &"").field("expires_ms", &self.expires_ms).finish() + } +} + +/// The answer to a set-up: the secret as Base32 without padding and as an `otpauth://` URI (for a QR code), good until +/// `expires_ms`. Shown once. Its `Debug` output hides both. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorSetupResponse { + pub secret: String, + pub otpauth_uri: String, + pub expires_ms: i64, +} + +impl std::fmt::Debug for TwoFactorSetupResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorSetupResponse") + .field("secret", &"") + .field("otpauth_uri", &"") + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// Ten recovery codes, shown once (in groups of four, like `ABCD-EFGH-JKMN-PQRS`; any case, with or without dashes is +/// accepted back). Each works once. Its `Debug` output hides them. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecoveryCodes { + pub recovery_codes: Vec, +} + +impl std::fmt::Debug for RecoveryCodes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RecoveryCodes").field("recovery_codes", &format_args!("<{} redacted>", self.recovery_codes.len())).finish() + } +} + +/// `GET /account/two-factor`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorStatus { + pub enabled: bool, + /// When it was turned on. + #[serde(default)] + pub enabled_ms: Option, + /// Recovery codes not used yet (0 while it is off). + pub recovery_codes_left: u32, + /// A set-up is running (it ends after ten minutes or five wrong codes). + pub pending: bool, +} + +/// `POST /api/v1/devices/revoke-others` (device auth): how many other devices were signed out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct RevokedOthers { + pub revoked: u32, } /// Where an add-a-computer code stands, in [`PairingStatus`]. @@ -64,7 +167,8 @@ pub struct PairingStatus { pub device_name: Option, } -/// One of the signed-in user's devices. +/// One of the signed-in user's devices. The fields after `current` are newer than the first release of the API: an +/// older server leaves them out, and a client that doesn't know them ignores them. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DeviceInfo { pub id: DeviceId, @@ -74,16 +178,766 @@ pub struct DeviceInfo { pub revoked: bool, /// The device making this request. pub current: bool, + /// What the device said about its client when it signed in: `agent`, `desktop` or `cli`, or empty. Display only. + #[serde(default)] + pub client_kind: String, + /// The platform the device said it runs on, free text, empty when it didn't say. + #[serde(default)] + pub platform: String, + /// The version of the client the device said it runs, free text, empty when it didn't say. + #[serde(default)] + pub app_version: String, + /// When the device was revoked, milliseconds since the Unix epoch, by the server's clock. + #[serde(default)] + pub revoked_ms: Option, + /// Why it was revoked: `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`. A client + /// treats a reason it doesn't know as "revoked". + #[serde(default)] + pub revoke_reason: Option, +} + +/// `GET /api/v1/account` (device auth): the signed-in account. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountInfo { + pub user_id: Uuid, + /// The username, in lowercase. + pub username: String, + /// `admin` or `member`. + pub role: String, + pub created_ms: i64, + /// When the password last changed through the server; `None` while it is still the first one. + #[serde(default)] + pub password_changed_ms: Option, + /// Whether signing in asks for a second factor. + #[serde(default)] + pub two_factor_enabled: bool, +} + +// ---- the admin API (`/api/v1/admin/…`, an admin's device token) ---- +// +// Every change asks for a fresh confirmation in its body: the acting admin's `password`. A device token never expires, +// so a stolen admin token alone must not be enough. A missing or mistyped field is `400 invalid`, a wrong password `403 +// wrong_password` (never 401). An admin with two-factor sign-in adds `totp_code` (or `recovery_code`) to the same +// bodies; a body without them keeps working for an admin without it, and fields a server does not know are ignored. + +/// The body of an admin change that needs nothing but the confirmation: `POST …/disable`, `POST …/enable`, +/// `DELETE /admin/users/{id}`, `POST` and `DELETE …/invite`, `POST …/reset-two-factor` and `DELETE …/devices/{id}`. Its +/// `Debug` output hides the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminConfirm { + /// The acting admin's current password. Missing: `400 invalid`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// The acting admin's code, when their account has two-factor sign-in. Missing or wrong then: `403 + /// two_factor_invalid`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + /// A recovery code instead of `totp_code`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for AdminConfirm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminConfirm") + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /admin/users/{id}/password-reset`: the confirmation as in [`AdminConfirm`], and whether redeeming the code also +/// turns the account's two-factor sign-in off (`reset_two_factor`, default `false`: an account that has it then needs +/// its second factor to redeem the code). Its `Debug` output hides the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminPasswordReset { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, + #[serde(default)] + pub reset_two_factor: bool, +} + +impl std::fmt::Debug for AdminPasswordReset { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminPasswordReset") + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .field("reset_two_factor", &self.reset_two_factor) + .finish() + } +} + +/// `GET /admin/server`: counters and kinds, nothing about paths, addresses or settings. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminServerInfo { + pub version: String, + /// Accounts that are not marked for removal. + pub users: u64, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + pub live_bytes: u64, + /// `disk`, `s3` or `memory`. + pub storage_kind: String, + /// `sqlite` or `postgres`. + pub database_kind: String, +} + +/// `GET /admin/storage`: server-wide storage counts and the ten largest accounts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminStorageOverview { + pub accounts: u64, + pub stored_bytes: u64, + pub default_quota_bytes: Option, + pub accounts_with_override: u64, + /// Accounts at or above 90% of their effective quota. + pub accounts_near_quota: u64, + /// Accounts at or above their effective quota. + pub accounts_over_quota: u64, + pub top_accounts: Vec, + pub manifest_index: AdminManifestIndex, +} + +/// One account in [`AdminStorageOverview`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminStorageAccount { + pub id: Uuid, + pub username: String, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +/// Global manifest-index counts from the metadata database. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminManifestIndex { + pub complete: u64, + pub building: u64, + pub not_indexed: u64, +} + +/// `GET /account/usage` and `GET /admin/users/{id}/usage`: stored bytes and labeled estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountUsage { + pub stored_bytes: u64, + pub quota_bytes: Option, + pub trash_bytes: Option, + pub history_bytes: Option, + pub estimates: UsageEstimates, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UsageEstimates { + pub index_complete: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UsageEstimateReason { + IndexIncomplete, +} + +/// An account in the admin API. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUserInfo { + pub id: Uuid, + pub username: String, + /// `admin` or `member`. + pub role: String, + pub created_ms: i64, + /// When an admin disabled it (`null`: enabled). + pub disabled_ms: Option, + /// Whether the account has two-factor sign-in on. + pub two_factor_enabled: bool, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + /// The quota an admin set for the account (`null`: the server-wide cap, if any, applies). + pub quota_bytes: Option, + /// An unexpired invite is open. + pub invite_pending: bool, +} + +/// `GET /admin/users?cursor&limit&query`: a page of accounts by username. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUserList { + pub users: Vec, + /// Opaque; give it back as `cursor` for the next page. Missing on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /admin/users`: a new account without a password, for an invite. Its `Debug` output hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCreateUser { + pub username: String, + /// `admin` or `member`. + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for AdminCreateUser { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminCreateUser") + .field("username", &self.username) + .field("role", &self.role) + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// A join code the admin made, shown once. `join_url` is the whole join string when the server knows its public +/// address, and missing otherwise. Its `Debug` output hides the code and the join string. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCode { + pub code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub join_url: Option, + pub expires_ms: i64, +} + +impl std::fmt::Debug for AdminCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminCode") + .field("code", &"") + .field("join_url", &self.join_url.as_ref().map(|_| "")) + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// The answer to `POST /admin/users`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCreatedUser { + pub user: AdminUserInfo, + pub invite: AdminCode, +} + +/// `PATCH /admin/users/{id}`: a new role and/or quota. At least one of them; a missing `quota_bytes` leaves the quota, +/// `null` removes it. Its `Debug` output hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUpdateUser { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, deserialize_with = "present", skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +/// A field that is there, `null` or not: `Some(None)` for `null`, `Some(Some(x))` for a value (a missing field is +/// `None`, from `#[serde(default)]`). +fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +impl std::fmt::Debug for AdminUpdateUser { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminUpdateUser") + .field("role", &self.role) + .field("quota_bytes", &self.quota_bytes) + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `GET /admin/users/{id}/devices`: every device of the account, revoked ones with when and why. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminDevices { + pub devices: Vec, +} + +/// The typed values of a security event (`details`). Every field is a closed list of words or a number: nothing a client +/// typed is ever in it. Fields an event does not have are missing, and fields a later release adds are ignored. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditDetails { + /// `agent`, `desktop` or `cli`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + /// `windows`, `macos`, `linux`, `android`, `ios` or `other`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + /// Why a device was signed out: `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// How far a failed sign-in got: `password`, `totp`, `recovery` or `challenge_expired`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step: Option, + /// `admin` or `member` for an account's role, `viewer`, `editor` or `owner` for a role in a team space. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_from: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_to: Option, + /// The quota an admin set: `null` when they removed it, missing when the event is not about a quota. + #[serde(default, deserialize_with = "present", skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option>, + /// How many devices an event signed out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revoked: Option, + /// The device an admin or the command line signed out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device: Option, + /// Set (to `true`) when redeeming a password reset code also turned the account's two-factor sign-in off. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor_removed: Option, + /// The account a team space event is about (the invited, changed or removed member, or the first owner). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user: Option, + /// Set when the values were too long for the log and whole fields were left out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncated: Option, +} + +/// One security event, as `GET /account/activity`, `GET /admin/audit` and the export show it. No address, name of a file or +/// device, or secret is ever in it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditEvent { + pub id: Uuid, + pub at_ms: i64, + /// The account that acted; `null` for the command line, and for a sign-in that failed (nobody proved who they were). + pub actor_user: Option, + pub actor_device: Option, + /// A name from the closed list in `docs/design/sync.md`, like `auth.login`. + pub action: String, + /// `user` or `device`, or empty. + pub target_kind: String, + pub target_id: Option, + /// `success` or `failure`. + pub outcome: String, + pub details: AuditDetails, + /// How many tries the event stands for: failed sign-ins are one event per account and minute. 1 for every other event. + pub count: u64, +} + +/// A page of `GET /account/activity` or `GET /admin/audit`, newest first. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditPage { + pub events: Vec, + /// Opaque; give it back as `cursor` for the next (older) page. Missing on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// The last line of `GET /admin/audit/export` (the lines before it are [`AuditEvent`]s). `complete` says the export holds +/// every event up to the moment it started; `truncated` says there were more than the export holds. A stream that ends +/// without either line was cut off and is not a complete export. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditExportEnd { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub complete: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncated: Option, + /// The events before this line. + pub exported: u64, +} + +/// What an account may do in a team space, from least to most: a viewer reads, an editor also writes, an owner also +/// manages the members. The order is the permission order, so `role >= SpaceRole::Editor` reads as "may write". +/// A personal space's owner has `Owner` in it implicitly. The words are the ones the `space_members` table and the +/// API use. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SpaceRole { + Viewer, + Editor, + Owner, +} + +impl SpaceRole { + /// Every role, least permission first. + pub const ALL: [SpaceRole; 3] = [SpaceRole::Viewer, SpaceRole::Editor, SpaceRole::Owner]; + + pub fn as_str(self) -> &'static str { + match self { + SpaceRole::Viewer => "viewer", + SpaceRole::Editor => "editor", + SpaceRole::Owner => "owner", + } + } + + /// The role with this exact word. + pub fn parse(word: &str) -> Option { + Self::ALL.into_iter().find(|role| role.as_str() == word) + } + + /// Whether the role may change files (commit, restore, upload). + pub fn may_write(self) -> bool { + self >= SpaceRole::Editor + } +} + +/// The confirmation an administrator's change carries: the password, and the second factor when the account has one (see +/// [`AdminConfirm`]). The space routes flatten it into their bodies, where an owner acting by their own role leaves it out. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceConfirm { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for SpaceConfirm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpaceConfirm") + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// One space in `GET /spaces`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceListItem { + pub id: SpaceId, + /// `personal` or `team`. + pub kind: String, + /// The team space's name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub role: SpaceRole, + /// `active`, or `invited` for an invitation not yet accepted. + pub state: String, + pub archived: bool, + pub created_ms: i64, + /// Accepted members. + pub members: u64, + /// Not given for an invitation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epoch: Option, + /// When the account accepted its membership of a team space (milliseconds). Someone who is removed and invited again + /// gets a new one, so a client can tell that its access was interrupted. Not given for a personal space or an + /// invitation, nor by older servers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub joined_ms: Option, +} + +/// `GET /spaces`: the personal space first, then the team spaces by name. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceList { + pub spaces: Vec, +} + +/// A member of a team space, or an account invited to it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceMember { + pub user: Uuid, + pub username: String, + pub role: SpaceRole, + /// `active` or `invited`. + pub state: String, + /// The palette index (0 to 11) the account is shown in, the same on every machine. + pub colour: u8, +} + +/// `GET /spaces/{id}/members`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceMembers { + pub members: Vec, +} + +/// `POST /spaces/{id}/members`: invite an account. An owner leaves the confirmation out; an administrator who is not an +/// owner gives it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AddSpaceMember { + pub username: String, + pub role: SpaceRole, + #[serde(flatten)] + pub confirm: SpaceConfirm, +} + +/// `PATCH /spaces/{id}/members/{user}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangeSpaceMember { + pub role: SpaceRole, + #[serde(flatten)] + pub confirm: SpaceConfirm, +} + +/// `GET /spaces/{id}/usage`: as `/account/usage` for the space, with what its live documents hold. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceUsage { + pub stored_bytes: u64, + pub quota_bytes: Option, + pub trash_bytes: Option, + pub history_bytes: Option, + pub live_bytes: u64, + pub estimates: UsageEstimates, +} + +/// `PATCH /spaces/{id}`: a new name (an owner, or an administrator with the confirmation) and/or a new limit (an +/// administrator, with the confirmation; `null` removes the space's own limit). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct UpdateSpace { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, deserialize_with = "present", skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option>, + #[serde(flatten)] + pub confirm: SpaceConfirm, +} + +/// `POST /spaces/{id}/archive` and `/unarchive` with the owner's password, or an administrator's confirmation. +pub type SpaceArchive = SpaceConfirm; + +/// `POST /admin/spaces`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCreateSpace { + pub name: String, + /// The first owner's account ID: added as an accepted owner. + pub owner: Uuid, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option, + #[serde(flatten)] + pub confirm: SpaceConfirm, +} + +/// `PUT /admin/spaces/{id}/owner`: make an account an accepted owner. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminSetSpaceOwner { + pub user: Uuid, + #[serde(flatten)] + pub confirm: SpaceConfirm, +} + +/// A team space in the administrator's list: counts and sizes, never content. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminSpaceInfo { + pub id: SpaceId, + pub name: String, + pub created_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub archived_ms: Option, + pub members: u64, + pub owners: u64, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +/// `GET /admin/spaces`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminSpaceList { + pub spaces: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// The names of the security events, the closed list the server writes and `GET /admin/audit` filters by (`docs/design/sync.md`, +/// "Security audit log"). A filter is one of these, or a prefix of some of them that ends in a dot, like `auth.`. +pub const AUDIT_ACTIONS: [&str; 44] = [ + "auth.login", + "auth.login_failed", + "auth.second_factor_failed", + "auth.recovery_code_used", + "auth.two_factor_locked", + "device.revoked", + "device.revoked_others", + "account.password_changed", + "account.two_factor_enabled", + "account.two_factor_disabled", + "account.recovery_codes_regenerated", + "admin.user_created", + "admin.user_disabled", + "admin.user_enabled", + "admin.user_removed", + "admin.role_changed", + "admin.quota_changed", + "admin.invite_issued", + "admin.invite_revoked", + "admin.password_reset_issued", + "admin.two_factor_reset", + "admin.device_revoked", + "join.invite_redeemed", + "join.reset_redeemed", + "join.pairing_redeemed", + "cli.password_reset", + "cli.two_factor_reset", + "cli.device_revoked", + "cli.devices_revoked_all", + "cli.user_removed", + "space.created", + "space.renamed", + "space.archived", + "space.unarchived", + "space.quota_changed", + "space.member_invited", + "space.member_joined", + "space.member_declined", + "space.member_role_changed", + "space.member_removed", + "space.member_left", + "cli.space_created", + "cli.space_member_changed", + "cli.space_archived", +]; + +/// Whether `text` is an audit filter the server takes for `action`: an event name, or a prefix ending in a dot that at least +/// one name starts with. Anything else is refused by the server as `400 invalid`; checking here keeps it from being sent. +pub fn valid_audit_action(text: &str) -> bool { + AUDIT_ACTIONS.contains(&text) + || (text.ends_with('.') + && text.bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.') + && AUDIT_ACTIONS.iter().any(|a| a.starts_with(text))) } #[cfg(test)] mod tests { use super::*; + #[test] + fn a_device_from_an_older_server_reads_with_empty_new_fields() { + let id = DeviceId::generate(); + let json = serde_json::json!({ "id": id, "name": "pc", "created_ms": 1, "last_used_ms": 2, "revoked": false, "current": true }); + let device: DeviceInfo = serde_json::from_value(json).unwrap(); + assert_eq!((device.client_kind.as_str(), device.platform.as_str(), device.app_version.as_str()), ("", "", "")); + assert_eq!((device.revoked_ms, device.revoke_reason), (None, None)); + } + + #[test] + fn the_new_error_codes_are_stable() { + assert_eq!(codes::WRONG_PASSWORD, "wrong_password"); + assert_eq!(codes::FORBIDDEN, "forbidden"); + assert_eq!(codes::ACCOUNT_DISABLED, "account_disabled"); + assert_eq!(codes::LAST_ADMIN, "last_admin"); + assert_eq!(codes::LAST_OWNER, "last_owner"); + assert_eq!(codes::READ_ONLY, "read_only"); + } + + #[test] + fn the_two_factor_codes_are_stable_and_a_challenge_hides_itself() { + assert_eq!(codes::TWO_FACTOR_REQUIRED, "two_factor_required"); + assert_eq!(codes::TWO_FACTOR_INVALID, "two_factor_invalid"); + assert_eq!(codes::TWO_FACTOR_UNAVAILABLE, "two_factor_unavailable"); + assert_eq!(codes::TWO_FACTOR_PENDING, "two_factor_pending"); + assert_eq!(codes::TWO_FACTOR_LOCKED, "two_factor_locked"); + let shown = format!("{:?}", TwoFactorChallenge { challenge: "challenge-secret".into(), expires_ms: 5 }); + assert!(!shown.contains("challenge-secret") && shown.contains(""), "{shown}"); + } + #[test] fn an_error_answer_reads_any_error_body() { let json = serde_json::json!({ "error": "conflict", "message": "m", "conflicts": [{ "op": 0, "kind": "root" }], "quota": { "used": 1, "limit": 2 } }); let answer: ErrorAnswer = serde_json::from_value(json).unwrap(); - assert_eq!(answer, ErrorAnswer { error: "conflict".into(), message: "m".into() }); + assert_eq!(answer, ErrorAnswer { error: "conflict".into(), message: "m".into(), name_taken: None }); + } + + #[test] + fn an_error_answer_keeps_the_optional_trash_name_conflict() { + let answer: ErrorAnswer = serde_json::from_value(serde_json::json!({ + "error": "conflict", "message": "no free conflict name", "name_taken": true + })) + .unwrap(); + assert_eq!(answer.name_taken, Some(true)); + let old: ErrorAnswer = serde_json::from_value(serde_json::json!({ "error": "conflict", "message": "m" })).unwrap(); + assert_eq!(old.name_taken, None, "older agents omit the optional field"); + } + + #[test] + fn admin_bodies_carry_the_second_factor_and_hide_every_secret() { + let confirm = + AdminConfirm { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), recovery_code: Some("ABCD-EFGH".into()) }; + let shown = format!("{confirm:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && !shown.contains("ABCD-EFGH"), "{shown}"); + assert_eq!(shown.matches("").count(), 3); + let reset = AdminPasswordReset { + password: Some("admin-secret-pw".into()), + totp_code: Some("918273".into()), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + let shown = format!("{reset:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && shown.contains("reset_two_factor: true"), "{shown}"); + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), totp_code: Some("918273".into()), ..AdminCreateUser::default() }; + assert!(!format!("{create:?}").contains("918273")); + let update = AdminUpdateUser { recovery_code: Some("ABCD-EFGH".into()), ..AdminUpdateUser::default() }; + assert!(!format!("{update:?}").contains("ABCD-EFGH")); + // Every factor field is optional and absent from a body that doesn't have it; the reset flag defaults to false. + assert_eq!( + serde_json::to_value(AdminConfirm { password: Some("p".into()), ..AdminConfirm::default() }).unwrap(), + serde_json::json!({ "password": "p" }) + ); + let reset: AdminPasswordReset = serde_json::from_value(serde_json::json!({ "password": "p" })).unwrap(); + assert!(!reset.reset_two_factor && reset.totp_code.is_none()); + let user: AdminUserInfo = serde_json::from_value(serde_json::json!({ + "id": Uuid::nil(), "username": "ana", "role": "member", "created_ms": 1, "disabled_ms": null, + "two_factor_enabled": true, "devices_active": 0, "stored_bytes": 0, "quota_bytes": null, "invite_pending": false + })) + .unwrap(); + assert!(user.two_factor_enabled); + } + + #[test] + fn an_update_tells_a_missing_quota_from_a_null_one() { + let missing: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "role": "admin", "password": "p" })).unwrap(); + assert_eq!(missing.quota_bytes, None); + let null: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": null })).unwrap(); + assert_eq!(null.quota_bytes, Some(None)); + let set: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": 5_000_000 })).unwrap(); + assert_eq!(set.quota_bytes, Some(Some(5_000_000))); + assert!(serde_json::from_value::(serde_json::json!({ "quota_bytes": -1 })).is_err()); + } + + #[test] + fn audit_bodies_read_what_the_server_writes_and_ignore_what_is_newer() { + let details: AuditDetails = serde_json::from_str(r#"{"revoked":2,"a_later_field":"x","quota_bytes":null}"#).unwrap(); + assert_eq!((details.revoked, details.quota_bytes), (Some(2), Some(None))); + let end: AuditExportEnd = serde_json::from_str(r#"{"complete":true,"exported":3}"#).unwrap(); + assert_eq!((end.complete, end.truncated, end.exported), (Some(true), None, 3)); + let page: AuditPage = serde_json::from_str(r#"{"events":[]}"#).unwrap(); + assert_eq!(page.next, None); + } + + #[test] + fn space_roles_are_ordered_by_permission_and_keep_their_words() { + assert!(SpaceRole::Viewer < SpaceRole::Editor && SpaceRole::Editor < SpaceRole::Owner); + assert!(!SpaceRole::Viewer.may_write() && SpaceRole::Editor.may_write() && SpaceRole::Owner.may_write()); + for role in SpaceRole::ALL { + assert_eq!(SpaceRole::parse(role.as_str()), Some(role)); + assert_eq!(serde_json::to_string(&role).unwrap(), format!("\"{}\"", role.as_str())); + } + for bad in ["", "Viewer", "admin", "owner "] { + assert_eq!(SpaceRole::parse(bad), None, "{bad:?}"); + } + } + + #[test] + fn audit_filters_are_the_closed_list_or_a_prefix_of_it() { + assert_eq!(AUDIT_ACTIONS.len(), 44); + assert!(AUDIT_ACTIONS.iter().all(|a| valid_audit_action(a))); + for prefix in ["auth.", "admin.", "cli.", "join.", "device.", "account.", "space."] { + assert!(valid_audit_action(prefix), "{prefix}"); + } + for bad in ["", "auth", "auth.log", "nope.", "AUTH.", "auth.login ", "admin.user_created.", "x' OR 1=1 --", "a.b."] { + assert!(!valid_audit_action(bad), "{bad:?}"); + } } } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index bb5682d0..32a591af 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -13,4 +13,4 @@ pub mod local; pub mod name; pub use ids::{DeviceId, DocId, Epoch, NodeId, SpaceId, VersionId}; -pub use name::{FoldKey, Name, NameError, UNICODE_VERSION}; +pub use name::{FoldKey, MAX_SPACE_NAME_CHARS, Name, NameError, SpaceName, SpaceNameError, UNICODE_VERSION, fold_search}; diff --git a/crates/api/src/local.rs b/crates/api/src/local.rs index c2b98742..622a2e46 100644 --- a/crates/api/src/local.rs +++ b/crates/api/src/local.rs @@ -13,10 +13,15 @@ //! `POST /v1/pairings` → [`PairingInfo`], `GET /v1/pairings/{id}` → [`PairingStatus`](crate::api::PairingStatus), //! `DELETE /v1/pairings/{id}` → 204; `GET /v1/devices` → a list of [`DeviceInfo`](crate::api::DeviceInfo), //! `DELETE /v1/devices/{id}` → the list after; `POST /v1/pause`, `/v1/resume` and `/v1/quit` → [`Status`]; +//! `GET /v1/usage` → [`AccountUsage`](crate::api::AccountUsage) from the signed-in account's server; //! `POST /v1/autostart` ([`AutostartRequest`]) → [`Status`]; `GET /v1/recents`, upgraded to a WebSocket of JSON //! text messages: [`RecentsRequest`] in, [`RecentsEvent`] out; `GET /v1/docs`, upgraded to a WebSocket of live //! documents ([`docs`]: [`DocsRequest`] and binary frames in, [`DocsEvent`] and binary frames out); `PUT /v1/blobs` -//! and `GET /v1/blobs/{hash}`: binary content kept out of live documents ([`blobs`]). Errors are the HTTP API's error +//! and `GET /v1/blobs/{hash}`: binary content kept out of live documents ([`blobs`]); the admin calls under +//! `/v1/admin` ([`paths::ADMIN_SERVER`], [`paths::ADMIN_USERS`]), which the agent passes on to the server's admin API +//! with the same bodies and answers (see `docs/agent.md`); `GET /v1/spaces` → [`SpaceList`] and, under +//! `/v1/spaces/{id}`, `POST …/accept`, `…/decline` (passed on to the server) and `…/sync` ([`SpaceSyncRequest`]): the +//! team spaces of the account and what this computer does with them. Errors are the HTTP API's error //! body ([`ErrorAnswer`](crate::api::ErrorAnswer) reads its code and message) with its codes and [`codes`]. //! - **Versions**: [`Status::api`] is the local API version the agent speaks; a client refuses one outside //! [`LOCAL_API_MIN`]..=[`LOCAL_API_MAX`] when it connects. @@ -26,7 +31,8 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use crate::ids::NodeId; +use crate::api::SpaceRole; +use crate::ids::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; use crate::limits::MAX_TREE_DEPTH; use crate::name::Name; @@ -52,14 +58,16 @@ pub const MAX_TITLE_BYTES: usize = 256; pub const MAX_MACHINE_BYTES: usize = 128; /// Largest JSON body or WebSocket message of the local API, either way. pub const MAX_LOCAL_JSON_BYTES: usize = 1024 * 1024; +/// Largest body of an admin call (the server's own limit is tiny too: a confirmation and a few fields). +pub const MAX_ADMIN_JSON_BYTES: usize = 8 * 1024; /// Error codes of the local API (the `error` field of [`ErrorAnswer`](crate::api::ErrorAnswer)), beside the HTTP API's /// `invalid`, `unauthorized` (the token), `not_found`, `too_large` and `internal`. pub mod codes { /// A `Host` that isn't `127.0.0.1:`, or an `Origin` header: the request didn't come from a local program. - pub const FORBIDDEN: &str = "forbidden"; + pub const FORBIDDEN: &str = crate::api::codes::FORBIDDEN; /// `login`: the server refused the username or password. - pub const WRONG_PASSWORD: &str = "wrong_password"; + pub const WRONG_PASSWORD: &str = crate::api::codes::WRONG_PASSWORD; /// `login` while signed in: sign out first. pub const ALREADY_SIGNED_IN: &str = "already_signed_in"; /// `logout`, or recents, while signed out. @@ -86,6 +94,23 @@ pub mod codes { pub const FOLDER_HAS_FILES: &str = "folder_has_files"; /// `GET /v1/blobs/{hash}`: neither this machine nor the account's server has that blob. pub const BLOB_NOT_FOUND: &str = "blob_not_found"; + /// `login` or `join`: the account has two-factor sign-in. The answer's `two_factor` (see [`ErrorExtras`](super::ErrorExtras)) + /// holds the challenge for `POST /v1/login/second-factor` (empty for `join`: send the same join again with the code). + pub const TWO_FACTOR_REQUIRED: &str = crate::api::codes::TWO_FACTOR_REQUIRED; + /// A code or recovery code was wrong (HTTP 401 for `login`, `login/second-factor` and `join`, 403 on the account calls). + /// It never means the device was revoked. + pub const TWO_FACTOR_INVALID: &str = crate::api::codes::TWO_FACTOR_INVALID; + /// Too many wrong codes: try again after the answer's `retry_after_secs` (HTTP 429, also the `Retry-After` header). + pub const TWO_FACTOR_LOCKED: &str = crate::api::codes::TWO_FACTOR_LOCKED; + /// The server has no key for two-factor secrets, so it can't be set up (HTTP 503). + pub const TWO_FACTOR_UNAVAILABLE: &str = crate::api::codes::TWO_FACTOR_UNAVAILABLE; + /// A set-up is open on another computer (HTTP 409). + pub const TWO_FACTOR_PENDING: &str = crate::api::codes::TWO_FACTOR_PENDING; + /// `login/second-factor`: the challenge doesn't work any more (it lasts five minutes and takes five wrong codes, and a + /// password change ends it): sign in again from the first step (HTTP 409; a 401 would read as a bad local token). + pub const TWO_FACTOR_EXPIRED: &str = "two_factor_expired"; + /// A state clash on the server (two-factor already on, no set-up to confirm, a password that changed meanwhile), HTTP 409. + pub const CONFLICT: &str = crate::api::codes::CONFLICT; /// Never sent by an agent: what a client reports when the agent answers a call it doesn't know (`not_found` for /// the call itself), as one from before that call answers it. Update Craft Cloud. pub const AGENT_TOO_OLD: &str = "agent_too_old"; @@ -101,11 +126,34 @@ pub mod paths { pub const DOCS: &str = "/v1/docs"; /// `POST`: join with a join string ([`super::JoinRequest`]) → [`super::Status`]. pub const JOIN: &str = "/v1/join"; + /// `POST` ([`super::SecondFactorLoginRequest`]): the second step of a sign-in that answered `two_factor_required` → + /// [`super::Status`]. + pub const LOGIN_SECOND_FACTOR: &str = "/v1/login/second-factor"; + /// `GET` → [`crate::api::TwoFactorStatus`]; with `/setup`, `/confirm`, `/disable` and `/recovery-codes` below it: the + /// signed-in account's two-factor sign-in, passed on to the server ([`super::PasswordRequest`], [`super::TotpRequest`], + /// [`super::TwoFactorProofRequest`]). Nothing is stored or logged on the way. + pub const TWO_FACTOR: &str = "/v1/account/two-factor"; + pub const TWO_FACTOR_SETUP: &str = "/v1/account/two-factor/setup"; + pub const TWO_FACTOR_CONFIRM: &str = "/v1/account/two-factor/confirm"; + pub const TWO_FACTOR_DISABLE: &str = "/v1/account/two-factor/disable"; + pub const TWO_FACTOR_RECOVERY_CODES: &str = "/v1/account/two-factor/recovery-codes"; + /// `GET` → [`crate::api::AccountInfo`]: the signed-in account (its role, and whether it has two-factor sign-in), as the + /// server says it. + pub const ACCOUNT: &str = "/v1/account"; + /// `GET /v1/account/activity?cursor&before_ms&limit` → [`crate::api::AuditPage`]: the signed-in account's own security + /// events, for every account. + pub const ACCOUNT_ACTIVITY: &str = "/v1/account/activity"; + /// `POST` ([`super::ChangePasswordRequest`]) → 204. + pub const ACCOUNT_PASSWORD: &str = "/v1/account/password"; + /// `POST` → [`crate::api::RevokedOthers`]: sign every other computer of the account out. + pub const DEVICES_REVOKE_OTHERS: &str = "/v1/devices/revoke-others"; /// `POST`: a new add-a-computer code → [`super::PairingInfo`]; `GET`/`DELETE` `/v1/pairings/{id}`: its state, or /// cancel it. pub const PAIRINGS: &str = "/v1/pairings"; /// `GET`: the account's computers; `DELETE /v1/devices/{id}`: remove one (never this one). pub const DEVICES: &str = "/v1/devices"; + /// `GET`: the signed-in account's stored bytes and labeled estimates, fetched from its server. + pub const USAGE: &str = "/v1/usage"; /// `POST`: hold syncing (kept across restarts) → [`super::Status`]. pub const PAUSE: &str = "/v1/pause"; /// `POST`: sync again after a pause → [`super::Status`]. @@ -116,6 +164,135 @@ pub mod paths { pub const AUTOSTART: &str = "/v1/autostart"; /// `PUT`: a blob → [`super::blobs::Stored`]; `GET /v1/blobs/{hash}`: its bytes ([`super::blobs`]). pub const BLOBS: &str = "/v1/blobs"; + /// `GET`: the server's counters and kinds (an admin's; any other account is refused with `forbidden`) → + /// [`AdminServerInfo`](crate::api::AdminServerInfo). + pub const ADMIN_SERVER: &str = "/v1/admin/server"; + /// `GET /v1/admin/storage` → [`AdminStorageOverview`](crate::api::AdminStorageOverview). + pub const ADMIN_STORAGE: &str = "/v1/admin/storage"; + /// `GET /v1/admin/users?cursor&limit&query` → [`AdminUserList`](crate::api::AdminUserList); `POST` (an + /// [`AdminCreateUser`](crate::api::AdminCreateUser)) → [`AdminCreatedUser`](crate::api::AdminCreatedUser). + /// Under `/v1/admin/users/{id}`: `PATCH` ([`AdminUpdateUser`](crate::api::AdminUpdateUser)) → the account's + /// [`AdminUserInfo`](crate::api::AdminUserInfo); `DELETE`, and `POST` to `…/disable`, `…/enable`, `…/invite` and + /// `…/password-reset`, and `DELETE …/invite` and `…/devices/{device_id}`, each with an + /// [`AdminConfirm`](crate::api::AdminConfirm) (the invite and the reset code answer an + /// [`AdminCode`](crate::api::AdminCode), the others have no body); `GET …/devices` → + /// [`AdminDevices`](crate::api::AdminDevices). The server's status and error code are the agent's. + pub const ADMIN_USERS: &str = "/v1/admin/users"; + /// `GET /v1/admin/users/{id}/usage` → [`AccountUsage`](crate::api::AccountUsage). + pub const ADMIN_USER_USAGE: &str = "/v1/admin/users"; + /// `GET /v1/admin/audit?cursor&before_ms&limit&user&action` → [`AuditPage`](crate::api::AuditPage): the whole security log, + /// newest first, filtered (an admin's). + pub const ADMIN_AUDIT: &str = "/v1/admin/audit"; + /// `GET /v1/admin/audit/export?before_ms&user&action`: the log as NDJSON, streamed through (chunked, never held whole by the + /// agent): [`AuditEvent`](crate::api::AuditEvent) lines and a last [`AuditExportEnd`](crate::api::AuditExportEnd) line. + /// A stream that ends without that last line was cut off. + pub const ADMIN_AUDIT_EXPORT: &str = "/v1/admin/audit/export"; + /// `GET`: version history for a synced file, addressed by its path relative to the sync root. + pub const FILE_HISTORY: &str = "/v1/files/history"; + /// `POST`: restore a prior version of a synced file, addressed by its path relative to the sync root. + pub const FILE_RESTORE: &str = "/v1/files/restore"; + /// `GET`: the server's deleted nodes; `POST` restore and remove calls address node IDs. + pub const TRASH: &str = "/v1/trash"; + pub const TRASH_RESTORE: &str = "/v1/trash/restore"; + pub const TRASH_REMOVE: &str = "/v1/trash/remove"; + /// `GET` → [`super::SpaceList`]: the personal space and the team spaces of the account, invitations among them, with + /// what this computer does with each. Under `/v1/spaces/{id}`: `POST …/accept` → the space as the server lists it + /// ([`SpaceListItem`](crate::api::SpaceListItem)), `POST …/decline` → 204 (both passed on to the server), and + /// `POST …/sync` ([`super::SpaceSyncRequest`]) → the space as [`super::SpaceInfo`], and `POST …/approve-deletes` + /// ([`super::ApproveDeletesRequest`]) → [`super::ApproveDeletesAnswer`]. + pub const SPACES: &str = "/v1/spaces"; +} + +/// One version in a file's newest-first history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryVersion { + pub id: VersionId, + pub parent_version: Option, + pub epoch: Epoch, + pub seq: u64, + pub size: u64, + pub mtime_ms: i64, + pub device: DeviceId, + pub created_ms: i64, + pub save: bool, + pub pruned: bool, +} + +/// A page of a file's newest-first version history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryResponse { + pub versions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// The path and page cursor for `GET /v1/files/history`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileHistoryRequest { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +/// `POST /v1/files/restore`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreRequest { + pub path: String, + pub version: VersionId, + pub expected_version: VersionId, +} + +/// The new version made by a successful restore. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreResponse { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// One deleted node from the server's trash. `path` is a local display hint and can be absent. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashItem { + pub id: NodeId, + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + pub deleted_ms: i64, + pub purge_after_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// A page of the server's trash, newest deletion first. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashResponse { + pub items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `{id}` in a local trash restore or remove request. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashNodeRequest { + pub id: String, +} + +/// The server's answer after restoring a node from trash. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashRestoreResponse { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub seq: u64, + pub epoch: Epoch, } /// Binary content an app keeps out of a live document (images, audio, a font): `PUT /v1/blobs` with the bytes as the @@ -271,6 +448,171 @@ pub struct Status { /// admin): nothing syncs until it signs out and joins again. Its files stay. #[serde(default)] pub removed: bool, + /// The team spaces this computer syncs, or lost access to, one entry each (the fields above describe the personal + /// space). An older agent says nothing. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub spaces: Vec, +} + +/// What this computer does about one space, in [`SpaceInfo`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SpaceSync { + /// The space syncs to a folder on this computer. + On, + /// It doesn't (a team space starts so on every computer: the person chooses). + Off, + /// It can't: the server is too old for team spaces, or this computer lost access to the space. + Unavailable, +} + +/// How the sync of one team space is going, in [`SpaceStatus`] and [`SpaceInfo`]: [`RunState`] for a space, and the two +/// ways a space stops that a whole agent doesn't. The `Debug` output leaves out the words of `NoAccess`, which name the +/// space. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum SpaceRun { + Idle, + Syncing, + /// The last pass failed for a reason that may pass by itself; it is tried again. + Retrying { + message: String, + }, + /// Stopped until someone puts something right (the folder is gone, say). + Paused { + reason: String, + }, + /// Syncing is paused by the user for every space. + PausedByUser, + /// The account no longer has access (it was removed, or the invitation withdrawn; an archived space is not this: its members + /// keep read access). The local copy is + /// kept and no longer syncs; if access comes back it syncs again. `message` says so in words. + NoAccess { + message: String, + }, + /// The server can't do team spaces yet (it needs an update). + Unavailable { + message: String, + }, +} + +impl std::fmt::Debug for SpaceRun { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Idle => f.write_str("Idle"), + Self::Syncing => f.write_str("Syncing"), + Self::Retrying { message } => f.debug_struct("Retrying").field("message", message).finish(), + Self::Paused { reason } => f.debug_struct("Paused").field("reason", reason).finish(), + Self::PausedByUser => f.write_str("PausedByUser"), + Self::NoAccess { .. } => f.debug_struct("NoAccess").finish_non_exhaustive(), + Self::Unavailable { message } => f.debug_struct("Unavailable").field("message", message).finish(), + } + } +} + +/// One synced team space in [`Status::spaces`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceStatus { + pub space: SpaceId, + pub run_state: SpaceRun, + /// When the last pass that didn't fail finished (milliseconds since the Unix epoch). + #[serde(default)] + pub last_sync_ms: Option, + /// The account is a viewer here (or the space is archived): the folder follows the space, and nothing of this + /// computer goes up. + #[serde(default)] + pub read_only: bool, +} + +/// One space in `GET /v1/spaces`: the server's view of it (its name, the account's role, whether it is an invitation) +/// and this computer's (the folder, whether it syncs, how that goes). The `Debug` output leaves out the name and the +/// folder. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpaceInfo { + pub space: SpaceId, + /// `personal` or `team`. + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub role: SpaceRole, + /// `active`, or `invited` for an invitation not yet accepted. + pub state: String, + #[serde(default)] + pub archived: bool, + /// The folder on this computer (absolute): where the space syncs, or would. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub folder: Option, + pub sync: SpaceSync, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_sync_ms: Option, + #[serde(default)] + pub read_only: bool, +} + +impl std::fmt::Debug for SpaceInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpaceInfo") + .field("space", &self.space) + .field("kind", &self.kind) + .field("role", &self.role) + .field("state", &self.state) + .field("archived", &self.archived) + .field("sync", &self.sync) + .field("run_state", &self.run_state) + .field("read_only", &self.read_only) + .finish_non_exhaustive() + } +} + +/// `GET /v1/spaces`. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct SpaceList { + pub spaces: Vec, +} + +/// `POST /v1/spaces/{id}/sync`: turn the sync of a team space on or off on this computer. `folder` (absolute) picks +/// another folder than the default one beside the personal folder; the folder must not lie inside another synced folder or +/// the agent's data folder. A folder that already has files in it is synced only with `adopt` (files that differ from +/// the space's become conflict copies; nothing is deleted). +#[derive(Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct SpaceSyncRequest { + pub on: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub folder: Option, + #[serde(default)] + pub adopt: bool, +} + +impl std::fmt::Debug for SpaceSyncRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpaceSyncRequest").field("on", &self.on).field("adopt", &self.adopt).finish_non_exhaustive() + } +} + +/// `POST /v1/spaces/{id}/approve-deletes`: approve the deletions of a team space that the mass-delete guard holds back, as +/// `craft-cloud-agent sync --approve-deletes --space` does (it writes the space's `approval.json`, which the running agent +/// takes). With a `set` (the identifier the waiting list was shown with) only that set is approved if it is the one waiting; +/// without, the set is approved if it is the one last shown by this call (the first call lists it and approves nothing). +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ApproveDeletesRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub set: Option, +} + +/// The answer to [`ApproveDeletesRequest`]: whether an approval was handed to the agent, and what the command line +/// would have said. Its `Debug` output leaves out the words, which list file names. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ApproveDeletesAnswer { + pub approved: bool, + pub message: String, +} + +impl std::fmt::Debug for ApproveDeletesAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ApproveDeletesAnswer").field("approved", &self.approved).finish_non_exhaustive() + } } /// `POST /v1/login`: what `craft-cloud-agent login` asks for. The password goes to the agent and from there to the @@ -324,6 +666,12 @@ pub struct JoinRequest { /// The new account's password, for an invite that needs one (answered `password_required` without it). #[serde(default)] pub password: Option, + /// The second factor of an account that has two-factor sign-in (answered `two_factor_required` without it, the code not + /// used up): a code from the authenticator app, or a recovery code instead. Send the same join again with one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, } impl std::fmt::Debug for JoinRequest { @@ -334,10 +682,128 @@ impl std::fmt::Debug for JoinRequest { .field("device_name", &self.device_name) .field("adopt", &self.adopt) .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// What an error answer carries beyond `error` and `message` ([`ErrorAnswer`](crate::api::ErrorAnswer) reads those two): the +/// challenge of `two_factor_required` and the wait of `two_factor_locked` (also in the `Retry-After` header). A client reads +/// it from the same body; every field is missing from answers that have no use for it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ErrorExtras { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_after_secs: Option, +} + +/// `POST /v1/login/second-factor`: the second step of a sign-in the agent answered `two_factor_required`. The agent keeps +/// nothing between the two steps, so the first step's `server`, `folder`, `device_name` and `adopt` come again (`user` is +/// the first step's and is only checked to be sane), with the `challenge` from the answer and a code (`totp_code`) or a recovery +/// code. Answered with [`Status`]. Its `Debug` output hides the challenge and the codes. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SecondFactorLoginRequest { + pub server: String, + pub user: String, + #[serde(default)] + pub folder: Option, + #[serde(default)] + pub device_name: Option, + #[serde(default)] + pub adopt: bool, + pub challenge: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for SecondFactorLoginRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorLoginRequest") + .field("server", &self.server) + .field("user", &self.user) + .field("folder", &self.folder) + .field("device_name", &self.device_name) + .field("adopt", &self.adopt) + .field("challenge", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) .finish() } } +/// `POST /v1/account/two-factor/setup`: the account's password, to start setting two-factor sign-in up. Answered with +/// [`TwoFactorSetupResponse`](crate::api::TwoFactorSetupResponse). Its `Debug` output hides the password. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PasswordRequest { + pub password: String, +} + +impl std::fmt::Debug for PasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PasswordRequest").field("password", &"").finish() + } +} + +/// `POST /v1/account/two-factor/confirm`: the first code of the set-up; answered with +/// [`RecoveryCodes`](crate::api::RecoveryCodes), shown once. Its `Debug` output hides the code. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TotpRequest { + pub totp_code: String, +} + +impl std::fmt::Debug for TotpRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TotpRequest").field("totp_code", &"").finish() + } +} + +/// `POST /v1/account/two-factor/disable` (answered 204) and `/recovery-codes` (answered with +/// [`RecoveryCodes`](crate::api::RecoveryCodes), with `totp_code` only): the password and one second factor. Its `Debug` output +/// hides all three. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorProofRequest { + pub password: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for TwoFactorProofRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorProofRequest") + .field("password", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /v1/account/password`: the account's password change; answered 204, and every other computer is signed out. Its +/// `Debug` output hides both passwords. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangePasswordRequest { + pub current_password: String, + pub new_password: String, +} + +impl std::fmt::Debug for ChangePasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChangePasswordRequest").field("current_password", &"").field("new_password", &"").finish() + } +} + +/// `PATCH /v1/devices/{id}`: a new name for one of the account's computers; answered with its +/// [`DeviceInfo`](crate::api::DeviceInfo). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenameDeviceRequest { + pub name: String, +} + /// `POST /v1/pairings`: a new add-a-computer code of this account, to enter on the computer being added. It /// replaces this computer's earlier one. Its `Debug` output hides the code. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -736,9 +1202,29 @@ mod tests { device_name: None, adopt: false, password: Some("hunter2-secret".into()), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERY-SECRET".into()), }; let shown = format!("{join:?}"); - assert!(!shown.contains("K7M4") && !shown.contains("hunter2-secret"), "{shown}"); + assert!(!shown.contains("K7M4") && !shown.contains("hunter2-secret") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let second = SecondFactorLoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + folder: None, + device_name: None, + adopt: false, + challenge: "challenge-secret".into(), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERY-SECRET".into()), + }; + let shown = format!("{second:?}"); + assert!(!shown.contains("challenge-secret") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let proof = + TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: Some("918273".into()), recovery_code: Some("RECOVERY-SECRET".into()) }; + let shown = format!("{proof:?} {:?} {:?}", PasswordRequest { password: "pw-secret-1".into() }, TotpRequest { totp_code: "918273".into() }); + assert!(!shown.contains("pw-secret-1") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let change = format!("{:?}", ChangePasswordRequest { current_password: "old-secret-pw".into(), new_password: "new-secret-pw".into() }); + assert!(!change.contains("secret-pw"), "{change}"); let pairing = PairingInfo { id: uuid::Uuid::nil(), join: "cloud.example.com/K7M4-Q9TX".into(), @@ -747,7 +1233,20 @@ mod tests { }; assert!(!format!("{pairing:?}").contains("K7M4")); let minimal: JoinRequest = serde_json::from_value(serde_json::json!({ "join": "x/y" })).unwrap(); - assert!(!minimal.adopt && minimal.password.is_none()); + assert!(!minimal.adopt && minimal.password.is_none() && minimal.totp_code.is_none() && minimal.recovery_code.is_none()); + // An older app's body reads, and a new body without a second factor is the old one. + assert_eq!( + serde_json::to_value(&minimal).unwrap(), + serde_json::json!({ "join": "x/y", "folder": null, "device_name": null, "adopt": false, "password": null }) + ); + // The extras of an error answer are optional and ignored by a reader that doesn't know them. + let extras: ErrorExtras = serde_json::from_value(serde_json::json!({ "error": "x", "message": "m" })).unwrap(); + assert_eq!(extras, ErrorExtras::default()); + let extras: ErrorExtras = serde_json::from_value( + serde_json::json!({ "error": "two_factor_required", "two_factor": { "challenge": "c", "expires_ms": 5 }, "retry_after_secs": 7 }), + ) + .unwrap(); + assert_eq!((extras.two_factor.map(|t| t.expires_ms), extras.retry_after_secs), (Some(5), Some(7))); } #[test] @@ -770,6 +1269,7 @@ mod tests { autostart: None, last_sync_ms: Some(1_790_000_000_000), removed: false, + spaces: Vec::new(), }; let json = serde_json::to_value(&status).unwrap(); assert_eq!(json["run_state"], serde_json::json!({ "kind": "signed_out" })); @@ -783,10 +1283,49 @@ mod tests { assert_eq!(older.autostart, None, "an older agent says nothing about autostart"); assert_eq!(older.last_sync_ms, None, "nor about the last sync"); assert!(!older.removed); + assert!(older.spaces.is_empty(), "nor about team spaces"); let login: LoginRequest = serde_json::from_value(serde_json::json!({ "server": "s", "user": "u", "password": "p", "extra": 1 })).unwrap(); assert!(!login.adopt && login.folder.is_none(), "missing optional fields default; unknown ones are ignored"); } + #[test] + fn spaces_have_a_plain_form_and_their_debug_output_names_nothing() { + let space = SpaceId::generate(); + let info = SpaceInfo { + space, + kind: "team".into(), + name: Some("Studio".into()), + role: SpaceRole::Viewer, + state: "active".into(), + archived: false, + folder: Some("/home/ana/Craft Cloud Shared/Studio".into()), + sync: SpaceSync::On, + run_state: Some(SpaceRun::NoAccess { message: "You no longer have access to Studio. Your local copy is kept.".into() }), + last_sync_ms: None, + read_only: true, + }; + let json = serde_json::to_value(&info).unwrap(); + assert_eq!(json["sync"], "on"); + assert_eq!(json["run_state"]["kind"], "no_access"); + assert!(json.get("last_sync_ms").is_none()); + assert_eq!(serde_json::from_value::(json).unwrap(), info); + let shown = format!("{info:?}"); + assert!(!shown.contains("Studio") && !shown.contains("Craft Cloud Shared"), "{shown}"); + assert!(shown.contains("NoAccess"), "the variant is still named: {shown}"); + let status = SpaceStatus { space, run_state: info.run_state.clone().unwrap(), last_sync_ms: None, read_only: false }; + let shown = format!("{status:?}"); + assert!(!shown.contains("Studio"), "the status leaves the name out too: {shown}"); + let answer = ApproveDeletesAnswer { approved: true, message: "delete on the server: Studio/plan.txt".into() }; + assert!(!format!("{answer:?}").contains("plan.txt")); + let older: Status = serde_json::from_value(serde_json::json!({ + "api": 1, "signed_in": true, "online": true, "run_state": { "kind": "idle" } + })) + .unwrap(); + assert!(older.spaces.is_empty()); + let request: SpaceSyncRequest = serde_json::from_value(serde_json::json!({ "on": true })).unwrap(); + assert_eq!(request, SpaceSyncRequest { on: true, folder: None, adopt: false }); + } + #[test] fn docs_messages_have_a_plain_form() { let frame = docs::encode_binary(docs::UPDATE, 0x0102_0304_0506_0708, b"abc"); diff --git a/crates/api/src/name.rs b/crates/api/src/name.rs index 76dbf09c..2ade8cbc 100644 --- a/crates/api/src/name.rs +++ b/crates/api/src/name.rs @@ -91,6 +91,91 @@ impl From for String { } } +/// The longest team space name, in characters. +pub const MAX_SPACE_NAME_CHARS: usize = 64; + +/// Why a team space name was rejected. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SpaceNameError { + #[error("the name is empty")] + Empty, + #[error("the name is {len} characters long; the limit is {MAX_SPACE_NAME_CHARS}")] + TooLong { len: usize }, + #[error("the name contains the control character U+{:04X}", u32::from(*ch))] + ControlChar { ch: char }, + #[error("the name starts or ends with a space")] + EdgeSpace, + /// A name shaped like a space ID would be taken for one wherever a space is named by ID or by name (the command line). + #[error("the name looks like a space ID")] + LooksLikeId, +} + +/// A team space's display name: NFC-normalised, 1 to [`MAX_SPACE_NAME_CHARS`] characters, no control characters, and +/// no space at either end. Unlike a [`Name`] it may hold characters a file name may not (`/`, `:`), because it is not +/// a path; the folder an agent makes for it goes through the node name rules separately. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct SpaceName(String); + +impl SpaceName { + /// Normalises `raw` to NFC and validates it. + pub fn new(raw: &str) -> Result { + let nfc: String = raw.nfc().collect(); + let len = nfc.chars().count(); + if len == 0 { + return Err(SpaceNameError::Empty); + } + if len > MAX_SPACE_NAME_CHARS { + return Err(SpaceNameError::TooLong { len }); + } + if let Some(ch) = nfc.chars().find(|ch| ch.is_control()) { + return Err(SpaceNameError::ControlChar { ch }); + } + if nfc.starts_with(char::is_whitespace) || nfc.ends_with(char::is_whitespace) { + return Err(SpaceNameError::EdgeSpace); + } + if uuid::Uuid::try_parse(&nfc).is_ok() { + return Err(SpaceNameError::LooksLikeId); + } + Ok(Self(nfc)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The caseless form two names must not share on one server. + pub fn fold_key(&self) -> FoldKey { + FoldKey(fold(&self.0)) + } +} + +impl std::fmt::Display for SpaceName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl TryFrom for SpaceName { + type Error = SpaceNameError; + + fn try_from(raw: String) -> Result { + Self::new(&raw) + } +} + +impl From for String { + fn from(name: SpaceName) -> Self { + name.0 + } +} + +/// The caseless form of search text for team space names: trimmed, folded as names are, but not validated as a name (a +/// search may be a single letter, or text no name could start with). +pub fn fold_search(text: &str) -> FoldKey { + FoldKey(fold(text.trim())) +} + /// Canonical caseless form: NFD, Unicode full case folding, then NFC. fn fold(s: &str) -> String { let nfd: String = s.nfd().collect(); @@ -206,6 +291,23 @@ mod tests { assert!(!same("a.psd", "b.psd")); } + #[test] + fn space_names_are_nfc_text_of_one_to_64_characters() { + assert_eq!(SpaceName::new("Client: ACME / 2026").unwrap().as_str(), "Client: ACME / 2026"); + assert_eq!(SpaceName::new("cafe\u{301}").unwrap().as_str(), "caf\u{e9}"); + assert_eq!(SpaceName::new(""), Err(SpaceNameError::Empty)); + assert!(SpaceName::new(&"日".repeat(64)).is_ok()); + assert_eq!(SpaceName::new(&"日".repeat(65)), Err(SpaceNameError::TooLong { len: 65 })); + assert_eq!(SpaceName::new("a\tb"), Err(SpaceNameError::ControlChar { ch: '\t' })); + assert_eq!(SpaceName::new(" a"), Err(SpaceNameError::EdgeSpace)); + assert_eq!(SpaceName::new("a "), Err(SpaceNameError::EdgeSpace)); + assert_eq!(SpaceName::new(" "), Err(SpaceNameError::EdgeSpace)); + // A name shaped like a space ID would be taken for one by `space create` / `space add `. + assert_eq!(SpaceName::new("0194a2d0-1c3e-7b5a-9d41-2f6e8a7b3c10"), Err(SpaceNameError::LooksLikeId)); + assert!(SpaceName::new("0194a2d0-studio").is_ok()); + assert_eq!(SpaceName::new("Straße").unwrap().fold_key(), SpaceName::new("STRASSE").unwrap().fold_key()); + } + #[test] fn the_unicode_version_is_pinned() { // A dependency bump that changes this changes stored fold keys: update it deliberately. diff --git a/crates/app/src/admin.rs b/crates/app/src/admin.rs new file mode 100644 index 00000000..a7c0c050 --- /dev/null +++ b/crates/app/src/admin.rs @@ -0,0 +1,166 @@ +//! The admin calls of the agent's local API (`/v1/admin/…`), which the agent passes on to the server's admin API for the +//! Craft Cloud app's admin pages. Every one asks the server (an Argon2 check of the admin's password among it), so call +//! them **off the UI thread**. +//! +//! The server decides everything: a member's call is `Refused { code: "forbidden" }`, a changing call without the +//! admin's current password, or with a wrong one, is `invalid` or `wrong_password` (HTTP 403, never 401), and a change +//! that would leave the server without an enabled admin is `last_admin`. A `rate_limited` or `two_factor_locked` refusal +//! carries the seconds to wait ([`Error::retry_after_secs`]). The password is sent in the body of each change +//! ([`AdminConfirm::password`] and the same field of the other bodies), goes to the server and is kept by neither the +//! agent nor this crate; the types' `Debug` output hides it. + +use craft_cloud_api::local::paths; + +use crate::{ + AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, Client, DeviceId, Error, Uuid, http, +}; + +/// Which accounts [`Client::admin_users`] lists: a page by username, optionally those whose name contains `query`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UserQuery { + /// The `next` of the page before (opaque); none for the first page. + pub cursor: Option, + /// Accounts on the page, 1 to 200; none: the server's default (50). + pub limit: Option, + /// Only usernames containing this text (case doesn't matter, at most 64 bytes); none or empty: all. + pub query: Option, +} + +impl UserQuery { + /// The path with its query string. + fn path(&self) -> String { + let mut pairs = Vec::new(); + if let Some(cursor) = self.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(format!("cursor={}", encode(cursor))); + } + if let Some(limit) = self.limit { + pairs.push(format!("limit={limit}")); + } + if let Some(query) = self.query.as_deref().filter(|q| !q.is_empty()) { + pairs.push(format!("query={}", encode(query))); + } + if pairs.is_empty() { paths::ADMIN_USERS.to_owned() } else { format!("{}?{}", paths::ADMIN_USERS, pairs.join("&")) } + } +} + +/// `text` percent-encoded for a query string (everything but unreserved characters). +pub(crate) fn encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn body(value: &T) -> Result, Error> { + serde_json::to_vec(value).map_err(|e| Error::Protocol(e.to_string())) +} + +fn user_path(id: Uuid) -> String { + format!("{}/{id}", paths::ADMIN_USERS) +} + +impl Client { + /// The server's counters and kinds. Only for an admin: any other account is `Refused { code: "forbidden" }`, which + /// is how the app tells whether to show its admin pages. + pub fn admin_server(&self) -> Result { + self.call("GET", paths::ADMIN_SERVER, None, http::SLOW_TIMEOUT) + } + + /// The server-wide storage overview; admins only. + pub fn admin_storage(&self) -> Result { + self.call("GET", paths::ADMIN_STORAGE, None, http::SLOW_TIMEOUT) + } + + /// One account's stored bytes and labeled estimates; admins only. + pub fn admin_user_usage(&self, id: Uuid) -> Result { + self.call("GET", &format!("{}/{id}/usage", paths::ADMIN_USER_USAGE), None, http::SLOW_TIMEOUT) + } + + /// A page of accounts by username ([`UserQuery`]); `next` of the answer is the cursor of the following page. + pub fn admin_users(&self, query: &UserQuery) -> Result { + self.call("GET", &query.path(), None, http::SLOW_TIMEOUT) + } + + /// A new account without a password, with an invite (its code is in the answer, once). `request.password` is the + /// acting admin's current password. + pub fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.call("POST", paths::ADMIN_USERS, Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// A new role and/or quota for an account (`request.password` is the acting admin's current password). + pub fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.call("PATCH", &user_path(id), Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// Disables an account: its devices are signed out, and it can't sign in. + pub fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/disable", user_path(id)), confirm) + } + + /// Enables a disabled account again (its signed-out devices stay signed out). + pub fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/enable", user_path(id)), confirm) + } + + /// Marks an account for removal; nothing it stored is deleted. + pub fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &user_path(id), confirm) + } + + /// A new invite for an account that has no password yet; the code is in the answer, once. + pub fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result { + self.call("POST", &format!("{}/invite", user_path(id)), Some(body(confirm)?), http::SLOW_TIMEOUT) + } + + /// Ends the account's open invite or reset code. + pub fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &format!("{}/invite", user_path(id)), confirm) + } + + /// A password reset code for an account (good for an hour); it is in the answer, once. `request.reset_two_factor` says + /// whether redeeming it also turns the account's two-factor sign-in off. + pub fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result { + self.call("POST", &format!("{}/password-reset", user_path(id)), Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// Takes an account's two-factor sign-in away and signs every device of it out (the server refuses it for the + /// admin's own account). + pub fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/reset-two-factor", user_path(id)), confirm) + } + + /// Every device of an account, revoked ones with when and why. + pub fn admin_devices(&self, id: Uuid) -> Result { + self.call("GET", &format!("{}/devices", user_path(id)), None, http::SLOW_TIMEOUT) + } + + /// Signs one device of an account out. + pub fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &format!("{}/devices/{device}", user_path(id)), confirm) + } + + /// A change whose only body is the confirmation, answered without one. + fn confirmed(&self, method: &str, path: &str, confirm: &AdminConfirm) -> Result<(), Error> { + self.answer(method, path, Some(body(confirm)?), http::SLOW_TIMEOUT).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_list_query_is_written_as_a_query_string() { + assert_eq!(UserQuery::default().path(), "/v1/admin/users"); + let all = UserQuery { cursor: Some("YW5h".into()), limit: Some(25), query: Some("a b&c=é".into()) }; + assert_eq!(all.path(), "/v1/admin/users?cursor=YW5h&limit=25&query=a%20b%26c%3D%C3%A9"); + let empty = UserQuery { cursor: Some(String::new()), limit: None, query: Some(String::new()) }; + assert_eq!(empty.path(), "/v1/admin/users", "empty means none"); + } +} diff --git a/crates/app/src/audit.rs b/crates/app/src/audit.rs new file mode 100644 index 00000000..1b95d9cf --- /dev/null +++ b/crates/app/src/audit.rs @@ -0,0 +1,368 @@ +//! The security audit log through the agent's local API: the signed-in account's own activity +//! ([`Client::account_activity`], for every account), and for an admin the whole log ([`Client::admin_audit`]) and its export +//! ([`Client::admin_audit_export`]). The server decides who may read what; this crate only asks. Every call asks the server: +//! call them **off the UI thread**. +//! +//! **The export is streamed**: the agent passes the server's NDJSON through as it comes (chunked), and this client writes each +//! line to the writer it is given as it reads it, with nothing held but one line (at most [`MAX_LINE_BYTES`]) and a small +//! buffer. It counts the export as whole only with the server's last line, `{"complete":true,"exported":n}`, when `n` is the +//! number of event lines it saw; a stream that ends without that line, breaks in the middle, or writes a line this crate can't +//! read is an error ([`Error::Protocol`] or [`Error::Io`]), and what was written to the writer is the caller's to throw away. +//! `{"truncated":true,…}` is a good export of the newest events that is not the whole log ([`ExportOutcome::truncated`]). + +use std::io::{BufRead, Read, Write}; + +use craft_cloud_api::api::{AuditEvent, AuditExportEnd, AuditPage, valid_audit_action}; +use craft_cloud_api::local::paths; + +use crate::{Client, Error, Uuid, http}; + +/// The longest line of an export this client reads, in bytes (an event is far smaller: its details are capped at 2 KiB). +pub const MAX_LINE_BYTES: usize = 64 * 1024; +/// The most events one export may hold before this client gives up (the server's own limit is 100,000). +pub const MAX_EXPORT_EVENTS: u64 = 1_000_000; + +/// Which events a list or an export asks for. A `None` leaves the filter out. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditQuery { + /// The `next` of the page before (opaque, given back as it came); none for the first page. Lists only. + pub cursor: Option, + /// Only events before this time (milliseconds since the Unix epoch); it limits the first page, a cursor wins over it. + pub before_ms: Option, + /// Events on the page, 1 to 200 (an account's own activity: 100); none: the server's default (50). Lists only. + pub limit: Option, + /// Only events about this account (an admin's filter). + pub user: Option, + /// Only this event name, or a prefix ending in a dot like `auth.` ([`craft_cloud_api::api::AUDIT_ACTIONS`]; an admin's filter). + pub action: Option, +} + +impl AuditQuery { + /// What is wrong with the query, before anything is sent: an action that is not in the closed list, a time before 1970, a + /// limit outside 1 to 200. + pub fn check(&self) -> Result<(), Error> { + let bad = + |message: &str| Error::Refused { code: craft_cloud_api::api::codes::INVALID.to_owned(), message: message.to_owned(), name_taken: None }; + if self.action.as_deref().is_some_and(|a| !valid_audit_action(a)) { + return Err(bad("action is an event name like auth.login, or a prefix like auth.")); + } + if self.before_ms.is_some_and(|t| t < 0) { + return Err(bad("before_ms is a time in milliseconds")); + } + if self.limit.is_some_and(|l| !(1..=200).contains(&l)) { + return Err(bad("limit is 1 to 200")); + } + Ok(()) + } + + /// `path` with the query string. + fn path(&self, path: &str) -> String { + let mut pairs = Vec::new(); + if let Some(cursor) = self.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(format!("cursor={}", crate::admin::encode(cursor))); + } + if let Some(before) = self.before_ms { + pairs.push(format!("before_ms={before}")); + } + if let Some(limit) = self.limit { + pairs.push(format!("limit={limit}")); + } + if let Some(user) = self.user { + pairs.push(format!("user={user}")); + } + if let Some(action) = self.action.as_deref().filter(|a| !a.is_empty()) { + pairs.push(format!("action={}", crate::admin::encode(action))); + } + if pairs.is_empty() { path.to_owned() } else { format!("{path}?{}", pairs.join("&")) } + } +} + +/// How an export ended, when it ended with the server's last line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExportOutcome { + /// Event lines written. + pub events: u64, + /// The log holds more events than an export does (the server's `truncated` line): these are the newest ones. + pub truncated: bool, +} + +impl Client { + /// The signed-in account, as the server says it: its role and whether it has two-factor sign-in. + pub fn account(&self) -> Result { + self.call("GET", paths::ACCOUNT, None, http::SLOW_TIMEOUT) + } + + /// A page of the signed-in account's own security events, newest first. Any account may; `user` and `action` of `query` + /// are not sent (they are the admin's filters). + pub fn account_activity(&self, query: &AuditQuery) -> Result { + let own = AuditQuery { user: None, action: None, ..query.clone() }; + own.check()?; + self.call("GET", &own.path(paths::ACCOUNT_ACTIVITY), None, http::SLOW_TIMEOUT) + } + + /// A page of the whole security log, newest first, filtered (an admin's: a member is `Refused { code: "forbidden" }`). + pub fn admin_audit(&self, query: &AuditQuery) -> Result { + query.check()?; + self.call("GET", &query.path(paths::ADMIN_AUDIT), None, http::SLOW_TIMEOUT) + } + + /// Writes the log, filtered by `query.user`, `query.action` and `query.before_ms`, to `out` as NDJSON (one event a line, + /// the server's own text), reading the stream as it comes (see the module's docs). Whole only when it returns + /// `Ok`; on any `Err`, `out` holds an unfinished export. A refusal before the stream starts is the usual `Refused`. + pub fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + let export = AuditQuery { cursor: None, limit: None, ..query.clone() }; + export.check()?; + let streamed = http::request_stream(self.addr(), self.token(), "GET", &export.path(paths::ADMIN_AUDIT_EXPORT), http::SLOW_TIMEOUT)?; + match streamed { + http::Streamed::Refused(answer) => Err(crate::refusal(answer).err().unwrap_or_else(|| Error::Protocol("the export was refused".into()))), + http::Streamed::Body(body) => copy_export(body, out), + } + } +} + +/// Reads `body` (the NDJSON of an export) line by line into `out`. +fn copy_export(body: impl Read, out: &mut dyn Write) -> Result { + let mut lines = std::io::BufReader::with_capacity(8 * 1024, body); + let mut events = 0u64; + let mut end: Option = None; + let mut line = Vec::new(); + loop { + line.clear(); + // One more byte than the longest line, to tell a line that is too long from one that just fits. + let read = + (&mut lines).take(MAX_LINE_BYTES as u64 + 1).read_until(b'\n', &mut line).map_err(|e| Error::Io(format!("the export broke off: {e}")))?; + if read == 0 { + break; + } + if line.last() != Some(&b'\n') { + return if read > MAX_LINE_BYTES { + Err(Error::Protocol("the export has a line that is too long".into())) + } else { + Err(Error::Protocol("the export was cut off in the middle of a line".into())) + }; + } + let text = line.strip_suffix(b"\n").unwrap_or(&line); + let text = text.strip_suffix(b"\r").unwrap_or(text); + if text.is_empty() { + continue; + } + if end.is_some() { + return Err(Error::Protocol("the export goes on after its last line".into())); + } + let value: serde_json::Value = + serde_json::from_slice(text).map_err(|e| Error::Protocol(format!("the export has a line that doesn't read: {e}")))?; + if value.get("exported").is_some() { + end = Some(serde_json::from_value(value).map_err(|e| Error::Protocol(format!("the export's last line doesn't read: {e}")))?); + continue; + } + // An event: read as one, so a stream of anything else is no export; what is written is the server's own line. + serde_json::from_value::(value).map_err(|e| Error::Protocol(format!("the export has an event that doesn't read: {e}")))?; + events = events.saturating_add(1); + if events > MAX_EXPORT_EVENTS { + return Err(Error::Protocol("the export has more events than this client takes".into())); + } + out.write_all(text).and_then(|()| out.write_all(b"\n")).map_err(|e| Error::Io(format!("can't write the export: {e}")))?; + } + out.flush().map_err(|e| Error::Io(format!("can't write the export: {e}")))?; + let Some(end) = end else { + return Err(Error::Protocol(format!("the export was cut off after {events} events: it has no last line"))); + }; + if end.exported != events { + return Err(Error::Protocol(format!("the export says {} events and has {events}", end.exported))); + } + match (end.complete, end.truncated) { + (Some(true), _) => Ok(ExportOutcome { events, truncated: false }), + (_, Some(true)) => Ok(ExportOutcome { events, truncated: true }), + _ => Err(Error::Protocol("the export's last line says neither complete nor truncated".into())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(n: u128) -> String { + format!( + r#"{{"id":"{}","at_ms":{n},"actor_user":null,"actor_device":null,"action":"auth.login","target_kind":"","target_id":null,"outcome":"success","details":{{}},"count":1}}"#, + Uuid::from_u128(n) + ) + } + + fn copy(text: &str) -> (Result, String) { + let mut out = Vec::new(); + let result = copy_export(text.as_bytes(), &mut out); + (result, String::from_utf8(out).unwrap()) + } + + #[test] + fn a_query_is_written_as_a_query_string_and_checked_first() { + let q = + AuditQuery { cursor: Some("YWJj".into()), before_ms: Some(5), limit: Some(20), user: Some(Uuid::nil()), action: Some("auth.".into()) }; + assert_eq!( + q.path("/v1/admin/audit"), + "/v1/admin/audit?cursor=YWJj&before_ms=5&limit=20&user=00000000-0000-0000-0000-000000000000&action=auth." + ); + assert_eq!(AuditQuery::default().path("/v1/admin/audit"), "/v1/admin/audit"); + assert!(q.check().is_ok()); + for bad in [ + AuditQuery { action: Some("auth.x".into()), ..AuditQuery::default() }, + AuditQuery { before_ms: Some(-1), ..AuditQuery::default() }, + AuditQuery { limit: Some(0), ..AuditQuery::default() }, + AuditQuery { limit: Some(201), ..AuditQuery::default() }, + ] { + assert!(matches!(bad.check(), Err(Error::Refused { ref code, .. }) if code == "invalid"), "{bad:?}"); + } + } + + #[test] + fn a_whole_export_is_copied_line_by_line() { + let text = format!("{}\n{}\n{{\"complete\":true,\"exported\":2}}\n", event(2), event(1)); + let (result, out) = copy(&text); + assert_eq!(result.unwrap(), ExportOutcome { events: 2, truncated: false }); + assert_eq!(out, format!("{}\n{}\n", event(2), event(1)), "the end line is not part of the export"); + let (result, _) = copy("{\"complete\":true,\"exported\":0}\n"); + assert_eq!(result.unwrap(), ExportOutcome { events: 0, truncated: false }); + } + + #[test] + fn a_truncated_export_is_good_but_not_whole() { + let text = format!("{}\n{{\"truncated\":true,\"exported\":1}}\n", event(1)); + assert_eq!(copy(&text).0.unwrap(), ExportOutcome { events: 1, truncated: true }); + } + + #[test] + fn a_stream_that_is_cut_off_or_wrong_is_no_export() { + // No last line (the server broke off, or the connection did). + let (result, _) = copy(&format!("{}\n{}\n", event(2), event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("cut off after 2 events")), "{result:?}"); + // Cut in the middle of a line. + let (result, _) = copy(&format!("{}\n{}", event(2), &event(1)[..20])); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("middle of a line")), "{result:?}"); + // A last line that disagrees with what came. + let (result, _) = copy(&format!("{}\n{{\"complete\":true,\"exported\":2}}\n", event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("says 2 events and has 1")), "{result:?}"); + // Neither complete nor truncated; lines after the end; lines that are not events; a line that is too long. + let (result, _) = copy("{\"exported\":0}\n"); + assert!(result.is_err()); + let (result, _) = copy(&format!("{{\"complete\":true,\"exported\":0}}\n{}\n", event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("goes on")), "{result:?}"); + let (result, _) = copy("{\"hello\":1}\n{\"complete\":true,\"exported\":0}\n"); + assert!(matches!(result, Err(Error::Protocol(_))), "{result:?}"); + let (result, _) = copy("not json\n"); + assert!(matches!(result, Err(Error::Protocol(_))), "{result:?}"); + let long = format!("{}\n", "x".repeat(MAX_LINE_BYTES + 10)); + let (result, _) = copy(&long); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("too long")), "{result:?}"); + } + + /// A reader that breaks after what it has (a connection that dropped): the export fails with an I/O error; a writer that + /// fails (a full disk) fails it too. + #[test] + fn a_broken_connection_or_a_full_disk_fails_the_export() { + struct Breaks(std::io::Cursor>); + impl Read for Breaks { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self.0.read(buf)? { + 0 => Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset")), + n => Ok(n), + } + } + } + let mut out = Vec::new(); + let result = copy_export(Breaks(std::io::Cursor::new(format!("{}\n", event(1)).into_bytes())), &mut out); + assert!(matches!(result, Err(Error::Io(ref m)) if m.contains("broke off")), "{result:?}"); + struct Full; + impl Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let result = copy_export(format!("{}\n", event(1)).as_bytes(), &mut Full); + assert!(matches!(result, Err(Error::Io(ref m)) if m.contains("can't write")), "{result:?}"); + } + + /// One answer from a listener of this test: the bytes it writes (the request is read first), then it closes. + fn serve_once(answer: Vec) -> std::net::SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut seen = Vec::new(); + let mut chunk = [0u8; 1024]; + while !seen.windows(4).any(|w| w == b"\r\n\r\n") { + let n = stream.read(&mut chunk).unwrap(); + seen.extend_from_slice(&chunk[..n]); + } + stream.write_all(&answer).unwrap(); + }); + addr + } + + fn chunk(text: &str) -> Vec { + format!("{:x}\r\n{text}\r\n", text.len()).into_bytes() + } + + fn exported(answer: Vec) -> (Result, String) { + let addr = serve_once(answer); + let mut out = Vec::new(); + let streamed = http::request_stream(addr, "token", "GET", "/v1/admin/audit/export", std::time::Duration::from_secs(5)).unwrap(); + let result = match streamed { + http::Streamed::Body(body) => copy_export(body, &mut out), + http::Streamed::Refused(answer) => Err(crate::refusal(answer).unwrap_err()), + }; + (result, String::from_utf8(out).unwrap()) + } + + fn head_again() -> Vec { + b"HTTP/1.1 200 OK\r\ncontent-type: application/x-ndjson\r\ntransfer-encoding: chunked\r\n\r\n".to_vec() + } + + /// Over a real connection: a chunked export that ends properly, one whose connection drops in the middle (no + /// terminating chunk: a failed export, whatever came before), and a refusal before it starts. + #[test] + fn an_export_over_a_connection_is_whole_only_when_it_ends_properly() { + let head = b"HTTP/1.1 200 OK\r\ncontent-type: application/x-ndjson\r\ntransfer-encoding: chunked\r\n\r\n".to_vec(); + let mut whole = head.clone(); + whole.extend(chunk(&format!("{}\n", event(2)))); + whole.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":2}}\n", event(1)))); + whole.extend(b"0\r\n\r\n"); + let (result, out) = exported(whole); + assert_eq!(result.unwrap(), ExportOutcome { events: 2, truncated: false }); + assert_eq!(out, format!("{}\n{}\n", event(2), event(1))); + + // The last line came, but the stream was cut before its terminating chunk: not whole (the end is not the server's). + let mut cut = head.clone(); + cut.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":1}}\n", event(1)))); + let (result, _) = exported(cut); + assert!(matches!(result, Err(Error::Io(_))), "{result:?}"); + // Cut off in the middle of a chunk. + let mut broken = head.clone(); + broken.extend(b"ff\r\n"); + broken.extend(event(1).as_bytes()); + let (result, out) = exported(broken); + assert!(result.is_err(), "{result:?}"); + assert!(out.is_empty(), "no half line is written: {out:?}"); + // Whole lines but no last line, and a clean end: still no export. + let mut no_end = head; + no_end.extend(chunk(&format!("{}\n", event(1)))); + no_end.extend(b"0\r\n\r\n"); + let (result, _) = exported(no_end); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("no last line")), "{result:?}"); + // The last chunk, then the connection ends before the body's last line break, or inside a trailer: not complete. + for tail in [&b"0\r\n"[..], &b"0\r\nX-Trailer: y"[..], &b"0\r\nX-Trailer: y\r\n"[..]] { + let mut cut_at_the_end = head_again(); + cut_at_the_end.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":1}}\n", event(1)))); + cut_at_the_end.extend(tail); + let (result, _) = exported(cut_at_the_end); + assert!(matches!(result, Err(Error::Io(_))), "{tail:?}: {result:?}"); + } + // A refusal: the agent's JSON, with the server's wait. + let body = r#"{"error":"rate_limited","message":"too many tries"}"#; + let refused = format!("HTTP/1.1 429 Too Many Requests\r\nretry-after: 30\r\ncontent-length: {}\r\n\r\n{body}", body.len()); + let (result, _) = exported(refused.into_bytes()); + assert!(matches!(&result, Err(e) if e.code() == Some("rate_limited") && e.retry_after_secs() == Some(30)), "{result:?}"); + } +} diff --git a/crates/app/src/blobs.rs b/crates/app/src/blobs.rs index 27698701..f3dd75be 100644 --- a/crates/app/src/blobs.rs +++ b/crates/app/src/blobs.rs @@ -28,7 +28,7 @@ pub struct Blobs { /// What an agent from before blobs answers. fn not_found() -> Error { - Error::Refused { code: craft_cloud_api::api::codes::NOT_FOUND.into(), message: "no such call".into() } + Error::Refused { code: craft_cloud_api::api::codes::NOT_FOUND.into(), message: "no such call".into(), name_taken: None } } /// An older agent's "no such call" as [`codes::AGENT_TOO_OLD`]. @@ -37,6 +37,7 @@ fn too_old(e: Error) -> Error { Error::Refused { code, .. } if code == craft_cloud_api::api::codes::NOT_FOUND => Error::Refused { code: codes::AGENT_TOO_OLD.into(), message: "the Craft Cloud agent on this computer is too old for binary content in live documents: update Craft Cloud".into(), + name_taken: None, }, other => other, } @@ -69,7 +70,11 @@ impl Blobs { /// agent too old for blobs is found out before a byte of them is sent ([`codes::AGENT_TOO_OLD`]). pub fn put(&self, bytes: &[u8]) -> Result { if bytes.is_empty() || bytes.len() > MAX_BLOB_BYTES { - return Err(Error::Refused { code: "too_large".into(), message: format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes") }); + return Err(Error::Refused { + code: "too_large".into(), + message: format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes"), + name_taken: None, + }); } self.check_agent()?; let answer = diff --git a/crates/app/src/fake.rs b/crates/app/src/fake.rs index ee5b8de2..eb0e3e5f 100644 --- a/crates/app/src/fake.rs +++ b/crates/app/src/fake.rs @@ -258,6 +258,7 @@ impl Hub { autostart: None, last_sync_ms: None, removed: false, + spaces: Vec::new(), }; respond_json(&mut stream, 200, &status); } @@ -876,7 +877,7 @@ mod tests { let stored = a.client().blobs().put(b"blob bytes").unwrap(); assert_eq!( b.client().blobs().get(&stored.hash, stored.len).unwrap_err(), - crate::Error::Refused { code: "offline".into(), message: "the server can't be reached to fetch the blob".into() } + crate::Error::Refused { code: "offline".into(), message: "the server can't be reached to fetch the blob".into(), name_taken: None } ); b.set_online(true); assert_eq!(b.client().blobs().get(&stored.hash, stored.len).unwrap(), b"blob bytes"); diff --git a/crates/app/src/http.rs b/crates/app/src/http.rs index 48aad8d5..fdd4cd0c 100644 --- a/crates/app/src/http.rs +++ b/crates/app/src/http.rs @@ -1,7 +1,7 @@ //! One HTTP/1.1 request to the agent's local API and its answer, over a plain loopback TCP connection: written out by //! hand (the agent is the only peer, and it answers with a `Content-Length`), read with `httparse`, every size capped. -use std::io::{Read, Write}; +use std::io::{BufRead, Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::time::Duration; @@ -20,11 +20,12 @@ pub(crate) const SLOW_TIMEOUT: Duration = Duration::from_secs(90); /// Most bytes of an answer's head. const MAX_HEAD_BYTES: usize = 16 * 1024; -/// An answer: status and body. +/// An answer: status and body, and the wait in seconds its `Retry-After` header names (a number of seconds; a date isn't read). #[derive(Debug)] pub(crate) struct Answer { pub status: u16, pub body: Vec, + pub retry_after_secs: Option, } /// Sends `method path` with the token and an optional JSON body to the agent at `addr`; the answer must come within @@ -74,7 +75,7 @@ pub(crate) fn read_answer(stream: &mut impl Read) -> Result { pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result { let mut buf = Vec::with_capacity(4096); let mut chunk = [0u8; 4096]; - let (status, length, head_len) = loop { + let (status, length, head_len, retry_after_secs) = loop { let n = stream.read(&mut chunk).map_err(io)?; if n == 0 { return Err(Error::Protocol("the agent closed the connection before answering".into())); @@ -86,15 +87,18 @@ pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result { let status = response.code.ok_or_else(|| Error::Protocol("an answer without a status".into()))?; let mut length = None; + let mut retry_after_secs = None; for header in response.headers.iter() { if header.name.eq_ignore_ascii_case("content-length") { let text = std::str::from_utf8(header.value).map_err(|_| Error::Protocol("a bad Content-Length".into()))?; length = Some(text.trim().parse::().map_err(|_| Error::Protocol("a bad Content-Length".into()))?); + } else if header.name.eq_ignore_ascii_case("retry-after") { + retry_after_secs = std::str::from_utf8(header.value).ok().and_then(|text| text.trim().parse::().ok()); } else if header.name.eq_ignore_ascii_case("transfer-encoding") { return Err(Error::Protocol("the agent's answer is chunked, which this client doesn't read".into())); } } - break (status, length, head_len); + break (status, length, head_len, retry_after_secs); } Ok(httparse::Status::Partial) if buf.len() <= MAX_HEAD_BYTES => continue, Ok(httparse::Status::Partial) => return Err(Error::Protocol("the answer's head is too long".into())), @@ -124,7 +128,192 @@ pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result>), +} + +/// Most bytes of a refusal's body read from a stream request. +const MAX_REFUSAL_BYTES: usize = 64 * 1024; +/// Longest chunk-size line (hex digits and extensions) read. +const MAX_CHUNK_LINE: usize = 64; + +/// `method path` with the token to the agent at `addr`, and its answer read as it comes: for a long body (an audit export) +/// that must not be held whole. A good answer (2xx) is its [`BodyReader`] (chunked, with a `Content-Length`, or to the end of +/// the connection); any other is read whole (at most [`MAX_REFUSAL_BYTES`]) as an [`Answer`]. `timeout` is how long a +/// single read or write may wait, not the whole transfer. +pub(crate) fn request_stream(addr: SocketAddr, token: &str, method: &str, path: &str, timeout: Duration) -> Result { + let mut stream = TcpStream::connect_timeout(&addr, CONNECT_TIMEOUT).map_err(|e| Error::NotRunning(format!("can't reach it: {e}")))?; + stream.set_read_timeout(Some(timeout)).map_err(io)?; + stream.set_write_timeout(Some(timeout)).map_err(io)?; + let head = format!("{method} {path} HTTP/1.1\r\nHost: {addr}\r\nAuthorization: Bearer {token}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + stream.write_all(head.as_bytes()).map_err(io)?; + stream.flush().map_err(io)?; + let mut reader = std::io::BufReader::with_capacity(8 * 1024, stream); + // The head: lines up to the empty one, at most MAX_HEAD_BYTES. + let mut raw = Vec::new(); + loop { + let mut line = Vec::new(); + let n = (&mut reader).take(MAX_HEAD_BYTES as u64).read_until(b'\n', &mut line).map_err(io)?; + if n == 0 { + return Err(Error::Protocol("the agent closed the connection before answering".into())); + } + raw.extend_from_slice(&line); + if raw.len() > MAX_HEAD_BYTES { + return Err(Error::Protocol("the answer's head is too long".into())); + } + if line == b"\r\n" || line == b"\n" { + break; + } + } + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut response = httparse::Response::new(&mut headers); + match response.parse(&raw) { + Ok(httparse::Status::Complete(_)) => {} + Ok(httparse::Status::Partial) => return Err(Error::Protocol("the answer's head is cut short".into())), + Err(e) => return Err(Error::Protocol(format!("the answer doesn't parse: {e}"))), + } + let status = response.code.ok_or_else(|| Error::Protocol("an answer without a status".into()))?; + let (mut length, mut chunked, mut retry_after_secs) = (None, false, None); + for header in response.headers.iter() { + let text = std::str::from_utf8(header.value).unwrap_or_default().trim(); + if header.name.eq_ignore_ascii_case("content-length") { + length = Some(text.parse::().map_err(|_| Error::Protocol("a bad Content-Length".into()))?); + } else if header.name.eq_ignore_ascii_case("transfer-encoding") { + chunked = text.eq_ignore_ascii_case("chunked"); + } else if header.name.eq_ignore_ascii_case("retry-after") { + retry_after_secs = text.parse::().ok(); + } + } + let mut body = if chunked { + BodyReader::chunked(reader) + } else if let Some(n) = length { + BodyReader::with_length(reader, n) + } else { + BodyReader::to_end(reader) + }; + if (200..300).contains(&status) { + return Ok(Streamed::Body(body)); + } + let mut bytes = Vec::new(); + // A refusal's body is small; a longer one is cut (the code and message are at its start). + let _ = (&mut body).take(MAX_REFUSAL_BYTES as u64).read_to_end(&mut bytes); + Ok(Streamed::Refused(Answer { status, body: bytes, retry_after_secs })) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Framing { + Length, + Chunked, + ToEnd, +} + +/// The body of an answer as it arrives, with its framing (chunks, or a length) taken off. A body that ends before its framing +/// says it should is an error, not an end. +pub(crate) struct BodyReader { + inner: R, + framing: Framing, + /// Bytes left of the length, or of the chunk being read. + left: u64, + done: bool, +} + +impl BodyReader { + pub(crate) fn chunked(inner: R) -> Self { + Self { inner, framing: Framing::Chunked, left: 0, done: false } + } + + pub(crate) fn with_length(inner: R, length: u64) -> Self { + Self { inner, framing: Framing::Length, left: length, done: length == 0 } + } + + pub(crate) fn to_end(inner: R) -> Self { + Self { inner, framing: Framing::ToEnd, left: 0, done: false } + } + + /// The next chunk's size (a line of hex digits, extensions ignored); `0` ends the body, after its trailers. + fn next_chunk(&mut self) -> std::io::Result { + let bad = |why: &str| std::io::Error::new(std::io::ErrorKind::InvalidData, why.to_owned()); + let mut line = Vec::new(); + let n = (&mut self.inner).take(MAX_CHUNK_LINE as u64 + 1).read_until(b'\n', &mut line)?; + if n == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body ended inside its chunks")); + } + if line.last() != Some(&b'\n') { + return Err(bad("a chunk size line is too long or cut off")); + } + let text = std::str::from_utf8(&line).map_err(|_| bad("a chunk size isn't text"))?; + let size = text.trim().split(';').next().unwrap_or_default().trim(); + let size = u64::from_str_radix(size, 16).map_err(|_| bad("a chunk size isn't a number"))?; + if size == 0 { + // The trailers, up to the empty line that ends the body: a stream that ends before it, or inside a trailer, is cut + // off (an error, not an end), and the trailers are bounded as the head is. + let mut seen = 0usize; + loop { + let mut trailer = Vec::new(); + let n = (&mut self.inner).take(MAX_HEAD_BYTES as u64 + 1).read_until(b'\n', &mut trailer)?; + if n == 0 || trailer.last() != Some(&b'\n') { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body ended before its last line break")); + } + if trailer == b"\r\n" || trailer == b"\n" { + break; + } + seen = seen.saturating_add(n); + if seen > MAX_HEAD_BYTES { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "the body's trailers are too long")); + } + } + } + Ok(size) + } + + /// The line break after a chunk's data. + fn end_of_chunk(&mut self) -> std::io::Result<()> { + let mut line = Vec::new(); + (&mut self.inner).take(3).read_until(b'\n', &mut line)?; + if line == b"\r\n" || line == b"\n" { + Ok(()) + } else { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "a chunk doesn't end where it should")) + } + } +} + +impl Read for BodyReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if buf.is_empty() || self.done { + return Ok(0); + } + if self.framing == Framing::ToEnd { + return self.inner.read(buf); + } + if self.framing == Framing::Chunked && self.left == 0 { + let size = self.next_chunk()?; + if size == 0 { + self.done = true; + return Ok(0); + } + self.left = size; + } + let max = usize::try_from(self.left).unwrap_or(usize::MAX).min(buf.len()); + let n = self.inner.read(buf.get_mut(..max).unwrap_or_default())?; + if n == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body is shorter than its framing says")); + } + self.left = self.left.saturating_sub(n as u64); + if self.left == 0 { + match self.framing { + Framing::Chunked => self.end_of_chunk()?, + _ => self.done = true, + } + } + Ok(n) + } } #[cfg(test)] @@ -146,5 +335,51 @@ mod tests { assert!(read_answer(&mut &b""[..]).is_err()); let no_length = b"HTTP/1.1 401 Unauthorized\r\n\r\n{\"error\":\"unauthorized\"}"; assert_eq!(read_answer(&mut &no_length[..]).unwrap().status, 401); + let slow = b"HTTP/1.1 429 Too Many Requests +Retry-After: 30 +content-length: 2 + +{}"; + assert_eq!(read_answer(&mut &slow[..]).unwrap().retry_after_secs, Some(30)); + assert_eq!(read_answer(&mut &no_length[..]).unwrap().retry_after_secs, None); + } + + #[test] + fn a_retry_after_header_is_kept_when_it_is_a_number_of_seconds() { + let text = b"HTTP/1.1 429 Too Many Requests\r\nRetry-After: 840\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &text[..]).unwrap().retry_after_secs, Some(840)); + let date = b"HTTP/1.1 429 Too Many Requests\r\nretry-after: Wed, 21 Oct 2026 07:28:00 GMT\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &date[..]).unwrap().retry_after_secs, None); + let none = b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &none[..]).unwrap().retry_after_secs, None); + } + + #[test] + fn bodies_are_read_by_their_framing() { + let read = |mut r: BodyReader<&[u8]>| { + let mut out = Vec::new(); + r.read_to_end(&mut out).map(|_| out) + }; + let chunked = b"5\r\nhello\r\n6;ext=1\r\n world\r\n0\r\nTrailer: x\r\n\r\nrest"; + assert_eq!(read(BodyReader::chunked(&chunked[..])).unwrap(), b"hello world"); + assert_eq!(read(BodyReader::with_length(&b"abcdef"[..], 4)).unwrap(), b"abcd"); + assert_eq!(read(BodyReader::to_end(&b"abc"[..])).unwrap(), b"abc"); + // Cut off inside a chunk, between chunks, or before the length: errors, never an end. + assert!(read(BodyReader::chunked(&b"5\r\nhel"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello"[..])).is_err()); + // The last chunk without the line break that ends the body, or cut inside a trailer: cut off, not complete. + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\n"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\nTrailer: x"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\nTrailer: x\r\n"[..])).is_err()); + assert_eq!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\n\r\n"[..])).unwrap(), b"hello"); + let long = format!("5\r\nhello\r\n0\r\n{}\r\n\r\n", "X: y\r\n".repeat(5_000)); + assert!(read(BodyReader::chunked(long.as_bytes())).is_err(), "trailers are bounded"); + assert!(read(BodyReader::with_length(&b"abc"[..], 4)).is_err()); + // Garbage and over-long size lines. + assert!(read(BodyReader::chunked(&b"zz\r\n"[..])).is_err()); + let long = format!("{}\r\n", "1".repeat(200)); + assert!(read(BodyReader::chunked(long.as_bytes())).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhelloXX0\r\n\r\n"[..])).is_err()); } } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 3f5d9af6..b9a46cac 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -13,7 +13,7 @@ //! [`Client::set_autostart`] (a few seconds at most). **Off the UI thread**, since they wait for the server (up to //! 90 s): [`Client::login`], [`Client::join`], [`Client::logout`], and the add-a-computer and computer-list calls //! ([`Client::pairing`], [`Client::pairing_state`], [`Client::cancel_pairing`], [`Client::devices`], -//! [`Client::remove_device`]), and opening a live document ([`live::Documents::open`], [`live::Importer::import`]: up +//! [`Client::remove_device`]), the admin calls ([`admin`]), and opening a live document ([`live::Documents::open`], [`live::Importer::import`]: up //! to 30 s). A live document's own calls ([`live::Document::send`], [`live::Document::try_recv`]) don't wait for an //! answer. //! @@ -49,23 +49,44 @@ //! TLS: the agent is on this machine. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod admin; +mod audit; pub mod blobs; #[cfg(feature = "fake")] pub mod fake; mod http; pub mod live; mod recents; +mod spaces; use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::time::Duration; use craft_cloud_api::api::ErrorAnswer; -use craft_cloud_api::local::{AutostartRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, paths}; +use craft_cloud_api::local::{ + AutostartRequest, ChangePasswordRequest, ErrorExtras, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, PasswordRequest, + RenameDeviceRequest, TotpRequest, paths, +}; +pub use admin::UserQuery; +pub use audit::{AuditQuery, ExportOutcome}; pub use craft_cloud_api::DeviceId; -pub use craft_cloud_api::api::{DeviceInfo, PairingState, PairingStatus}; -pub use craft_cloud_api::local::{JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RunState, Status, codes}; +pub use craft_cloud_api::api::{ + AUDIT_ACTIONS, AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminManifestIndex, + AdminPasswordReset, AdminServerInfo, AdminStorageAccount, AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditEvent, + AuditPage, DeviceInfo, PairingState, PairingStatus, RecoveryCodes, RevokedOthers, TwoFactorChallenge, TwoFactorSetupResponse, TwoFactorStatus, + UsageEstimateReason, UsageEstimates, valid_audit_action, +}; +pub use craft_cloud_api::api::{SpaceListItem, SpaceRole}; +pub use craft_cloud_api::local::{ + ApproveDeletesAnswer, ApproveDeletesRequest, SpaceInfo, SpaceList, SpaceRun, SpaceStatus, SpaceSync, SpaceSyncRequest, +}; +pub use craft_cloud_api::local::{ + HistoryResponse, HistoryVersion, JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RestoreResponse, RunState, + SecondFactorLoginRequest, Status, TrashItem, TrashNodeRequest, TrashResponse, TrashRestoreResponse, TwoFactorProofRequest, codes, +}; +pub use craft_cloud_api::{NodeId, SpaceId, VersionId}; pub use recents::{RecentsEvent, RecentsWatch}; pub use uuid::Uuid; @@ -78,9 +99,15 @@ pub enum Error { /// The agent speaks another local API version: one of the two needs an update. #[error("the Craft Cloud agent speaks local API version {0}, this app {LOCAL_API_MIN}-{LOCAL_API_MAX}: update the older one")] Version(u32), - /// The agent refused the request: `code` is one of [`codes`] (or the HTTP API's), `message` is for people. + /// The agent refused the request: `code` is one of [`codes`] (or the HTTP API's), `message` is for people, and + /// `name_taken` identifies a trash-restore conflict when the server provides it. #[error("{message}")] - Refused { code: String, message: String }, + Refused { code: String, message: String, name_taken: Option }, + /// A refusal that carries more: the challenge of `two_factor_required` and/or the wait of `two_factor_locked` + /// (`retry_after_secs`). Only answers with one of those are this; every other refusal is [`Error::Refused`], so code that + /// matches `Refused { code, .. }` goes on working. [`Error::code`] reads the code of both. + #[error("{message}")] + RefusedWith { code: String, message: String, two_factor: Option, retry_after_secs: Option }, /// The connection broke, or the agent didn't answer in time. #[error("the connection to the Craft Cloud agent broke: {0}")] Io(String), @@ -89,6 +116,32 @@ pub enum Error { Protocol(String), } +impl Error { + /// The code of a refusal ([`Error::Refused`] or [`Error::RefusedWith`]); `None` for any other error. + pub fn code(&self) -> Option<&str> { + match self { + Error::Refused { code, .. } | Error::RefusedWith { code, .. } => Some(code), + _ => None, + } + } + + /// The challenge of a `two_factor_required` refusal. + pub fn challenge(&self) -> Option<&TwoFactorChallenge> { + match self { + Error::RefusedWith { two_factor, .. } => two_factor.as_ref(), + _ => None, + } + } + + /// How many seconds a `two_factor_locked` (or other) refusal says to wait. + pub fn retry_after_secs(&self) -> Option { + match self { + Error::RefusedWith { retry_after_secs, .. } => *retry_after_secs, + _ => None, + } + } +} + /// The running agent, as `local-api.json` names it. #[derive(Debug, Clone)] pub struct Client { @@ -147,6 +200,11 @@ impl Client { serde_json::from_slice(&answer.body).map_err(|e| Error::Protocol(e.to_string())) } + /// A call with a body that is answered with no body (`204`). + fn call_unit(&self, method: &str, path: &str, body: Option>, timeout: Duration) -> Result<(), Error> { + self.answer(method, path, body, timeout).map(|_| ()) + } + /// A call answered without a body (`204`). fn call_empty(&self, method: &str, path: &str, timeout: Duration) -> Result<(), Error> { self.answer(method, path, None, timeout).map(|_| ()) @@ -183,6 +241,67 @@ impl Client { self.call("POST", paths::LOGIN, Some(body), http::SLOW_TIMEOUT) } + /// The second step of a sign-in that answered `Refused`/`RefusedWith { code: "two_factor_required" }`: the first step's + /// server, folder, computer name and `adopt` again, with the answer's challenge ([`Error::challenge`]) and a code from + /// the authenticator app or a recovery code. A wrong code is `two_factor_invalid` (the challenge takes five), too many + /// is `two_factor_locked` with [`Error::retry_after_secs`]; neither signs anything out. Waits for the server: call it off + /// the UI thread. + pub fn login_second_factor(&self, request: &SecondFactorLoginRequest) -> Result { + let body = serde_json::to_vec(request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::LOGIN_SECOND_FACTOR, Some(body), http::SLOW_TIMEOUT) + } + + /// Whether the signed-in account has two-factor sign-in on, how many recovery codes are left and whether a set-up is + /// open. Asks the server: call it off the UI thread. A server from before two-factor sign-in answers `not_found`. + pub fn two_factor_status(&self) -> Result { + self.call("GET", paths::TWO_FACTOR, None, http::SLOW_TIMEOUT) + } + + /// Starts setting two-factor sign-in up with the account's password: the secret (Base32 and as an `otpauth://` URI, for + /// a QR code), good for 10 minutes. `wrong_password` (403) is no sign-out. The secret is shown once and never stored. + pub fn two_factor_setup(&self, password: &str) -> Result { + let body = serde_json::to_vec(&PasswordRequest { password: password.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_SETUP, Some(body), http::SLOW_TIMEOUT) + } + + /// Turns two-factor sign-in on with the first code; the ten recovery codes, shown once (the account's other computers + /// are signed out). + pub fn two_factor_confirm(&self, totp_code: &str) -> Result { + let body = serde_json::to_vec(&TotpRequest { totp_code: totp_code.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_CONFIRM, Some(body), http::SLOW_TIMEOUT) + } + + /// Turns two-factor sign-in off: the password and a code or a recovery code. + pub fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + let body = serde_json::to_vec(request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call_unit("POST", paths::TWO_FACTOR_DISABLE, Some(body), http::SLOW_TIMEOUT) + } + + /// Ten new recovery codes (the old ones stop working): the password and a code from the authenticator app. + pub fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + let request = TwoFactorProofRequest { password: password.to_owned(), totp_code: Some(totp_code.to_owned()), recovery_code: None }; + let body = serde_json::to_vec(&request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_RECOVERY_CODES, Some(body), http::SLOW_TIMEOUT) + } + + /// Changes the account's password; the account's other computers are signed out. + pub fn change_password(&self, current_password: &str, new_password: &str) -> Result<(), Error> { + let request = ChangePasswordRequest { current_password: current_password.to_owned(), new_password: new_password.to_owned() }; + let body = serde_json::to_vec(&request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call_unit("POST", paths::ACCOUNT_PASSWORD, Some(body), http::SLOW_TIMEOUT) + } + + /// Signs every other computer of the account out; how many were. + pub fn revoke_other_devices(&self) -> Result { + self.call("POST", paths::DEVICES_REVOKE_OTHERS, None, http::SLOW_TIMEOUT) + } + + /// Renames one of the account's computers that isn't signed out. + pub fn rename_device(&self, id: DeviceId, name: &str) -> Result { + let body = serde_json::to_vec(&RenameDeviceRequest { name: name.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("PATCH", &format!("{}/{id}", paths::DEVICES), Some(body), http::SLOW_TIMEOUT) + } + /// Signs the agent out (the files stay). `local`: without reaching the server (the device stays valid there). /// Waits for the server (up to 90 s): call it off the UI thread. pub fn logout(&self, local: bool) -> Result { @@ -221,6 +340,11 @@ impl Client { self.call("GET", paths::DEVICES, None, http::SLOW_TIMEOUT) } + /// The signed-in account's storage usage and labeled estimates. Asks the server each time. + pub fn usage(&self) -> Result { + self.call("GET", paths::USAGE, None, http::SLOW_TIMEOUT) + } + /// Removes another computer from the account (its next request is refused; its files stay there), and gives the /// list after. This computer is `Refused { code: "cannot_remove_self" }`: that is [`Client::logout`]. Asks the /// server: call it off the UI thread. @@ -228,6 +352,52 @@ impl Client { self.call("DELETE", &format!("{}/{id}", paths::DEVICES), None, http::SLOW_TIMEOUT) } + /// The newest-first server history for a file in the synced folder. `cursor` is the opaque `next` value from the + /// previous page. Asks the server: call it off the UI thread. + pub fn file_history(&self, path: &str, cursor: Option<&str>, limit: Option) -> Result { + let mut query = format!("path={}", query_component(path)); + if let Some(cursor) = cursor { + query.push_str(&format!("&cursor={}", query_component(cursor))); + } + if let Some(limit) = limit { + query.push_str(&format!("&limit={limit}")); + } + self.call("GET", &format!("{}?{query}", paths::FILE_HISTORY), None, http::SLOW_TIMEOUT) + } + + /// Restores `version` of a file in the synced folder, if `expected_version` is still current. Asks the server: + /// call it off the UI thread. + pub fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result { + let body = serde_json::to_vec(&craft_cloud_api::local::RestoreRequest { path: path.to_owned(), version, expected_version }) + .map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::FILE_RESTORE, Some(body), http::SLOW_TIMEOUT) + } + + /// The server's newest-first trash page. `cursor` is the opaque `next` value from the preceding page. + pub fn trash(&self, cursor: Option<&str>, limit: Option) -> Result { + let mut query = Vec::new(); + if let Some(cursor) = cursor { + query.push(format!("cursor={}", query_component(cursor))); + } + if let Some(limit) = limit { + query.push(format!("limit={limit}")); + } + let path = if query.is_empty() { paths::TRASH.to_owned() } else { format!("{}?{}", paths::TRASH, query.join("&")) }; + self.call("GET", &path, None, http::SLOW_TIMEOUT) + } + + /// Restores a node from the server's trash. Call off the UI thread. + pub fn restore_from_trash(&self, id: NodeId) -> Result { + let body = serde_json::to_vec(&TrashNodeRequest { id: id.to_string() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TRASH_RESTORE, Some(body), http::SLOW_TIMEOUT) + } + + /// Marks a node as removed from the server's trash. This does not free its stored data yet. + pub fn remove_from_trash(&self, id: NodeId) -> Result<(), Error> { + let body = serde_json::to_vec(&TrashNodeRequest { id: id.to_string() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.answer("POST", paths::TRASH_REMOVE, Some(body), http::SLOW_TIMEOUT).map(|_| ()) + } + /// Pauses syncing until [`Client::resume`], also across restarts; recents keep working. Answered by the agent /// itself (a few seconds at most): fine from a UI thread. pub fn pause(&self) -> Result { @@ -283,13 +453,34 @@ impl Client { } } +fn query_component(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b'/') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + /// `answer` if it succeeded, else the agent's refusal (its code and message). fn refusal(answer: http::Answer) -> Result { if (200..300).contains(&answer.status) { Ok(answer) } else { match serde_json::from_slice::(&answer.body) { - Ok(body) => Err(Error::Refused { code: body.error, message: body.message }), + Ok(body) => { + let extras = serde_json::from_slice::(&answer.body).unwrap_or_default(); + // The wait is in the answer's JSON; the `Retry-After` header is the same wait, used when the JSON lacks it. + let retry_after_secs = extras.retry_after_secs.or(answer.retry_after_secs); + if extras.two_factor.is_none() && retry_after_secs.is_none() { + Err(Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }) + } else { + Err(Error::RefusedWith { code: body.error, message: body.message, two_factor: extras.two_factor, retry_after_secs }) + } + } Err(_) => Err(Error::Protocol(format!("HTTP {}", answer.status))), } } @@ -300,6 +491,23 @@ mod tests { use super::*; use std::io::{Read, Write}; + #[test] + fn a_refusal_takes_its_wait_from_the_retry_after_header_when_the_json_has_none() { + let body = |text: &str| text.as_bytes().to_vec(); + let only_header = http::Answer { status: 429, body: body(r#"{"error":"two_factor_locked","message":"wait"}"#), retry_after_secs: Some(900) }; + assert_eq!(refusal(only_header).unwrap_err().retry_after_secs(), Some(900)); + // The JSON wins when both are there. + let both = http::Answer { + status: 429, + body: body(r#"{"error":"two_factor_locked","message":"wait","retry_after_secs":840}"#), + retry_after_secs: Some(900), + }; + assert_eq!(refusal(both).unwrap_err().retry_after_secs(), Some(840)); + // Neither: a plain refusal. + let neither = http::Answer { status: 403, body: body(r#"{"error":"two_factor_invalid","message":"no"}"#), retry_after_secs: None }; + assert!(matches!(refusal(neither).unwrap_err(), Error::Refused { .. })); + } + /// What the fake agent was asked: method, path and body. type Asked = (String, String, String); @@ -365,6 +573,7 @@ mod tests { autostart: Some(signed_in), last_sync_ms: None, removed: false, + spaces: Vec::new(), } } @@ -379,8 +588,20 @@ mod tests { let device = craft_cloud_api::DeviceId::generate(); let info = PairingInfo { id, join: "c.example/K7M4-Q9TX".into(), link: "https://c.example/join/K7M4-Q9TX".into(), expires_ms: 9 }; let used = PairingStatus { state: PairingState::Used, device_name: Some("lptp1".into()) }; - let list = vec![DeviceInfo { id: device, name: "lptp1".into(), created_ms: 1, last_used_ms: 2, revoked: false, current: false }]; - let refused = json(&ErrorAnswer { error: "password_required".into(), message: "choose a password".into() }); + let list = vec![DeviceInfo { + id: device, + name: "lptp1".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }]; + let refused = json(&ErrorAnswer { error: "password_required".into(), message: "choose a password".into(), name_taken: None }); let script = vec![ (400, refused), (200, json(&status(true))), @@ -397,8 +618,15 @@ mod tests { let (_dir, agent, asked) = fake_agent(script); let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); - let request = - JoinRequest { join: "c.example/K7M4-Q9TX".into(), folder: None, device_name: Some("lptp1".into()), adopt: true, password: None }; + let request = JoinRequest { + join: "c.example/K7M4-Q9TX".into(), + folder: None, + device_name: Some("lptp1".into()), + adopt: true, + password: None, + totp_code: None, + recovery_code: None, + }; let refused = agent.join(&request); assert!(matches!(refused, Err(Error::Refused { ref code, .. }) if code == "password_required"), "{refused:?}"); let (method, path, body) = next(); @@ -430,6 +658,350 @@ mod tests { assert_eq!(serde_json::from_str::(&body).unwrap(), AutostartRequest { on: true }); } + /// The two-factor calls send what the local API wants and read what it answers; a refusal keeps its challenge or its wait. + #[test] + fn the_two_factor_calls_send_their_requests_and_keep_what_a_refusal_carries() { + let recovery = RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into(); 10] }; + let setup = TwoFactorSetupResponse { secret: "JBSWY3DPEHPK3PXP".into(), otpauth_uri: "otpauth://totp/x".into(), expires_ms: 5 }; + let state = TwoFactorStatus { enabled: true, enabled_ms: Some(3), recovery_codes_left: 9, pending: false }; + let device = craft_cloud_api::DeviceId::generate(); + let renamed = DeviceInfo { + id: device, + name: "new name".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }; + let required = + json(&serde_json::json!({ "error": "two_factor_required", "message": "m", "two_factor": { "challenge": "chal", "expires_ms": 9 } })); + let locked = json(&serde_json::json!({ "error": "two_factor_locked", "message": "wait", "retry_after_secs": 840 })); + let plain = json(&ErrorAnswer { error: "two_factor_invalid".into(), message: "wrong".into(), name_taken: None }); + let script = vec![ + (401, required), + (200, json(&status(true))), + (429, locked), + (403, plain), + (200, json(&state)), + (200, json(&setup)), + (200, json(&recovery)), + (204, String::new()), + (200, json(&recovery)), + (204, String::new()), + (200, json(&RevokedOthers { revoked: 2 })), + (200, json(&renamed)), + ]; + let (_dir, agent, asked) = fake_agent(script); + let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); + + let login = LoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + password: "pw-secret-1".into(), + folder: None, + device_name: None, + adopt: false, + }; + let refused = agent.login(&login).unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required")); + assert_eq!(refused.challenge().map(|c| (c.challenge.as_str(), c.expires_ms)), Some(("chal", 9))); + assert!(!format!("{refused:?}").contains("\"chal\""), "the challenge is hidden in Debug"); + let _ = next(); + let second = SecondFactorLoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + folder: None, + device_name: None, + adopt: false, + challenge: "chal".into(), + totp_code: Some("123456".into()), + recovery_code: None, + }; + assert!(agent.login_second_factor(&second).unwrap().signed_in); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::LOGIN_SECOND_FACTOR)); + assert_eq!(serde_json::from_str::(&body).unwrap(), second); + + // A lock keeps its wait; a plain refusal stays a plain `Refused` (code and message only). + let err = agent.login_second_factor(&second).unwrap_err(); + assert_eq!((err.code(), err.retry_after_secs(), err.challenge()), (Some("two_factor_locked"), Some(840), None)); + let _ = next(); + let err = agent.login_second_factor(&second).unwrap_err(); + assert!(matches!(&err, Error::Refused { code, .. } if code == "two_factor_invalid"), "{err:?}"); + let _ = next(); + + assert_eq!(agent.two_factor_status().unwrap(), state); + assert_eq!(next(), ("GET".into(), paths::TWO_FACTOR.to_owned(), String::new())); + assert_eq!(agent.two_factor_setup("pw-secret-1").unwrap(), setup); + let (_, path, body) = next(); + assert_eq!( + (path.as_str(), serde_json::from_str::(&body).unwrap().password.as_str()), + (paths::TWO_FACTOR_SETUP, "pw-secret-1") + ); + assert_eq!(agent.two_factor_confirm("123456").unwrap(), recovery); + let (_, path, body) = next(); + assert_eq!((path.as_str(), serde_json::from_str::(&body).unwrap().totp_code.as_str()), (paths::TWO_FACTOR_CONFIRM, "123456")); + let off = TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: None, recovery_code: Some("ABCD-EFGH-JKMN-PQRS".into()) }; + agent.two_factor_disable(&off).unwrap(); + let (_, path, body) = next(); + assert_eq!((path.as_str(), serde_json::from_str::(&body).unwrap()), (paths::TWO_FACTOR_DISABLE, off)); + assert_eq!(agent.two_factor_recovery_codes("pw-secret-1", "654321").unwrap(), recovery); + let (_, path, body) = next(); + let sent = serde_json::from_str::(&body).unwrap(); + assert_eq!((path.as_str(), sent.totp_code.as_deref(), sent.recovery_code), (paths::TWO_FACTOR_RECOVERY_CODES, Some("654321"), None)); + agent.change_password("old-secret-pw", "new-secret-pw").unwrap(); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::ACCOUNT_PASSWORD)); + assert_eq!(serde_json::from_str::(&body).unwrap().new_password, "new-secret-pw"); + assert_eq!(agent.revoke_other_devices().unwrap(), RevokedOthers { revoked: 2 }); + assert_eq!(next(), ("POST".into(), paths::DEVICES_REVOKE_OTHERS.to_owned(), String::new())); + assert_eq!(agent.rename_device(device, "new name").unwrap(), renamed); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path), ("PATCH", format!("{}/{device}", paths::DEVICES))); + assert_eq!(serde_json::from_str::(&body).unwrap().name, "new name"); + } + + /// The admin calls: each sends its request to its path with the confirmation in the body, and reads its answer; a + /// refusal comes back with the server's code, and the wait of a `Retry-After` is kept. + #[test] + fn each_admin_call_sends_its_request_and_reads_its_answer() { + let id = uuid::Uuid::now_v7(); + let device = craft_cloud_api::DeviceId::generate(); + let user = AdminUserInfo { + id, + username: "ana".into(), + role: "member".into(), + created_ms: 1, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 1, + stored_bytes: 5, + quota_bytes: None, + invite_pending: false, + }; + let code = AdminCode { code: "ABCDEFGHJKMNPQRS".into(), join_url: None, expires_ms: 9 }; + let list = AdminUserList { users: vec![user.clone()], next: Some("YW5h".into()) }; + let forbidden = json(&ErrorAnswer { error: "wrong_password".into(), message: "that is not your password".into(), name_taken: None }); + let script = vec![ + (200, json(&list)), + (200, json(&list)), + (201, json(&AdminCreatedUser { user: user.clone(), invite: code.clone() })), + (200, json(&user)), + (204, String::new()), + (204, String::new()), + (202, String::new()), + (200, json(&code)), + (204, String::new()), + (200, json(&code)), + (204, String::new()), + (200, json(&AdminDevices { devices: Vec::new() })), + (204, String::new()), + (403, forbidden), + ]; + let (_dir, agent, asked) = fake_agent(script); + let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); + let confirm = AdminConfirm { password: Some("admin-secret-pw".into()), ..AdminConfirm::default() }; + let password = |body: &str| serde_json::from_str::(body).unwrap()["password"] == "admin-secret-pw"; + + assert_eq!(agent.admin_users(&UserQuery::default()).unwrap(), list); + assert_eq!(next(), ("GET".into(), "/v1/admin/users".into(), String::new())); + let page = UserQuery { cursor: Some("YW5h".into()), limit: Some(10), query: Some("an".into()) }; + agent.admin_users(&page).unwrap(); + assert_eq!(next().1, "/v1/admin/users?cursor=YW5h&limit=10&query=an"); + let create = + AdminCreateUser { username: "bo".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + assert_eq!(agent.admin_create_user(&create).unwrap().invite, code); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", "/v1/admin/users")); + assert!(password(&sent) && sent.contains("\"bo\""), "{sent}"); + let update = AdminUpdateUser { quota_bytes: Some(None), password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + assert_eq!(agent.admin_update_user(id, &update).unwrap(), user); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("PATCH", format!("/v1/admin/users/{id}").as_str())); + assert!(password(&sent) && sent.contains("\"quota_bytes\":null"), "{sent}"); + agent.admin_disable_user(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/disable")); + agent.admin_enable_user(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/enable")); + agent.admin_remove_user(id, &confirm).unwrap(); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("DELETE", format!("/v1/admin/users/{id}").as_str())); + assert!(password(&sent)); + assert_eq!(agent.admin_issue_invite(id, &confirm).unwrap(), code); + assert_eq!(next().1, format!("/v1/admin/users/{id}/invite")); + agent.admin_revoke_invite(id, &confirm).unwrap(); + assert_eq!(next(), ("DELETE".into(), format!("/v1/admin/users/{id}/invite"), json(&confirm))); + let reset = AdminPasswordReset { + password: Some("admin-secret-pw".into()), + totp_code: Some("123456".into()), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + assert_eq!(agent.admin_issue_reset(id, &reset).unwrap(), code); + let (_, path, sent) = next(); + assert_eq!(path, format!("/v1/admin/users/{id}/password-reset")); + assert_eq!(serde_json::from_str::(&sent).unwrap(), reset, "the factor and the reset flag go with it"); + agent.admin_reset_two_factor(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/reset-two-factor")); + assert!(agent.admin_devices(id).unwrap().devices.is_empty()); + assert_eq!(next(), ("GET".into(), format!("/v1/admin/users/{id}/devices"), String::new())); + agent.admin_revoke_device(id, device, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/devices/{device}")); + let wrong = agent.admin_disable_user(id, &confirm); + assert!(matches!(wrong, Err(Error::Refused { ref code, .. }) if code == "wrong_password"), "{wrong:?}"); + } + + #[test] + fn storage_admin_calls_use_the_local_admin_routes() { + let id = uuid::Uuid::now_v7(); + let overview = AdminStorageOverview { + accounts: 1, + stored_bytes: 17, + default_quota_bytes: Some(99), + accounts_with_override: 0, + accounts_near_quota: 0, + accounts_over_quota: 0, + top_accounts: vec![AdminStorageAccount { id, username: "ana".into(), stored_bytes: 17, quota_bytes: Some(99) }], + manifest_index: AdminManifestIndex { complete: 3, building: 1, not_indexed: 2 }, + }; + let usage = AccountUsage { + stored_bytes: 17, + quota_bytes: Some(99), + trash_bytes: Some(4), + history_bytes: Some(8), + estimates: UsageEstimates { index_complete: true, reason: None }, + }; + let (_dir, agent, asked) = fake_agent(vec![(200, json(&overview)), (200, json(&usage))]); + assert_eq!(agent.admin_storage().unwrap(), overview); + assert_eq!(asked.recv_timeout(Duration::from_secs(5)).unwrap(), ("GET".into(), "/v1/admin/storage".into(), String::new())); + assert_eq!(agent.admin_user_usage(id).unwrap(), usage); + assert_eq!(asked.recv_timeout(Duration::from_secs(5)).unwrap(), ("GET".into(), format!("/v1/admin/users/{id}/usage"), String::new())); + } + + #[test] + fn history_and_restore_use_relative_paths_and_typed_bodies() { + let restored = RestoreResponse { + version: VersionId::generate(), + parent_version: VersionId::generate(), + seq: 3, + epoch: craft_cloud_api::Epoch::generate(), + }; + let (_dir, agent, asked) = fake_agent(vec![ + (200, json(&HistoryResponse { versions: vec![], next: Some("7.0192f0c4-0000-7000-8000-000000000000".into()) })), + (200, json(&restored)), + ]); + let history = agent.file_history("Art Folder/paint #1.craft", None, Some(25)).unwrap(); + assert_eq!(history.next.as_deref(), Some("7.0192f0c4-0000-7000-8000-000000000000")); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(method, "GET"); + assert_eq!(path, "/v1/files/history?path=Art%20Folder/paint%20%231.craft&limit=25"); + assert!(body.is_empty()); + + let expected = VersionId::generate(); + assert_eq!(agent.restore_file("Art Folder/paint #1.craft", restored.version, expected).unwrap(), restored); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::FILE_RESTORE)); + assert_eq!( + serde_json::from_str::(&body).unwrap(), + craft_cloud_api::local::RestoreRequest { + path: "Art Folder/paint #1.craft".into(), + version: restored.version, + expected_version: expected, + } + ); + } + + #[test] + fn trash_calls_use_paging_and_typed_node_ids() { + let id = NodeId::generate(); + let restored = TrashRestoreResponse { + node: id, + parent: NodeId::generate(), + name: "restored.txt".into(), + renamed: true, + version: Some(VersionId::generate()), + seq: 4, + epoch: craft_cloud_api::Epoch::generate(), + }; + let item = craft_cloud_api::local::TrashItem { + id, + parent: restored.parent, + name: "gone.txt".into(), + is_dir: false, + size: Some(3), + deleted_ms: 10, + purge_after_ms: 20, + path: Some("folder/gone.txt".into()), + }; + let (_dir, agent, asked) = fake_agent(vec![ + (200, json(&TrashResponse { items: vec![item], next: Some("cursor value".into()) })), + (200, json(&restored)), + (204, String::new()), + ]); + assert_eq!(agent.trash(None, Some(25)).unwrap().next.as_deref(), Some("cursor value")); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("GET", "/v1/trash?limit=25")); + assert!(body.is_empty()); + + assert_eq!(agent.restore_from_trash(id).unwrap(), restored); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::TRASH_RESTORE)); + assert_eq!(serde_json::from_str::(&body).unwrap(), TrashNodeRequest { id: id.to_string() }); + + agent.remove_from_trash(id).unwrap(); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::TRASH_REMOVE)); + assert_eq!(serde_json::from_str::(&body).unwrap(), TrashNodeRequest { id: id.to_string() }); + } + + #[test] + fn trash_cursor_with_reserved_characters_is_percent_encoded() { + let (_dir, agent, asked) = fake_agent(vec![(200, json(&TrashResponse { items: vec![], next: None }))]); + agent.trash(Some("a+b/c=d&e f%g"), Some(25)).unwrap(); + let (method, path, _body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(method, "GET"); + assert_eq!(path, "/v1/trash?cursor=a%2Bb/c%3Dd%26e%20f%25g&limit=25"); + // The agent reads the query as URL-encoded pairs: decoding the value must give the cursor back. + let value = path.split_once("cursor=").and_then(|(_, rest)| rest.split_once('&')).map(|(value, _)| value).unwrap(); + let mut decoded = Vec::new(); + let bytes = value.as_bytes(); + let mut at = 0; + while at < bytes.len() { + if bytes[at] == b'%' { + decoded.push(u8::from_str_radix(&value[at + 1..at + 3], 16).unwrap()); + at += 3; + } else { + decoded.push(bytes[at]); + at += 1; + } + } + assert_eq!(String::from_utf8(decoded).unwrap(), "a+b/c=d&e f%g"); + } + + #[test] + fn trash_restore_keeps_the_server_name_taken_detail() { + let id = NodeId::generate(); + let refused = json(&ErrorAnswer { error: "conflict".into(), message: "no free conflict name".into(), name_taken: Some(true) }); + let (_dir, agent, _asked) = fake_agent(vec![(409, refused)]); + let result = agent.restore_from_trash(id); + assert!(matches!(result, Err(Error::Refused { ref code, name_taken: Some(true), .. }) if code == "conflict"), "{result:?}"); + } + + #[test] + fn usage_maps_unauthorized_and_uses_the_local_route() { + let answer = json(&ErrorAnswer { error: "unauthorized".into(), message: "not signed in".into(), name_taken: None }); + let (_dir, agent, asked) = fake_agent(vec![(401, answer)]); + let result = agent.usage(); + assert!(matches!(result, Err(Error::Refused { ref code, .. }) if code == "unauthorized"), "{result:?}"); + let (method, path, _) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("GET", paths::USAGE)); + } + /// An agent that takes the connection but never answers (hung, or something else on its port) costs a /// UI thread at most a couple of seconds; it used to wait 90 s, the sign-in's time. #[test] @@ -464,4 +1036,67 @@ mod tests { let Err(Error::NotRunning(why)) = Client::connect_in(dir.path()) else { panic!("connected to nothing") }; assert!(!why.contains("its agent stopped"), "told by the pid: {why}"); } + /// The space calls send their requests to their paths and read the agent's answers; an older agent's "no such call" is + /// `agent_too_old`, while the server's own `not_found` (no such invitation) passes through. + #[test] + fn space_calls_send_their_requests_and_an_older_agent_is_too_old() { + let space = SpaceId::generate(); + let info = SpaceInfo { + space, + kind: "team".into(), + name: Some("Studio".into()), + role: SpaceRole::Editor, + state: "active".into(), + archived: false, + folder: Some("/home/ana/Craft Cloud Shared/Studio".into()), + sync: SpaceSync::Off, + run_state: None, + last_sync_ms: None, + read_only: false, + }; + let item = SpaceListItem { + id: space, + kind: "team".into(), + name: Some("Studio".into()), + role: SpaceRole::Editor, + state: "active".into(), + archived: false, + created_ms: 1, + members: 2, + root: Some(NodeId::generate()), + epoch: None, + joined_ms: None, + }; + let no_such_call = json(&ErrorAnswer { error: "not_found".into(), message: "no such call".into(), name_taken: None }); + let no_invitation = json(&ErrorAnswer { error: "not_found".into(), message: "server: not found".into(), name_taken: None }); + let script = vec![ + (200, json(&SpaceList { spaces: vec![info.clone()] })), + (200, json(&item)), + (204, String::new()), + (200, json(&SpaceInfo { sync: SpaceSync::On, ..info.clone() })), + (200, json(&ApproveDeletesAnswer { approved: true, message: "the running agent will carry out these deletions".into() })), + (404, no_invitation), + (404, no_such_call), + ]; + let (_dir, agent, asked) = fake_agent(script); + let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); + + assert_eq!(agent.spaces().unwrap().spaces, vec![info.clone()]); + assert_eq!(next(), ("GET".to_owned(), "/v1/spaces".to_owned(), String::new())); + assert_eq!(agent.accept_space(space).unwrap(), item); + assert_eq!(next(), ("POST".to_owned(), format!("/v1/spaces/{space}/accept"), String::new())); + agent.decline_space(space).unwrap(); + assert_eq!(next(), ("POST".to_owned(), format!("/v1/spaces/{space}/decline"), String::new())); + let on = agent.set_space_sync(space, &SpaceSyncRequest { on: true, folder: None, adopt: true }).unwrap(); + assert_eq!(on.sync, SpaceSync::On); + assert_eq!(next(), ("POST".to_owned(), format!("/v1/spaces/{space}/sync"), r#"{"on":true,"adopt":true}"#.to_owned())); + let approved = agent.approve_space_deletes(space, Some("3f9a0b1c2d3e")).unwrap(); + assert!(approved.approved); + assert_eq!(next(), ("POST".to_owned(), format!("/v1/spaces/{space}/approve-deletes"), r#"{"set":"3f9a0b1c2d3e"}"#.to_owned())); + let gone = agent.decline_space(space).unwrap_err(); + assert!(matches!(&gone, Error::Refused { code, .. } if code == "not_found"), "the server's answer passes through: {gone:?}"); + next(); + let old = agent.spaces().unwrap_err(); + assert!(matches!(&old, Error::Refused { code, .. } if code == codes::AGENT_TOO_OLD), "{old:?}"); + } } diff --git a/crates/app/src/live.rs b/crates/app/src/live.rs index 1e3eba68..18f371c6 100644 --- a/crates/app/src/live.rs +++ b/crates/app/src/live.rs @@ -214,7 +214,9 @@ impl Shared { } } Ok(DocsEvent::Error { id: Some(id), seq: Some(seq), code, message }) => self.route(id, Event::Refused { seq, code, message }), - Ok(DocsEvent::Error { id: Some(id), seq: None, code, message }) => self.answer(id, Answer::Refused(Error::Refused { code, message })), + Ok(DocsEvent::Error { id: Some(id), seq: None, code, message }) => { + self.answer(id, Answer::Refused(Error::Refused { code, message, name_taken: None })) + } Ok(DocsEvent::Error { id: None, .. }) | Err(_) => {} }, Message::Binary(bytes) => { @@ -317,7 +319,7 @@ impl Documents { let status = response.status().as_u16(); let body = response.into_body().unwrap_or_default(); return Err(match serde_json::from_slice::(&body) { - Ok(body) => Error::Refused { code: body.error, message: body.message }, + Ok(body) => Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }, Err(_) => Error::Protocol(format!("HTTP {status}")), }); } @@ -451,7 +453,7 @@ impl Importer { Ok(Document { id: self.id, head, snapshot, updates, events: self.events, shared: self.shared, seq: AtomicU64::new(0) }) } Answer::NotLive(reason) => { - Err(Error::Refused { code: "not_live".into(), message: format!("the file opens as a plain file ({reason:?})") }) + Err(Error::Refused { code: "not_live".into(), message: format!("the file opens as a plain file ({reason:?})"), name_taken: None }) } Answer::NeedsImport => Err(Error::Protocol("asked for the import again".into())), Answer::Refused(e) => Err(e), diff --git a/crates/app/src/recents.rs b/crates/app/src/recents.rs index 30552fef..e769abd1 100644 --- a/crates/app/src/recents.rs +++ b/crates/app/src/recents.rs @@ -75,7 +75,7 @@ type Socket = WebSocket; fn refused(event: Wire) -> Error { match event { - Wire::Error { code, message, .. } => Error::Refused { code, message }, + Wire::Error { code, message, .. } => Error::Refused { code, message, name_taken: None }, other => Error::Protocol(format!("unexpected {other:?}")), } } @@ -110,7 +110,7 @@ fn open(client: &Client, app: &str, follow: bool) -> Result<(Socket, (Vec(&body) { - Ok(body) => Error::Refused { code: body.error, message: body.message }, + Ok(body) => Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }, Err(_) => Error::Protocol(format!("HTTP {status}")), }); } @@ -142,7 +142,7 @@ fn open(client: &Client, app: &str, follow: bool) -> Result<(Socket, (Vec bool { match e { Error::NotRunning(_) | Error::Io(_) => true, - Error::Refused { code, .. } => code == codes::NOT_SIGNED_IN, + Error::Refused { code, .. } | Error::RefusedWith { code, .. } => code == codes::NOT_SIGNED_IN, Error::Version(_) | Error::Protocol(_) => false, } } @@ -332,7 +332,7 @@ impl Link { } Ok(Some(Wire::Error { id: None, code, message })) => { if code == codes::NOT_SIGNED_IN { - break Error::Refused { code, message }; + break Error::Refused { code, message, name_taken: None }; } } Ok(None) => {} diff --git a/crates/app/src/spaces.rs b/crates/app/src/spaces.rs new file mode 100644 index 00000000..5c84c93d --- /dev/null +++ b/crates/app/src/spaces.rs @@ -0,0 +1,64 @@ +//! Team spaces through the agent's local API (`/v1/spaces`): the spaces of the account with what this computer does with +//! each, accepting and declining an invitation (passed on to the server), and turning the sync of a team space on or +//! off here. A call that asks the server ([`Client::accept_space`], [`Client::decline_space`]) waits for it: call it off +//! the UI thread. An agent from before team spaces answers [`codes::AGENT_TOO_OLD`] to all of them. + +use craft_cloud_api::SpaceId; +use craft_cloud_api::api::SpaceListItem; +use craft_cloud_api::local::{ApproveDeletesAnswer, ApproveDeletesRequest, SpaceInfo, SpaceList, SpaceSyncRequest, paths}; + +use crate::{Client, Error, codes, http}; + +/// An older agent's "no such call" as [`codes::AGENT_TOO_OLD`]. The server's own `not_found` (no such invitation) has a +/// message of its own and passes through. +fn too_old(e: Error) -> Error { + match e { + Error::Refused { code, message, .. } if code == craft_cloud_api::api::codes::NOT_FOUND && message == "no such call" => Error::Refused { + code: codes::AGENT_TOO_OLD.into(), + message: "the Craft Cloud agent on this computer is too old for team spaces: update Craft Cloud".into(), + name_taken: None, + }, + other => other, + } +} + +fn space_path(id: SpaceId, call: &str) -> String { + format!("{}/{id}/{call}", paths::SPACES) +} + +impl Client { + /// The personal space and the team spaces of the account, invitations among them, as the agent last saw the + /// server's list, with the folder, the sync and its state on this computer. Answered by the agent itself (a few + /// seconds at most): fine from a UI thread. + pub fn spaces(&self) -> Result { + self.call("GET", paths::SPACES, None, http::COMMAND_TIMEOUT).map_err(too_old) + } + + /// Accepts an invitation to a team space (the server's answer: the space as it lists it). The space then starts with + /// syncing off on this computer: [`Client::set_space_sync`] turns it on. + pub fn accept_space(&self, id: SpaceId) -> Result { + self.call("POST", &space_path(id, "accept"), None, http::SLOW_TIMEOUT).map_err(too_old) + } + + /// Declines an invitation to a team space. + pub fn decline_space(&self, id: SpaceId) -> Result<(), Error> { + self.answer("POST", &space_path(id, "decline"), None, http::SLOW_TIMEOUT).map(|_| ()).map_err(too_old) + } + + /// Turns the sync of a team space on this computer on or off (optionally with another folder, see + /// [`SpaceSyncRequest`]). Answers the space as `GET /v1/spaces` lists it. + pub fn set_space_sync(&self, id: SpaceId, request: &SpaceSyncRequest) -> Result { + let body = serde_json::to_vec(request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", &space_path(id, "sync"), Some(body), http::SLOW_TIMEOUT).map_err(too_old) + } + + /// Approves the deletions the mass-delete guard holds back in a team space, as `craft-cloud-agent sync + /// --approve-deletes --space` does. With `set`, only that set (the identifier the list was shown with) and only if it is + /// the one waiting; without, the set the previous call of this kind listed (the first call lists and approves nothing). + /// The running agent carries the approved deletions out in its next pass. `approved` in the answer says whether an + /// approval was handed over. + pub fn approve_space_deletes(&self, id: SpaceId, set: Option<&str>) -> Result { + let body = serde_json::to_vec(&ApproveDeletesRequest { set: set.map(str::to_owned) }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", &space_path(id, "approve-deletes"), Some(body), http::COMMAND_TIMEOUT).map_err(too_old) + } +} diff --git a/crates/egui/src/lib.rs b/crates/egui/src/lib.rs index 38711739..39d4131c 100644 --- a/crates/egui/src/lib.rs +++ b/crates/egui/src/lib.rs @@ -270,6 +270,7 @@ mod tests { autostart: None, last_sync_ms: None, removed: false, + spaces: Vec::new(), }; assert!(matches!(AgentState::from_status(&status), AgentState::SignedIn { paused: true, ref server, .. } if server == "cloud.example.com")); } diff --git a/crates/kit/src/session.rs b/crates/kit/src/session.rs index 3a55e985..459791be 100644 --- a/crates/kit/src/session.rs +++ b/crates/kit/src/session.rs @@ -506,7 +506,7 @@ impl Session { let file_hash = file_bytes.map(|b| *blake3::hash(b).as_bytes()); let opened = match docs.open_with(path, D::APP, &OpenOptions { file_hash }) { Ok(o) => o, - Err(craft_cloud_app::Error::Refused { code, message }) => return Err(KitError::Agent(format!("{code}: {message}"))), + Err(craft_cloud_app::Error::Refused { code, message, .. }) => return Err(KitError::Agent(format!("{code}: {message}"))), Err(e) => return not_running(e), }; let (document, snapshot) = match opened { diff --git a/crates/kit/src/value.rs b/crates/kit/src/value.rs index faa47539..6d7a0281 100644 --- a/crates/kit/src/value.rs +++ b/crates/kit/src/value.rs @@ -95,7 +95,7 @@ impl BlobStore for AgentBlobs { /// An agent's refusal, its code kept in the message (`agent_too_old: …`). fn agent_error(e: craft_cloud_app::Error) -> KitError { match e { - craft_cloud_app::Error::Refused { code, message } => KitError::Agent(format!("{code}: {message}")), + craft_cloud_app::Error::Refused { code, message, .. } => KitError::Agent(format!("{code}: {message}")), other => KitError::from(other), } } diff --git a/crates/meta/Cargo.toml b/crates/meta/Cargo.toml index 318a1996..72c435c3 100644 --- a/crates/meta/Cargo.toml +++ b/crates/meta/Cargo.toml @@ -14,6 +14,7 @@ serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true tokio.workspace = true +tracing.workspace = true uuid.workspace = true [features] diff --git a/crates/meta/migrations/postgres/0007_devices_v2.sql b/crates/meta/migrations/postgres/0007_devices_v2.sql new file mode 100644 index 00000000..21abd244 --- /dev/null +++ b/crates/meta/migrations/postgres/0007_devices_v2.sql @@ -0,0 +1,14 @@ +-- Device sessions v2. A device now says what kind of client it is (`agent`, `desktop` or `cli`, empty when the +-- client didn't say), on what platform and in what version: shown in the device list and never a basis for any +-- permission. A revoked device keeps when and why it was revoked (`user`, `others`, `password`, `admin`, +-- `disabled`, `two_factor`, `idle` or `reset`), set together with `revoked` and only by the first revoke. The +-- `revoked` flag stays the one the server checks. A user keeps when the password last changed through the +-- server (`NULL` when never) and, for a later release, when the account was disabled (`NULL` while it is not). +-- Existing rows stay valid: the new text columns read as empty and the new times as `NULL`. +ALTER TABLE devices ADD COLUMN client_kind TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN platform TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN app_version TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN revoked_ms BIGINT; +ALTER TABLE devices ADD COLUMN revoke_reason TEXT; +ALTER TABLE users ADD COLUMN password_changed_ms BIGINT; +ALTER TABLE users ADD COLUMN disabled_ms BIGINT; diff --git a/crates/meta/migrations/postgres/0008_admin_resets.sql b/crates/meta/migrations/postgres/0008_admin_resets.sql new file mode 100644 index 00000000..fec1d19f --- /dev/null +++ b/crates/meta/migrations/postgres/0008_admin_resets.sql @@ -0,0 +1,12 @@ +-- Admin API. A row of `invites` now says what it is for: `invite` (a new computer for an account, as before) or +-- `reset` (a password reset code the admin made, good for an hour and redeemed through the same join route). A +-- space may carry a quota of its own that replaces the server-wide cap for that account (`NULL`: the cap applies), +-- and a user may be marked for removal (`removed_ms`, `NULL` while it is not): the account is disabled and left +-- out of the lists, and its data stays until a later release frees it. Existing rows stay valid: every old +-- invite reads as `invite`, and no space has a quota or user a removal mark. `users.auth_gen` counts the changes +-- that end what a sign-in checked before them (a disable, an enable, a removal, a password change or reset): a +-- sign-in that read it before verifying the password creates its device only if it is still the same afterwards. +ALTER TABLE invites ADD COLUMN purpose TEXT NOT NULL DEFAULT 'invite'; +ALTER TABLE spaces ADD COLUMN quota_bytes BIGINT; +ALTER TABLE users ADD COLUMN removed_ms BIGINT; +ALTER TABLE users ADD COLUMN auth_gen BIGINT NOT NULL DEFAULT 0; diff --git a/crates/meta/migrations/postgres/0009_audit_events.sql b/crates/meta/migrations/postgres/0009_audit_events.sql new file mode 100644 index 00000000..62e3a055 --- /dev/null +++ b/crates/meta/migrations/postgres/0009_audit_events.sql @@ -0,0 +1,26 @@ +-- The security audit log. One row per event: when, who did it (`actor_user`, `actor_device`, `NULL` for the command +-- line or a guess at a name nobody has), what (`action`, a name from a closed list in the code), what it was done to +-- (`target_kind`, `target_id`), whether it worked, and a small JSON text of typed values (`details`) that never holds +-- an address, a name of a file or device, a secret or anything a client typed. There are no foreign keys, so an +-- event outlives the account it is about. A failed sign-in is one row per account and minute: `agg_key` names +-- that row and `agg_count` counts the tries in it (rows of other events have a `NULL` key, which a unique index +-- allows many times). Nothing updates or deletes a row except the pruning of old events and that counter. +CREATE TABLE audit_events ( + id BYTEA NOT NULL PRIMARY KEY, + at_ms BIGINT NOT NULL, + actor_user BYTEA, + actor_device BYTEA, + action TEXT NOT NULL, + target_kind TEXT NOT NULL DEFAULT '', + target_id BYTEA, + outcome TEXT NOT NULL, + details TEXT NOT NULL DEFAULT '{}', + agg_key TEXT, + agg_count BIGINT NOT NULL DEFAULT 1 +); + +CREATE INDEX audit_events_at ON audit_events (at_ms); + +CREATE INDEX audit_events_actor ON audit_events (actor_user, at_ms); + +CREATE UNIQUE INDEX audit_events_agg ON audit_events (agg_key); diff --git a/crates/meta/migrations/postgres/0010_two_factor.sql b/crates/meta/migrations/postgres/0010_two_factor.sql new file mode 100644 index 00000000..961a0d28 --- /dev/null +++ b/crates/meta/migrations/postgres/0010_two_factor.sql @@ -0,0 +1,32 @@ +CREATE TABLE user_two_factor ( + user_id BYTEA NOT NULL PRIMARY KEY REFERENCES users(id), + secret_ciphertext BYTEA, + enabled_ms BIGINT, + pending_ciphertext BYTEA, + pending_device BYTEA, + pending_expires_ms BIGINT, + pending_attempts BIGINT NOT NULL DEFAULT 0, + last_accepted_step BIGINT NOT NULL DEFAULT 0, + failed_attempts BIGINT NOT NULL DEFAULT 0, + locked_until_ms BIGINT +); +CREATE TABLE user_recovery_codes ( + user_id BYTEA NOT NULL REFERENCES users(id), + code_hash BYTEA NOT NULL, + used_ms BIGINT, + PRIMARY KEY (user_id, code_hash) +); +CREATE TABLE login_challenges ( + id_hash BYTEA NOT NULL PRIMARY KEY, + user_id BYTEA NOT NULL REFERENCES users(id), + kind TEXT NOT NULL, + client_key TEXT NOT NULL, + device_name TEXT NOT NULL, + client_kind TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + app_version TEXT NOT NULL DEFAULT '', + attempts BIGINT NOT NULL DEFAULT 0, + expires_ms BIGINT NOT NULL +); +CREATE INDEX login_challenges_expiry ON login_challenges (expires_ms); +ALTER TABLE invites ADD COLUMN reset_two_factor BIGINT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/crates/meta/migrations/postgres/0011_web_sessions.sql b/crates/meta/migrations/postgres/0011_web_sessions.sql new file mode 100644 index 00000000..999659b1 --- /dev/null +++ b/crates/meta/migrations/postgres/0011_web_sessions.sql @@ -0,0 +1,18 @@ +-- Browser sessions of the web admin. A session is a random token in a cookie, of which only the BLAKE3 hash is kept +-- here (`id_hash`). `auth_gen` is the account's generation when the session was made, so every change that raises +-- `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off) +-- ends the session with it. `last_seen_ms` is moved at most once a minute and carries the idle expiry, +-- `absolute_expires_ms` the hard one, `revoked_ms` is a logout. `mfa_verified_ms` is when a second factor was given +-- at sign-in (`NULL` for an account without one). There is no password, no token and no address in a row. +CREATE TABLE web_sessions ( + id_hash BYTEA NOT NULL PRIMARY KEY, + user_id BYTEA NOT NULL REFERENCES users(id), + auth_gen BIGINT NOT NULL, + created_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL, + absolute_expires_ms BIGINT NOT NULL, + revoked_ms BIGINT, + mfa_verified_ms BIGINT +); +CREATE INDEX web_sessions_user ON web_sessions (user_id); +CREATE INDEX web_sessions_expiry ON web_sessions (absolute_expires_ms) diff --git a/crates/meta/migrations/postgres/0012_versions_node_seq.sql b/crates/meta/migrations/postgres/0012_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/postgres/0012_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/migrations/postgres/0013_trash.sql b/crates/meta/migrations/postgres/0013_trash.sql new file mode 100644 index 00000000..7fd42245 --- /dev/null +++ b/crates/meta/migrations/postgres/0013_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/migrations/postgres/0014_manifest_index.sql b/crates/meta/migrations/postgres/0014_manifest_index.sql new file mode 100644 index 00000000..340a97ac --- /dev/null +++ b/crates/meta/migrations/postgres/0014_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BYTEA NOT NULL, + ordinal BIGINT NOT NULL, + chunk BYTEA NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BYTEA NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/migrations/postgres/0015_team_spaces.sql b/crates/meta/migrations/postgres/0015_team_spaces.sql new file mode 100644 index 00000000..6b763b32 --- /dev/null +++ b/crates/meta/migrations/postgres/0015_team_spaces.sql @@ -0,0 +1,42 @@ +-- Team spaces. A space gets a kind (`personal` or `team`), a display name (team spaces only, with its case-folded +-- form for the unique index), a creation time and an archive mark, and its owner becomes optional: set exactly for +-- a personal space. Every existing space becomes a personal one with the same id, owner, epoch and counters, and +-- its creation time is its owner's. Postgres names the column-level UNIQUE on `owner` `spaces_owner_key`. +-- `space_members` says who has which role in a team space. A row with no `accepted_ms` is an invitation and gives no +-- access. `invited_by` has no foreign key, as in `audit_events`. A declined invitation, a member who leaves and a +-- removed member are deleted rows. +ALTER TABLE spaces ADD COLUMN kind TEXT NOT NULL DEFAULT 'personal' CHECK (kind IN ('personal', 'team')); + +ALTER TABLE spaces ADD COLUMN name TEXT; + +ALTER TABLE spaces ADD COLUMN name_fold TEXT; + +ALTER TABLE spaces ADD COLUMN created_ms BIGINT NOT NULL DEFAULT 0; + +ALTER TABLE spaces ADD COLUMN archived_ms BIGINT; + +UPDATE spaces SET created_ms = COALESCE((SELECT u.created_ms FROM users u WHERE u.id = spaces.owner), 0); + +ALTER TABLE spaces ALTER COLUMN owner DROP NOT NULL; + +ALTER TABLE spaces DROP CONSTRAINT spaces_owner_key; + +ALTER TABLE spaces ADD CONSTRAINT spaces_owner_by_kind CHECK ((kind = 'personal') = (owner IS NOT NULL)); + +ALTER TABLE spaces ADD CONSTRAINT spaces_name_by_kind CHECK ((kind = 'team') = (name IS NOT NULL AND name_fold IS NOT NULL)); + +CREATE UNIQUE INDEX spaces_personal_owner ON spaces (owner) WHERE owner IS NOT NULL; + +CREATE UNIQUE INDEX spaces_team_name ON spaces (name_fold) WHERE kind = 'team'; + +CREATE TABLE space_members ( + space_id BYTEA NOT NULL REFERENCES spaces (id), + user_id BYTEA NOT NULL REFERENCES users (id), + role TEXT NOT NULL CHECK (role IN ('viewer', 'editor', 'owner')), + invited_ms BIGINT NOT NULL, + invited_by BYTEA, + accepted_ms BIGINT, + PRIMARY KEY (space_id, user_id) +); + +CREATE INDEX space_members_user ON space_members (user_id, space_id) diff --git a/crates/meta/migrations/sqlite/0007_devices_v2.sql b/crates/meta/migrations/sqlite/0007_devices_v2.sql new file mode 100644 index 00000000..21abd244 --- /dev/null +++ b/crates/meta/migrations/sqlite/0007_devices_v2.sql @@ -0,0 +1,14 @@ +-- Device sessions v2. A device now says what kind of client it is (`agent`, `desktop` or `cli`, empty when the +-- client didn't say), on what platform and in what version: shown in the device list and never a basis for any +-- permission. A revoked device keeps when and why it was revoked (`user`, `others`, `password`, `admin`, +-- `disabled`, `two_factor`, `idle` or `reset`), set together with `revoked` and only by the first revoke. The +-- `revoked` flag stays the one the server checks. A user keeps when the password last changed through the +-- server (`NULL` when never) and, for a later release, when the account was disabled (`NULL` while it is not). +-- Existing rows stay valid: the new text columns read as empty and the new times as `NULL`. +ALTER TABLE devices ADD COLUMN client_kind TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN platform TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN app_version TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN revoked_ms BIGINT; +ALTER TABLE devices ADD COLUMN revoke_reason TEXT; +ALTER TABLE users ADD COLUMN password_changed_ms BIGINT; +ALTER TABLE users ADD COLUMN disabled_ms BIGINT; diff --git a/crates/meta/migrations/sqlite/0008_admin_resets.sql b/crates/meta/migrations/sqlite/0008_admin_resets.sql new file mode 100644 index 00000000..fec1d19f --- /dev/null +++ b/crates/meta/migrations/sqlite/0008_admin_resets.sql @@ -0,0 +1,12 @@ +-- Admin API. A row of `invites` now says what it is for: `invite` (a new computer for an account, as before) or +-- `reset` (a password reset code the admin made, good for an hour and redeemed through the same join route). A +-- space may carry a quota of its own that replaces the server-wide cap for that account (`NULL`: the cap applies), +-- and a user may be marked for removal (`removed_ms`, `NULL` while it is not): the account is disabled and left +-- out of the lists, and its data stays until a later release frees it. Existing rows stay valid: every old +-- invite reads as `invite`, and no space has a quota or user a removal mark. `users.auth_gen` counts the changes +-- that end what a sign-in checked before them (a disable, an enable, a removal, a password change or reset): a +-- sign-in that read it before verifying the password creates its device only if it is still the same afterwards. +ALTER TABLE invites ADD COLUMN purpose TEXT NOT NULL DEFAULT 'invite'; +ALTER TABLE spaces ADD COLUMN quota_bytes BIGINT; +ALTER TABLE users ADD COLUMN removed_ms BIGINT; +ALTER TABLE users ADD COLUMN auth_gen BIGINT NOT NULL DEFAULT 0; diff --git a/crates/meta/migrations/sqlite/0009_audit_events.sql b/crates/meta/migrations/sqlite/0009_audit_events.sql new file mode 100644 index 00000000..7b0d86e0 --- /dev/null +++ b/crates/meta/migrations/sqlite/0009_audit_events.sql @@ -0,0 +1,26 @@ +-- The security audit log. One row per event: when, who did it (`actor_user`, `actor_device`, `NULL` for the command +-- line or a guess at a name nobody has), what (`action`, a name from a closed list in the code), what it was done to +-- (`target_kind`, `target_id`), whether it worked, and a small JSON text of typed values (`details`) that never holds +-- an address, a name of a file or device, a secret or anything a client typed. There are no foreign keys, so an +-- event outlives the account it is about. A failed sign-in is one row per account and minute: `agg_key` names +-- that row and `agg_count` counts the tries in it (rows of other events have a `NULL` key, which a unique index +-- allows many times). Nothing updates or deletes a row except the pruning of old events and that counter. +CREATE TABLE audit_events ( + id BLOB NOT NULL PRIMARY KEY, + at_ms BIGINT NOT NULL, + actor_user BLOB, + actor_device BLOB, + action TEXT NOT NULL, + target_kind TEXT NOT NULL DEFAULT '', + target_id BLOB, + outcome TEXT NOT NULL, + details TEXT NOT NULL DEFAULT '{}', + agg_key TEXT, + agg_count BIGINT NOT NULL DEFAULT 1 +); + +CREATE INDEX audit_events_at ON audit_events (at_ms); + +CREATE INDEX audit_events_actor ON audit_events (actor_user, at_ms); + +CREATE UNIQUE INDEX audit_events_agg ON audit_events (agg_key); diff --git a/crates/meta/migrations/sqlite/0010_two_factor.sql b/crates/meta/migrations/sqlite/0010_two_factor.sql new file mode 100644 index 00000000..308e19d9 --- /dev/null +++ b/crates/meta/migrations/sqlite/0010_two_factor.sql @@ -0,0 +1,32 @@ +CREATE TABLE user_two_factor ( + user_id BLOB NOT NULL PRIMARY KEY REFERENCES users(id), + secret_ciphertext BLOB, + enabled_ms BIGINT, + pending_ciphertext BLOB, + pending_device BLOB, + pending_expires_ms BIGINT, + pending_attempts INTEGER NOT NULL DEFAULT 0, + last_accepted_step BIGINT NOT NULL DEFAULT 0, + failed_attempts INTEGER NOT NULL DEFAULT 0, + locked_until_ms BIGINT +); +CREATE TABLE user_recovery_codes ( + user_id BLOB NOT NULL REFERENCES users(id), + code_hash BLOB NOT NULL, + used_ms BIGINT, + PRIMARY KEY (user_id, code_hash) +); +CREATE TABLE login_challenges ( + id_hash BLOB NOT NULL PRIMARY KEY, + user_id BLOB NOT NULL REFERENCES users(id), + kind TEXT NOT NULL, + client_key TEXT NOT NULL, + device_name TEXT NOT NULL, + client_kind TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + app_version TEXT NOT NULL DEFAULT '', + attempts INTEGER NOT NULL DEFAULT 0, + expires_ms BIGINT NOT NULL +); +CREATE INDEX login_challenges_expiry ON login_challenges (expires_ms); +ALTER TABLE invites ADD COLUMN reset_two_factor INTEGER NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/crates/meta/migrations/sqlite/0011_web_sessions.sql b/crates/meta/migrations/sqlite/0011_web_sessions.sql new file mode 100644 index 00000000..a2ee36a0 --- /dev/null +++ b/crates/meta/migrations/sqlite/0011_web_sessions.sql @@ -0,0 +1,18 @@ +-- Browser sessions of the web admin. A session is a random token in a cookie, of which only the BLAKE3 hash is kept +-- here (`id_hash`). `auth_gen` is the account's generation when the session was made, so every change that raises +-- `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off) +-- ends the session with it. `last_seen_ms` is moved at most once a minute and carries the idle expiry, +-- `absolute_expires_ms` the hard one, `revoked_ms` is a logout. `mfa_verified_ms` is when a second factor was given +-- at sign-in (`NULL` for an account without one). There is no password, no token and no address in a row. +CREATE TABLE web_sessions ( + id_hash BLOB NOT NULL PRIMARY KEY, + user_id BLOB NOT NULL REFERENCES users(id), + auth_gen BIGINT NOT NULL, + created_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL, + absolute_expires_ms BIGINT NOT NULL, + revoked_ms BIGINT, + mfa_verified_ms BIGINT +); +CREATE INDEX web_sessions_user ON web_sessions (user_id); +CREATE INDEX web_sessions_expiry ON web_sessions (absolute_expires_ms) diff --git a/crates/meta/migrations/sqlite/0012_versions_node_seq.sql b/crates/meta/migrations/sqlite/0012_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/sqlite/0012_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/migrations/sqlite/0013_trash.sql b/crates/meta/migrations/sqlite/0013_trash.sql new file mode 100644 index 00000000..6da29e88 --- /dev/null +++ b/crates/meta/migrations/sqlite/0013_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = CAST(strftime('%s', 'now') AS BIGINT) * 1000 WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/migrations/sqlite/0014_manifest_index.sql b/crates/meta/migrations/sqlite/0014_manifest_index.sql new file mode 100644 index 00000000..cd491700 --- /dev/null +++ b/crates/meta/migrations/sqlite/0014_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BLOB NOT NULL, + ordinal BIGINT NOT NULL, + chunk BLOB NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BLOB NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/migrations/sqlite/0015_team_spaces.sql b/crates/meta/migrations/sqlite/0015_team_spaces.sql new file mode 100644 index 00000000..b4361eb6 --- /dev/null +++ b/crates/meta/migrations/sqlite/0015_team_spaces.sql @@ -0,0 +1,54 @@ +-- craft: foreign-keys-off +-- Team spaces. A space gets a kind (`personal` or `team`), a display name (team spaces only, with its case-folded +-- form for the unique index), a creation time and an archive mark, and its owner becomes optional: set exactly for +-- a personal space. SQLite cannot drop the column-level UNIQUE on `owner` or add the checks in place, so `spaces` is +-- rebuilt as its documentation describes: create the new table, copy every row, drop the old one, rename the new +-- one. The marker on the first line makes the runner turn foreign-key enforcement off for this migration (other +-- tables reference `spaces`) and run `PRAGMA foreign_key_check` before it commits. Every existing space becomes a +-- personal one with the same id, owner, epoch and counters. +-- `space_members` says who has which role in a team space. A row with no `accepted_ms` is an invitation and gives no +-- access. `invited_by` has no foreign key, as in `audit_events`. A declined invitation, a member who leaves and a +-- removed member are deleted rows. +CREATE TABLE spaces_new ( + id BLOB PRIMARY KEY, + owner BLOB REFERENCES users (id), + epoch BLOB NOT NULL, + seq BIGINT NOT NULL, + oldest_seq BIGINT NOT NULL, + root BLOB NOT NULL, + stored_bytes BIGINT NOT NULL DEFAULT 0, + live_bytes BIGINT NOT NULL DEFAULT 0, + quota_bytes BIGINT, + kind TEXT NOT NULL DEFAULT 'personal' CHECK (kind IN ('personal', 'team')), + name TEXT, + name_fold TEXT, + created_ms BIGINT NOT NULL DEFAULT 0, + archived_ms BIGINT, + CHECK ((kind = 'personal') = (owner IS NOT NULL)), + CHECK ((kind = 'team') = (name IS NOT NULL AND name_fold IS NOT NULL)) +); + +INSERT INTO spaces_new (id, owner, epoch, seq, oldest_seq, root, stored_bytes, live_bytes, quota_bytes, kind, created_ms) +SELECT s.id, s.owner, s.epoch, s.seq, s.oldest_seq, s.root, s.stored_bytes, s.live_bytes, s.quota_bytes, 'personal', + COALESCE((SELECT u.created_ms FROM users u WHERE u.id = s.owner), 0) +FROM spaces s; + +DROP TABLE spaces; + +ALTER TABLE spaces_new RENAME TO spaces; + +CREATE UNIQUE INDEX spaces_personal_owner ON spaces (owner) WHERE owner IS NOT NULL; + +CREATE UNIQUE INDEX spaces_team_name ON spaces (name_fold) WHERE kind = 'team'; + +CREATE TABLE space_members ( + space_id BLOB NOT NULL REFERENCES spaces (id), + user_id BLOB NOT NULL REFERENCES users (id), + role TEXT NOT NULL CHECK (role IN ('viewer', 'editor', 'owner')), + invited_ms BIGINT NOT NULL, + invited_by BLOB, + accepted_ms BIGINT, + PRIMARY KEY (space_id, user_id) +); + +CREATE INDEX space_members_user ON space_members (user_id, space_id) diff --git a/crates/meta/src/admin.rs b/crates/meta/src/admin.rs new file mode 100644 index 00000000..56dbbcc6 --- /dev/null +++ b/crates/meta/src/admin.rs @@ -0,0 +1,623 @@ +//! What an administrator does through the admin API: server counters, listing and creating accounts, changing a role or +//! a quota, disabling, enabling and removing an account, invites and password reset codes, and a user's devices. +//! +//! - **One `Meta` method per change, with a typed result**, and each writes its audit event in its own transaction, before +//! the commit, so a change whose event cannot be written is not made. A change that changes nothing (disabling an account +//! that is disabled, signing out a device that is out already, giving the role an account has) writes no event. +//! - **The acting admin is checked again inside every change** ([`AdminActor`], `begin_admin`): the role the middleware +//! read and the password the handler verified (a slow Argon2id check) can be stale by the time the transaction runs. +//! The device (or the browser session) must still be signed in, the account neither disabled nor marked for removal, the role still `admin` +//! and the password hash still the one that was verified. +//! - **Lock order** in every change: (1) the `last_admin` transaction lock, if the change can take away an +//! administrator; (2) the rows of the acting admin and of the target, in ascending ID; (3) rows derived from them +//! (devices, invites, pairings). Sign-ins, redeems and pairings take the user's row first as well (see +//! `users::lock_user`), so none of these can deadlock with another, and a disable can never be followed by a device +//! that an earlier check let in. +//! - **The last administrator** cannot be demoted, disabled or removed. A count cannot be locked with `FOR UPDATE`, and +//! locking the rows of two different targets does not serialise two changes that each look at the other's row. So +//! every change that can take an administrator away takes one transaction-wide lock first and counts after it: on +//! Postgres `pg_advisory_xact_lock`, on SQLite the write transaction (`BEGIN IMMEDIATE`, held by one writer at a +//! time) already is that lock. + +use craft_cloud_proto::limits::MAX_CHUNK_BYTES; +use craft_cloud_proto::{DeviceId, NodeId, SpaceId}; +use sqlx::Row; +use uuid::Uuid; + +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; +use crate::codec::{to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::join::{Purpose, put_code}; +use crate::two_factor; +use crate::users::{delete_join_codes, insert_user, lock_user, revoke_devices}; +use crate::web::session_is_live; +use crate::{DeviceRecord, Dialect, Meta, MetaError, RevokeReason, Role, UserId, WriteTx, valid_username}; + +/// The key of the Postgres advisory lock that serialises changes which can take an administrator away. Arbitrary and +/// fixed, and different from the migration lock's. +const LAST_ADMIN_LOCK_KEY: i64 = 7240318150662; + +/// The smallest quota an admin may give an account: one maximal chunk, like the server-wide cap. +pub const MIN_QUOTA_BYTES: u64 = MAX_CHUNK_BYTES as u64; + +/// The longest username search an admin may make, in bytes. +pub const MAX_SEARCH_BYTES: usize = 64; + +/// What an admin's request came with: the device token of the JSON API, or the browser session of the web admin. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum AdminVia { + /// The device the request came with; it must still be signed in. + Device(DeviceId), + /// The hash of the session token the request came with and the idle lifetime of sessions; the session must still be valid + /// when the change runs (not logged out, under the account's current `auth_gen`, inside its absolute lifetime and not idle + /// for `idle_ms`, all judged at the time the change is made, so a request that began before the session ended cannot + /// change anything after). No device acts, so the events it writes name none. + Web { id_hash: [u8; 32], idle_ms: i64 }, +} + +impl std::fmt::Debug for AdminVia { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AdminVia::Device(device) => f.debug_tuple("Device").field(device).finish(), + AdminVia::Web { idle_ms, .. } => f.debug_struct("Web").field("id_hash", &"").field("idle_ms", idle_ms).finish(), + } + } +} + +/// The admin who asks, as the handler knew them when it verified their password. +#[derive(Clone, PartialEq, Eq)] +pub struct AdminActor { + pub user: UserId, + /// How the request signed in; it must still be so. + pub via: AdminVia, + /// The stored password hash the confirmation was verified against; it must still be the account's. + pub password_hash: String, + /// Whether the acting admin's account had two-factor sign-in on when their confirmation was checked + /// ([`Meta::confirm_second_factor`]); it must still be so (turning it on or off meanwhile is `Conflict`). + pub two_factor: bool, +} + +impl std::fmt::Debug for AdminActor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminActor") + .field("user", &self.user) + .field("via", &self.via) + .field("password_hash", &"") + .field("two_factor", &self.two_factor) + .finish() + } +} + +/// An account as the admin lists it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdminUser { + pub id: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + pub disabled_ms: Option, + /// When the account was marked for removal (it is disabled too); the lists leave such accounts out, a single lookup does not. + pub removed_ms: Option, + /// Two-factor sign-in is on. + pub two_factor_enabled: bool, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + /// The quota an admin set for the account (`None`: the server-wide cap, if any, applies). + pub quota_bytes: Option, + /// An unexpired invite is open (a reset code is not an invite). + pub invite_pending: bool, +} + +/// The counters `GET /admin/server` shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServerCounts { + pub users: u64, + pub devices_active: u64, + pub stored_bytes: u64, + pub live_bytes: u64, +} + +/// One page of [`Meta::admin_list_users`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserPage { + pub users: Vec, + /// The username to continue after, when there are more. + pub next_after: Option, +} + +/// A code an admin made: when it stops working. The code itself is never stored, only its hash. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Issued { + pub expires_ms: i64, +} + +/// How long a password reset code works: one hour. +pub const RESET_TTL_MS: i64 = 60 * 60 * 1000; + +struct Target { + role: Role, + disabled: bool, + removed: bool, + has_password: bool, +} + +const USER_SELECT: &str = "SELECT u.id, u.username, u.role, u.created_ms, u.disabled_ms, u.removed_ms, s.stored_bytes, s.quota_bytes, \ + (SELECT COUNT(*) FROM user_two_factor t WHERE t.user_id = u.id AND t.enabled_ms IS NOT NULL) AS two_factor, \ + (SELECT COUNT(*) FROM devices d WHERE d.user_id = u.id AND d.revoked = 0) AS devices_active, \ + (SELECT COUNT(*) FROM invites i WHERE i.user_id = u.id AND i.purpose = 'invite' AND i.expires_ms > $1) AS invites \ + FROM users u JOIN spaces s ON s.owner = u.id"; + +fn admin_user_from(row: &sqlx::any::AnyRow) -> Result { + let quota: Option = row.try_get("quota_bytes")?; + Ok(AdminUser { + id: UserId(uuid_from(&row.try_get::, _>("id")?)?), + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + disabled_ms: row.try_get("disabled_ms")?, + removed_ms: row.try_get("removed_ms")?, + two_factor_enabled: row.try_get::("two_factor")? > 0, + devices_active: to_u64(row.try_get("devices_active")?)?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + quota_bytes: quota.map(to_u64).transpose()?, + invite_pending: row.try_get::("invites")? > 0, + }) +} + +/// `text` as a `LIKE` pattern that matches it literally (`\` is the escape character, see the queries). +fn escape_like(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + if matches!(c, '\\' | '%' | '_') { + out.push('\\'); + } + out.push(c); + } + out +} + +/// An event the acting admin makes to `target`. +fn by_admin(actor: &AdminActor, action: AuditAction, target: UserId) -> NewEvent { + let device = match actor.via { + AdminVia::Device(device) => Some(device), + AdminVia::Web { .. } => None, + }; + NewEvent::new(action).by(actor.user, device).on_user(target) +} + +fn same_user(a: UserId, b: UserId) -> bool { + a.0 == b.0 +} + +impl Meta { + /// Starts the write transaction of a change the admin `actor` makes (to `target`, if it has one): takes the + /// locks in the order the module describes and checks the actor again. `reduces_admins`: the change can take an + /// administrator away, so it takes the `last_admin` lock first. + pub(crate) async fn begin_admin(&self, actor: &AdminActor, target: Option, reduces_admins: bool) -> Result { + let mut w = self.begin_write().await?; + // SQLite: the write transaction already serialises every change, so nothing more is needed there. + if reduces_admins && self.dialect == Dialect::Postgres { + sqlx::query("SELECT pg_advisory_xact_lock($1)").bind(LAST_ADMIN_LOCK_KEY).execute(&mut *w.tx).await?; + } + let mut ids = vec![actor.user]; + if let Some(target) = target + && !same_user(target, actor.user) + { + ids.push(target); + } + ids.sort_by_key(|id| *id.0.as_bytes()); + for id in ids { + // A target that isn't there is the caller's `NotFound`; the actor's own row is checked just below. + lock_user(&mut w.tx, id).await?; + } + let row = sqlx::query("SELECT role, password_hash, disabled_ms, removed_ms FROM users WHERE id = $1") + .bind(uuid_bytes(&actor.user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(row) = row else { return Err(MetaError::DeviceRevoked) }; + let still_signed_in = match actor.via { + AdminVia::Device(device) => sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&actor.user.0)) + .fetch_optional(&mut *w.tx) + .await? + .is_some(), + // The time is read here, after every lock wait above and in the order the module describes, so a session that ended + // while this change waited does not authorise it. + AdminVia::Web { id_hash, idle_ms } => session_is_live(&mut w, actor.user, &id_hash, self.clock_ms(), idle_ms).await?, + }; + if !still_signed_in || row.try_get::, _>("disabled_ms")?.is_some() || row.try_get::, _>("removed_ms")?.is_some() { + return Err(MetaError::DeviceRevoked); + } + if Role::parse(&row.try_get::("role")?)? != Role::Admin { + return Err(MetaError::Forbidden("you are no longer an administrator")); + } + if row.try_get::("password_hash")? != actor.password_hash { + return Err(MetaError::Conflict("your password changed while this request was running; try again")); + } + // The confirmation asked for a second factor, or didn't, by what the account had then. + if two_factor::is_enabled(&mut w.tx, actor.user).await? != actor.two_factor { + return Err(MetaError::Conflict("your two-factor sign-in changed while this request was running; try again")); + } + Ok(w) + } + + async fn target(w: &mut WriteTx, user: UserId) -> Result { + let row = sqlx::query("SELECT role, password_hash, disabled_ms, removed_ms FROM users WHERE id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("user"))?; + Ok(Target { + role: Role::parse(&row.try_get::("role")?)?, + disabled: row.try_get::, _>("disabled_ms")?.is_some(), + removed: row.try_get::, _>("removed_ms")?.is_some(), + has_password: !row.try_get::("password_hash")?.is_empty(), + }) + } + + /// `LastAdmin` unless an administrator who is not disabled and not marked for removal is left, counted in `w` + /// after the `last_admin` lock was taken and the change was made. + async fn keep_an_admin(w: &mut WriteTx) -> Result<(), MetaError> { + let row = sqlx::query("SELECT COUNT(*) AS n FROM users WHERE role = 'admin' AND disabled_ms IS NULL AND removed_ms IS NULL") + .fetch_one(&mut *w.tx) + .await?; + if row.try_get::("n")? < 1 { Err(MetaError::LastAdmin) } else { Ok(()) } + } + + /// The counters of `GET /admin/server`: accounts that are not marked for removal, devices that are not revoked, and + /// the bytes accounts store and their live documents hold. + pub async fn admin_server_counts(&self) -> Result { + let count = |row: sqlx::any::AnyRow, column: &str| -> Result { to_u64(row.try_get(column)?) }; + // Postgres sums BIGINT as NUMERIC, which the `Any` driver can't read: cast back. + let users = sqlx::query("SELECT COUNT(*) AS n FROM users WHERE removed_ms IS NULL").fetch_one(&self.pool).await?; + let devices = sqlx::query("SELECT COUNT(*) AS n FROM devices WHERE revoked = 0").fetch_one(&self.pool).await?; + let bytes = sqlx::query( + "SELECT CAST(COALESCE(SUM(stored_bytes), 0) AS BIGINT) AS stored, CAST(COALESCE(SUM(live_bytes), 0) AS BIGINT) AS live FROM spaces", + ) + .fetch_one(&self.pool) + .await?; + Ok(ServerCounts { + users: count(users, "n")?, + devices_active: count(devices, "n")?, + stored_bytes: to_u64(bytes.try_get("stored")?)?, + live_bytes: to_u64(bytes.try_get("live")?)?, + }) + } + + /// One page of accounts that are not marked for removal, by username ascending: those after `after` (exclusive) + /// whose username contains `search` (case does not matter, `%` and `_` are ordinary characters), at most `limit`. + /// `next_after` is set when there are more. A `search` over [`MAX_SEARCH_BYTES`] is `Invalid`. + pub async fn admin_list_users(&self, after: Option<&str>, search: Option<&str>, limit: u32, now_ms: i64) -> Result { + let search = search.unwrap_or_default(); + if search.len() > MAX_SEARCH_BYTES { + return Err(MetaError::Invalid("the search is at most 64 bytes")); + } + let pattern = format!("%{}%", escape_like(&search.to_ascii_lowercase())); + let sql = + format!("{USER_SELECT} WHERE u.removed_ms IS NULL AND u.username > $2 AND u.username LIKE $3 ESCAPE '\\' ORDER BY u.username LIMIT $4"); + let fetch = i64::from(limit).saturating_add(1); + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(now_ms) + .bind(after.unwrap_or_default()) + .bind(pattern) + .bind(fetch) + .fetch_all(&self.pool) + .await?; + let mut users = rows.iter().map(admin_user_from).collect::, _>>()?; + let more = users.len() > limit as usize; + users.truncate(limit as usize); + let next_after = if more { users.last().map(|u| u.username.clone()) } else { None }; + Ok(UserPage { users, next_after }) + } + + /// One account, marked for removal or not; `None` if there is no such user. + pub async fn admin_user(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("{USER_SELECT} WHERE u.id = $2"))) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(admin_user_from).transpose() + } + + /// Makes an account for `username` with `role`, no password and its own space, and an invite under `code_hash` + /// (good for `ttl_ms`) the person redeems to choose a password. A name that is taken, by any account (one marked for + /// removal too), is `UsernameTaken`. + pub async fn admin_create_user( + &self, + actor: &AdminActor, + username: &str, + role: Role, + code_hash: &[u8; 32], + now_ms: i64, + ttl_ms: i64, + ) -> Result<(AdminUser, Issued), MetaError> { + if !valid_username(username) { + return Err(MetaError::Invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); + } + let username = username.to_ascii_lowercase(); + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the invite's lifetime is too long"))?; + let mut w = self.begin_admin(actor, None, false).await?; + let taken = sqlx::query("SELECT 1 FROM users WHERE username = $1").bind(username.as_str()).fetch_optional(&mut *w.tx).await?; + if taken.is_some() { + return Err(MetaError::UsernameTaken); + } + let ids = (UserId(Uuid::now_v7()), SpaceId::generate(), NodeId::generate()); + let user = insert_user(&mut w.tx, &username, "", role, now_ms, ids).await?.id; + put_code(&mut w.tx, user, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; + let created = by_admin(actor, AuditAction::AdminUserCreated, user).with(AuditDetails { role_to: Some(role), ..AuditDetails::default() }); + record_in(&mut w.tx, now_ms, created).await?; + self.commit_audited(w, now_ms).await?; + let made = self.admin_user(user, now_ms).await?.ok_or(MetaError::Corrupt("a new account is missing"))?; + Ok((made, Issued { expires_ms })) + } + + /// Changes an account's role and/or its quota: `role` `None` leaves the role, `quota` `None` leaves the quota, + /// `Some(None)` removes it, `Some(Some(n))` sets it (at least [`MIN_QUOTA_BYTES`], at most `i64::MAX`). Demoting the + /// last administrator is `LastAdmin`; an account marked for removal is `Conflict`. + pub async fn admin_update_user( + &self, + actor: &AdminActor, + target: UserId, + role: Option, + quota: Option>, + now_ms: i64, + ) -> Result { + if role.is_none() && quota.is_none() { + return Err(MetaError::Invalid("give a role or a quota_bytes to change")); + } + if let Some(Some(bytes)) = quota + && !(MIN_QUOTA_BYTES..=i64::MAX as u64).contains(&bytes) + { + return Err(MetaError::Invalid("quota_bytes is at least one chunk (4194304) and at most 9223372036854775807")); + } + let mut w = self.begin_admin(actor, Some(target), role == Some(Role::Member)).await?; + let found = Self::target(&mut w, target).await?; + if found.removed { + return Err(MetaError::Conflict("this account is marked for removal")); + } + if let Some(role) = role { + sqlx::query("UPDATE users SET role = $1 WHERE id = $2").bind(role.as_str()).bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if found.role == Role::Admin && role == Role::Member { + Self::keep_an_admin(&mut w).await?; + } + // One event for the role and one for the quota when both change, in this transaction. A role the account has + // already is no change and has none. + if role != found.role { + let details = AuditDetails { role_from: Some(found.role), role_to: Some(role), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminRoleChanged, target).with(details)).await?; + } + } + if let Some(quota) = quota { + let quota = quota.map(to_i64).transpose()?; + let before: Option = sqlx::query("SELECT quota_bytes FROM spaces WHERE owner = $1") + .bind(uuid_bytes(&target.0)) + .fetch_optional(&mut *w.tx) + .await? + .map(|row| row.try_get("quota_bytes")) + .transpose()? + .flatten(); + sqlx::query("UPDATE spaces SET quota_bytes = $1 WHERE owner = $2").bind(quota).bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if quota != before { + let details = AuditDetails { quota_bytes: Some(quota.map(to_u64).transpose()?), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminQuotaChanged, target).with(details)).await?; + } + } + self.commit_audited(w, now_ms).await?; + self.admin_user(target, now_ms).await?.ok_or(MetaError::NotFound("user")) + } + + /// Disables an account: sets `disabled_ms`, signs every device out ([`RevokeReason::Disabled`]) and deletes the + /// account's invite, reset code and pairings. Disabling one's own account is `Invalid`, the last enabled + /// administrator `LastAdmin`; an account that is disabled already is left as it is. Returns the devices signed out. + pub async fn admin_disable_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("you can't disable your own account")); + } + let mut w = self.begin_admin(actor, Some(target), true).await?; + let found = Self::target(&mut w, target).await?; + sqlx::query("UPDATE users SET disabled_ms = COALESCE(disabled_ms, $1), auth_gen = auth_gen + 1 WHERE id = $2") + .bind(now_ms) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + Self::keep_an_admin(&mut w).await?; + let revoked = revoke_devices(&mut w.tx, target, None, RevokeReason::Disabled, now_ms).await?; + delete_join_codes(&mut w.tx, target).await?; + // The devices signed out with it are counted in the event. An account that was disabled already is no change. + if !found.disabled { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserDisabled, target).with(AuditDetails::revoked(revoked.len()))).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Enables a disabled account. Devices that were signed out stay signed out. One marked for removal is `Conflict`. + /// An account that is not disabled is left as it is and writes no event. + pub async fn admin_enable_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed { + return Err(MetaError::Conflict("this account is marked for removal")); + } + sqlx::query("UPDATE users SET disabled_ms = NULL, auth_gen = auth_gen + 1 WHERE id = $1") + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + if found.disabled { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserEnabled, target)).await?; + } + self.commit_audited(w, now_ms).await + } + + /// Marks an account for removal: as [`Meta::admin_disable_user`], and also sets `removed_ms`, which leaves it out of + /// the lists and cannot be undone here. Nothing it stored is deleted; its username stays taken. Removing one's + /// own account is `Invalid`, the last enabled administrator `LastAdmin`. Returns the devices signed out. + pub async fn admin_remove_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("you can't remove your own account")); + } + let mut w = self.begin_admin(actor, Some(target), true).await?; + let found = Self::target(&mut w, target).await?; + sqlx::query( + "UPDATE users SET removed_ms = COALESCE(removed_ms, $1), disabled_ms = COALESCE(disabled_ms, $1), auth_gen = auth_gen + 1 WHERE id = $2", + ) + .bind(now_ms) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + Self::keep_an_admin(&mut w).await?; + let revoked = revoke_devices(&mut w.tx, target, None, RevokeReason::Disabled, now_ms).await?; + delete_join_codes(&mut w.tx, target).await?; + // The event outlives the account, which stays only as a mark. Marked already: no change, no event. + if !found.removed { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserRemoved, target).with(AuditDetails::revoked(revoked.len()))).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// An invite under `code_hash` for an account that has no password yet (one nobody redeemed an invite for), replacing + /// its open invite or reset code. An account with a password is `Conflict`: an invite adds a computer without asking + /// for the password, which would give whoever holds it the account's files; a reset code is the way in there. A + /// disabled account, or one marked for removal, is `Conflict` too. + pub async fn admin_issue_invite( + &self, + actor: &AdminActor, + target: UserId, + code_hash: &[u8; 32], + now_ms: i64, + ttl_ms: i64, + ) -> Result { + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the invite's lifetime is too long"))?; + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed || found.disabled { + return Err(MetaError::Conflict("this account is disabled or marked for removal")); + } + if found.has_password { + return Err(MetaError::Conflict("this account has a password: make a password reset code for it instead")); + } + put_code(&mut w.tx, target, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminInviteIssued, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Issued { expires_ms }) + } + + /// Deletes the account's open invite or reset code; `NotFound` when there is none (so nothing changed and nothing is + /// written). + pub async fn admin_revoke_invite(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + Self::target(&mut w, target).await?; + let done = sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("invite")); + } + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminInviteRevoked, target)).await?; + self.commit_audited(w, now_ms).await + } + + /// A password reset code under `code_hash`, good for `ttl_ms`, replacing the account's open invite or reset code. + /// Redeeming it sets a new password and signs every device of the account out ([`Meta::redeem`]). Nothing changes + /// until then. With `reset_two_factor`, redeeming it also turns the account's two-factor sign-in off; without, an + /// account that has it needs its second factor to redeem the code. Resetting one's own password is `Invalid` (change + /// it), another administrator's `Forbidden` unless the acting admin has two-factor sign-in on themselves. A disabled + /// account, or one marked for removal, is `Conflict`. + pub async fn admin_issue_reset( + &self, + actor: &AdminActor, + target: UserId, + code_hash: &[u8; 32], + reset_two_factor: bool, + now_ms: i64, + ttl_ms: i64, + ) -> Result { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("change your own password in your account instead")); + } + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the code's lifetime is too long"))?; + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed || found.disabled { + return Err(MetaError::Conflict("this account is disabled or marked for removal")); + } + Self::admin_target_needs_two_factor(&found, actor)?; + put_code(&mut w.tx, target, code_hash, Purpose::Reset, reset_two_factor, now_ms, expires_ms).await?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminPasswordResetIssued, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Issued { expires_ms }) + } + + /// Takes away another account's two-factor sign-in: the secret, the codes and the challenges go, every device is signed + /// out ([`RevokeReason::TwoFactor`]) and the invite and pairings are deleted, in one transaction (the target's lock + /// and the acting admin's checks are as for every admin change). Returns the devices signed out, for the caller to end + /// their live connections. Your own account is `Invalid` (turn it off in your account), an account without two-factor + /// sign-in `Conflict` (nothing is signed out), another administrator's `Forbidden` unless the acting admin has + /// two-factor sign-in on themselves. + pub async fn admin_reset_two_factor(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("turn your own two-factor sign-in off in your account instead")); + } + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + Self::admin_target_needs_two_factor(&found, actor)?; + let revoked = two_factor::reset_in(&mut w.tx, target, now_ms).await?.ok_or(MetaError::Conflict("this account has no two-factor sign-in"))?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminTwoFactorReset, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Resetting another **administrator's** password or two-factor sign-in takes the acting admin's own two-factor + /// sign-in: a stolen admin token and password alone can't take over the other admins' accounts. (`begin_admin` has + /// compared `actor.two_factor` with the account's state under the lock, so it is the truth here.) + fn admin_target_needs_two_factor(target: &Target, actor: &AdminActor) -> Result<(), MetaError> { + if target.role == Role::Admin && !actor.two_factor { + return Err(MetaError::Forbidden( + "resetting another administrator needs two-factor sign-in on your own account: turn it on first, or use the command line on the server", + )); + } + Ok(()) + } + + /// All of an account's devices, revoked ones too; `NotFound` for a user that isn't there. + pub async fn admin_devices(&self, target: UserId) -> Result, MetaError> { + let found = sqlx::query("SELECT 1 FROM users WHERE id = $1").bind(uuid_bytes(&target.0)).fetch_optional(&self.pool).await?; + if found.is_none() { + return Err(MetaError::NotFound("user")); + } + self.list_devices(target).await + } + + /// Signs one of `target`'s devices out ([`RevokeReason::Admin`]) and deletes the pairing it made. A device that is + /// revoked already stays as it is. A device of another account, or an unknown user, is `NotFound`. + pub async fn admin_revoke_device(&self, actor: &AdminActor, target: UserId, device: DeviceId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + Self::target(&mut w, target).await?; + let done = sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE id = $3 AND user_id = $4 AND revoked = 0") + .bind(now_ms) + .bind(RevokeReason::Admin.as_str()) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + let owned = sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&target.0)) + .fetch_optional(&mut *w.tx) + .await?; + if owned.is_none() { + return Err(MetaError::NotFound("device")); + } + } + sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; + // The event is about the account, whose own activity then shows it, and names the device. A device signed out + // already is no change. + if done.rows_affected() > 0 { + let details = AuditDetails { device: Some(device), reason: Some(RevokeReason::Admin), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminDeviceRevoked, target).with(details)).await?; + } + self.commit_audited(w, now_ms).await + } +} diff --git a/crates/meta/src/audit.rs b/crates/meta/src/audit.rs new file mode 100644 index 00000000..8f605e05 --- /dev/null +++ b/crates/meta/src/audit.rs @@ -0,0 +1,1079 @@ +//! The security audit log: events in the `audit_events` table, written in the transaction of the change they record, +//! and read back by the account's own activity, the admin's list and export, and the command line. +//! +//! - **Names are a closed list** ([`AuditAction`]); no event has a name a client typed. +//! - **`details` holds typed values only** ([`AuditDetails`]): every field is a closed enumeration or a number, each +//! event allows only its own fields ([`AuditAction::fields`]), and nothing a client sent as text ever reaches it. In +//! particular `platform` is reduced to [`Platform`] (anything unknown is `other`), because a device may call its +//! platform any printable text of 32 bytes, an address or a file name included. What is written is at most +//! [`MAX_DETAILS_BYTES`] of JSON: when it would be more, whole fields go (the least important first) and +//! `"truncated": true` says so; bytes are never cut off, which would leave invalid JSON. +//! - **No addresses, file or device names, user names of unknown accounts, secrets** are stored anywhere in a row. +//! An account is named by its ID, which outlives the account (there are no foreign keys). +//! - **Changes write their event in their own transaction, before the commit, and fail with it** (fail closed): an event +//! that cannot be written means the change is not made. A failed sign-in changes nothing, so its event is written +//! after the fact in a transaction of its own ([`Meta::audit_record_failure`]), and a failure to write it is for the +//! caller to log: it must never change the answer or undo the abuse protection around it. +//! - **Append-only, in the code.** Nothing here updates or deletes a row, with two named exceptions: the counter of an +//! aggregated failure (one row per account, action, client kind and minute, `agg_count + 1` on a collision) and +//! [`Meta::audit_prune`], which deletes events older than the retention in batches of their own, never inside the +//! transaction of a security change. A test reads this crate's sources and allows exactly those two statements. + +use craft_cloud_proto::api::SpaceRole; +use craft_cloud_proto::{DeviceId, SpaceId}; +use serde_json::{Map, Value}; +use sqlx::Row; +use uuid::Uuid; + +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::users::{DeviceClient, RevokeReason, Role, UserId}; +use crate::{Meta, MetaError}; + +/// The most serialized UTF-8 bytes of an event's `details`. +pub const MAX_DETAILS_BYTES: usize = 2048; + +/// Rows one pruning statement deletes. +pub const PRUNE_BATCH: i64 = 1000; +/// Batches one opportunistic pruning (after a write) runs before it leaves the rest to the next one. +pub const OPPORTUNISTIC_PRUNE_BATCHES: usize = 20; +/// How often at most a write starts a pruning. +pub const PRUNE_EVERY_MS: i64 = 60 * 60 * 1000; +/// The shortest and longest retention `[audit] retention_days` may name. +pub const MIN_RETENTION_DAYS: u32 = 1; +pub const MAX_RETENTION_DAYS: u32 = 3650; + +const MINUTE_MS: i64 = 60_000; + +/// What an event is. The names are the ones in `docs/design/sync.md`; a name is never reused for something else. +/// +/// Not every name has a writer yet: the two-factor events (`auth.second_factor_failed`, `auth.recovery_code_used`, +/// `auth.two_factor_locked`, `account.two_factor_*`, `account.recovery_codes_regenerated`, `admin.two_factor_reset`, +/// `cli.two_factor_reset`) and `cli.user_removed` are written by the changes that add those operations. The names are +/// listed here so that the list a client may filter by does not change then. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AuditAction { + AuthLogin, + AuthLoginFailed, + AuthSecondFactorFailed, + AuthRecoveryCodeUsed, + AuthTwoFactorLocked, + DeviceRevoked, + DeviceRevokedOthers, + AccountPasswordChanged, + AccountTwoFactorEnabled, + AccountTwoFactorDisabled, + AccountRecoveryCodesRegenerated, + AdminUserCreated, + AdminUserDisabled, + AdminUserEnabled, + AdminUserRemoved, + AdminRoleChanged, + AdminQuotaChanged, + AdminInviteIssued, + AdminInviteRevoked, + AdminPasswordResetIssued, + AdminTwoFactorReset, + AdminDeviceRevoked, + JoinInviteRedeemed, + JoinResetRedeemed, + JoinPairingRedeemed, + CliPasswordReset, + CliTwoFactorReset, + CliDeviceRevoked, + CliDevicesRevokedAll, + CliUserRemoved, + SpaceCreated, + SpaceRenamed, + SpaceArchived, + SpaceUnarchived, + SpaceQuotaChanged, + SpaceMemberInvited, + SpaceMemberJoined, + SpaceMemberDeclined, + SpaceMemberRoleChanged, + SpaceMemberRemoved, + SpaceMemberLeft, + CliSpaceCreated, + CliSpaceMemberChanged, + CliSpaceArchived, +} + +impl AuditAction { + /// Every action, in the order the documentation lists them. + pub const ALL: [AuditAction; 44] = [ + AuditAction::AuthLogin, + AuditAction::AuthLoginFailed, + AuditAction::AuthSecondFactorFailed, + AuditAction::AuthRecoveryCodeUsed, + AuditAction::AuthTwoFactorLocked, + AuditAction::DeviceRevoked, + AuditAction::DeviceRevokedOthers, + AuditAction::AccountPasswordChanged, + AuditAction::AccountTwoFactorEnabled, + AuditAction::AccountTwoFactorDisabled, + AuditAction::AccountRecoveryCodesRegenerated, + AuditAction::AdminUserCreated, + AuditAction::AdminUserDisabled, + AuditAction::AdminUserEnabled, + AuditAction::AdminUserRemoved, + AuditAction::AdminRoleChanged, + AuditAction::AdminQuotaChanged, + AuditAction::AdminInviteIssued, + AuditAction::AdminInviteRevoked, + AuditAction::AdminPasswordResetIssued, + AuditAction::AdminTwoFactorReset, + AuditAction::AdminDeviceRevoked, + AuditAction::JoinInviteRedeemed, + AuditAction::JoinResetRedeemed, + AuditAction::JoinPairingRedeemed, + AuditAction::CliPasswordReset, + AuditAction::CliTwoFactorReset, + AuditAction::CliDeviceRevoked, + AuditAction::CliDevicesRevokedAll, + AuditAction::CliUserRemoved, + AuditAction::SpaceCreated, + AuditAction::SpaceRenamed, + AuditAction::SpaceArchived, + AuditAction::SpaceUnarchived, + AuditAction::SpaceQuotaChanged, + AuditAction::SpaceMemberInvited, + AuditAction::SpaceMemberJoined, + AuditAction::SpaceMemberDeclined, + AuditAction::SpaceMemberRoleChanged, + AuditAction::SpaceMemberRemoved, + AuditAction::SpaceMemberLeft, + AuditAction::CliSpaceCreated, + AuditAction::CliSpaceMemberChanged, + AuditAction::CliSpaceArchived, + ]; + + pub fn as_str(self) -> &'static str { + match self { + AuditAction::AuthLogin => "auth.login", + AuditAction::AuthLoginFailed => "auth.login_failed", + AuditAction::AuthSecondFactorFailed => "auth.second_factor_failed", + AuditAction::AuthRecoveryCodeUsed => "auth.recovery_code_used", + AuditAction::AuthTwoFactorLocked => "auth.two_factor_locked", + AuditAction::DeviceRevoked => "device.revoked", + AuditAction::DeviceRevokedOthers => "device.revoked_others", + AuditAction::AccountPasswordChanged => "account.password_changed", + AuditAction::AccountTwoFactorEnabled => "account.two_factor_enabled", + AuditAction::AccountTwoFactorDisabled => "account.two_factor_disabled", + AuditAction::AccountRecoveryCodesRegenerated => "account.recovery_codes_regenerated", + AuditAction::AdminUserCreated => "admin.user_created", + AuditAction::AdminUserDisabled => "admin.user_disabled", + AuditAction::AdminUserEnabled => "admin.user_enabled", + AuditAction::AdminUserRemoved => "admin.user_removed", + AuditAction::AdminRoleChanged => "admin.role_changed", + AuditAction::AdminQuotaChanged => "admin.quota_changed", + AuditAction::AdminInviteIssued => "admin.invite_issued", + AuditAction::AdminInviteRevoked => "admin.invite_revoked", + AuditAction::AdminPasswordResetIssued => "admin.password_reset_issued", + AuditAction::AdminTwoFactorReset => "admin.two_factor_reset", + AuditAction::AdminDeviceRevoked => "admin.device_revoked", + AuditAction::JoinInviteRedeemed => "join.invite_redeemed", + AuditAction::JoinResetRedeemed => "join.reset_redeemed", + AuditAction::JoinPairingRedeemed => "join.pairing_redeemed", + AuditAction::CliPasswordReset => "cli.password_reset", + AuditAction::CliTwoFactorReset => "cli.two_factor_reset", + AuditAction::CliDeviceRevoked => "cli.device_revoked", + AuditAction::CliDevicesRevokedAll => "cli.devices_revoked_all", + AuditAction::CliUserRemoved => "cli.user_removed", + AuditAction::SpaceCreated => "space.created", + AuditAction::SpaceRenamed => "space.renamed", + AuditAction::SpaceArchived => "space.archived", + AuditAction::SpaceUnarchived => "space.unarchived", + AuditAction::SpaceQuotaChanged => "space.quota_changed", + AuditAction::SpaceMemberInvited => "space.member_invited", + AuditAction::SpaceMemberJoined => "space.member_joined", + AuditAction::SpaceMemberDeclined => "space.member_declined", + AuditAction::SpaceMemberRoleChanged => "space.member_role_changed", + AuditAction::SpaceMemberRemoved => "space.member_removed", + AuditAction::SpaceMemberLeft => "space.member_left", + AuditAction::CliSpaceCreated => "cli.space_created", + AuditAction::CliSpaceMemberChanged => "cli.space_member_changed", + AuditAction::CliSpaceArchived => "cli.space_archived", + } + } + + /// The action with this exact name. + pub fn parse(name: &str) -> Option { + Self::ALL.into_iter().find(|action| action.as_str() == name) + } + + /// The `details` fields this event may have; the writer drops any other. Listed in the order of importance, which is + /// the order they are written in and the reverse of the order they are dropped in when the text is too long. + pub fn fields(self) -> &'static [Field] { + use Field::*; + match self { + AuditAction::AuthLogin | AuditAction::JoinInviteRedeemed | AuditAction::JoinPairingRedeemed => &[ClientKind, Platform], + AuditAction::JoinResetRedeemed => &[Revoked, TwoFactorRemoved, ClientKind, Platform], + AuditAction::AuthLoginFailed | AuditAction::AuthSecondFactorFailed => &[ClientKind, Step], + AuditAction::AuthRecoveryCodeUsed => &[ClientKind], + AuditAction::DeviceRevoked => &[Reason], + AuditAction::DeviceRevokedOthers => &[Revoked, Reason], + AuditAction::AdminUserCreated => &[RoleTo], + AuditAction::AdminRoleChanged => &[RoleFrom, RoleTo], + AuditAction::AdminQuotaChanged => &[QuotaBytes], + AuditAction::AdminDeviceRevoked => &[Device, Reason], + AuditAction::CliDeviceRevoked => &[Device, Reason], + AuditAction::AccountPasswordChanged + | AuditAction::AdminUserDisabled + | AuditAction::AdminUserRemoved + | AuditAction::CliPasswordReset + | AuditAction::CliTwoFactorReset + | AuditAction::CliDevicesRevokedAll + | AuditAction::CliUserRemoved => &[Revoked], + AuditAction::AccountTwoFactorEnabled + | AuditAction::AccountTwoFactorDisabled + | AuditAction::AccountRecoveryCodesRegenerated + | AuditAction::AdminUserEnabled + | AuditAction::AdminInviteIssued + | AuditAction::AdminInviteRevoked + | AuditAction::AdminPasswordResetIssued + | AuditAction::AdminTwoFactorReset => &[], + AuditAction::AuthTwoFactorLocked => &[ClientKind], + AuditAction::SpaceCreated | AuditAction::CliSpaceCreated => &[User, QuotaBytes], + AuditAction::SpaceRenamed + | AuditAction::SpaceArchived + | AuditAction::SpaceUnarchived + | AuditAction::SpaceMemberLeft + | AuditAction::CliSpaceArchived => &[], + AuditAction::SpaceQuotaChanged => &[QuotaBytes], + AuditAction::SpaceMemberInvited => &[User, SpaceRoleTo], + AuditAction::SpaceMemberJoined | AuditAction::SpaceMemberDeclined | AuditAction::SpaceMemberRemoved => &[User], + AuditAction::SpaceMemberRoleChanged | AuditAction::CliSpaceMemberChanged => &[User, SpaceRoleFrom, SpaceRoleTo], + } + } +} + +/// What a `details` field is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Field { + ClientKind, + Platform, + Reason, + Step, + RoleFrom, + RoleTo, + QuotaBytes, + Revoked, + Device, + /// A reset code redeemed for an account that had two-factor sign-in turned it off. + TwoFactorRemoved, + /// The account a team space event is about. + User, + /// A role in a team space, written as `role_from` / `role_to` like an account's server role. + SpaceRoleFrom, + SpaceRoleTo, +} + +/// The platforms an event names. A device may call its platform any printable text; only these reach the log. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + Windows, + Macos, + Linux, + Android, + Ios, + Other, +} + +impl Platform { + pub fn as_str(self) -> &'static str { + match self { + Platform::Windows => "windows", + Platform::Macos => "macos", + Platform::Linux => "linux", + Platform::Android => "android", + Platform::Ios => "ios", + Platform::Other => "other", + } + } + + /// The platform a client named: `None` for none at all, one of the list when it is that (case does not matter), + /// `Other` for anything else, whatever it looks like. + pub fn normalize(platform: &str) -> Option { + if platform.is_empty() { + return None; + } + Some( + [Platform::Windows, Platform::Macos, Platform::Linux, Platform::Android, Platform::Ios] + .into_iter() + .find(|known| known.as_str().eq_ignore_ascii_case(platform)) + .unwrap_or(Platform::Other), + ) + } +} + +/// The client kinds an event names. Device kinds are [`crate::CLIENT_KINDS`]; `Web` is assigned only by web sign-in code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientKind { + Agent, + Desktop, + Cli, + Web, +} + +impl ClientKind { + pub fn as_str(self) -> &'static str { + match self { + ClientKind::Agent => "agent", + ClientKind::Desktop => "desktop", + ClientKind::Cli => "cli", + ClientKind::Web => "web", + } + } + + /// Parses device-sent kinds only. Web is assigned by the server, never accepted from a device. + fn parse(kind: &str) -> Option { + [ClientKind::Agent, ClientKind::Desktop, ClientKind::Cli].into_iter().find(|known| known.as_str() == kind) + } +} + +/// How far a failed sign-in got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Step { + Password, + Totp, + Recovery, + ChallengeExpired, +} + +impl Step { + pub fn as_str(self) -> &'static str { + match self { + Step::Password => "password", + Step::Totp => "totp", + Step::Recovery => "recovery", + Step::ChallengeExpired => "challenge_expired", + } + } +} + +/// The typed values of an event. Which of them an event keeps is decided by [`AuditAction::fields`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AuditDetails { + pub client_kind: Option, + pub platform: Option, + pub reason: Option, + pub step: Option, + pub role_from: Option, + pub role_to: Option, + /// `Some(None)`: the quota was removed. + pub quota_bytes: Option>, + pub revoked: Option, + pub device: Option, + /// Redeeming a reset code also took the account's two-factor sign-in away (only ever set to `true`). + pub two_factor_removed: bool, + /// The account a team space event is about (the member, or the first owner). + pub user: Option, + pub space_role_from: Option, + pub space_role_to: Option, +} + +impl AuditDetails { + /// What a device says about its client, reduced to the closed lists. + pub fn client(client: &DeviceClient) -> Self { + Self { client_kind: ClientKind::parse(&client.kind), platform: Platform::normalize(&client.platform), ..Self::default() } + } + + pub fn revoked(count: usize) -> Self { + Self { revoked: Some(u32::try_from(count).unwrap_or(u32::MAX)), ..Self::default() } + } + + pub fn reason(reason: RevokeReason) -> Self { + Self { reason: Some(reason), ..Self::default() } + } + + pub fn step(step: Step) -> Self { + Self { step: Some(step), ..Self::default() } + } + + pub fn web() -> Self { + Self { client_kind: Some(ClientKind::Web), ..Self::default() } + } + + /// The value of `field`, if it is set. + fn value(&self, field: Field) -> Option { + match field { + Field::ClientKind => self.client_kind.map(|v| Value::from(v.as_str())), + Field::Platform => self.platform.map(|v| Value::from(v.as_str())), + Field::Reason => self.reason.map(|v| Value::from(v.as_str())), + Field::Step => self.step.map(|v| Value::from(v.as_str())), + Field::RoleFrom => self.role_from.map(|v| Value::from(v.as_str())), + Field::RoleTo => self.role_to.map(|v| Value::from(v.as_str())), + Field::QuotaBytes => self.quota_bytes.map(|v| v.map_or(Value::Null, Value::from)), + Field::Revoked => self.revoked.map(Value::from), + Field::Device => self.device.map(|v| Value::from(v.to_string())), + Field::TwoFactorRemoved => self.two_factor_removed.then_some(Value::Bool(true)), + Field::User => self.user.map(|v| Value::from(v.0.to_string())), + Field::SpaceRoleFrom => self.space_role_from.map(|v| Value::from(v.as_str())), + Field::SpaceRoleTo => self.space_role_to.map(|v| Value::from(v.as_str())), + } + } + + fn name(field: Field) -> &'static str { + match field { + Field::ClientKind => "client_kind", + Field::Platform => "platform", + Field::Reason => "reason", + Field::Step => "step", + Field::RoleFrom => "role_from", + Field::RoleTo => "role_to", + Field::QuotaBytes => "quota_bytes", + Field::Revoked => "revoked", + Field::Device => "device", + Field::TwoFactorRemoved => "two_factor_removed", + Field::User => "user", + Field::SpaceRoleFrom => "role_from", + Field::SpaceRoleTo => "role_to", + } + } + + /// The JSON text stored for `action`: the fields it allows that are set, within [`MAX_DETAILS_BYTES`]. + pub fn to_json(&self, action: AuditAction) -> String { + let fields = action.fields().iter().filter_map(|field| self.value(*field).map(|value| (Self::name(*field), value))).collect(); + serialize_fields(fields, MAX_DETAILS_BYTES) + } +} + +/// `fields` (most important first) as a JSON object of at most `max` bytes. When it would be longer, fields are dropped +/// from the end, one at a time, and `"truncated": true` is added; no text is ever cut short. +pub(crate) fn serialize_fields(mut fields: Vec<(&'static str, Value)>, max: usize) -> String { + let mut truncated = false; + loop { + let mut object = Map::new(); + for (name, value) in &fields { + object.insert((*name).to_owned(), value.clone()); + } + if truncated { + object.insert("truncated".to_owned(), Value::Bool(true)); + } + let text = Value::Object(object).to_string(); + if text.len() <= max { + return text; + } + if fields.pop().is_none() { + // Not reachable for any `max` this crate uses (the flag alone is 18 bytes), but never invalid JSON. + return "{\"truncated\":true}".to_owned(); + } + truncated = true; + } +} + +/// Whether the event is about something that worked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditOutcome { + Success, + Failure, +} + +impl AuditOutcome { + pub fn as_str(self) -> &'static str { + match self { + AuditOutcome::Success => "success", + AuditOutcome::Failure => "failure", + } + } + + fn parse(s: &str) -> Result { + match s { + "success" => Ok(AuditOutcome::Success), + "failure" => Ok(AuditOutcome::Failure), + _ => Err(MetaError::Corrupt("unknown audit outcome")), + } + } +} + +/// What an event was done to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetKind { + User, + Device, + Space, +} + +impl TargetKind { + pub fn as_str(self) -> &'static str { + match self { + TargetKind::User => "user", + TargetKind::Device => "device", + TargetKind::Space => "space", + } + } +} + +/// Who made a change that a signed-in account or the command line can make, for its event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Origin { + /// The account's own act, from a device the caller does not say. + Account, + /// The account's own act, from this device. + Device(DeviceId), + /// The admin's command line on the server's host: no account acts. + Cli, +} + +/// An event on its way into the log. +#[derive(Debug, Clone, Copy)] +pub(crate) struct NewEvent { + pub action: AuditAction, + pub actor_user: Option, + pub actor_device: Option, + pub target: Option<(TargetKind, Uuid)>, + pub details: AuditDetails, +} + +impl NewEvent { + pub fn new(action: AuditAction) -> Self { + Self { action, actor_user: None, actor_device: None, target: None, details: AuditDetails::default() } + } + + pub fn by(mut self, user: UserId, device: Option) -> Self { + self.actor_user = Some(user); + self.actor_device = device; + self + } + + pub fn on_user(mut self, user: UserId) -> Self { + self.target = Some((TargetKind::User, user.0)); + self + } + + pub fn on_space(mut self, space: SpaceId) -> Self { + self.target = Some((TargetKind::Space, *space.as_uuid())); + self + } + + pub fn on_device(mut self, device: DeviceId) -> Self { + self.target = Some((TargetKind::Device, *device.as_uuid())); + self + } + + pub fn with(mut self, details: AuditDetails) -> Self { + self.details = details; + self + } +} + +const INSERT: &str = concat!( + "INSERT INTO audit_events (id, at_ms, actor_user, actor_device, action, target_kind, target_id, outcome, details, agg_key, agg_count) ", + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 1)" +); + +/// Inserts `event` on `conn` (a transaction's connection). `aggregate`: the key of an aggregated failure, whose row +/// only counts up when the key is there already; `None` for every other event, whose key is `NULL`. +async fn insert( + conn: &mut sqlx::AnyConnection, + event: &NewEvent, + outcome: AuditOutcome, + now_ms: i64, + aggregate: Option, +) -> Result<(), MetaError> { + // Qualified: on Postgres a bare `agg_count` is ambiguous between the table's column and `excluded`'s. Only the + // counter changes, so the row keeps the identity, time and details of its first try. + let sql = if aggregate.is_some() { + format!("{INSERT} ON CONFLICT (agg_key) DO UPDATE SET agg_count = audit_events.agg_count + 1") + } else { + INSERT.to_owned() + }; + sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(uuid_bytes(&Uuid::now_v7())) + .bind(now_ms) + .bind(event.actor_user.map(|u| uuid_bytes(&u.0))) + .bind(event.actor_device.map(|d| uuid_bytes(d.as_uuid()))) + .bind(event.action.as_str()) + .bind(event.target.map_or("", |(kind, _)| kind.as_str())) + .bind(event.target.map(|(_, id)| uuid_bytes(&id))) + .bind(outcome.as_str()) + .bind(event.details.to_json(event.action)) + .bind(aggregate) + .execute(conn) + .await?; + Ok(()) +} + +/// Writes `event` as a success in `tx`, the transaction of the change it records. Its error fails the change. +pub(crate) async fn record_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, now_ms: i64, event: NewEvent) -> Result<(), MetaError> { + insert(tx, &event, AuditOutcome::Success, now_ms, None).await +} + +/// A failed sign-in step, counted by action, client kind and minute. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditFailure { + /// `AuthLoginFailed` or `AuthSecondFactorFailed`; any other action is `Invalid`. + pub action: AuditAction, + /// The account the guess was about; `None` for a name no account has, which is all the log may say of it. + pub target: Option, + pub step: Step, +} + +/// Where a page of the log starts: strictly after (older than) this event, in the order `(at_ms, id)` descending. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditCursor { + pub at_ms: i64, + pub id: Uuid, +} + +impl AuditCursor { + pub const BYTES: usize = 24; + + pub fn to_bytes(self) -> [u8; Self::BYTES] { + let mut out = [0u8; Self::BYTES]; + out[..8].copy_from_slice(&self.at_ms.to_be_bytes()); + out[8..].copy_from_slice(self.id.as_bytes()); + out + } + + /// `None` unless `bytes` is exactly what [`AuditCursor::to_bytes`] makes (and a time that can be one). + pub fn from_bytes(bytes: &[u8]) -> Option { + let bytes: &[u8; Self::BYTES] = bytes.try_into().ok()?; + let at_ms = i64::from_be_bytes(bytes.get(..8)?.try_into().ok()?); + let id = Uuid::from_slice(bytes.get(8..)?).ok()?; + (at_ms >= 0).then_some(Self { at_ms, id }) + } +} + +/// Which actions a list is limited to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ActionFilter { + /// This action only. + Exact(AuditAction), + /// The actions whose name starts with this (it ends in a dot, like `auth.`). + Prefix(&'static str), +} + +impl ActionFilter { + /// An exact name, or a prefix ending in a dot that at least one name starts with. Anything else is `None`, so + /// nothing a client typed ever reaches a query. + pub fn parse(text: &str) -> Option { + if let Some(action) = AuditAction::parse(text) { + return Some(ActionFilter::Exact(action)); + } + if !text.ends_with('.') || !text.bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.') { + return None; + } + // The prefix is kept as the `'static` start of the first name it matches: the same characters, from the closed list. + AuditAction::ALL + .iter() + .find(|action| action.as_str().starts_with(text)) + .and_then(|action| action.as_str().get(..text.len())) + .map(ActionFilter::Prefix) + } + + /// The names the filter lets through. + pub fn actions(self) -> Vec { + match self { + ActionFilter::Exact(action) => vec![action], + ActionFilter::Prefix(prefix) => AuditAction::ALL.into_iter().filter(|action| action.as_str().starts_with(prefix)).collect(), + } + } +} + +/// One page of the log, asked for. +#[derive(Debug, Clone, Copy, Default)] +pub struct AuditQuery { + /// Events the account acted in or was the target of. + pub user: Option, + pub action: Option, + /// Only events before this time (the first call of a listing). + pub before_ms: Option, + /// Only events up to and including this time (an export's fixed end). + pub until_ms: Option, + /// Only events strictly after this one in the listing's order. + pub after: Option, + pub limit: u32, +} + +/// An event as read back. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditEvent { + pub id: Uuid, + pub at_ms: i64, + pub actor_user: Option, + pub actor_device: Option, + pub action: AuditAction, + /// `user`, `device`, `space`, or empty. + pub target_kind: String, + pub target_id: Option, + pub outcome: AuditOutcome, + /// A JSON object text, as written. + pub details: String, + /// The tries one aggregated row counts (1 for every other event). + pub count: u64, +} + +impl AuditEvent { + /// Where a page continues after this event. + pub fn cursor(&self) -> AuditCursor { + AuditCursor { at_ms: self.at_ms, id: self.id } + } +} + +/// What a pruning did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Pruned { + pub deleted: u64, + /// There was more to delete than the batches allowed. + pub more: bool, +} + +const SELECT: &str = "SELECT id, at_ms, actor_user, actor_device, action, target_kind, target_id, outcome, details, agg_count FROM audit_events"; + +fn event_from(row: &sqlx::any::AnyRow) -> Result { + let actor_user: Option> = row.try_get("actor_user")?; + let actor_device: Option> = row.try_get("actor_device")?; + let target_id: Option> = row.try_get("target_id")?; + Ok(AuditEvent { + id: uuid_from(&row.try_get::, _>("id")?)?, + at_ms: row.try_get("at_ms")?, + actor_user: actor_user.map(|b| uuid_from(&b).map(UserId)).transpose()?, + actor_device: actor_device.map(|b| uuid_from(&b).map(DeviceId::from_uuid)).transpose()?, + action: AuditAction::parse(&row.try_get::("action")?).ok_or(MetaError::Corrupt("unknown audit action"))?, + target_kind: row.try_get("target_kind")?, + target_id: target_id.map(|b| uuid_from(&b)).transpose()?, + outcome: AuditOutcome::parse(&row.try_get::("outcome")?)?, + details: row.try_get("details")?, + count: to_u64(row.try_get("agg_count")?)?, + }) +} + +enum Arg { + Int(i64), + Bytes(Vec), + Text(String), +} + +/// How long a pruning that is running holds the right to run, and what the last complete one's time was. +#[derive(Debug)] +pub(crate) struct PruneState { + last_done_ms: std::sync::atomic::AtomicI64, + running: std::sync::atomic::AtomicBool, +} + +impl PruneState { + const NEVER: i64 = i64::MIN; + + pub fn new() -> Self { + Self { last_done_ms: std::sync::atomic::AtomicI64::new(Self::NEVER), running: std::sync::atomic::AtomicBool::new(false) } + } +} + +/// Gives the right to run back when dropped, whether the pruning finished, failed or was cancelled. +struct Running<'a>(&'a PruneState); + +impl Drop for Running<'_> { + fn drop(&mut self) { + self.0.running.store(false, std::sync::atomic::Ordering::Release); + } +} + +impl Meta { + /// The store that also prunes events older than `days` days after a write, at most once an hour and in bounded + /// batches ([`Meta::audit_prune_if_due`]). Without it nothing is pruned but by [`Meta::audit_prune`]. + pub fn with_audit_retention_days(mut self, days: u32) -> Self { + self.audit_retention_ms = Some(i64::from(days).saturating_mul(24 * 60 * 60 * 1000)); + self + } + + /// Writes a failed sign-in step in a transaction of its own, after the fact: one row per account (or per minute, for + /// names no account has), action, client kind and minute, whose `count` goes up by one with every further try. The caller logs a + /// failure and goes on: the answer to the client and the limits around the sign-in do not depend on this. + pub async fn audit_record_failure(&self, failure: AuditFailure, now_ms: i64) -> Result<(), MetaError> { + self.audit_record_failure_kind(failure, None, now_ms).await + } + + pub async fn audit_record_web_failure(&self, failure: AuditFailure, now_ms: i64) -> Result<(), MetaError> { + self.audit_record_failure_kind(failure, Some(ClientKind::Web), now_ms).await + } + + async fn audit_record_failure_kind(&self, failure: AuditFailure, kind: Option, now_ms: i64) -> Result<(), MetaError> { + if !matches!(failure.action, AuditAction::AuthLoginFailed | AuditAction::AuthSecondFactorFailed) { + return Err(MetaError::Invalid("only failed sign-in steps are aggregated")); + } + let mut details = AuditDetails::step(failure.step); + details.client_kind = kind; + let mut event = NewEvent::new(failure.action).with(details); + if let Some(user) = failure.target { + event = event.on_user(user); + } + let who = failure.target.map_or_else(|| "-".to_owned(), |u| u.0.simple().to_string()); + let kind = kind.map_or("device", ClientKind::as_str); + let key = format!("{}:{kind}:{who}:{}", failure.action.as_str(), now_ms.div_euclid(MINUTE_MS)); + let mut w = self.begin_write().await?; + insert(&mut w.tx, &event, AuditOutcome::Failure, now_ms, Some(key)).await?; + w.commit().await?; + self.prune_soon(now_ms); + Ok(()) + } + + /// One page of events, newest first, in the order `(at_ms, id)` descending: at most `query.limit` (and never more + /// than 1000). + pub async fn audit_list(&self, query: &AuditQuery) -> Result, MetaError> { + let mut sql = format!("{SELECT} WHERE 1 = 1"); + let mut args = Vec::new(); + let mut next = |arg: Arg| -> usize { + args.push(arg); + args.len() + }; + if let Some(user) = query.user { + let n = next(Arg::Bytes(uuid_bytes(&user.0))); + // An event also concerns the account it names as `user` in its details (a space member invited, removed ...). + // Details are compact JSON, and an ID has no character that `LIKE` reads. + let m = next(Arg::Text(format!("%\"user\":\"{}\"%", user.0.hyphenated()))); + sql.push_str(&format!(" AND (actor_user = ${n} OR target_id = ${n} OR details LIKE ${m})")); + } + if let Some(filter) = query.action { + // The names come from the closed list, never from a request, so they are safe to write into the text. + let names: Vec = filter.actions().into_iter().map(|a| format!("'{}'", a.as_str())).collect(); + sql.push_str(&format!(" AND action IN ({})", names.join(", "))); + } + if let Some(before) = query.before_ms { + let n = next(Arg::Int(before)); + sql.push_str(&format!(" AND at_ms < ${n}")); + } + if let Some(until) = query.until_ms { + let n = next(Arg::Int(until)); + sql.push_str(&format!(" AND at_ms <= ${n}")); + } + if let Some(after) = query.after { + let at = next(Arg::Int(after.at_ms)); + let id = next(Arg::Bytes(uuid_bytes(&after.id))); + sql.push_str(&format!(" AND (at_ms < ${at} OR (at_ms = ${at} AND id < ${id}))")); + } + let limit = next(Arg::Int(i64::from(query.limit.min(1000)))); + sql.push_str(&format!(" ORDER BY at_ms DESC, id DESC LIMIT ${limit}")); + let mut q = sqlx::query(sqlx::AssertSqlSafe(sql)); + for arg in args { + q = match arg { + Arg::Int(i) => q.bind(i), + Arg::Bytes(b) => q.bind(b), + Arg::Text(t) => q.bind(t), + }; + } + q.fetch_all(&self.pool).await?.iter().map(event_from).collect() + } + + /// How many events are older than `before_ms` (what [`Meta::audit_prune`] would delete). + pub async fn audit_count_before(&self, before_ms: i64) -> Result { + let row = sqlx::query("SELECT COUNT(*) AS n FROM audit_events WHERE at_ms < $1").bind(before_ms).fetch_one(&self.pool).await?; + to_u64(row.try_get("n")?) + } + + /// Deletes every event older than `before_ms`, in batches of [`PRUNE_BATCH`] rows, each in a transaction of its own. + /// Returns how many went. It is the one way a row is deleted (a test checks the sources). + pub async fn audit_prune(&self, before_ms: i64) -> Result { + let mut total = 0u64; + loop { + let done = self.audit_prune_batches(before_ms, 64).await?; + total = total.saturating_add(done.deleted); + if !done.more { + return Ok(total); + } + } + } + + /// [`Meta::audit_prune`] for at most `max_batches` batches: `more` says whether events older than `before_ms` are + /// left. A batch holds the write lock only for its own statement, so a sign-in never waits for a long pruning. + pub async fn audit_prune_batches(&self, before_ms: i64, max_batches: usize) -> Result { + let mut deleted = 0u64; + for _ in 0..max_batches { + let mut w = self.begin_write().await?; + let done = sqlx::query("DELETE FROM audit_events WHERE id IN (SELECT id FROM audit_events WHERE at_ms < $1 ORDER BY at_ms LIMIT $2)") + .bind(before_ms) + .bind(PRUNE_BATCH) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + deleted = deleted.saturating_add(done.rows_affected()); + if done.rows_affected() < PRUNE_BATCH as u64 { + return Ok(Pruned { deleted, more: false }); + } + } + Ok(Pruned { deleted, more: true }) + } + + /// Prunes events past the retention ([`Meta::with_audit_retention_days`]) if the last complete pruning was an hour or + /// more before `now_ms` (or there was none), in at most [`OPPORTUNISTIC_PRUNE_BATCHES`] batches. The time of the last + /// one is set only when it deleted everything that was due, so a backlog or an error lets the next write go on. + /// Returns whether a pruning ran. Called after the commit of a write, never inside a transaction. + pub async fn audit_prune_if_due(&self, now_ms: i64) -> Result { + self.audit_prune_if_due_in(now_ms, OPPORTUNISTIC_PRUNE_BATCHES).await + } + + /// [`Meta::audit_prune_if_due`] with its own limit of batches, so that a test can make a backlog without thousands of rows. + pub async fn audit_prune_if_due_in(&self, now_ms: i64, max_batches: usize) -> Result { + use std::sync::atomic::Ordering; + let Some(keep_ms) = self.audit_retention_ms else { return Ok(false) }; + let state = &self.prune_state; + let last = state.last_done_ms.load(Ordering::Acquire); + if last != PruneState::NEVER && now_ms.saturating_sub(last) < PRUNE_EVERY_MS { + return Ok(false); + } + if state.running.swap(true, Ordering::AcqRel) { + return Ok(false); + } + let _running = Running(state); + let done = self.audit_prune_batches(now_ms.saturating_sub(keep_ms), max_batches).await?; + if !done.more { + state.last_done_ms.store(now_ms, Ordering::Release); + } + Ok(true) + } + + /// Commits a change that wrote an audit event, then lets the write start a pruning ([`Meta::prune_soon`]). + pub(crate) async fn commit_audited(&self, w: crate::WriteTx, now_ms: i64) -> Result<(), MetaError> { + w.commit().await?; + self.prune_soon(now_ms); + Ok(()) + } + + /// Starts [`Meta::audit_prune_if_due`] in the background, so the write that asked does not wait for it. + pub(crate) fn prune_soon(&self, now_ms: i64) { + if self.audit_retention_ms.is_none() { + return; + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { return }; + let meta = self.clone(); + runtime.spawn(async move { + if let Err(e) = meta.audit_prune_if_due(now_ms).await { + tracing::warn!(error = %e, "couldn't prune old audit events"); + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_action_has_a_unique_name_that_parses_back() { + let mut seen = std::collections::HashSet::new(); + for action in AuditAction::ALL { + assert!(seen.insert(action.as_str()), "{} twice", action.as_str()); + assert_eq!(AuditAction::parse(action.as_str()), Some(action)); + assert!(action.as_str().bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.')); + } + assert_eq!(seen.len(), 44); + assert_eq!(AuditAction::parse("auth.login "), None); + assert_eq!(AuditAction::parse("AUTH.LOGIN"), None); + } + + #[test] + fn a_filter_is_a_name_or_a_prefix_of_names() { + assert_eq!(ActionFilter::parse("auth.login"), Some(ActionFilter::Exact(AuditAction::AuthLogin))); + let auth = ActionFilter::parse("auth.").unwrap(); + assert_eq!(auth.actions().len(), 5); + assert!(auth.actions().iter().all(|a| a.as_str().starts_with("auth."))); + assert_eq!(ActionFilter::parse("admin.").unwrap().actions().len(), 11); + for bad in ["", "auth", "au.", "auth.%", "auth._", "nope.", "AUTH.", "auth.login'", "auth. ", "auth.logi", "'; DROP TABLE x; --"] { + assert_eq!(ActionFilter::parse(bad), None, "{bad:?}"); + } + } + + #[test] + fn a_platform_is_reduced_to_the_closed_list() { + assert_eq!(Platform::normalize(""), None); + assert_eq!(Platform::normalize("Windows"), Some(Platform::Windows)); + assert_eq!(Platform::normalize("macos"), Some(Platform::Macos)); + // Hostile text a device may legally send: 32 bytes of printable characters or less. + for hostile in [ + "10.0.0.1", + "secret.psd", + "fe80::1", + "\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}", + "windows 11", + "linux\u{7f}", + ] { + assert_eq!(Platform::normalize(hostile), Some(Platform::Other), "{hostile:?}"); + } + } + + #[test] + fn details_keep_only_the_fields_their_event_allows() { + let all = AuditDetails { + client_kind: Some(ClientKind::Agent), + platform: Some(Platform::Linux), + reason: Some(RevokeReason::Admin), + step: Some(Step::Password), + role_from: Some(Role::Member), + role_to: Some(Role::Admin), + quota_bytes: Some(None), + revoked: Some(3), + device: Some(DeviceId::from_uuid(Uuid::nil())), + two_factor_removed: true, + user: Some(UserId(Uuid::nil())), + space_role_from: Some(SpaceRole::Viewer), + space_role_to: Some(SpaceRole::Owner), + }; + let keys = |action: AuditAction| -> Vec { + let value: Value = serde_json::from_str(&all.to_json(action)).unwrap(); + let mut keys: Vec = value.as_object().unwrap().keys().cloned().collect(); + keys.sort(); + keys + }; + assert_eq!(keys(AuditAction::AuthLogin), ["client_kind", "platform"]); + assert_eq!(keys(AuditAction::AuthLoginFailed), ["client_kind", "step"]); + assert_eq!(keys(AuditAction::AuthRecoveryCodeUsed), ["client_kind"]); + assert_eq!(keys(AuditAction::AdminRoleChanged), ["role_from", "role_to"]); + assert_eq!(keys(AuditAction::AdminQuotaChanged), ["quota_bytes"]); + assert_eq!(keys(AuditAction::AdminUserEnabled), Vec::::new()); + assert_eq!(keys(AuditAction::CliDeviceRevoked), ["device", "reason"]); + assert_eq!(keys(AuditAction::JoinResetRedeemed), ["client_kind", "platform", "revoked", "two_factor_removed"]); + assert_eq!(keys(AuditAction::AccountTwoFactorEnabled), Vec::::new()); + assert_eq!(keys(AuditAction::SpaceCreated), ["quota_bytes", "user"]); + assert_eq!(keys(AuditAction::SpaceMemberInvited), ["role_to", "user"]); + assert_eq!(keys(AuditAction::SpaceMemberRoleChanged), ["role_from", "role_to", "user"]); + assert_eq!(keys(AuditAction::CliSpaceMemberChanged), ["role_from", "role_to", "user"]); + assert_eq!(keys(AuditAction::SpaceMemberRemoved), ["user"]); + assert_eq!(keys(AuditAction::SpaceRenamed), Vec::::new()); + assert_eq!(all.to_json(AuditAction::AdminQuotaChanged), r#"{"quota_bytes":null}"#); + assert_eq!(AuditDetails::revoked(2).to_json(AuditAction::AdminUserDisabled), r#"{"revoked":2}"#); + assert_eq!(AuditDetails::default().to_json(AuditAction::AdminInviteIssued), "{}"); + } + + #[test] + fn details_over_the_limit_lose_whole_fields_and_stay_valid_json() { + // Multi-byte characters, so that cutting bytes anywhere would break the text. + let long = "\u{20ac}".repeat(300); + let fields = vec![ + ("first", Value::from("kept")), + ("second", Value::from(long.clone())), + ("third", Value::from(long.clone())), + ("last", Value::from(long)), + ]; + let text = serialize_fields(fields.clone(), MAX_DETAILS_BYTES); + assert!(text.len() <= MAX_DETAILS_BYTES, "{}", text.len()); + let value: Value = serde_json::from_str(&text).unwrap(); + let object = value.as_object().unwrap(); + assert_eq!(object.get("truncated"), Some(&Value::Bool(true))); + assert_eq!(object.get("first"), Some(&Value::from("kept"))); + // The least important go first, and a field that stays is whole. + assert!(object.get("last").is_none()); + for (name, value) in &fields { + if let Some(kept) = object.get(*name) { + assert_eq!(kept, value); + } + } + // Under the limit nothing is dropped and nothing is flagged. + let small = serialize_fields(vec![("a", Value::from(1))], MAX_DETAILS_BYTES); + assert_eq!(small, r#"{"a":1}"#); + // Even one field over the limit leaves a valid object. + let one = serialize_fields(vec![("only", Value::from("\u{20ac}".repeat(1000)))], MAX_DETAILS_BYTES); + assert_eq!(one, r#"{"truncated":true}"#); + } + + #[test] + fn a_cursor_round_trips_and_refuses_other_bytes() { + let cursor = AuditCursor { at_ms: 1_700_000_000_123, id: Uuid::now_v7() }; + assert_eq!(AuditCursor::from_bytes(&cursor.to_bytes()), Some(cursor)); + assert_eq!(AuditCursor::from_bytes(&[]), None); + assert_eq!(AuditCursor::from_bytes(&cursor.to_bytes()[..23]), None); + let mut longer = cursor.to_bytes().to_vec(); + longer.push(0); + assert_eq!(AuditCursor::from_bytes(&longer), None); + assert_eq!(AuditCursor::from_bytes(&AuditCursor { at_ms: -1, id: Uuid::nil() }.to_bytes()), None); + } +} diff --git a/crates/meta/src/blobs.rs b/crates/meta/src/blobs.rs index 45fbd781..70dffc14 100644 --- a/crates/meta/src/blobs.rs +++ b/crates/meta/src/blobs.rs @@ -11,10 +11,10 @@ use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId, SpaceId}; use sqlx::Row; use crate::codec::{hash_from, to_i64, to_u64, uuid_bytes}; -use crate::{Meta, MetaError}; +use crate::{Dialect, Meta, MetaError}; /// How a blob's kind is stored in the `kind` columns. -trait KindCode { +pub(crate) trait KindCode { fn code(self) -> i16; } @@ -41,7 +41,7 @@ async fn space_has(tx: &mut sqlx::AnyConnection, space: SpaceId, kind: BlobKind, const BATCH: usize = 500; /// `$first, $first+1, …` for `count` placeholders. -fn placeholders(first: usize, count: usize) -> String { +pub(crate) fn placeholders(first: usize, count: usize) -> String { (first..first + count).map(|n| format!("${n}")).collect::>().join(", ") } @@ -104,27 +104,42 @@ pub(crate) async fn record(tx: &mut sqlx::AnyConnection, space: SpaceId, blob: B charge(tx, space, size, cap).await } -/// Adds `size` to the space's stored bytes, refusing (`QuotaExceeded`) when that goes over `cap`. On Postgres the -/// update locks the space's row until the transaction ends, so two uploads to one space can't both slip under. +/// Adds `size` to the space's stored bytes, refusing (`QuotaExceeded`) when that goes over the space's limit: its own +/// quota (set by an admin), else `cap`. On Postgres the update locks the space's row until the transaction ends, so two +/// uploads to one space can't both slip under. async fn charge(tx: &mut sqlx::AnyConnection, space: SpaceId, size: i64, cap: Option) -> Result<(), MetaError> { - let total: i64 = sqlx::query("UPDATE spaces SET stored_bytes = stored_bytes + $1 WHERE id = $2 RETURNING stored_bytes") + let row = sqlx::query("UPDATE spaces SET stored_bytes = stored_bytes + $1 WHERE id = $2 RETURNING stored_bytes, quota_bytes") .bind(size) .bind(uuid_bytes(space.as_uuid())) .fetch_one(&mut *tx) - .await? - .try_get("stored_bytes")?; - let total = to_u64(total)?; - match cap { + .await?; + let total = to_u64(row.try_get("stored_bytes")?)?; + let quota: Option = row.try_get("quota_bytes")?; + match effective_quota(quota.map(to_u64).transpose()?, cap) { Some(limit) if total > limit => Err(MetaError::QuotaExceeded { used: total.saturating_sub(to_u64(size)?), limit }), _ => Ok(()), } } +/// The account's quota override, else the server-wide cap. +pub(crate) fn effective_quota(quota: Option, cap: Option) -> Option { + quota.or(cap) +} + /// The bytes `space` stores. async fn stored_bytes(conn: &mut sqlx::AnyConnection, space: SpaceId) -> Result { - let row = sqlx::query("SELECT stored_bytes FROM spaces WHERE id = $1").bind(uuid_bytes(space.as_uuid())).fetch_optional(&mut *conn).await?; + Ok(stored_and_quota(conn, space).await?.0) +} + +/// The bytes `space` stores, and the quota an admin set for it (`None`: the server-wide cap, if any, applies). +async fn stored_and_quota(conn: &mut sqlx::AnyConnection, space: SpaceId) -> Result<(u64, Option), MetaError> { + let row = sqlx::query("SELECT stored_bytes, quota_bytes FROM spaces WHERE id = $1") + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *conn) + .await?; let row = row.ok_or(MetaError::NotFound("space"))?; - to_u64(row.try_get("stored_bytes")?) + let quota: Option = row.try_get("quota_bytes")?; + Ok((to_u64(row.try_get("stored_bytes")?)?, quota.map(to_u64).transpose()?)) } /// The file size recorded for a manifest this space has, if it has it. @@ -157,19 +172,20 @@ impl Meta { stored_bytes(&mut conn, space).await } - /// Whether `space` has room for a blob of `stored_size` bytes under the cap ([`Meta::with_stored_bytes_cap`]): + /// Whether `space` has room for a blob of `stored_size` bytes under its limit: the quota an admin set for the + /// space, else the cap ([`Meta::with_stored_bytes_cap`]), else none (so a quota holds without a cap): /// `QuotaExceeded` if it doesn't have the blob yet and taking it would go over. A cheap look before the bytes are /// stored, so a space over its cap can't fill the disk with blobs that are then refused; recording the blob /// checks again, under the write lock. Two uploads under the cap can both pass this look and one then be refused /// after its bytes are stored: at most a few unrecorded blobs per account (bounded by its body slots), which GC /// (M4) sweeps. pub async fn check_room(&self, space: SpaceId, kind: BlobKind, id: &[u8; 32], stored_size: u64) -> Result<(), MetaError> { - let Some(limit) = self.stored_cap else { return Ok(()) }; let mut conn = self.pool.acquire().await?; + let (used, quota) = stored_and_quota(&mut conn, space).await?; + let Some(limit) = effective_quota(quota, self.stored_cap) else { return Ok(()) }; if space_has(&mut conn, space, kind, id).await? { return Ok(()); } - let used = stored_bytes(&mut conn, space).await?; if used.saturating_add(stored_size) > limit { Err(MetaError::QuotaExceeded { used, limit }) } else { Ok(()) } } @@ -177,6 +193,7 @@ impl Meta { /// `QuotaExceeded`, recording nothing, if it would take the space over the cap. pub async fn add_chunk(&self, space: SpaceId, id: &ChunkId, stored_size: u64, now_ms: i64) -> Result<(), MetaError> { let mut w = self.begin_write().await?; + self.check_guard(&mut w.tx, space).await?; let blob = Blob { kind: BlobKind::Chunk, id: id.as_bytes(), stored_size: to_i64(stored_size)?, file_size: None }; record(&mut w.tx, space, blob, now_ms, self.stored_cap).await?; w.commit().await @@ -191,6 +208,7 @@ impl Meta { sorted.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); sorted.dedup_by(|a, b| a.0 == b.0); let mut w = self.begin_write().await?; + self.check_guard(&mut w.tx, space).await?; for part in sorted.chunks(BATCH) { let rows = |width: usize| (0..part.len()).map(|i| format!("({})", placeholders(i * width + 1, width))).collect::>().join(", "); let mut blobs = sqlx::query(sqlx::AssertSqlSafe(format!( @@ -247,9 +265,13 @@ impl Meta { return Err(MetaError::MissingChunks(missing)); } let mut w = self.begin_write().await?; + self.check_guard(&mut w.tx, space).await?; let id = manifest.id(); let blob = Blob { kind: BlobKind::Manifest, id: id.as_bytes(), stored_size: to_i64(stored_size)?, file_size: Some(to_i64(manifest.size())?) }; record(&mut w.tx, space, blob, now_ms, self.stored_cap).await?; + // The index of the manifest's chunks is kept in the same transaction (see `index`): an accepted manifest is never + // without an index row. + crate::index::register(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; w.commit().await } diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index bd8911b9..f0597a0d 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -13,12 +13,16 @@ use std::collections::HashMap; +use craft_cloud_proto::conflict_name; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; -use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; +use craft_cloud_proto::{ + Conflict, ConflictKind, ConflictStamp, DeviceId, Epoch, ManifestId, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId, +}; use sqlx::Row; use crate::blobs::manifest_file_size; -use crate::codec::{flag, to_i64, uuid_bytes, uuid_from}; +use crate::codec::{flag, hash_from, to_i64, uuid_bytes, uuid_from}; +use crate::trash::TRASH_BELOW; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; @@ -29,6 +33,61 @@ pub struct CommitOutcome { pub epoch: Epoch, } +/// A successful restore of an old version: the version it made, the one that was current, and the space's new +/// sequence number and epoch (unchanged by a restore). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RestoreOutcome { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// A node that came back from the trash: where it sits now, whether it took a conflict name, and the space's new +/// sequence number and epoch (unchanged). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashRestoreOutcome { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + /// The file's new version (a copy of its last one); none for a folder. + pub version: Option, + pub seq: u64, + pub epoch: Epoch, +} + +/// The most conflict names tried for one restored node. +const MAX_CONFLICT_COUNTER: u32 = 100; + +fn rejected(kind: ConflictKind) -> MetaError { + MetaError::Rejected(vec![Conflict { op: 0, kind }]) +} + +/// `ms` since the Unix epoch as a UTC minute, for a conflict name (the server doesn't know a user's time zone). +fn conflict_stamp(ms: i64) -> ConflictStamp { + let days = ms.div_euclid(86_400_000); + let minutes = ms.rem_euclid(86_400_000) / 60_000; + // Days to a civil date in the proleptic Gregorian calendar, counted from 0000-03-01. + let z = days.saturating_add(719_468); + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let shifted_month = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * shifted_month + 2) / 5 + 1; + let month = if shifted_month < 10 { shifted_month + 3 } else { shifted_month - 9 }; + let year = year_of_era.saturating_add(era.saturating_mul(400)).saturating_add(i64::from(month <= 2)); + ConflictStamp { + year: u16::try_from(year).unwrap_or(0), + month: u8::try_from(month).unwrap_or(1), + day: u8::try_from(day).unwrap_or(1), + hour: u8::try_from(minutes / 60).unwrap_or(0), + minute: u8::try_from(minutes % 60).unwrap_or(0), + utc: true, + } +} + struct NodeRow { parent: Option, name: String, @@ -116,8 +175,12 @@ impl Ctx<'_> { if deleted { self.failed_restores.clear(); } - sqlx::query("UPDATE nodes SET deleted = $1, changed_seq = $2 WHERE space_id = $3 AND id = $4") + // The time in the trash starts at the change from live to deleted and ends when the node comes back: a later + // deletion starts it afresh, and a pending request to remove the node for good dies with the deletion it was for. + let deleted_ms = deleted.then_some(self.now_ms); + sqlx::query("UPDATE nodes SET deleted = $1, deleted_ms = $2, purge_requested_ms = NULL, changed_seq = $3 WHERE space_id = $4 AND id = $5") .bind(i64::from(deleted)) + .bind(deleted_ms) .bind(self.seq) .bind(self.space()) .bind(uuid_bytes(id.as_uuid())) @@ -344,6 +407,194 @@ impl Ctx<'_> { } } } + /// The first of 100 conflict names for `original` in `parent` that no live node has (the first has no counter, + /// then `, 2` to `, 100`), or `None`. + async fn free_conflict_name(&mut self, parent: NodeId, original: &Name, is_dir: bool, device: &str) -> Result, MetaError> { + let stamp = conflict_stamp(self.now_ms); + for counter in 1..=MAX_CONFLICT_COUNTER { + let candidate = conflict_name(original, is_dir, device, stamp, counter).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + if self.live_child(parent, &candidate).await?.is_none() { + return Ok(Some(candidate)); + } + } + Ok(None) + } + + /// Writes where a deleted node comes back and makes it live: one statement, one sequence number. + async fn revive(&mut self, id: NodeId, parent: NodeId, name: &Name) -> Result<(), MetaError> { + sqlx::query( + "UPDATE nodes SET parent = $1, name = $2, fold_key = $3, deleted = 0, deleted_ms = NULL, purge_requested_ms = NULL, changed_seq = $4 \ + WHERE space_id = $5 AND id = $6", + ) + .bind(uuid_bytes(parent.as_uuid())) + .bind(name.as_str().to_owned()) + .bind(name.fold_key().as_str().to_owned()) + .bind(self.seq) + .bind(self.space()) + .bind(uuid_bytes(id.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + /// Brings a node of the trash back, with the deleted folders above it, root-most first and with their IDs. Where + /// a deleted folder's name is taken by a live folder, that folder is used instead (the deleted one stays deleted) + /// and what is below it moves in. Where a name is taken by a file, or the node's own name is taken by anything, the + /// node comes back under a conflict name. Never overwrites, never creates a node. A conflict that can't be solved + /// is an `Err(MetaError::Rejected)`, and the caller drops the transaction. + async fn restore_from_trash(&mut self, node: NodeId, tree_cap: u64) -> Result { + let found = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await?; + let Some(found) = found else { return Err(MetaError::NotFound("node in the trash")) }; + // A node that was removed from the trash is gone from it, whatever the tombstone still holds. + if !flag(&found, "deleted")? || found.try_get::, _>("purge_requested_ms")?.is_some() { + return Err(MetaError::NotFound("node in the trash")); + } + let is_dir = flag(&found, "is_dir")?; + let device = sqlx::query("SELECT name FROM devices WHERE id = $1") + .bind(uuid_bytes(self.device.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .map(|row| row.try_get::("name")) + .transpose()? + .unwrap_or_default(); + let up = self.ancestors(node).await?; + let deleted_run: Vec<&Ancestor> = up.iter().take_while(|a| a.deleted).collect(); + let Some(base) = up.get(deleted_run.len()) else { + // The walk ended inside the deleted run: the depth bound (too deep) or a deleted root (damaged data). + return if up.last().is_some_and(|a| a.parent.is_some()) { + Err(rejected(ConflictKind::TooDeep)) + } else { + Err(MetaError::Corrupt("the root is deleted")) + }; + }; + let mut parent = base.id; + for (index, a) in deleted_run.iter().enumerate().rev() { + let own = Name::new(&a.name).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let taken = self.live_child(parent, &own).await?; + let is_target = index == 0; + if !is_target && let Some(existing) = taken { + // A live folder of that name stands in for the deleted one; a file there means this folder is renamed. + if self.node(existing).await?.is_some_and(|n| n.is_dir) { + parent = existing; + continue; + } + } + let final_name = match taken { + None => own, + Some(existing) => { + let folder = !is_target || is_dir; + match self.free_conflict_name(parent, &own, folder, &device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + } + } + }; + // A folder comes back where it did not stand before (below a live folder that has moved, or merged into + // another): it must not take the tree past its depth limit. A file may sit in the deepest folder. + if (!is_target || is_dir) + && let Err(kind) = self.check_placement(parent, None).await? + { + return Err(rejected(kind)); + } + self.revive(a.id, parent, &final_name).await?; + parent = a.id; + } + // A file comes back as a new version with the content of its last one. A machine that sent the delete and + // never heard the answer still names the old version in it, and so can't delete the restored file again: the + // server refuses that delete as a version mismatch, and the machine's engine already treats a node that came + // back at another version as one whose delete went through. + let version = if is_dir { None } else { Some(self.renew_version(node).await?) }; + if is_dir { + self.restore_below(node, tree_cap, &device).await?; + } + let placed = self.node(node).await?.ok_or(MetaError::Corrupt("a restored node vanished"))?; + let parent = placed.parent.ok_or(MetaError::Corrupt("a restored node has no parent"))?; + let original = up.first().map(|a| a.name.as_str()).unwrap_or_default(); + Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, version, seq: 0, epoch: self.epoch }) + } + + /// Brings back what is in the trash below the folder `top`, which is live again: its deleted children and theirs, + /// whenever they were deleted, at most `cap` of them, shallowest first (then newest deletion first, then by ID), so a + /// node never comes back without its folder. The rest stays in the trash; its folder is live or still in the trash + /// under a live one, so it is listed on its own and a further restore brings it back. A child keeps its name; one + /// whose name another child that came back took (a name deleted twice, the newest deletion first) gets a conflict + /// name. A removed node, and what is only below a removed folder, stays deleted. The work done row by row under the + /// space's write lock is bounded by `cap`. + async fn restore_below(&mut self, top: NodeId, cap: u64, device: &str) -> Result<(), MetaError> { + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let limit = i64::try_from(cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} SELECT n.id, n.parent, n.name, n.is_dir FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id \ + ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $4" + ))) + .bind(self.space()) + .bind(uuid_bytes(top.as_uuid())) + .bind(bound) + .bind(limit) + .fetch_all(&mut *self.conn) + .await?; + for row in &rows { + let id = NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?); + let parent = NodeId::from_uuid(uuid_from(&row.try_get::, _>("parent")?)?); + let is_dir = flag(row, "is_dir")?; + let own = Name::new(&row.try_get::("name")?).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let name = match self.live_child(parent, &own).await? { + None => own, + Some(existing) => match self.free_conflict_name(parent, &own, is_dir, device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + }, + }; + if is_dir && let Err(kind) = self.check_placement(parent, None).await? { + return Err(rejected(kind)); + } + self.revive(id, parent, &name).await?; + if !is_dir { + self.renew_version(id).await?; + } + } + Ok(()) + } + + /// Gives the file `node` a new current version with the manifest, modification time and executable bit of its last + /// one (no save record), after it. Returns the new version's ID. + async fn renew_version(&mut self, node: NodeId) -> Result { + let row = sqlx::query( + "SELECT n.version AS last, v.manifest, v.mtime_ms, v.exec FROM nodes n \ + JOIN versions v ON v.space_id = n.space_id AND v.id = n.version WHERE n.space_id = $1 AND n.id = $2", + ) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .ok_or(MetaError::Corrupt("a file has no current version"))?; + let last = VersionId::from_uuid(uuid_from(&row.try_get::, _>("last")?)?); + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&row.try_get::, _>("manifest")?)?), + mtime_ms: row.try_get("mtime_ms")?, + exec: flag(&row, "exec")?, + save: None, + }; + match self.insert_version(node, Some(last), &new).await? { + Ok(()) => {} + // The space no longer holds the manifest: the content is not there to bring back. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(rejected(kind)), + } + sqlx::query("UPDATE nodes SET version = $1 WHERE space_id = $2 AND id = $3") + .bind(uuid_bytes(new.id.as_uuid())) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(new.id) + } + /// `node` and its ancestors, nearest first; stops after `MAX_TREE_DEPTH + 2` rows, so a walk that did not /// reach the root (last parent is `Some`) is too deep. async fn ancestors(&mut self, node: NodeId) -> Result, MetaError> { @@ -401,6 +652,7 @@ impl Meta { let mut w = self.begin_write().await?; // On Postgres this row lock makes commits to one space run one at a time. let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + self.check_guard(&mut w.tx, space).await?; let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; let mut conflicts = Vec::new(); @@ -417,6 +669,86 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } + /// Brings the node `node` of the trash back in `space`, as one change of the space (one sequence number, all or + /// nothing): see `Ctx::restore_from_trash` for the rules. A folder brings back what is in the trash below it too. `NotFound` for a node that is not in the trash (live, + /// unknown, another space's, or removed from the trash); `Rejected` (`NameTaken`) when no conflict name is free. + pub async fn restore_from_trash(&self, space: SpaceId, device: DeviceId, node: NodeId, now_ms: i64) -> Result { + let mut w = self.begin_write().await?; + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + self.check_guard(&mut w.tx, space).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let outcome = ctx.restore_from_trash(node, self.trash_tree_cap).await?; + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(TrashRestoreOutcome { seq, ..outcome }) + } + + /// Makes `source`, an older version of the file `node`, its content again as a new version: an ordinary + /// `Update` after `expected`, which must still be the current version. The new version has a fresh ID, copies the + /// manifest, modification time and executable bit of `source`, and gets no save record (a save record names a + /// state of a live document, and a restore changes the file only). The epoch stays. Everything is checked under the + /// space's write lock, in the transaction that writes the version. + /// + /// `NotFound` for a node or source version the space does not have (a source of another file included) or whose + /// manifest the space no longer holds. `Invalid` for a directory, a deleted file, or a file that has a live + /// document. `Rejected` (a `VersionMismatch` for operation 0) when `expected` is no longer current. + pub async fn restore_version( + &self, + space: SpaceId, + device: DeviceId, + node: NodeId, + source: VersionId, + expected: VersionId, + now_ms: i64, + ) -> Result { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the restore with every other commit and save of the space. + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + self.check_guard(&mut w.tx, space).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let row = ctx.node(node).await?.ok_or(MetaError::NotFound("node"))?; + if row.is_dir { + return Err(MetaError::Invalid("only files have versions to restore")); + } + if row.deleted { + return Err(MetaError::Invalid("the file is deleted: bring it back with POST /trash/{id}/restore first, then restore a version")); + } + let live = sqlx::query("SELECT 1 FROM live_docs WHERE space_id = $1 AND node = $2 LIMIT 1") + .bind(ctx.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await?; + if live.is_some() { + return Err(MetaError::Invalid("this file has a live document: restoring a version applies to plain file versions only")); + } + let found = sqlx::query("SELECT manifest, mtime_ms, exec FROM versions WHERE space_id = $1 AND id = $2 AND node = $3") + .bind(ctx.space()) + .bind(uuid_bytes(source.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await? + .ok_or(MetaError::NotFound("version"))?; + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&found.try_get::, _>("manifest")?)?), + mtime_ms: found.try_get("mtime_ms")?, + exec: flag(&found, "exec")?, + save: None, + }; + let version = new.id; + match ctx.apply(&Op::Update { node, expected, version: new }).await? { + Ok(()) => {} + // The space holds no manifest of that ID any more: the data is not there to restore. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(MetaError::Rejected(vec![Conflict { op: 0, kind }])), + } + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(RestoreOutcome { version, parent_version: expected, seq, epoch: info.epoch }) + } + /// Adds a save record to `node`'s version `version`, which went up without one (the app's record reached the /// agent only after the upload: it was saved while the agent was away). Appended only: a version that has a /// record keeps it, and the same call again changes nothing. When it is added, the node is in the change feed @@ -431,6 +763,7 @@ impl Meta { let json = serde_json::to_string(save).map_err(|_| MetaError::Invalid("a save record that can't be written"))?; let mut w = self.begin_write().await?; let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + self.check_guard(&mut w.tx, space).await?; let found = sqlx::query("SELECT save FROM versions WHERE space_id = $1 AND id = $2 AND node = $3") .bind(uuid_bytes(space.as_uuid())) .bind(uuid_bytes(version.as_uuid())) @@ -459,3 +792,25 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_millisecond_count_reads_as_a_utc_minute() { + let at = |ms: i64| { + let s = conflict_stamp(ms); + (s.year, s.month, s.day, s.hour, s.minute, s.utc) + }; + assert_eq!(at(0), (1970, 1, 1, 0, 0, true)); + assert_eq!(at(1_760_000_000_000), (2025, 10, 9, 8, 53, true)); + // A leap day, and the last minute of a year. + assert_eq!(at(1_709_164_800_000 + 59 * 60_000), (2024, 2, 29, 0, 59, true)); + assert_eq!(at(1_767_225_540_000), (2025, 12, 31, 23, 59, true)); + // Before the epoch and absurd values do not panic. + assert_eq!(at(-1).0, 1969); + let _ = conflict_stamp(i64::MAX); + let _ = conflict_stamp(i64::MIN); + } +} diff --git a/crates/meta/src/history.rs b/crates/meta/src/history.rs new file mode 100644 index 00000000..2c0dd98b --- /dev/null +++ b/crates/meta/src/history.rs @@ -0,0 +1,82 @@ +//! A file's version history: the versions of one file, newest first, a page at a time. +//! +//! Versions are never removed, so every row of `versions` for the node is reported. The `pruned` flag of the answer is +//! part of the contract for the day versions can be thinned; until then it is always false. + +use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; +use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; +use sqlx::Row; + +use crate::codec::{flag, to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::{Meta, MetaError}; + +/// One page of a file's history. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HistoryPage { + pub versions: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, +} + +impl Meta { + /// One page of the versions of the file `node` in `space`, ordered by `(seq, id)` descending. `after` is the + /// previous page's `next` (exclusive). `NotFound` when the space has no such node; `Invalid` for a directory or + /// a `limit` outside 1 to [`HISTORY_MAX_LIMIT`]. A deleted file still has a history. Each call reads one snapshot. + pub async fn history(&self, space: SpaceId, node: NodeId, after: Option, limit: u32) -> Result { + if limit == 0 || limit > HISTORY_MAX_LIMIT { + return Err(MetaError::Invalid("a history page has 1 to 200 versions")); + } + let mut tx = self.read_snapshot().await?; + let found = sqlx::query("SELECT is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("node"))?; + if flag(&found, "is_dir")? { + return Err(MetaError::Invalid("only files have a history")); + } + let (cursor_seq, cursor_version) = match after { + Some(cursor) => (to_i64(cursor.seq)?, uuid_bytes(cursor.version.as_uuid())), + // Above every sequence number and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The cursor is exclusive. The row-value + // comparison and the order match the index `versions_node_seq`, so the database seeks to the cursor and stops + // after the page. + let rows = sqlx::query( + "SELECT id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, \ + CAST(CASE WHEN save IS NULL THEN 0 ELSE 1 END AS BIGINT) AS has_save \ + FROM versions WHERE space_id = $1 AND node = $2 AND (seq, id) < ($3, $4) \ + ORDER BY seq DESC, id DESC LIMIT $5", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(cursor_seq) + .bind(cursor_version) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut versions = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent_version")?; + versions.push(HistoryVersion { + id: VersionId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent_version: parent.as_deref().map(uuid_from).transpose()?.map(VersionId::from_uuid), + epoch: Epoch::from_uuid(uuid_from(&row.try_get::, _>("epoch")?)?), + seq: to_u64(row.try_get("seq")?)?, + size: to_u64(row.try_get("size")?)?, + mtime_ms: row.try_get("mtime_ms")?, + device: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("device")?)?), + created_ms: row.try_get("created_ms")?, + save: flag(row, "has_save")?, + pruned: false, + }); + } + let next = if more { versions.last().map(|v| HistoryCursor { seq: v.seq, version: v.id }) } else { None }; + Ok(HistoryPage { versions, next }) + } +} diff --git a/crates/meta/src/index.rs b/crates/meta/src/index.rs new file mode 100644 index 00000000..8d8bcf95 --- /dev/null +++ b/crates/meta/src/index.rs @@ -0,0 +1,640 @@ +//! The reference index: which chunks each manifest names, kept in the database next to the manifests themselves. +//! +//! A manifest is a file's content: a list of chunks. The list lives in the manifest's object in the store, which is +//! the authority; this index is a copy of it that the database can query ("which chunks does this manifest need", +//! "which manifests use this chunk"). Because it is a copy it is only trusted when it is *complete*: every manifest has +//! a row in `manifest_index`, and its state is 1 only after the whole list of edges was compared, ordinal by ordinal, +//! with the manifest it was made from. A row count is never enough. +//! +//! The index is kept in the transaction that records a manifest (`Meta::add_manifest`): up to a budget of rows per +//! transaction. A longer manifest is recorded as being built, and the builder ([`Meta::index_complete`]) writes the +//! rest in further transactions of at most that many rows, so no one holds the write lock for a million rows. The same +//! builder indexes manifests that existed before the index did; it resumes where an interrupted run stopped. +//! +//! When the edges of a manifest do not match it, they are thrown away and written again, in batches of the same size +//! (state 2, "resetting", marks it): an index of a million edges is never deleted in one transaction, and a run that stops +//! between two batches is resumed by the next stage. Rows are compared from the lowest ordinal there is, so an edge at a +//! negative ordinal is found too. +//! +//! Nothing here frees anything. The index is read by the checker (`verify`) and by whatever later decides what is +//! garbage; until it says a manifest is complete, that manifest's chunks are unknown, and unknown means kept. + +use std::collections::HashMap; + +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use sqlx::Row; + +use crate::blobs::{KindCode, placeholders}; +use crate::codec::{hash_from, to_i64, to_u64}; +use crate::{Dialect, Meta, MetaError}; + +/// Edges one transaction writes by default (and how many rows a manifest may have to be indexed whole in the +/// transaction that records it). +pub const INDEX_BATCH_ROWS: usize = 5_000; + +/// Edges per `INSERT` statement. +const ROWS_PER_STATEMENT: usize = 500; + +/// How often a builder starts over for one manifest whose edges do not match it before giving up. +const REBUILDS: usize = 2; + +const BUILDING: i64 = 0; +const COMPLETE: i64 = 1; +/// The edges are being thrown away to start the index again, a batch of rows to a transaction. Nothing is written +/// and nothing is marked complete until the last edge is gone and the state is back to [`BUILDING`]. A run that stops +/// halfway leaves this marker, and the next one carries on from the rows that are left. +const RESETTING: i64 = 2; + +/// What the database records of a blob: its ID, the bytes it is stored as (the format byte included) and, for a +/// manifest, the size of its file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobRecord { + pub id: [u8; 32], + pub stored_size: u64, + pub file_size: Option, +} + +/// How far a manifest's index has come. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexState { + /// No row: the manifest was recorded before the index existed and nothing has indexed it yet. + Absent, + /// Some edges are written (`done` of `total`), or all of them but not yet compared with the manifest. + Building { done: u64, total: u64 }, + /// Every edge was compared with the manifest. + Complete { total: u64 }, +} + +/// What a comparison of a manifest's stored edges with the manifest found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeCheck { + /// The edges are exactly the manifest's chunks, in order, nothing more. + Exact, + /// The first place they differ. + Mismatch(EdgeProblem), +} + +/// Where and how stored edges differ from their manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EdgeProblem { + pub ordinal: u64, + pub kind: EdgeProblemKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeProblemKind { + /// The edge at the ordinal names another chunk than the manifest does there. + WrongChunk, + /// The manifest has a chunk at the ordinal that has no edge. + MissingEdge, + /// There is an edge at an ordinal the manifest has no chunk for. + ExtraEdge, + /// There is an edge at a negative ordinal, which no manifest has. The ordinal of the problem is its absolute value. + NegativeOrdinal, +} + +impl EdgeProblemKind { + pub fn as_str(self) -> &'static str { + match self { + EdgeProblemKind::WrongChunk => "wrong_chunk", + EdgeProblemKind::MissingEdge => "missing_edge", + EdgeProblemKind::ExtraEdge => "extra_edge", + EdgeProblemKind::NegativeOrdinal => "negative_ordinal", + } + } +} + +/// The result of checking a manifest's edges and, when they are exact, marking its index complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Validation { + Complete, + Mismatch(EdgeProblem), + /// Another builder changed the index meanwhile; look again. + Raced, +} + +/// What the database alone says about the index (the checker adds what only the store can tell). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IndexAudit { + /// Recorded chunks (`blobs` of kind chunk). + pub chunks: u64, + /// Stored bytes of chunks no manifest index edge names. + pub unreferenced_chunk_bytes: u64, + /// Manifests recorded (`blobs` of kind manifest), and how many of them are in each state of the index. + pub manifests: u64, + pub complete: u64, + pub building: u64, + pub absent: u64, + /// Rows of `manifest_index` whose manifest is not recorded. + pub index_without_manifest: Finding, + /// Manifests that have edges but no `manifest_index` row. + pub edges_without_index: Finding, + /// Complete manifests whose number of edges is not the number of chunks they were indexed with. + pub complete_with_wrong_count: Finding, + /// Edges that name a chunk that is not recorded: (manifest, chunk). + pub unknown_dependencies: Finding, + /// Versions and snapshots that name a manifest that is not recorded. + pub roots_without_manifest: Finding, + /// Manifests named by a version or snapshot whose index is not complete. + pub roots_not_indexed: Finding, + /// Recorded manifests that no version, snapshot or import snapshot names. These may be live-document assets, whose + /// references sit inside the documents (the server cannot read them); they are reported, never judged. + pub unrooted_manifests: Finding, + /// Recorded chunks no complete manifest names (only meaningful once every manifest is complete). + pub unreferenced_chunks: Finding, + /// Space memberships of a blob that is not recorded. + pub space_blobs_without_blob: Finding, +} + +/// A count and a few examples (hex IDs, `manifest` or `manifest/chunk`). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Finding { + pub count: u64, + pub examples: Vec, +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A row of `manifest_index`. +struct IndexRow { + chunk_count: i64, + indexed: i64, + state: i64, +} + +async fn read_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, for_update: bool) -> Result, MetaError> { + let sql = if for_update { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1 FOR UPDATE" + } else { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1" + }; + let row = sqlx::query(sql).bind(id.as_bytes().to_vec()).fetch_optional(&mut *conn).await?; + row.map(|r| Ok(IndexRow { chunk_count: r.try_get("chunk_count")?, indexed: r.try_get("indexed_ordinals")?, state: r.try_get("state")? })) + .transpose() +} + +/// The row of a manifest, made (as being built, nothing written) if there is none. On Postgres the row stays locked until the +/// transaction ends, so two builders of one manifest take turns. +async fn ensure_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64, for_update: bool) -> Result { + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, $2, 0, 0) ON CONFLICT DO NOTHING") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + read_state(conn, id, for_update).await?.ok_or(MetaError::Corrupt("an index row vanished")) +} + +/// Marks a manifest's index as starting again: the row takes the manifest's chunk count, writes nothing, and is not +/// complete (the edges are derived: the manifest is the authority). The edges themselves go in [`reset_step`]. +async fn begin_reset(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result<(), MetaError> { + sqlx::query("UPDATE manifest_index SET chunk_count = $1, indexed_ordinals = 0, state = $2 WHERE manifest = $3") + .bind(total) + .bind(RESETTING) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Throws away at most `budget` of the edges of a manifest whose index starts again, the lowest ordinals first (a +/// negative one included), so no transaction holds the write lock for the up to a million rows a manifest can have. +/// Returns whether none are left: then the row is building again. Fewer rows deleted than the budget means none are. +async fn reset_step(conn: &mut sqlx::AnyConnection, id: &ManifestId, budget: usize) -> Result { + let limit = i64::try_from(budget.max(1)).map_err(|_| MetaError::Invalid("batch too large"))?; + let deleted = sqlx::query( + "DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal IN (SELECT ordinal FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal LIMIT $2)", + ) + .bind(id.as_bytes().to_vec()) + .bind(limit) + .execute(&mut *conn) + .await? + .rows_affected(); + let finished = i64::try_from(deleted).is_ok_and(|n| n < limit); + if finished { + sqlx::query("UPDATE manifest_index SET state = $1 WHERE manifest = $2 AND state = $3") + .bind(BUILDING) + .bind(id.as_bytes().to_vec()) + .bind(RESETTING) + .execute(&mut *conn) + .await?; + } + Ok(finished) +} + +/// Writes the edges for the ordinals `from..to` of `manifest`, 500 to a statement. Edges already there are left alone +/// (the comparison catches one that is wrong). +async fn write_edges(conn: &mut sqlx::AnyConnection, manifest: &Manifest, from: usize, to: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let chunks = manifest.chunks().get(from..to).ok_or(MetaError::Corrupt("an index range outside its manifest"))?; + for (part_index, part) in chunks.chunks(ROWS_PER_STATEMENT).enumerate() { + let rows = (0..part.len()).map(|i| format!("({})", placeholders(i * 3 + 1, 3))).collect::>().join(", "); + let mut query = + sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES {rows} ON CONFLICT DO NOTHING"))); + for (i, chunk) in part.iter().enumerate() { + let ordinal = from + .checked_add(part_index.saturating_mul(ROWS_PER_STATEMENT)) + .and_then(|n| n.checked_add(i)) + .ok_or(MetaError::Corrupt("ordinal overflow"))?; + query = query.bind(id.as_bytes().to_vec()).bind(to_i64(ordinal as u64)?).bind(chunk.id.as_bytes().to_vec()); + } + query.execute(&mut *conn).await?; + } + Ok(()) +} + +/// Compares the stored edges of `manifest`, a page of `page` rows at a time, with its chunks. +async fn compare(conn: &mut sqlx::AnyConnection, manifest: &Manifest, page: usize) -> Result { + let id = manifest.id(); + let chunks = manifest.chunks(); + let limit = i64::try_from(page.max(1)).map_err(|_| MetaError::Invalid("page too large"))?; + // Pagination starts at the lowest number there is, so an edge at a negative ordinal is not skipped. + let (mut next, mut from): (usize, i64) = (0, i64::MIN); + loop { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 AND ordinal >= $2 ORDER BY ordinal LIMIT $3") + .bind(id.as_bytes().to_vec()) + .bind(from) + .bind(limit) + .fetch_all(&mut *conn) + .await?; + for row in &rows { + let raw: i64 = row.try_get("ordinal")?; + if raw < 0 { + // Rows come lowest first, so a negative ordinal is the first thing seen: no manifest has such an edge. + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: raw.unsigned_abs(), kind: EdgeProblemKind::NegativeOrdinal })); + } + let ordinal = to_u64(raw)?; + let chunk = hash_from(&row.try_get::, _>("chunk")?)?; + let expected = next as u64; + if ordinal > expected { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: expected, kind: EdgeProblemKind::MissingEdge })); + } + match chunks.get(next) { + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + Some(c) if *c.id.as_bytes() != chunk => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::WrongChunk })), + Some(_) => {} + } + next += 1; + from = match raw.checked_add(1) { + Some(n) => n, + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + }; + } + if rows.len() < page.max(1) { + break; + } + } + if next < chunks.len() { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: next as u64, kind: EdgeProblemKind::MissingEdge })); + } + Ok(EdgeCheck::Exact) +} + +/// Marks a manifest's index complete if it is still as it was when its edges were compared (`indexed_ordinals` and +/// `chunk_count` both `total`, state still 0). Returns whether it did. +async fn mark_complete(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result { + let done = sqlx::query("UPDATE manifest_index SET state = 1 WHERE manifest = $1 AND state = 0 AND chunk_count = $2 AND indexed_ordinals = $2") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + Ok(done.rows_affected() == 1) +} + +/// Writes the next batch of edges of `manifest` (at most `budget`) and moves the frontier. Returns the state after it, +/// and whether the index is still being thrown away to start again (then this step only deleted edges, at most +/// `budget` of them, and `done` says nothing about edges being written). +pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(IndexState, bool), MetaError> { + let id = manifest.id(); + let total = manifest.chunks().len(); + let total_i = to_i64(total as u64)?; + let row = ensure_state(conn, &id, total_i, pg).await?; + // A row that does not belong to this manifest's list starts again: the manifest is the authority. + let foreign = row.chunk_count != total_i || row.indexed < 0 || row.indexed > total_i; + if foreign { + begin_reset(conn, &id, total_i).await?; + } else if row.state == COMPLETE { + return Ok((IndexState::Complete { total: total as u64 }, false)); + } + if foreign || row.state == RESETTING { + let finished = reset_step(conn, &id, budget).await?; + return Ok((IndexState::Building { done: 0, total: total as u64 }, !finished)); + } + let start = usize::try_from(row.indexed).map_err(|_| MetaError::Corrupt("a negative index frontier"))?; + let end = start.saturating_add(budget.max(1)).min(total); + if end > start { + write_edges(conn, manifest, start, end).await?; + sqlx::query("UPDATE manifest_index SET indexed_ordinals = $1 WHERE manifest = $2 AND indexed_ordinals < $1") + .bind(to_i64(end as u64)?) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + } + Ok((IndexState::Building { done: end as u64, total: total as u64 }, false)) +} + +/// Called in the transaction that records a manifest: makes its index row and writes up to a budget of edges, and when +/// that is all of them compares them with the manifest and marks the index complete. A longer manifest is left being +/// built for [`Meta::index_complete`]. A row that already exists is not touched beyond that. +pub(crate) async fn register(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + if let (IndexState::Building { done, total: n }, false) = stage(conn, pg, manifest, budget).await? + && done >= n + && compare(conn, manifest, budget).await? == EdgeCheck::Exact + { + mark_complete(conn, &id, total).await?; + } + Ok(()) +} + +impl Meta { + /// The store with another budget of edges per index transaction (the default is [`INDEX_BATCH_ROWS`]). For tests. + pub fn with_index_batch_rows(mut self, rows: usize) -> Self { + self.index_rows = rows.max(1); + self + } + + /// Writes the next batch of `manifest`'s edges, at most the index budget, in one transaction, and says how far the + /// index has come. Call it again until it says all edges are written, then [`Meta::index_validate`]; or call + /// [`Meta::index_complete`], which does both. Each call is safe to repeat and to run beside commits and other builders. + pub async fn index_stage(&self, manifest: &Manifest) -> Result { + let mut w = self.begin_write().await?; + let (state, _) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + Ok(state) + } + + /// Compares every edge of `manifest` with it, in a read of one snapshot, and if they are exact marks its index + /// complete (provided every edge was written and nothing changed meanwhile). Nothing else marks an index complete. + pub async fn index_validate(&self, manifest: &Manifest) -> Result { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + let check = { + let mut tx = self.read_snapshot().await?; + let state = read_state(&mut tx, &id, false).await?; + let ready = state.is_some_and(|s| s.chunk_count == total && s.indexed == total && s.state != RESETTING); + let check = if ready { Some(compare(&mut tx, manifest, self.index_rows).await?) } else { None }; + tx.commit().await?; + check + }; + match check { + None => Ok(Validation::Raced), + Some(EdgeCheck::Mismatch(problem)) => Ok(Validation::Mismatch(problem)), + Some(EdgeCheck::Exact) => { + let mut w = self.begin_write().await?; + let marked = mark_complete(&mut w.tx, &id, total).await?; + w.commit().await?; + // Already complete (another builder was first) is as good. + if marked || matches!(self.index_state(&id).await?, IndexState::Complete { .. }) { + Ok(Validation::Complete) + } else { + Ok(Validation::Raced) + } + } + } + } + + /// Indexes `manifest` to the end: edges in batches of at most the index budget, each its own transaction, then the + /// comparison that marks it complete. Resumes where an earlier run stopped. Edges that do not match the manifest + /// are thrown away and written again (the manifest is the authority), at most twice; then it is `Corrupt`. + pub async fn index_complete(&self, manifest: &Manifest) -> Result<(), MetaError> { + let id = manifest.id(); + let total = manifest.chunks().len() as u64; + let mut rebuilds = 0; + for _ in 0..(REBUILDS + 1) * 4 { + loop { + let mut w = self.begin_write().await?; + let (state, resetting) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + match state { + IndexState::Complete { .. } => return Ok(()), + // Still throwing edges away: nothing is written yet, whatever the frontier says. + IndexState::Building { .. } if resetting => {} + IndexState::Building { done, .. } if done >= total => break, + IndexState::Building { .. } | IndexState::Absent => {} + } + } + match self.index_validate(manifest).await? { + Validation::Complete => return Ok(()), + Validation::Raced => {} + Validation::Mismatch(_) => { + if rebuilds >= REBUILDS { + return Err(MetaError::Corrupt("a manifest's index does not match it after being rebuilt")); + } + rebuilds += 1; + let mut w = self.begin_write().await?; + // Only a row that is not complete is rebuilt here: a complete one that is wrong is the checker's to report. + let row = read_state(&mut w.tx, &id, self.dialect == Dialect::Postgres).await?; + // The edges go in batches of the next stages, each its own transaction, and a stop between them resumes. + if row.is_some_and(|r| r.state != COMPLETE) { + begin_reset(&mut w.tx, &id, to_i64(total)?).await?; + } + w.commit().await?; + } + } + } + Err(MetaError::Corrupt("a manifest's index could not be completed")) + } + + /// How far `manifest`'s index has come. + pub async fn index_state(&self, manifest: &ManifestId) -> Result { + let mut conn = self.pool.acquire().await?; + Ok(match read_state(&mut conn, manifest, false).await? { + None => IndexState::Absent, + Some(r) if r.state == COMPLETE => IndexState::Complete { total: to_u64(r.chunk_count)? }, + Some(r) => IndexState::Building { done: to_u64(r.indexed)?, total: to_u64(r.chunk_count)? }, + }) + } + + /// Compares the edges stored for `manifest` with it, without changing anything. + pub async fn index_check(&self, manifest: &Manifest) -> Result { + let mut tx = self.read_snapshot().await?; + let check = compare(&mut tx, manifest, self.index_rows).await?; + tx.commit().await?; + Ok(check) + } + + /// The stored edges of a manifest, in order: `(ordinal, chunk)`. For tests and the checker. + pub async fn index_edges(&self, manifest: &ManifestId) -> Result, MetaError> { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal") + .bind(manifest.as_bytes().to_vec()) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok((to_u64(r.try_get("ordinal")?)?, ChunkId::from_bytes(hash_from(&r.try_get::, _>("chunk")?)?)))).collect() + } + + /// How many edges the index holds in all. + pub async fn index_edge_count(&self) -> Result { + let n: i64 = sqlx::query("SELECT COUNT(*) AS n FROM manifest_chunks").fetch_one(&self.pool).await?.try_get("n")?; + to_u64(n) + } + + /// Up to `limit` recorded manifests after `after` (in ID order) whose index is not complete: the work of a builder. + pub async fn manifests_to_index(&self, after: Option, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |m| m.as_bytes().to_vec()); + let rows = sqlx::query( + "SELECT b.id FROM blobs b LEFT JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = $1 AND b.id > $2 AND (i.manifest IS NULL OR i.state <> 1) \ + ORDER BY b.id LIMIT $3", + ) + .bind(BlobKind::Manifest.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok(ManifestId::from_bytes(hash_from(&r.try_get::, _>("id")?)?))).collect() + } + + /// Up to `limit` recorded blobs of `kind` after `after`, in ID order, with what the database says of their sizes. + pub async fn blob_records_after(&self, kind: BlobKind, after: Option<[u8; 32]>, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |id| id.to_vec()); + let rows = sqlx::query("SELECT id, stored_size, file_size FROM blobs WHERE kind = $1 AND id > $2 ORDER BY id LIMIT $3") + .bind(kind.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|r| { + let file_size: Option = r.try_get("file_size")?; + Ok(BlobRecord { + id: hash_from(&r.try_get::, _>("id")?)?, + stored_size: to_u64(r.try_get("stored_size")?)?, + file_size: file_size.map(to_u64).transpose()?, + }) + }) + .collect() + } + + /// Of `ids`, the recorded chunks and how many bytes each is stored as (the format byte included). + pub async fn recorded_chunk_sizes(&self, ids: &[ChunkId]) -> Result, MetaError> { + let mut found = HashMap::new(); + for part in ids.chunks(ROWS_PER_STATEMENT) { + let sql = format!("SELECT id, stored_size FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, part.len())); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(BlobKind::Chunk.code()); + for id in part { + query = query.bind(id.as_bytes().to_vec()); + } + for row in query.fetch_all(&self.pool).await? { + found.insert(ChunkId::from_bytes(hash_from(&row.try_get::, _>("id")?)?), to_u64(row.try_get("stored_size")?)?); + } + } + Ok(found) + } + + /// The recorded blobs among the IDs of one bounded store-list page, with the stored size the database records for each. + pub async fn recorded_blob_sizes(&self, kind: BlobKind, ids: &[[u8; 32]]) -> Result, MetaError> { + let mut found = std::collections::HashMap::new(); + if !ids.is_empty() { + let sql = format!("SELECT id, stored_size FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, ids.len())); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(kind.code()); + for id in ids { + query = query.bind(id.to_vec()); + } + for row in query.fetch_all(&self.pool).await? { + let size = u64::try_from(row.try_get::("stored_size")?).unwrap_or(0); + found.insert(hash_from(&row.try_get::, _>("id")?)?, size); + } + } + Ok(found) + } + + /// What the database says about the index, its roots and the blobs it names, in a read of one snapshot, with up to + /// `examples` examples of each problem. Changes nothing. + pub async fn index_audit(&self, examples: usize) -> Result { + let mut tx = self.read_snapshot().await?; + let mut audit = IndexAudit::default(); + audit.manifests = scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs WHERE kind = 1").await?; + audit.chunks = scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs WHERE kind = 0").await?; + audit.unreferenced_chunk_bytes = scalar( + &mut tx, + "SELECT CAST(COALESCE(SUM(b.stored_size), 0) AS BIGINT) AS n FROM blobs b WHERE b.kind = 0 AND NOT EXISTS (SELECT 1 FROM manifest_chunks mc WHERE mc.chunk = b.id)", + ) + .await?; + audit.complete = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state = 1").await?; + audit.building = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state <> 1").await?; + audit.absent = audit.manifests.saturating_sub(audit.complete).saturating_sub(audit.building); + // Every manifest a version, a live document's snapshot or its import snapshot names. + const ROOTS: &str = "(SELECT manifest AS a FROM versions UNION SELECT snapshot AS a FROM live_docs WHERE snapshot IS NOT NULL UNION SELECT import_snapshot AS a FROM live_docs WHERE import_snapshot IS NOT NULL) r"; + audit.index_without_manifest = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = i.manifest)", + ) + .await?; + audit.edges_without_index = finding( + &mut tx, + examples, + &["mc.manifest"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = mc.manifest)", + ) + .await?; + audit.complete_with_wrong_count = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE i.state = 1 AND i.chunk_count <> (SELECT COUNT(*) FROM manifest_chunks mc WHERE mc.manifest = i.manifest)", + ) + .await?; + audit.unknown_dependencies = finding( + &mut tx, + examples, + &["mc.manifest", "mc.chunk"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 0 AND b.id = mc.chunk)", + ) + .await?; + audit.roots_without_manifest = + finding(&mut tx, examples, &["r.a"], &format!("{ROOTS} WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a)")) + .await?; + audit.roots_not_indexed = finding( + &mut tx, + examples, + &["r.a"], + &format!( + "{ROOTS} WHERE EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a) AND NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = r.a AND i.state = 1)" + ), + ) + .await?; + audit.unrooted_manifests = + finding(&mut tx, examples, &["b.id"], &format!("blobs b WHERE b.kind = 1 AND NOT EXISTS (SELECT 1 FROM {ROOTS} WHERE r.a = b.id)")) + .await?; + audit.unreferenced_chunks = + finding(&mut tx, examples, &["b.id"], "blobs b WHERE b.kind = 0 AND NOT EXISTS (SELECT 1 FROM manifest_chunks mc WHERE mc.chunk = b.id)") + .await?; + audit.space_blobs_without_blob = + finding(&mut tx, examples, &["sb.id"], "space_blobs sb WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = sb.kind AND b.id = sb.id)") + .await?; + tx.commit().await?; + Ok(audit) + } +} + +/// A count query's number. +async fn scalar(conn: &mut sqlx::AnyConnection, sql: &'static str) -> Result { + to_u64(sqlx::query(sql).fetch_one(&mut *conn).await?.try_get("n")?) +} + +/// How many distinct `columns` the rows of `from_where` have, and the first few (hex, joined by `/`). Only constant +/// text goes into the SQL. +async fn finding(conn: &mut sqlx::AnyConnection, examples: usize, columns: &[&str], from_where: &str) -> Result { + let select = columns.iter().enumerate().map(|(i, c)| format!("{c} AS c{i}")).collect::>().join(", "); + let counted = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT COUNT(*) AS n FROM (SELECT DISTINCT {select} FROM {from_where}) q"))) + .fetch_one(&mut *conn) + .await?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT DISTINCT {select} FROM {from_where} ORDER BY 1 LIMIT {examples}"))) + .fetch_all(&mut *conn) + .await?; + let mut shown = Vec::with_capacity(rows.len()); + for row in &rows { + let mut parts = Vec::with_capacity(columns.len()); + for i in 0..columns.len() { + parts.push(hex(&row.try_get::, _>(format!("c{i}").as_str())?)); + } + shown.push(parts.join("/")); + } + Ok(Finding { count: to_u64(counted.try_get("n")?)?, examples: shown }) +} diff --git a/crates/meta/src/join.rs b/crates/meta/src/join.rs index 4a42a837..e5b6ca6a 100644 --- a/crates/meta/src/join.rs +++ b/crates/meta/src/join.rs @@ -7,8 +7,19 @@ //! Of two redeems of one code at the same moment, the second finds nothing ([`MetaError::InvalidCode`]): SQLite runs //! them one after the other, and on Postgres the second statement waits for the first's row lock and then sees the //! row gone or used. +//! - An invite row has a purpose: `invite` (a new computer for the account: the command line's `user invite` and the +//! admin API's invites) or `reset` (a password reset code made through the admin API, good for an hour). Redeeming +//! a reset code always sets a new password, signs out every device of the account and deletes its pairings, and +//! then, as every redeem does, creates the redeeming computer's device. +//! - A disabled account, or one marked for removal, redeems nothing ([`MetaError::AccountDisabled`]). That is decided +//! after the account's row is locked, so a disable that commits meanwhile is not followed by a device. //! - A pairing works only while the device that made it is signed in: revoking that device deletes it, and a redeem //! checks it too. +//! - An account with two-factor sign-in needs its second factor to redeem an invite or a reset code ([`Proof`]): the +//! factor is checked and used up in the same transaction as the invite and the device, a wrong one leaves the invite +//! as it was (its counters are committed on their own), and none is [`MetaError::TwoFactorRequired`] without using the +//! code up. A reset code the admin made with `reset_two_factor` takes the second factor away instead of asking for it. +//! A pairing needs none: only a signed-in device can make one. //! - Expired rows are deleted on the next write to their table (a new invite or a new pairing). use craft_cloud_proto::api::PairingState; @@ -17,9 +28,61 @@ use craft_cloud_proto::{DeviceId, NodeId, SpaceId}; use sqlx::Row; use uuid::Uuid; +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; use crate::codec::{uuid_bytes, uuid_from}; -use crate::users::{check_device_name, insert_device, insert_user}; -use crate::{Meta, MetaError, Role, UserId, valid_username}; +use crate::two_factor::{self, Proof}; +use crate::users::{DeviceClient, account_blocked, check_device_name, delete_join_codes, insert_device, insert_user, lock_user, revoke_devices}; +use crate::{Meta, MetaError, RevokeReason, Role, UserId, valid_username}; + +/// What a row of `invites` is for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Purpose { + /// Adds a computer to the account (and, for an account without a password, sets it). + Invite, + /// Sets a new password and signs the account's devices out. + Reset, +} + +impl Purpose { + pub(crate) fn as_str(self) -> &'static str { + match self { + Purpose::Invite => "invite", + Purpose::Reset => "reset", + } + } + + fn parse(s: &str) -> Result { + match s { + "invite" => Ok(Purpose::Invite), + "reset" => Ok(Purpose::Reset), + _ => Err(MetaError::Corrupt("unknown invite purpose")), + } + } +} + +/// Replaces `user`'s open invite or reset code with one under `code_hash`, in `tx`, after deleting every expired one. +/// `reset_two_factor`: redeeming a reset code also turns the account's two-factor sign-in off. +pub(crate) async fn put_code( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + code_hash: &[u8; 32], + purpose: Purpose, + reset_two_factor: bool, + now_ms: i64, + expires_ms: i64, +) -> Result<(), MetaError> { + sqlx::query("DELETE FROM invites WHERE expires_ms <= $1").bind(now_ms).execute(&mut **tx).await?; + sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("INSERT INTO invites (user_id, code_hash, expires_ms, purpose, reset_two_factor) VALUES ($1, $2, $3, $4, $5)") + .bind(uuid_bytes(&user.0)) + .bind(code_hash.to_vec()) + .bind(expires_ms) + .bind(purpose.as_str()) + .bind(i64::from(reset_two_factor)) + .execute(&mut **tx) + .await?; + Ok(()) +} /// The account an invite is for, from [`Meta::create_invite`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -31,10 +94,12 @@ pub struct InviteFor { } /// What [`Meta::redeem`] created: a device of `user`'s. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Redeemed { pub user: UserId, pub device: DeviceId, + /// The devices a reset code signed out (empty for every other code), for the caller to end their live connections. + pub revoked: Vec, } impl Meta { @@ -56,13 +121,8 @@ impl Meta { (insert_user(&mut w.tx, &username, "", Role::Member, now_ms, ids).await?.id, true) } }; - sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; - sqlx::query("INSERT INTO invites (user_id, code_hash, expires_ms) VALUES ($1, $2, $3)") - .bind(uuid_bytes(&user.0)) - .bind(code_hash.to_vec()) - .bind(expires_ms) - .execute(&mut *w.tx) - .await?; + // An open reset code of the account is replaced as well: the command line's invite is an invite. + put_code(&mut w.tx, user, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; w.commit().await?; Ok(InviteFor { user, new_account }) } @@ -86,6 +146,11 @@ impl Meta { let id = Uuid::now_v7(); let expires_ms = now_ms.checked_add(PAIRING_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; let mut w = self.begin_write().await?; + // Users first, as a sign-out does: a pairing is never made for a device a sign-out just revoked. + lock_user(&mut w.tx, user).await?; + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } sqlx::query("DELETE FROM pairings WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; let done = sqlx::query( @@ -139,23 +204,30 @@ impl Meta { w.commit().await } - /// Whether the invite under `code_hash` needs a new password (its account has none): `None` when no unexpired - /// invite has that hash (it may be a pairing's, or wrong). Lets the server answer `password_required` before it - /// spends time hashing a password. + /// Whether the invite under `code_hash` needs a new password (its account has none, or it is a reset code, which + /// always does): `None` when no unexpired invite has that hash (it may be a pairing's, or wrong). Lets the server + /// answer `password_required` before it spends time hashing a password. pub async fn invite_needs_password(&self, code_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { - let row = sqlx::query("SELECT u.password_hash FROM invites i JOIN users u ON u.id = i.user_id WHERE i.code_hash = $1 AND i.expires_ms > $2") - .bind(code_hash.to_vec()) - .bind(now_ms) - .fetch_optional(&self.pool) - .await?; - row.map(|row| row.try_get::("password_hash").map(|hash| hash.is_empty())).transpose().map_err(MetaError::from) + let row = sqlx::query( + "SELECT u.password_hash, i.purpose FROM invites i JOIN users u ON u.id = i.user_id WHERE i.code_hash = $1 AND i.expires_ms > $2", + ) + .bind(code_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + let purpose = Purpose::parse(&row.try_get::("purpose")?)?; + Ok(Some(purpose == Purpose::Reset || row.try_get::("password_hash")?.is_empty())) } /// Redeems the code under `code_hash` for a new device named `device_name` with `token_hash`, in one transaction: /// an unexpired invite (its row deleted) or an unused, unexpired pairing whose maker is still signed in (marked /// used by the new device). An invite to an account without a password needs `new_password_hash`, which becomes the /// account's ([`MetaError::PasswordRequired`] without it, and nothing changes); for any other code it is ignored. - /// Anything else is [`MetaError::InvalidCode`]. + /// A reset code always needs it: it becomes the account's password whatever the old one was, every device of the + /// account is signed out first ([`Redeemed::revoked`]) and its pairings are deleted, so the new device is the only + /// one signed in. A disabled account, or one marked for removal, is [`MetaError::AccountDisabled`] and nothing + /// changes. Anything else is [`MetaError::InvalidCode`]. pub async fn redeem( &self, code_hash: &[u8; 32], @@ -163,48 +235,180 @@ impl Meta { token_hash: &[u8; 32], new_password_hash: Option<&str>, now_ms: i64, + ) -> Result { + self.redeem_with(code_hash, device_name, &DeviceClient::default(), token_hash, new_password_hash, now_ms).await + } + + /// [`Meta::redeem`] for a device that says what client it is. An account with two-factor sign-in is + /// [`MetaError::TwoFactorRequired`] here (see [`Meta::redeem_with_second_factor`]). + pub async fn redeem_with( + &self, + code_hash: &[u8; 32], + device_name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + new_password_hash: Option<&str>, + now_ms: i64, + ) -> Result { + self.redeem_with_second_factor(code_hash, device_name, client, token_hash, new_password_hash, None, now_ms).await + } + + /// [`Meta::redeem_with`] with the second factor the person gave, for an invite or reset code of an account that has + /// two-factor sign-in. Under the account's lock, after the invite is found again and consumed, the account's + /// **current** state decides: with two-factor sign-in on, a reset code made with `reset_two_factor` takes it away and + /// asks for nothing; any other code needs `second_factor`. None is [`MetaError::TwoFactorRequired`]; a wrong one + /// [`MetaError::TwoFactorInvalid`], with the invite still there and the wrong-proof counters committed on their own; + /// a lock in force [`MetaError::TwoFactorLocked`]. A right one is used up in this transaction, with the invite and the + /// device. A pairing code never needs one, and a second factor given for an account that has none is ignored. + #[allow(clippy::too_many_arguments)] + pub async fn redeem_with_second_factor( + &self, + code_hash: &[u8; 32], + device_name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + new_password_hash: Option<&str>, + second_factor: Option<&Proof<'_>>, + now_ms: i64, ) -> Result { check_device_name(device_name)?; + client.check()?; let device = DeviceId::generate(); let mut w = self.begin_write().await?; - let invite = sqlx::query("DELETE FROM invites WHERE code_hash = $1 AND expires_ms > $2 RETURNING user_id") + // The account's row is locked before the invite is consumed (see `lock_user`): a password reset takes the + // user and then deletes the invite, so taking them the other way round could deadlock on Postgres. The invite + // is found with a plain read first, and consumed under the lock with its conditions checked again. + let found = sqlx::query("SELECT user_id, purpose, reset_two_factor FROM invites WHERE code_hash = $1 AND expires_ms > $2") .bind(code_hash.to_vec()) .bind(now_ms) .fetch_optional(&mut *w.tx) .await?; - let user = match invite { + let mut revoked = Vec::new(); + let mut two_factor_removed = false; + let (user, action) = match found { Some(row) => { let user = UserId(uuid_from(&row.try_get::, _>("user_id")?)?); + let purpose = Purpose::parse(&row.try_get::("purpose")?)?; + let reset_two_factor: i64 = row.try_get("reset_two_factor")?; + lock_user(&mut w.tx, user).await?; + let still_there = sqlx::query( + "SELECT 1 FROM invites WHERE code_hash = $1 AND user_id = $2 AND expires_ms > $3 AND purpose = $4 AND reset_two_factor = $5", + ) + .bind(code_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(purpose.as_str()) + .bind(reset_two_factor) + .fetch_optional(&mut *w.tx) + .await?; + if still_there.is_none() { + // Taken, replaced or deleted (a reset) while this waited for the lock. + return Err(MetaError::InvalidCode); + } + // Decided under the account's lock: a disable that committed while this waited is seen here, and one + // that follows waits for this transaction and then signs the new device out with the rest. + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } let current: String = sqlx::query("SELECT password_hash FROM users WHERE id = $1") .bind(uuid_bytes(&user.0)) .fetch_optional(&mut *w.tx) .await? .ok_or(MetaError::Corrupt("an invite's account is missing"))? .try_get("password_hash")?; - if current.is_empty() { - // Dropping the transaction puts the invite back. - let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; - sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2").bind(new).bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + // The account's two-factor state as it is under the lock now, whatever it was when the invite was made. + if two_factor::is_enabled(&mut w.tx, user).await? { + if purpose == Purpose::Reset && reset_two_factor == 1 { + two_factor::remove_in(&mut w.tx, user).await?; + two_factor_removed = true; + } else { + let Some(proof) = second_factor else { return Err(MetaError::TwoFactorRequired) }; + let row = two_factor::load_row(&mut w.tx, user).await?.ok_or(MetaError::Corrupt("two-factor row vanished under the lock"))?; + if let two_factor::Checked::Wrong = two_factor::check_proof(&mut w.tx, user, Some(device), false, &row, proof, now_ms).await? + { + // Nothing has been consumed yet: the wrong proof is counted and committed in this very + // transaction, under the account's lock, and the invite stays. + return Err(self.refuse_sign_in_proof(w, user, None, proof, now_ms, false).await); + } + } } - user + // Only now is the invite used up, with everything above decided under the lock. + let consumed = sqlx::query( + "DELETE FROM invites WHERE code_hash = $1 AND user_id = $2 AND expires_ms > $3 AND purpose = $4 AND reset_two_factor = $5", + ) + .bind(code_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(purpose.as_str()) + .bind(reset_two_factor) + .execute(&mut *w.tx) + .await?; + if consumed.rows_affected() == 0 { + return Err(MetaError::InvalidCode); + } + match purpose { + Purpose::Reset => { + // Dropping the transaction puts the code back. + let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; + sqlx::query("UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3") + .bind(new) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + // Every device goes, then the pairings: the new device is added after, so it is the only + // one signed in. + revoked = revoke_devices(&mut w.tx, user, None, RevokeReason::Reset, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + } + Purpose::Invite if current.is_empty() => { + // Dropping the transaction puts the invite back. + let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; + sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2") + .bind(new) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + } + Purpose::Invite => {} + } + (user, if purpose == Purpose::Reset { AuditAction::JoinResetRedeemed } else { AuditAction::JoinInviteRedeemed }) } None => { + // The same for a pairing: its account is locked before the device is added, so a reset that revokes + // the devices of the account cannot miss one that is being added. + let owner = + sqlx::query("SELECT user_id FROM pairings WHERE code_hash = $1").bind(code_hash.to_vec()).fetch_optional(&mut *w.tx).await?; + let Some(owner) = owner else { return Err(MetaError::InvalidCode) }; + let owner = UserId(uuid_from(&owner.try_get::, _>("user_id")?)?); + lock_user(&mut w.tx, owner).await?; + if account_blocked(&mut w.tx, owner).await? { + return Err(MetaError::AccountDisabled); + } let pairing = sqlx::query( "UPDATE pairings SET used_by = $1 \ - WHERE code_hash = $2 AND used_by IS NULL AND expires_ms > $3 \ + WHERE code_hash = $2 AND user_id = $3 AND used_by IS NULL AND expires_ms > $4 \ AND device_id IN (SELECT id FROM devices WHERE revoked = 0) RETURNING user_id", ) .bind(uuid_bytes(device.as_uuid())) .bind(code_hash.to_vec()) + .bind(uuid_bytes(&owner.0)) .bind(now_ms) .fetch_optional(&mut *w.tx) .await? .ok_or(MetaError::InvalidCode)?; - UserId(uuid_from(&pairing.try_get::, _>("user_id")?)?) + (UserId(uuid_from(&pairing.try_get::, _>("user_id")?)?), AuditAction::JoinPairingRedeemed) } }; - insert_device(&mut w.tx, device, user, device_name, token_hash, now_ms).await?; - w.commit().await?; - Ok(Redeemed { user, device }) + insert_device(&mut w.tx, device, user, device_name, client, token_hash, now_ms).await?; + let mut details = AuditDetails::client(client); + if action == AuditAction::JoinResetRedeemed { + details.revoked = Some(u32::try_from(revoked.len()).unwrap_or(u32::MAX)); + // The actor is the account that redeemed the code: the code does not say which admin made it. + details.two_factor_removed = two_factor_removed; + } + record_in(&mut w.tx, now_ms, NewEvent::new(action).by(user, Some(device)).on_device(device).with(details)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Redeemed { user, device, revoked }) } } diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 1784df29..19a7e2bd 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -7,14 +7,23 @@ //! row, so commits to one space run one at a time and commits to different spaces run in parallel. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod admin; +mod audit; mod blobs; mod codec; mod commit; +mod history; +mod index; mod join; mod live; mod schema; +mod spaces; +mod trash; mod tree; +mod two_factor; +mod usage; mod users; +mod web; use std::str::FromStr; use std::sync::Arc; @@ -24,15 +33,34 @@ use sqlx::any::{AnyConnectOptions, AnyPoolOptions}; use sqlx::{AnyPool, ConnectOptions}; use tokio::sync::{Mutex, OwnedMutexGuard}; -pub use commit::CommitOutcome; +pub use admin::{AdminActor, AdminUser, AdminVia, Issued, MAX_SEARCH_BYTES, MIN_QUOTA_BYTES, RESET_TTL_MS, ServerCounts, UserPage}; +pub use audit::{ + ActionFilter, AuditAction, AuditCursor, AuditDetails, AuditEvent, AuditFailure, AuditOutcome, AuditQuery, ClientKind, Field, MAX_DETAILS_BYTES, + MAX_RETENTION_DAYS, MIN_RETENTION_DAYS, OPPORTUNISTIC_PRUNE_BATCHES, Origin, PRUNE_BATCH, PRUNE_EVERY_MS, Platform, Pruned, Step, TargetKind, +}; +pub use commit::{CommitOutcome, RestoreOutcome, TrashRestoreOutcome}; pub use craft_cloud_proto::BlobKind; +pub use history::HistoryPage; +pub use index::{BlobRecord, EdgeCheck, EdgeProblem, EdgeProblemKind, Finding, INDEX_BATCH_ROWS, IndexAudit, IndexState, Validation}; pub use join::{InviteFor, Redeemed}; pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, MAX_LIVE_DOCUMENTS_PER_SPACE, MAX_LIVE_PAGE, SnapshotRef, }; +pub use spaces::{Member, SpaceActor, SpaceOwner, SpacePage, SpaceSummary, UserSpace, WriteGuard}; +pub use trash::{TrashEntry, TrashPage}; pub use tree::{MAX_PAGE_SAVE_BYTES, SpaceInfo}; -pub use users::{DeviceAuth, DeviceRecord, Role, UserId, UserRecord, valid_username}; +pub use two_factor::{ + CHALLENGE_TTL_MS, Completed, DeviceActor, LOCK_AFTER_FAILURES, LOCK_BASE_MS, LOCK_MAX_MS, MAX_CHALLENGE_ATTEMPTS, MAX_OPEN_CHALLENGES, + MAX_SETUP_ATTEMPTS, NewChallenge, Proof, RECOVERY_CODES, SETUP_TTL_MS, SignIn, StoredSecrets, TotpVerifier, TwoFactorInfo, UnreadableSecret, + retry_after_secs, +}; +pub use usage::{StorageAccount, StorageEstimate, StorageEstimateReason, StorageIndexCounts, StorageOverview}; +pub use users::{ + AccountRecord, CLIENT_KINDS, DeviceAuth, DeviceClient, DeviceRecord, MAX_CLIENT_FIELD_BYTES, RevokeReason, Role, UserId, UserRecord, + valid_username, +}; +pub use web::{NewWebSession, WebSession, WebSignIn}; #[derive(Debug, thiserror::Error)] pub enum MetaError { @@ -77,15 +105,67 @@ pub enum MetaError { /// An invite to an account without a password was redeemed without a new one; nothing was changed. #[error("this invite needs a new password for the account")] PasswordRequired, + /// The device that asked for a change was revoked while the request ran (it is no longer signed in); nothing was + /// changed. + #[error("this device is no longer signed in")] + DeviceRevoked, + /// What the request was based on changed while it ran (the password it checked is no longer the account's); + /// nothing was changed and the request may be made again. + #[error("{0}")] + Conflict(&'static str), + /// The admin who asked may not do this (their role changed while the request ran, or the rules about resetting + /// another admin); nothing was changed. + #[error("{0}")] + Forbidden(&'static str), + /// The change would leave the server without an administrator who can sign in; nothing was changed. + #[error("this would leave the server without an enabled administrator")] + LastAdmin, + /// The change would leave a team space without an accepted owner; nothing was changed. + #[error("this would leave the team space without an owner")] + LastOwner, + /// The account is a member of the team space but its role does not allow this write (it was lowered, or the space was + /// archived, after the request was let in); nothing was changed. + #[error("this space is read-only for your role")] + ReadOnly, + /// The account is disabled or marked for removal. Said only to someone who proved they hold its password or a + /// code of it, never to a guesser. + #[error("this account is disabled")] + AccountDisabled, + /// The account has two-factor sign-in and the request carried no second factor; nothing was changed (an invite or + /// reset code is not used up). + #[error("this account needs its second factor")] + TwoFactorRequired, + /// A code or recovery code was wrong, or a code that was used before; its counters were committed and nothing else + /// was changed. + #[error("the second factor is wrong")] + TwoFactorInvalid, + /// Too many wrong second factors: every check is refused until `retry_after_ms` have passed. + #[error("the second factor is locked after too many wrong codes")] + TwoFactorLocked { retry_after_ms: i64 }, + /// A set-up of two-factor sign-in is open on another device. + #[error("two-factor set-up is open on another device")] + TwoFactorPending, + /// A sign-in challenge that is unknown, expired, used or no longer fit, never saying which. + #[error("this sign-in step doesn't work any more")] + ChallengeGone, #[error("invalid input: {0}")] Invalid(&'static str), #[error("stored data is damaged: {0}")] Corrupt(&'static str), + /// A database that was to exist already ([`Meta::open_existing`]) is not there or not ready; nothing was created. + #[error("{0}")] + Missing(&'static str), } /// How long a SQLite connection waits for a lock another connection or program holds before giving up. pub const SQLITE_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +/// The most deleted items below one folder that restoring or removing the folder from the trash takes with it in one +/// call, shallowest first. What is beyond stays in the trash and, once its folder is live or removed from the trash, +/// appears in the trash list on its own, to be restored or removed by a further call. The bound keeps the work done +/// under the space's write lock by one call small. +pub const MAX_TRASH_TREE_NODES: u64 = 10_000; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Dialect { Sqlite, @@ -113,12 +193,48 @@ pub struct Meta { live_held_documents_cap: u64, /// The most bytes all of one space's live documents may hold ([`Meta::with_live_bytes_cap`]). live_bytes_cap: u64, + /// How long audit events are kept when a write prunes them ([`Meta::with_audit_retention_days`]); `None`: a write never prunes. + audit_retention_ms: Option, + /// When the last complete pruning ran, and whether one runs now; shared by the clones. + prune_state: Arc, + /// Where the store reads the time of day for what it judges itself (a browser session's expiry inside an admin change); + /// `None`: the system clock ([`Meta::with_clock`]). + clock: Option i64 + Send + Sync>>, + /// Edges of the reference index one transaction writes ([`Meta::with_index_batch_rows`]). + index_rows: usize, + /// The most deleted items below one folder that restoring or removing the folder from the trash takes with it + /// ([`Meta::with_trash_tree_cap`]). + trash_tree_cap: u64, + /// Set on a handle made by [`Meta::guarded`]: the write transactions of the space's writes check the caller's role + /// again before they change anything. + guard: Option, } impl Meta { + /// The configured stored-bytes cap for one space, if any. + pub fn stored_bytes_cap(&self) -> Option { + self.stored_cap + } /// Connects and brings the schema up to date. `url` is `sqlite:path/to/meta.db` (created if missing), /// `sqlite::memory:`, or `postgres://user:pass@host/db`. pub async fn open(url: &str) -> Result { + Self::start(url, false).await + } + + /// Connects to a database that already exists and creates nothing: a SQLite file that is not there is an error + /// (`mode=ro`, whatever options the URL had), not a new empty database, and a database whose schema is not the + /// current one is refused, not upgraded. For a command that only reads, so that a wrong path is reported instead of + /// being taken for an empty database that has no problems. `sqlite::memory:` has nothing to exist. + /// + /// A read-only SQLite connection never writes the database file or the WAL, and cannot checkpoint: closing the last + /// connection of a read-write one checkpoints the WAL of a crashed process into the database file and removes it, + /// which is a write a read-only command must not make. It still reads the WAL's committed data. Opening may leave + /// an empty `-wal` and a `-shm` file (SQLite's rebuilt index of the WAL, no data) when there were none. + pub async fn open_existing(url: &str) -> Result { + Self::start(url, true).await + } + + async fn start(url: &str, existing: bool) -> Result { sqlx::any::install_default_drivers(); let dialect = if url.starts_with("sqlite:") { Dialect::Sqlite @@ -128,6 +244,8 @@ impl Meta { return Err(MetaError::UnsupportedUrl); }; let url = match dialect { + Dialect::Sqlite if existing && url.contains(":memory:") => return Err(MetaError::Missing("an in-memory database has no existing data")), + Dialect::Sqlite if existing => format!("{}?mode=ro", url.split_once('?').map_or(url, |(path, _)| path)), Dialect::Sqlite if !url.contains('?') && !url.contains(":memory:") => format!("{url}?mode=rwc"), _ => url.to_owned(), }; @@ -154,10 +272,14 @@ impl Meta { Dialect::Postgres => pool_options, }; let pool = pool_options.connect_with(options).await?; - if dialect == Dialect::Sqlite && !url.contains(":memory:") { - sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + if existing { + schema::check_current(&pool, dialect).await?; + } else { + if dialect == Dialect::Sqlite && !url.contains(":memory:") { + sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + } + schema::migrate(&pool, dialect).await?; } - schema::migrate(&pool, dialect).await?; Ok(Self { pool, dialect, @@ -169,9 +291,35 @@ impl Meta { live_documents_cap: MAX_LIVE_DOCUMENTS_PER_SPACE, live_held_documents_cap: MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, live_bytes_cap: DEFAULT_LIVE_BYTES_PER_SPACE, + audit_retention_ms: None, + prune_state: Arc::new(audit::PruneState::new()), + clock: None, + index_rows: INDEX_BATCH_ROWS, + trash_tree_cap: MAX_TRASH_TREE_NODES, + guard: None, }) } + /// The store with another clock, for what it judges by the time of day itself: whether a browser session is still inside its + /// lifetimes when an admin change runs (read after the change's locks are taken). The server gives it the clock it injects + /// for tests; without one the system clock is read. The clock belongs to this handle and its clones. + pub fn with_clock(mut self, clock: Arc i64 + Send + Sync>) -> Self { + self.clock = Some(clock); + self + } + + /// Milliseconds since the Unix epoch, by the store's clock. + pub(crate) fn clock_ms(&self) -> i64 { + self.clock.as_ref().map_or_else(users::wall_clock_ms, |clock| clock()) + } + + /// The store with another cap on the deleted items below one folder that restoring or removing the folder from the + /// trash takes with it in one call (the default is [`MAX_TRASH_TREE_NODES`]); the rest stays in the trash. For tests. + pub fn with_trash_tree_cap(mut self, cap: u64) -> Self { + self.trash_tree_cap = cap; + self + } + /// The store with another cap on the bytes of updates one live document's log may hold (the default is /// [`MAX_LIVE_DOC_BYTES`]); an append over it is refused with [`MetaError::DocFull`]. For tests. pub fn with_live_doc_cap(mut self, cap: u64) -> Self { @@ -226,6 +374,14 @@ impl Meta { self } + /// `sqlite` or `postgres`, for the admin's server overview. + pub fn database_kind(&self) -> &'static str { + match self.dialect { + Dialect::Sqlite => "sqlite", + Dialect::Postgres => "postgres", + } + } + /// Whether a database connection is in use (a query under way, or a connection on its way back to the pool). /// A simulation on a simulated clock (`turmoil`) keeps time still while it is, since SQLite answers from a /// thread of its own, outside the simulation: otherwise how far simulated time got before an answer would depend diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index e4c5220d..5391a021 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -12,6 +12,15 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0004_join.sql"), include_str!("../migrations/sqlite/0005_live_documents.sql"), include_str!("../migrations/sqlite/0006_save_records.sql"), + include_str!("../migrations/sqlite/0007_devices_v2.sql"), + include_str!("../migrations/sqlite/0008_admin_resets.sql"), + include_str!("../migrations/sqlite/0009_audit_events.sql"), + include_str!("../migrations/sqlite/0010_two_factor.sql"), + include_str!("../migrations/sqlite/0011_web_sessions.sql"), + include_str!("../migrations/sqlite/0012_versions_node_seq.sql"), + include_str!("../migrations/sqlite/0013_trash.sql"), + include_str!("../migrations/sqlite/0014_manifest_index.sql"), + include_str!("../migrations/sqlite/0015_team_spaces.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -20,12 +29,62 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0004_join.sql"), include_str!("../migrations/postgres/0005_live_documents.sql"), include_str!("../migrations/postgres/0006_save_records.sql"), + include_str!("../migrations/postgres/0007_devices_v2.sql"), + include_str!("../migrations/postgres/0008_admin_resets.sql"), + include_str!("../migrations/postgres/0009_audit_events.sql"), + include_str!("../migrations/postgres/0010_two_factor.sql"), + include_str!("../migrations/postgres/0011_web_sessions.sql"), + include_str!("../migrations/postgres/0012_versions_node_seq.sql"), + include_str!("../migrations/postgres/0013_trash.sql"), + include_str!("../migrations/postgres/0014_manifest_index.sql"), + include_str!("../migrations/postgres/0015_team_spaces.sql"), ]; +/// The first line of a SQLite migration that must run with foreign-key enforcement off, because it rebuilds a table +/// other tables reference. The runner sets `PRAGMA foreign_keys = OFF` before the transaction begins (the pragma has no +/// effect inside one), checks `PRAGMA foreign_key_check` before the commit, and turns enforcement on again afterwards, +/// whether the migration worked or not. +const FOREIGN_KEYS_OFF: &str = "-- craft: foreign-keys-off"; + +fn wants_foreign_keys_off(sql: &str) -> bool { + sql.lines().next().is_some_and(|line| line.trim_end() == FOREIGN_KEYS_OFF) +} + /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one /// database. Every server must use the same key; no other code takes this lock. const MIGRATION_LOCK_KEY: i64 = 7240318150661; +/// Checks that the database has the current schema, and changes nothing: for a database that must exist already +/// ([`Meta::open_existing`](crate::Meta::open_existing)). +pub(crate) async fn check_current(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { + let migrations = match dialect { + Dialect::Sqlite => SQLITE, + Dialect::Postgres => POSTGRES, + }; + let known = i64::try_from(migrations.len()).map_err(|_| MetaError::Invalid("too many migrations"))?; + let table = match dialect { + Dialect::Sqlite => "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'", + Dialect::Postgres => { + "SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'schema_version'" + } + }; + let tables: i64 = sqlx::query(table).fetch_one(pool).await?.try_get("n")?; + if tables == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + let current: i64 = sqlx::query("SELECT COALESCE(MAX(version), 0) AS v FROM schema_version").fetch_one(pool).await?.try_get("v")?; + if current == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + if current > known { + return Err(MetaError::Corrupt("the database was created by a newer Craft Cloud")); + } + if current < known { + return Err(MetaError::Missing("the database schema is older than this program; start the server once (`serve`) to upgrade it")); + } + Ok(()) +} + pub(crate) async fn migrate(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { // One connection for the whole run. The Postgres advisory lock belongs to this session, so the lock, // the work and the unlock must all use it. @@ -49,13 +108,12 @@ pub(crate) async fn migrate(pool: &AnyPool, dialect: Dialect) -> Result<(), Meta } for (index, sql) in migrations.iter().enumerate().skip(usize::try_from(current).unwrap_or(usize::MAX)) { let version = i64::try_from(index).map_err(|_| MetaError::Invalid("too many migrations"))? + 1; - let mut tx = sqlx::Connection::begin(&mut *conn).await?; - // Statements are split on `;`, so a migration file must not contain `;` inside a string literal - // or a comment. - for statement in sql.split(';').map(str::trim).filter(|s| !s.is_empty()) { - sqlx::query(statement).execute(&mut *tx).await?; + if dialect == Dialect::Sqlite && wants_foreign_keys_off(sql) { + run_with_foreign_keys_off(&mut conn, sql, version).await?; + continue; } - sqlx::query("INSERT INTO schema_version (version) VALUES ($1)").bind(version).execute(&mut *tx).await?; + let mut tx = sqlx::Connection::begin(&mut *conn).await?; + run_statements(&mut tx, sql, version).await?; tx.commit().await?; } Ok(()) @@ -72,3 +130,119 @@ pub(crate) async fn migrate(pool: &AnyPool, dialect: Dialect) -> Result<(), Meta } work } + +/// Runs a migration's statements and records its version, in the transaction `tx`. +async fn run_statements(tx: &mut sqlx::Transaction<'_, sqlx::Any>, sql: &'static str, version: i64) -> Result<(), MetaError> { + // Statements are split on `;`, so a migration file must not contain `;` inside a string literal or a comment. + for statement in sql.split(';').map(str::trim).filter(|s| !s.is_empty()) { + sqlx::query(statement).execute(&mut **tx).await?; + } + sqlx::query("INSERT INTO schema_version (version) VALUES ($1)").bind(version).execute(&mut **tx).await?; + Ok(()) +} + +/// Runs a SQLite migration that rebuilds a table other tables reference. Foreign-key enforcement is off for the +/// whole transaction and `PRAGMA foreign_key_check` must come back empty before it commits, so a rebuild that left a +/// dangling reference is rolled back and refused. Enforcement is turned on again whatever happened. +async fn run_with_foreign_keys_off(conn: &mut sqlx::pool::PoolConnection, sql: &'static str, version: i64) -> Result<(), MetaError> { + sqlx::query("PRAGMA foreign_keys = OFF").execute(&mut **conn).await?; + let migrated = async { + let mut tx = sqlx::Connection::begin(&mut **conn).await?; + run_statements(&mut tx, sql, version).await?; + let dangling = sqlx::query("PRAGMA foreign_key_check").fetch_all(&mut *tx).await?; + if !dangling.is_empty() { + return Err(MetaError::Corrupt("a migration left rows that break a foreign key")); + } + tx.commit().await?; + Ok(()) + } + .await; + if let Err(e) = sqlx::query("PRAGMA foreign_keys = ON").execute(&mut **conn).await { + // The connection may still enforce nothing. Close it instead of returning it to the pool. + conn.close_on_drop(); + migrated?; + return Err(e.into()); + } + migrated +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Meta; + + async fn enforced(conn: &mut sqlx::AnyConnection) -> i64 { + sqlx::query("PRAGMA foreign_keys").fetch_one(conn).await.unwrap().try_get::(0).unwrap() + } + + async fn count(conn: &mut sqlx::AnyConnection, sql: &'static str) -> i64 { + sqlx::query(sql).fetch_one(conn).await.unwrap().try_get::(0).unwrap() + } + + #[test] + fn only_the_first_line_is_the_marker() { + assert!(wants_foreign_keys_off("-- craft: foreign-keys-off\nCREATE TABLE t (a INT)")); + assert!(wants_foreign_keys_off("-- craft: foreign-keys-off\r\nCREATE TABLE t (a INT)")); + assert!(!wants_foreign_keys_off("-- a comment\n-- craft: foreign-keys-off\nCREATE TABLE t (a INT)")); + assert!(!wants_foreign_keys_off("CREATE TABLE t (a INT)")); + assert!(!wants_foreign_keys_off("-- craft: foreign-keys-off-ish\n")); + assert!(!wants_foreign_keys_off("")); + // The team spaces migration rebuilds `spaces` and says so; the Postgres one needs no marker. + assert!(wants_foreign_keys_off(SQLITE.last().copied().unwrap_or_default())); + assert!(!wants_foreign_keys_off(POSTGRES.last().copied().unwrap_or_default())); + } + + #[tokio::test] + async fn migrating_leaves_foreign_keys_enforced() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let mut conn = meta.pool.acquire().await.unwrap(); + assert_eq!(enforced(&mut conn).await, 1); + assert_eq!(count(&mut conn, "SELECT MAX(version) FROM schema_version").await, i64::try_from(SQLITE.len()).unwrap()); + } + + #[tokio::test] + async fn a_migration_that_breaks_a_foreign_key_is_rolled_back_and_enforcement_comes_back() { + const DANGLING: &str = "-- craft: foreign-keys-off\n\ + CREATE TABLE probe_parent (id INTEGER PRIMARY KEY);\n\ + CREATE TABLE probe_child (parent INTEGER REFERENCES probe_parent (id));\n\ + INSERT INTO probe_child (parent) VALUES (42)"; + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let mut conn = meta.pool.acquire().await.unwrap(); + let failed = run_with_foreign_keys_off(&mut conn, DANGLING, 900).await; + assert!(matches!(failed, Err(MetaError::Corrupt(_))), "{failed:?}"); + assert_eq!(enforced(&mut conn).await, 1); + assert_eq!(count(&mut conn, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'probe_%'").await, 0, "the transaction was not rolled back"); + assert_eq!(count(&mut conn, "SELECT COUNT(*) FROM schema_version WHERE version = 900").await, 0); + } + + #[tokio::test] + async fn a_migration_that_fails_midway_changes_nothing_and_enforcement_comes_back() { + const BROKEN: &str = "-- craft: foreign-keys-off\n\ + CREATE TABLE probe_parent (id INTEGER PRIMARY KEY);\n\ + INSERT INTO probe_missing VALUES (1)"; + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let mut conn = meta.pool.acquire().await.unwrap(); + let failed = run_with_foreign_keys_off(&mut conn, BROKEN, 901).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + assert_eq!(enforced(&mut conn).await, 1); + assert_eq!(count(&mut conn, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'probe_%'").await, 0); + } + + #[tokio::test] + async fn a_rebuild_with_consistent_rows_commits_and_records_its_version() { + const REBUILD: &str = "-- craft: foreign-keys-off\n\ + CREATE TABLE probe_parent (id INTEGER PRIMARY KEY);\n\ + CREATE TABLE probe_child (parent INTEGER REFERENCES probe_parent (id));\n\ + INSERT INTO probe_parent (id) VALUES (1);\n\ + INSERT INTO probe_child (parent) VALUES (1);\n\ + DROP TABLE probe_parent;\n\ + CREATE TABLE probe_parent (id INTEGER PRIMARY KEY);\n\ + INSERT INTO probe_parent (id) VALUES (1)"; + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let mut conn = meta.pool.acquire().await.unwrap(); + run_with_foreign_keys_off(&mut conn, REBUILD, 902).await.unwrap(); + assert_eq!(enforced(&mut conn).await, 1); + assert_eq!(count(&mut conn, "SELECT COUNT(*) FROM probe_child").await, 1); + assert_eq!(count(&mut conn, "SELECT COUNT(*) FROM schema_version WHERE version = 902").await, 1); + } +} diff --git a/crates/meta/src/spaces.rs b/crates/meta/src/spaces.rs new file mode 100644 index 00000000..ee10ce3d --- /dev/null +++ b/crates/meta/src/spaces.rs @@ -0,0 +1,942 @@ +//! Team spaces and their members: who may do what in a space, and the changes that decide it. +//! +//! - **Access** is one function, [`Meta::space_access`], one indexed query: a personal space's owner is its `Owner`; in a +//! team space the role of an *accepted* member counts (an archived space gives at most `Viewer`); a disabled or +//! removed account has none anywhere. Nothing is cached. +//! - **Members** are rows of `space_members`. An invitation is a row without `accepted_ms` and gives no access. A +//! declined invitation, a member who leaves and a removed member are deleted rows, and the audit log keeps the history. +//! - **A team space always has an accepted owner.** Lowering, removing or letting the last accepted owner leave is +//! [`MetaError::LastOwner`]. Every change to a space's members locks the space's row first (`FOR UPDATE` on Postgres, +//! the single writer on SQLite), re-reads the actor's own role inside the transaction, and counts the owners after +//! that lock, so two changes that would each leave the other owner behind cannot both commit, and a change by someone +//! whose role was lowered meanwhile is refused. +//! - **Who acts** is a [`SpaceActor`]: an account by its own space role, an administrator (checked again in the +//! transaction like every admin change), or the command line. Only an administrator or the command line creates a +//! team space, and the command line is the only actor whose invitation needs no acceptance. +//! - **Audit**: every change writes its event in its own transaction ([`AuditAction::SpaceCreated`] and the rest). A +//! change that changes nothing writes none. Events hold IDs and role words only, never a space name. +//! +//! Lock order: users (by ID), then the space row, then its members. No other module takes them in another order. + +use craft_cloud_proto::api::SpaceRole; +use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId, SpaceName}; +use sqlx::Row; +use sqlx::any::AnyRow; + +use crate::admin::{AdminActor, AdminVia}; +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; +use crate::codec::{to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::users::{account_blocked, lock_user}; +use crate::{Dialect, MIN_QUOTA_BYTES, Meta, MetaError, UserId, WriteTx}; + +/// Who makes a change to a team space or its members. +#[derive(Debug, Clone)] +pub enum SpaceActor { + /// A signed-in account, acting by its role in the space (checked in the transaction). `device` is the device the + /// request came with; it must still be signed in. `None` is for the command line's tests and callers that have no + /// device (no session is checked); the JSON routes always pass the request's device, and no route of the space API takes + /// a browser session (the admin pages' own routes carry theirs in [`AdminVia::Web`]). + User { user: UserId, device: Option }, + /// An administrator, checked again in the transaction as for every admin change. + Admin(AdminActor), + /// The command line on the server's host: no account acts, and its events say so. + Cli, +} + +/// The role a write needs and the account that asks, for [`Meta::guarded`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WriteGuard { + pub user: UserId, + pub min: SpaceRole, +} + +/// A space as `GET /spaces` lists it for one account. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserSpace { + pub id: SpaceId, + pub personal: bool, + /// The team space's name; `None` for a personal space. + pub name: Option, + pub role: SpaceRole, + /// An invitation the account has not accepted. + pub invited: bool, + pub archived: bool, + pub created_ms: i64, + /// Accepted members (1 for a personal space). + pub members: u64, + /// `None` until the invitation is accepted. + pub root: Option, + pub epoch: Option, + /// When the account accepted its membership (milliseconds). A member who is removed and invited again gets a new one; + /// `None` for a personal space and for an invitation. + pub joined_ms: Option, +} + +/// A team space as the administrator's list shows it: counts, never content. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpaceSummary { + pub id: SpaceId, + pub name: String, + /// The folded name, which the list is ordered and paged by. + pub name_fold: String, + pub created_ms: i64, + pub archived_ms: Option, + /// Accepted members, owners included. + pub members: u64, + /// Accepted owners whose accounts can sign in: an owner who is disabled or marked for removal has no access (see + /// [`Meta::space_access`]) and does not count, so 0 means nobody can manage the space. + pub owners: u64, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +/// One page of [`Meta::admin_list_spaces`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpacePage { + pub spaces: Vec, + /// Where the next page starts (the last name key of this one); `None` on the last page. + pub next_after: Option, +} + +const SUMMARY_SELECT: &str = "SELECT s.id, s.name, s.name_fold, s.created_ms, s.archived_ms, s.stored_bytes, s.quota_bytes, \ + (SELECT CAST(COUNT(*) AS BIGINT) FROM space_members m WHERE m.space_id = s.id AND m.accepted_ms IS NOT NULL) AS members, \ + (SELECT CAST(COUNT(*) AS BIGINT) FROM space_members m JOIN users u ON u.id = m.user_id WHERE m.space_id = s.id AND m.accepted_ms IS NOT NULL AND m.role = 'owner' AND u.disabled_ms IS NULL AND u.removed_ms IS NULL) AS owners \ + FROM spaces s WHERE s.kind = 'team'"; + +fn summary_from(row: &AnyRow) -> Result { + Ok(SpaceSummary { + id: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + name: row.try_get::, _>("name")?.ok_or(MetaError::Corrupt("a team space has no name"))?, + name_fold: row.try_get::, _>("name_fold")?.ok_or(MetaError::Corrupt("a team space has no name key"))?, + created_ms: row.try_get("created_ms")?, + archived_ms: row.try_get("archived_ms")?, + members: to_u64(row.try_get("members")?)?, + owners: to_u64(row.try_get("owners")?)?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + quota_bytes: row.try_get::, _>("quota_bytes")?.map(to_u64).transpose()?, + }) +} + +/// An accepted owner of a team space, with the state of the account (for the administrator's pages). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpaceOwner { + pub space: SpaceId, + pub user: UserId, + pub username: String, + pub disabled_ms: Option, + pub removed_ms: Option, +} + +impl SpaceOwner { + /// Whether the account can sign in, which is what an owner's power needs. + pub fn active(&self) -> bool { + self.disabled_ms.is_none() && self.removed_ms.is_none() + } +} + +/// `text` as a `LIKE` pattern that matches it literally (`\` is the escape character in the queries above). +fn escape_like(text: &str) -> String { + text.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_") +} + +/// A member of a team space, or an account invited to it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Member { + pub user: UserId, + pub username: String, + pub role: SpaceRole, + pub invited_ms: i64, + /// The account that invited it; `None` for the command line. + pub invited_by: Option, + /// `None` while the invitation is pending. + pub accepted_ms: Option, +} + +impl Member { + /// Whether the account has accepted, which is what access needs. + pub fn accepted(&self) -> bool { + self.accepted_ms.is_some() + } +} + +/// What a change asks of a [`SpaceActor::User`] under the space's lock. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Need { + /// A row of their own in the space, any state (accept, decline, leave). + Row, + /// An accepted `Owner` row, with the archive's cap on roles applied. + Owner, + /// An accepted `Owner` row, whatever the space's state: an owner may archive and unarchive their space. + OwnerEvenIfArchived, +} + +/// A team space as a change finds it after locking its row. +struct Locked { + archived: bool, +} + +const MEMBER_SELECT: &str = "SELECT m.user_id, u.username, m.role, m.invited_ms, m.invited_by, m.accepted_ms \ + FROM space_members m JOIN users u ON u.id = m.user_id"; + +fn member_from(row: &AnyRow) -> Result { + Ok(Member { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + username: row.try_get("username")?, + role: parse_role(&row.try_get::("role")?)?, + invited_ms: row.try_get("invited_ms")?, + invited_by: row.try_get::>, _>("invited_by")?.as_deref().map(uuid_from).transpose()?.map(UserId), + accepted_ms: row.try_get("accepted_ms")?, + }) +} + +fn parse_role(word: &str) -> Result { + SpaceRole::parse(word).ok_or(MetaError::Corrupt("unknown space role")) +} + +/// The event of a change `actor` makes. The command line has no actor, and its events have names of their own: +/// `cli.space_created` for a creation, `cli.space_member_changed` for an invitation, a role change or a removal and +/// `cli.space_archived` for an archive or unarchive. It makes +/// no other change (a member's own acts are theirs), and `Corrupt` says so if one is ever asked of it. +fn event_for(actor: &SpaceActor, action: AuditAction, space: SpaceId) -> Result { + let event = match actor { + SpaceActor::User { user, device } => NewEvent::new(action).by(*user, *device), + SpaceActor::Admin(admin) => { + let device = match admin.via { + AdminVia::Device(device) => Some(device), + AdminVia::Web { .. } => None, + }; + NewEvent::new(action).by(admin.user, device) + } + SpaceActor::Cli => NewEvent::new(match action { + AuditAction::SpaceCreated => AuditAction::CliSpaceCreated, + AuditAction::SpaceMemberInvited | AuditAction::SpaceMemberRoleChanged | AuditAction::SpaceMemberRemoved => { + AuditAction::CliSpaceMemberChanged + } + AuditAction::SpaceArchived | AuditAction::SpaceUnarchived => AuditAction::CliSpaceArchived, + _ => return Err(MetaError::Corrupt("the command line makes no such change")), + }), + }; + Ok(event.on_space(space)) +} + +impl Meta { + /// What `user` may do in `space`: `None` for no access at all, which is also the answer for a space that does not + /// exist (a caller must not tell the two apart). See the module's rules. + pub async fn space_access(&self, user: UserId, space: SpaceId) -> Result, MetaError> { + let mut conn = self.pool.acquire().await?; + access_on(&mut conn, user, space).await + } + + /// A handle whose space writes (commit, restore, save, upload, removal from the trash) re-check `guard` inside their + /// write transaction, after the space's row is locked: a role lowered, a member removed or a space archived by a change + /// that committed between the server's access check and the write is respected. A refused write changes nothing and is + /// `NotFound("space")` (no access any more) or [`MetaError::ReadOnly`]. Cheap: a clone with one field set. Only the + /// space writes look at it. + pub fn guarded(&self, guard: WriteGuard) -> Meta { + Meta { guard: Some(guard), ..self.clone() } + } + + /// Runs the guard of a [`Meta::guarded`] handle, if there is one, on the write transaction's connection. On Postgres + /// the space's row is locked first, at the strength the write needs anyway (`FOR NO KEY UPDATE`: an upload updates the + /// row's byte count, and a weaker lock taken first would make two concurrent uploads wait for each other's upgrade and + /// deadlock); a change to the space's members, which takes `FOR UPDATE`, waits for it and the other way round. On SQLite + /// the single writer already orders everything. + pub(crate) async fn check_guard(&self, conn: &mut sqlx::AnyConnection, space: SpaceId) -> Result<(), MetaError> { + let Some(guard) = self.guard else { return Ok(()) }; + if self.dialect == Dialect::Postgres { + sqlx::query("SELECT 1 FROM spaces WHERE id = $1 FOR NO KEY UPDATE").bind(uuid_bytes(space.as_uuid())).fetch_optional(&mut *conn).await?; + } + match access_on(conn, guard.user, space).await? { + None => Err(MetaError::NotFound("space")), + Some(role) if role < guard.min => Err(MetaError::ReadOnly), + Some(_) => Ok(()), + } + } + + /// The members of a team space, accepted and invited, by username; `NotFound` for a space that is not a team space. + /// Who may ask is the caller's to decide ([`Meta::space_access`]). + pub async fn list_members(&self, space: SpaceId) -> Result, MetaError> { + let mut tx = self.read_snapshot().await?; + let kind = sqlx::query("SELECT kind FROM spaces WHERE id = $1").bind(uuid_bytes(space.as_uuid())).fetch_optional(&mut *tx).await?; + match kind { + Some(row) if row.try_get::("kind")? == "team" => {} + _ => return Err(MetaError::NotFound("space")), + } + let rows = sqlx::query(sqlx::AssertSqlSafe(format!("{MEMBER_SELECT} WHERE m.space_id = $1 ORDER BY u.username, m.user_id"))) + .bind(uuid_bytes(space.as_uuid())) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + rows.iter().map(member_from).collect() + } + + /// Creates a team space called `name` with `owner` as its first owner, who is a member from the start (naming the first + /// owner is the consent step for that one row), and a root directory like every space. `quota_bytes` is the space's + /// own limit (at least [`MIN_QUOTA_BYTES`]); `None`: the server's default applies. Only an administrator or the command + /// line may ([`SpaceActor::User`] is `Forbidden`). An owner who is not an account that can sign in is `NotFound("user")`; + /// a name another team space has, in any case, is `Conflict`. + pub async fn create_team_space( + &self, + actor: &SpaceActor, + name: &SpaceName, + owner: UserId, + quota_bytes: Option, + now_ms: i64, + ) -> Result { + if let Some(bytes) = quota_bytes + && !(MIN_QUOTA_BYTES..=i64::MAX as u64).contains(&bytes) + { + return Err(MetaError::Invalid("quota_bytes is at least one chunk (4194304) and at most 9223372036854775807")); + } + let mut w = match actor { + SpaceActor::User { .. } => return Err(MetaError::Forbidden("only an administrator creates team spaces")), + SpaceActor::Admin(admin) => self.begin_admin(admin, Some(owner), false).await?, + SpaceActor::Cli => { + let mut w = self.begin_write().await?; + lock_user(&mut w.tx, owner).await?; + w + } + }; + if !user_can_join(&mut w, owner).await? { + return Err(MetaError::NotFound("user")); + } + let fold = name.fold_key(); + let taken = sqlx::query("SELECT 1 FROM spaces WHERE kind = 'team' AND name_fold = $1").bind(fold.as_str()).fetch_optional(&mut *w.tx).await?; + if taken.is_some() { + return Err(MetaError::Conflict("a team space with this name exists")); + } + let (space, root) = (SpaceId::generate(), NodeId::generate()); + sqlx::query( + "INSERT INTO spaces (id, owner, epoch, seq, oldest_seq, root, quota_bytes, kind, name, name_fold, created_ms) \ + VALUES ($1, NULL, $2, 0, 0, $3, $4, 'team', $5, $6, $7)", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(Epoch::generate().as_uuid())) + .bind(uuid_bytes(root.as_uuid())) + .bind(quota_bytes.map(to_i64).transpose()?) + .bind(name.as_str()) + .bind(fold.as_str()) + .bind(now_ms) + .execute(&mut *w.tx) + .await + // The check above and this insert are in one transaction, but another process on one Postgres may have taken the + // name in between: the unique index turns the loser into the same answer. + .map_err(|e| match e { + sqlx::Error::Database(ref db) if db.is_unique_violation() => MetaError::Conflict("a team space with this name exists"), + other => MetaError::Db(other), + })?; + sqlx::query( + "INSERT INTO nodes (space_id, id, parent, name, fold_key, is_dir, version, deleted, changed_seq) \ + VALUES ($1, $2, NULL, '', '', 1, NULL, 0, 0)", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(root.as_uuid())) + .execute(&mut *w.tx) + .await?; + insert_member(&mut w, space, owner, SpaceRole::Owner, actor_user(actor), now_ms, true).await?; + let details = AuditDetails { user: Some(owner), quota_bytes: Some(quota_bytes), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(actor, AuditAction::SpaceCreated, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await?; + Ok(space) + } + + /// Invites `target` to the team space as `role`: a pending invitation that gives no access until the account accepts + /// ([`Meta::accept_invitation`]). The command line's invitation is accepted at once (whoever runs it has the files + /// anyway). Owners and administrators may; an account that does not exist, is disabled or is marked for removal is + /// `NotFound("user")` (one answer for all three); one that is a member or invited already is `Conflict`. + pub async fn invite_member(&self, actor: &SpaceActor, space: SpaceId, target: UserId, role: SpaceRole, now_ms: i64) -> Result { + let (mut w, _) = self.begin_space_change(actor, space, Some(target), Need::Owner).await?; + if !user_can_join(&mut w, target).await? { + return Err(MetaError::NotFound("user")); + } + if member_row(&mut w, space, target).await?.is_some() { + return Err(MetaError::Conflict("this account is a member of the space or invited to it already")); + } + insert_member(&mut w, space, target, role, actor_user(actor), now_ms, matches!(actor, SpaceActor::Cli)).await?; + let details = AuditDetails { user: Some(target), space_role_to: Some(role), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(actor, AuditAction::SpaceMemberInvited, space)?.with(details)).await?; + let member = member_row(&mut w, space, target).await?.ok_or(MetaError::Corrupt("a new member is missing"))?; + self.commit_audited(w, now_ms).await?; + Ok(member) + } + + /// The invited account accepts: it is a member with the invited role from now on, which is returned. No invitation + /// (never made, withdrawn, declined, or accepted already) is `NotFound("invitation")` and writes nothing. + pub async fn accept_invitation(&self, user: UserId, device: Option, space: SpaceId, now_ms: i64) -> Result { + let actor = SpaceActor::User { user, device }; + let (mut w, locked) = self.begin_space_change(&actor, space, None, Need::Row).await?; + // An archived space is not listed to an invitee (it is read-only for everyone): the answer is the one for no space. + if locked.archived { + return Err(MetaError::NotFound("space")); + } + let row = + sqlx::query("UPDATE space_members SET accepted_ms = $1 WHERE space_id = $2 AND user_id = $3 AND accepted_ms IS NULL RETURNING role") + .bind(now_ms) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("invitation"))?; + let role = parse_role(&row.try_get::("role")?)?; + let details = AuditDetails { user: Some(user), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(&actor, AuditAction::SpaceMemberJoined, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await?; + Ok(role) + } + + /// The invited account declines: the invitation is deleted. No invitation is `NotFound("invitation")` and writes + /// nothing; a member who has accepted leaves with [`Meta::leave_space`]. + pub async fn decline_invitation(&self, user: UserId, device: Option, space: SpaceId, now_ms: i64) -> Result<(), MetaError> { + let actor = SpaceActor::User { user, device }; + let (mut w, locked) = self.begin_space_change(&actor, space, None, Need::Row).await?; + if locked.archived { + return Err(MetaError::NotFound("space")); + } + let done = sqlx::query("DELETE FROM space_members WHERE space_id = $1 AND user_id = $2 AND accepted_ms IS NULL") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("invitation")); + } + let details = AuditDetails { user: Some(user), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(&actor, AuditAction::SpaceMemberDeclined, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await + } + + /// Gives a member (or an invitation) another role, and returns the row. The role it has already is no change and + /// writes no event, except that the command line also accepts a pending invitation (whoever runs it has the files, as + /// with [`Meta::invite_member`]). Lowering the last accepted owner is [`MetaError::LastOwner`]. A target without a row is + /// `NotFound("member")`. + pub async fn change_member_role( + &self, + actor: &SpaceActor, + space: SpaceId, + target: UserId, + role: SpaceRole, + now_ms: i64, + ) -> Result { + let (mut w, _) = self.begin_space_change(actor, space, Some(target), Need::Owner).await?; + let current = member_row(&mut w, space, target).await?.ok_or(MetaError::NotFound("member"))?; + let accept_now = matches!(actor, SpaceActor::Cli) && !current.accepted(); + if current.role == role && !accept_now { + return Ok(current); + } + if current.role == SpaceRole::Owner && role != SpaceRole::Owner && current.accepted() { + keep_an_owner(&mut w, space).await?; + } + sqlx::query("UPDATE space_members SET role = $1, accepted_ms = COALESCE(accepted_ms, $2) WHERE space_id = $3 AND user_id = $4") + .bind(role.as_str()) + .bind(accept_now.then_some(now_ms)) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + let details = AuditDetails { user: Some(target), space_role_from: Some(current.role), space_role_to: Some(role), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(actor, AuditAction::SpaceMemberRoleChanged, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Member { role, accepted_ms: current.accepted_ms.or(accept_now.then_some(now_ms)), ..current }) + } + + /// Removes a member, or withdraws an invitation. Nothing in the space is deleted. Removing the last accepted owner is + /// [`MetaError::LastOwner`]; a target without a row is `NotFound("member")`. An account removing its own row is + /// [`Meta::leave_space`]. + pub async fn remove_member(&self, actor: &SpaceActor, space: SpaceId, target: UserId, now_ms: i64) -> Result<(), MetaError> { + if let SpaceActor::User { user, device } = actor + && *user == target + { + return self.leave_space(*user, *device, space, now_ms).await; + } + let (mut w, _) = self.begin_space_change(actor, space, Some(target), Need::Owner).await?; + let current = member_row(&mut w, space, target).await?.ok_or(MetaError::NotFound("member"))?; + if current.role == SpaceRole::Owner && current.accepted() { + keep_an_owner(&mut w, space).await?; + } + delete_member(&mut w, space, target).await?; + let details = AuditDetails { user: Some(target), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(actor, AuditAction::SpaceMemberRemoved, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await + } + + /// A member leaves, whatever their role. The last accepted owner leaving is [`MetaError::LastOwner`]. An account + /// that is only invited declines instead ([`Meta::decline_invitation`]): `NotFound("member")` here. + pub async fn leave_space(&self, user: UserId, device: Option, space: SpaceId, now_ms: i64) -> Result<(), MetaError> { + let actor = SpaceActor::User { user, device }; + let (mut w, _) = self.begin_space_change(&actor, space, None, Need::Row).await?; + let current = member_row(&mut w, space, user).await?.filter(Member::accepted).ok_or(MetaError::NotFound("member"))?; + if current.role == SpaceRole::Owner { + keep_an_owner(&mut w, space).await?; + } + delete_member(&mut w, space, user).await?; + record_in(&mut w.tx, now_ms, event_for(&actor, AuditAction::SpaceMemberLeft, space)?).await?; + self.commit_audited(w, now_ms).await + } + + /// Renames a team space. Owners and administrators may; a name another team space has, in any case, is `Conflict`; the + /// name it has already, letter for letter, is no change and writes no event. The event holds no name. + pub async fn rename_team_space(&self, actor: &SpaceActor, space: SpaceId, name: &SpaceName, now_ms: i64) -> Result<(), MetaError> { + let (mut w, _) = self.begin_space_change(actor, space, None, Need::Owner).await?; + let current = sqlx::query("SELECT name FROM spaces WHERE id = $1").bind(uuid_bytes(space.as_uuid())).fetch_one(&mut *w.tx).await?; + if current.try_get::, _>("name")?.as_deref() == Some(name.as_str()) { + return Ok(()); + } + let fold = name.fold_key(); + let taken = sqlx::query("SELECT 1 FROM spaces WHERE kind = 'team' AND name_fold = $1 AND id <> $2") + .bind(fold.as_str()) + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *w.tx) + .await?; + if taken.is_some() { + return Err(MetaError::Conflict("a team space with this name exists")); + } + sqlx::query("UPDATE spaces SET name = $1, name_fold = $2 WHERE id = $3") + .bind(name.as_str()) + .bind(fold.as_str()) + .bind(uuid_bytes(space.as_uuid())) + .execute(&mut *w.tx) + .await + .map_err(|e| match e { + sqlx::Error::Database(ref db) if db.is_unique_violation() => MetaError::Conflict("a team space with this name exists"), + other => MetaError::Db(other), + })?; + record_in(&mut w.tx, now_ms, event_for(actor, AuditAction::SpaceRenamed, space)?).await?; + self.commit_audited(w, now_ms).await + } + + /// Archives (`archived`) or unarchives a team space: every member's access is capped at `Viewer` while it is archived, + /// and nothing is deleted. Owners and administrators may, and an owner may unarchive a space that caps their own role. + /// The state it has already is no change and writes no event. + pub async fn set_space_archived(&self, actor: &SpaceActor, space: SpaceId, archived: bool, now_ms: i64) -> Result<(), MetaError> { + let (mut w, locked) = self.begin_space_change(actor, space, None, Need::OwnerEvenIfArchived).await?; + if locked.archived == archived { + return Ok(()); + } + sqlx::query("UPDATE spaces SET archived_ms = $1 WHERE id = $2") + .bind(archived.then_some(now_ms)) + .bind(uuid_bytes(space.as_uuid())) + .execute(&mut *w.tx) + .await?; + let action = if archived { AuditAction::SpaceArchived } else { AuditAction::SpaceUnarchived }; + record_in(&mut w.tx, now_ms, event_for(actor, action, space)?).await?; + self.commit_audited(w, now_ms).await + } + + /// Sets a team space's own limit (`None`: the server's default applies again). Administrators only. The limit it has + /// already is no change and writes no event. + pub async fn set_space_quota(&self, admin: &AdminActor, space: SpaceId, quota_bytes: Option, now_ms: i64) -> Result<(), MetaError> { + if let Some(bytes) = quota_bytes + && !(MIN_QUOTA_BYTES..=i64::MAX as u64).contains(&bytes) + { + return Err(MetaError::Invalid("quota_bytes is at least one chunk (4194304) and at most 9223372036854775807")); + } + let actor = SpaceActor::Admin(admin.clone()); + let (mut w, _) = self.begin_space_change(&actor, space, None, Need::Row).await?; + let current: Option = sqlx::query("SELECT quota_bytes FROM spaces WHERE id = $1") + .bind(uuid_bytes(space.as_uuid())) + .fetch_one(&mut *w.tx) + .await? + .try_get("quota_bytes")?; + let wanted = quota_bytes.map(to_i64).transpose()?; + if current == wanted { + return Ok(()); + } + sqlx::query("UPDATE spaces SET quota_bytes = $1 WHERE id = $2").bind(wanted).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + let details = AuditDetails { quota_bytes: Some(quota_bytes), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, event_for(&actor, AuditAction::SpaceQuotaChanged, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await + } + + /// Makes `user` an accepted owner of the team space, whatever their row was (none, an invitation, any role): the + /// administrator naming an owner is the consent step, as for the first owner at creation. For a space whose owners are + /// all gone. An account that cannot sign in is `NotFound("user")`. An accepted owner already is no change and writes no + /// event; otherwise the event is `space.member_role_changed` with `role_to` owner (and `role_from` if there was a row). + pub async fn set_owner_accepted(&self, admin: &AdminActor, space: SpaceId, user: UserId, now_ms: i64) -> Result<(), MetaError> { + let actor = SpaceActor::Admin(admin.clone()); + let (mut w, _) = self.begin_space_change(&actor, space, Some(user), Need::Row).await?; + if !user_can_join(&mut w, user).await? { + return Err(MetaError::NotFound("user")); + } + let current = member_row(&mut w, space, user).await?; + match ¤t { + Some(member) if member.role == SpaceRole::Owner && member.accepted() => return Ok(()), + Some(_) => { + sqlx::query("UPDATE space_members SET role = 'owner', accepted_ms = COALESCE(accepted_ms, $1) WHERE space_id = $2 AND user_id = $3") + .bind(now_ms) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + } + None => insert_member(&mut w, space, user, SpaceRole::Owner, Some(admin.user), now_ms, true).await?, + } + let details = AuditDetails { + user: Some(user), + space_role_from: current.as_ref().map(|member| member.role), + space_role_to: Some(SpaceRole::Owner), + ..AuditDetails::default() + }; + record_in(&mut w.tx, now_ms, event_for(&actor, AuditAction::SpaceMemberRoleChanged, space)?.with(details)).await?; + self.commit_audited(w, now_ms).await + } + + /// The spaces `user` can see, for `GET /spaces`: their personal space first, then the team spaces they are a member of + /// or invited to, by name. A pending invitation has no root and no epoch yet. An archived space is listed for every + /// accepted member (with `archived`; its access is capped at viewer, so it is theirs to read); an invitation to an + /// archived space is not listed. + pub async fn user_spaces(&self, user: UserId) -> Result, MetaError> { + let mut tx = self.read_snapshot().await?; + let mut out = Vec::new(); + let personal = sqlx::query("SELECT id, created_ms, root, epoch FROM spaces WHERE owner = $1 AND kind = 'personal'") + .bind(uuid_bytes(&user.0)) + .fetch_all(&mut *tx) + .await?; + for row in &personal { + out.push(UserSpace { + id: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + personal: true, + name: None, + role: SpaceRole::Owner, + invited: false, + archived: false, + created_ms: row.try_get("created_ms")?, + members: 1, + root: Some(NodeId::from_uuid(uuid_from(&row.try_get::, _>("root")?)?)), + epoch: Some(Epoch::from_uuid(uuid_from(&row.try_get::, _>("epoch")?)?)), + joined_ms: None, + }); + } + let teams = sqlx::query( + "SELECT s.id, s.name, s.created_ms, s.archived_ms, s.root, s.epoch, m.role, m.accepted_ms, \ + (SELECT CAST(COUNT(*) AS BIGINT) FROM space_members c WHERE c.space_id = s.id AND c.accepted_ms IS NOT NULL) AS members \ + FROM space_members m JOIN spaces s ON s.id = m.space_id WHERE m.user_id = $1 AND s.kind = 'team' ORDER BY s.name_fold, s.id", + ) + .bind(uuid_bytes(&user.0)) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + for row in &teams { + let role = parse_role(&row.try_get::("role")?)?; + let joined_ms = row.try_get::, _>("accepted_ms")?; + let invited = joined_ms.is_none(); + let archived = row.try_get::, _>("archived_ms")?.is_some(); + if archived && invited { + continue; + } + out.push(UserSpace { + id: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + personal: false, + name: row.try_get("name")?, + role, + invited, + archived, + created_ms: row.try_get("created_ms")?, + members: to_u64(row.try_get("members")?)?, + root: if invited { None } else { Some(NodeId::from_uuid(uuid_from(&row.try_get::, _>("root")?)?)) }, + epoch: if invited { None } else { Some(Epoch::from_uuid(uuid_from(&row.try_get::, _>("epoch")?)?)) }, + joined_ms, + }); + } + Ok(out) + } + + /// One team space as the administrator's list shows it; `NotFound("space")` for a space that is not a team space. + pub async fn admin_space(&self, space: SpaceId) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("{SUMMARY_SELECT} AND s.id = $1"))) + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&self.pool) + .await? + .ok_or(MetaError::NotFound("space"))?; + summary_from(&row) + } + + /// The team spaces by name, for the administrator (no content, only counts): `limit` of them after the name key `after`, + /// whose name contains `search` (case does not matter). `SpacePage::next_after` continues the list. + pub async fn admin_list_spaces(&self, after: Option<&str>, search: Option<&str>, limit: u32) -> Result { + let pattern = match search.filter(|s| !s.is_empty()) { + Some(text) => { + if text.len() > crate::MAX_SEARCH_BYTES { + return Err(MetaError::Invalid("the search is too long")); + } + format!("%{}%", escape_like(craft_cloud_proto::fold_search(text).as_str())) + } + None => "%".to_owned(), + }; + let sql = format!("{SUMMARY_SELECT} AND s.name_fold > $1 AND s.name_fold LIKE $2 ESCAPE '\\' ORDER BY s.name_fold LIMIT $3"); + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(after.unwrap_or_default()) + .bind(pattern) + .bind(i64::from(limit).saturating_add(1)) + .fetch_all(&self.pool) + .await?; + let mut spaces = rows.iter().map(summary_from).collect::, _>>()?; + let more = spaces.len() > limit as usize; + spaces.truncate(limit as usize); + let next_after = if more { spaces.last().map(|s| s.name_fold.clone()) } else { None }; + Ok(SpacePage { spaces, next_after }) + } + + /// The accepted owners of `spaces` (team spaces; at most a page's worth, 200), by space and username, with the state of + /// their accounts, in one query. For the administrator's pages; who may ask is the caller's to decide. + pub async fn space_owners(&self, spaces: &[SpaceId]) -> Result, MetaError> { + if spaces.is_empty() { + return Ok(Vec::new()); + } + if spaces.len() > 200 { + return Err(MetaError::Invalid("at most 200 spaces at a time")); + } + let marks = (1..=spaces.len()).map(|n| format!("${n}")).collect::>().join(", "); + let sql = format!( + "SELECT m.space_id, m.user_id, u.username, u.disabled_ms, u.removed_ms FROM space_members m JOIN users u ON u.id = m.user_id WHERE m.role = 'owner' AND m.accepted_ms IS NOT NULL AND m.space_id IN ({marks}) ORDER BY m.space_id, u.username" + ); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)); + for space in spaces { + query = query.bind(uuid_bytes(space.as_uuid())); + } + let rows = query.fetch_all(&self.pool).await?; + rows.iter() + .map(|row| { + Ok(SpaceOwner { + space: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("space_id")?)?), + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + username: row.try_get("username")?, + disabled_ms: row.try_get("disabled_ms")?, + removed_ms: row.try_get("removed_ms")?, + }) + }) + .collect() + } + + /// The bytes all of a space's live documents hold, for the space's usage. + pub async fn space_live_bytes(&self, space: SpaceId) -> Result { + let row = sqlx::query("SELECT live_bytes FROM spaces WHERE id = $1") + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&self.pool) + .await? + .ok_or(MetaError::NotFound("space"))?; + to_u64(row.try_get("live_bytes")?) + } + + /// The role `user` has as an accepted member of the team space, whatever the space's state (an archived space does not + /// cap it here): for the server to decide whether the caller acts as an owner. `None` for no accepted row, a personal + /// space or a space that does not exist. + pub async fn space_member_role(&self, user: UserId, space: SpaceId) -> Result, MetaError> { + let row = sqlx::query( + "SELECT m.role FROM space_members m JOIN spaces s ON s.id = m.space_id \ + WHERE m.space_id = $1 AND m.user_id = $2 AND m.accepted_ms IS NOT NULL AND s.kind = 'team'", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + row.map(|row| parse_role(&row.try_get::("role")?)).transpose() + } + + /// The team space with this name (in any case), for the command line. + pub async fn team_space_by_name(&self, name: &SpaceName) -> Result, MetaError> { + let row = sqlx::query("SELECT id FROM spaces WHERE kind = 'team' AND name_fold = $1") + .bind(name.fold_key().as_str()) + .fetch_optional(&self.pool) + .await?; + row.map(|row| Ok(SpaceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?))).transpose() + } + + /// A team space's name; `None` for a personal space or a space that does not exist. + pub async fn team_space_name(&self, space: SpaceId) -> Result, MetaError> { + let row = sqlx::query("SELECT name FROM spaces WHERE id = $1 AND kind = 'team'") + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|row| row.try_get::, _>("name")).transpose()?.flatten()) + } + + /// The team spaces `user` is an accepted member of: whose long-polls to wake when the account is disabled or removed. + pub async fn member_space_ids(&self, user: UserId) -> Result, MetaError> { + let rows = sqlx::query("SELECT space_id FROM space_members WHERE user_id = $1 AND accepted_ms IS NOT NULL") + .bind(uuid_bytes(&user.0)) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|row| Ok(SpaceId::from_uuid(uuid_from(&row.try_get::, _>("space_id")?)?))).collect() + } + + /// Starts the write transaction of a change to a team space's members: checks the actor, takes the locks in the module's + /// order and, for a change only an owner may make (`need`), reads the actor's role again under the space's lock. + /// `target` is the account the change is about, if it has one. A space that is not a team space, or an account with + /// no role in it, gets `NotFound("space")`: the two are not told apart; a member without the right role is `Forbidden`. + async fn begin_space_change( + &self, + actor: &SpaceActor, + space: SpaceId, + target: Option, + need: Need, + ) -> Result<(WriteTx, Locked), MetaError> { + let mut w = match actor { + SpaceActor::Admin(admin) => self.begin_admin(admin, target, false).await?, + SpaceActor::User { user, device } => { + let mut w = self.begin_write().await?; + let mut ids = vec![*user]; + ids.extend(target.filter(|target| target != user)); + ids.sort_by_key(|id| *id.0.as_bytes()); + for id in ids { + lock_user(&mut w.tx, id).await?; + } + // The account and the device the request came with must still be signed in. + let known = sqlx::query("SELECT 1 FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut *w.tx).await?; + if known.is_none() || account_blocked(&mut w.tx, *user).await? { + return Err(MetaError::DeviceRevoked); + } + if let Some(device) = device { + let active = sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + if active.is_none() { + return Err(MetaError::DeviceRevoked); + } + } + w + } + SpaceActor::Cli => { + let mut w = self.begin_write().await?; + if let Some(target) = target { + lock_user(&mut w.tx, target).await?; + } + w + } + }; + let sql = match self.dialect { + Dialect::Postgres => "SELECT kind, archived_ms FROM spaces WHERE id = $1 FOR UPDATE", + Dialect::Sqlite => "SELECT kind, archived_ms FROM spaces WHERE id = $1", + }; + let row = sqlx::query(sql).bind(uuid_bytes(space.as_uuid())).fetch_optional(&mut *w.tx).await?; + let locked = match row { + Some(row) if row.try_get::("kind")? == "team" => Locked { archived: row.try_get::, _>("archived_ms")?.is_some() }, + _ => return Err(MetaError::NotFound("space")), + }; + if let SpaceActor::User { user, .. } = actor { + let member = member_row(&mut w, space, *user).await?; + match need { + // Accepting, declining and leaving are the caller's own rows: an account with no row at all learns nothing + // about the space (the answer is the one for a space that does not exist), and only the wrong state of its + // own row is `NotFound("invitation")` or `NotFound("member")`. + Need::Row => { + if member.is_none() { + return Err(MetaError::NotFound("space")); + } + } + // Under the space's lock, so a role lowered by a change that committed meanwhile is respected. + Need::Owner | Need::OwnerEvenIfArchived => { + let capped = locked.archived && need == Need::Owner; + let role = member.filter(Member::accepted).map(|member| if capped { member.role.min(SpaceRole::Viewer) } else { member.role }); + match role { + Some(SpaceRole::Owner) => {} + Some(_) => return Err(MetaError::ReadOnly), + None => return Err(MetaError::NotFound("space")), + } + } + } + } + Ok((w, locked)) + } +} + +/// [`Meta::space_access`] on a given connection (also a write transaction's). +async fn access_on(conn: &mut sqlx::AnyConnection, user: UserId, space: SpaceId) -> Result, MetaError> { + let row = sqlx::query( + "SELECT s.kind, s.owner, s.archived_ms, m.role, m.accepted_ms, u.disabled_ms, u.removed_ms \ + FROM spaces s JOIN users u ON u.id = $1 LEFT JOIN space_members m ON m.space_id = s.id AND m.user_id = u.id \ + WHERE s.id = $2", + ) + .bind(uuid_bytes(&user.0)) + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *conn) + .await?; + let Some(row) = row else { return Ok(None) }; + if row.try_get::, _>("disabled_ms")?.is_some() || row.try_get::, _>("removed_ms")?.is_some() { + return Ok(None); + } + match row.try_get::("kind")?.as_str() { + "personal" => { + let owner: Option> = row.try_get("owner")?; + Ok(owner.as_deref().map(uuid_from).transpose()?.filter(|owner| *owner == user.0).map(|_| SpaceRole::Owner)) + } + "team" => { + let role: Option = row.try_get("role")?; + let accepted: Option = row.try_get("accepted_ms")?; + let (Some(role), Some(_)) = (role, accepted) else { return Ok(None) }; + let role = parse_role(&role)?; + let archived = row.try_get::, _>("archived_ms")?.is_some(); + Ok(Some(if archived { role.min(SpaceRole::Viewer) } else { role })) + } + _ => Err(MetaError::Corrupt("unknown space kind")), + } +} + +/// The account that acts, for a row that names who invited: none for the command line. +fn actor_user(actor: &SpaceActor) -> Option { + match actor { + SpaceActor::User { user, .. } => Some(*user), + SpaceActor::Admin(admin) => Some(admin.user), + SpaceActor::Cli => None, + } +} + +/// Whether `user` exists and can sign in (not disabled, not marked for removal), read after its row was locked. +async fn user_can_join(w: &mut WriteTx, user: UserId) -> Result { + let found = sqlx::query("SELECT 1 FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut *w.tx).await?; + Ok(found.is_some() && !account_blocked(&mut w.tx, user).await?) +} + +async fn member_row(w: &mut WriteTx, space: SpaceId, user: UserId) -> Result, MetaError> { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("{MEMBER_SELECT} WHERE m.space_id = $1 AND m.user_id = $2"))) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + row.as_ref().map(member_from).transpose() +} + +async fn insert_member( + w: &mut WriteTx, + space: SpaceId, + user: UserId, + role: SpaceRole, + invited_by: Option, + now_ms: i64, + accepted: bool, +) -> Result<(), MetaError> { + sqlx::query("INSERT INTO space_members (space_id, user_id, role, invited_ms, invited_by, accepted_ms) VALUES ($1, $2, $3, $4, $5, $6)") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .bind(role.as_str()) + .bind(now_ms) + .bind(invited_by.map(|by| uuid_bytes(&by.0))) + .bind(accepted.then_some(now_ms)) + .execute(&mut *w.tx) + .await?; + Ok(()) +} + +async fn delete_member(w: &mut WriteTx, space: SpaceId, user: UserId) -> Result<(), MetaError> { + sqlx::query("DELETE FROM space_members WHERE space_id = $1 AND user_id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + Ok(()) +} + +/// Refuses with [`MetaError::LastOwner`] unless the space has an accepted owner besides the one the change takes away. +/// Called with the space's row locked, for a change to an accepted owner's row. +async fn keep_an_owner(w: &mut WriteTx, space: SpaceId) -> Result<(), MetaError> { + let owners: i64 = sqlx::query("SELECT COUNT(*) AS n FROM space_members WHERE space_id = $1 AND role = 'owner' AND accepted_ms IS NOT NULL") + .bind(uuid_bytes(space.as_uuid())) + .fetch_one(&mut *w.tx) + .await? + .try_get("n")?; + if owners < 2 { + return Err(MetaError::LastOwner); + } + Ok(()) +} diff --git a/crates/meta/src/trash.rs b/crates/meta/src/trash.rs new file mode 100644 index 00000000..f05c3a16 --- /dev/null +++ b/crates/meta/src/trash.rs @@ -0,0 +1,174 @@ +//! The trash: the deleted nodes of a space, and the request to remove one of them for good. +//! +//! Deleting stays what it always was, a tombstone in the change feed. The trash is that same set of tombstones seen +//! with the time of the deletion. Bringing a node back is [`Meta::restore_from_trash`] in `commit.rs`, because it +//! shares the commit path's rules for names. Removing a node from the trash here only marks it: nothing is freed, +//! so versions, chunks and the tombstone stay as they were. +//! +//! A node is in the trash when it is deleted and its removal was not asked for. What is below a folder in the trash +//! is listed only through the folder (the folder's restore and removal take it along), whenever it was deleted. + +use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; +use craft_cloud_proto::limits::MAX_TREE_DEPTH; +use craft_cloud_proto::{NodeId, SpaceId}; +use sqlx::Row; + +use crate::codec::{flag, to_u64, uuid_bytes, uuid_from}; +use crate::tree::space_info; +use crate::{Dialect, Meta, MetaError}; + +/// A deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashEntry { + pub id: NodeId, + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + pub size: Option, + pub deleted_ms: i64, +} + +/// One page of the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashPage { + pub entries: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, + /// How many deleted nodes without a deletion time were found and stamped with `now` (a damaged database: the + /// migration and the commit path always set it). They are never treated as expired. + pub repaired: u64, +} + +/// The deleted nodes below the node `$2` of space `$1` that are in the trash, found through nodes that are in the trash +/// themselves (a folder that was removed from the trash hides what is below it); `depth` is 1 for a child. `$3` bounds +/// the depth. A statement continues after it with `$4` and up. +pub(crate) const TRASH_BELOW: &str = "WITH RECURSIVE sub(id, depth) AS (\ + SELECT id, 1 FROM nodes WHERE space_id = $1 AND parent = $2 AND deleted = 1 AND purge_requested_ms IS NULL \ + UNION ALL \ + SELECT n.id, sub.depth + 1 FROM nodes n JOIN sub ON n.parent = sub.id \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND sub.depth < $3) "; + +impl Meta { + /// One page of the trash of `space`: its deleted nodes that were not removed from the trash, newest deletion + /// first by `(deleted_ms, id)`, after `after` (exclusive). `Invalid` for a `limit` outside 1 to + /// [`TRASH_MAX_LIMIT`]. A deleted node without a deletion time gets `now_ms` first, so the trash never lists a + /// node as older than it can be shown to be. + pub async fn trash(&self, space: SpaceId, after: Option, limit: u32, now_ms: i64) -> Result { + if limit == 0 || limit > TRASH_MAX_LIMIT { + return Err(MetaError::Invalid("a trash page has 1 to 200 items")); + } + let repaired = self.stamp_unstamped(space, now_ms).await?; + let mut tx = self.read_snapshot().await?; + let (cursor_ms, cursor_node) = match after { + Some(cursor) => (cursor.deleted_ms, uuid_bytes(cursor.node.as_uuid())), + // Above every time and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The row-value comparison and the order + // match the index `nodes_trash`. + let rows = sqlx::query( + "SELECT n.id, n.parent, n.name, n.is_dir, n.deleted_ms, v.size AS v_size \ + FROM nodes n LEFT JOIN versions v ON v.space_id = n.space_id AND v.id = n.version \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND (n.deleted_ms, n.id) < ($2, $3) \ + AND NOT EXISTS (SELECT 1 FROM nodes p WHERE p.space_id = n.space_id AND p.id = n.parent AND p.deleted = 1 AND p.purge_requested_ms IS NULL) \ + ORDER BY n.deleted_ms DESC, n.id DESC LIMIT $4", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(cursor_ms) + .bind(cursor_node) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut entries = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent")?; + let size: Option = row.try_get("v_size")?; + entries.push(TrashEntry { + id: NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent: NodeId::from_uuid(uuid_from(&parent.ok_or(MetaError::Corrupt("a deleted node has no parent"))?)?), + name: row.try_get("name")?, + is_dir: flag(row, "is_dir")?, + size: size.map(to_u64).transpose()?, + deleted_ms: row.try_get("deleted_ms")?, + }); + } + let next = if more { entries.last().map(|e| TrashCursor { deleted_ms: e.deleted_ms, node: e.id }) } else { None }; + Ok(TrashPage { entries, next, repaired }) + } + + /// Gives every deleted node of `space` that has no deletion time `now_ms`; returns how many. Looks first without + /// the write lock, since there are none in a sound database. + async fn stamp_unstamped(&self, space: SpaceId, now_ms: i64) -> Result { + let space_key = uuid_bytes(space.as_uuid()); + let any = sqlx::query("SELECT 1 FROM nodes WHERE space_id = $1 AND deleted = 1 AND deleted_ms IS NULL LIMIT 1") + .bind(space_key.clone()) + .fetch_optional(&self.pool) + .await?; + if any.is_none() { + return Ok(0); + } + let mut w = self.begin_write().await?; + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let done = sqlx::query("UPDATE nodes SET deleted_ms = $1 WHERE space_id = $2 AND deleted = 1 AND deleted_ms IS NULL") + .bind(now_ms) + .bind(space_key) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(done.rows_affected()) + } + + /// Takes the deleted node `node` out of the trash: it is no longer listed and can't be restored from the trash. + /// Only marks it (`purge_requested_ms`): no version, chunk or tombstone is touched, so the change feed and the + /// sequence number stay as they are. Asking again changes nothing. `NotFound` when `node` is not a deleted node of + /// `space` in the trash. Removing a folder marks the nodes in the trash below it too, whenever they were deleted: at + /// most [`crate::MAX_TRASH_TREE_NODES`] of them (see [`Meta::with_trash_tree_cap`]), shallowest first, in the order of a + /// restore. The rest is no longer below a folder in the trash, so it is listed and can be removed by a further call. + pub async fn request_purge(&self, space: SpaceId, node: NodeId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the mark with commits and restores of the space. + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + self.check_guard(&mut w.tx, space).await?; + let row = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("node in the trash"))?; + if !flag(&row, "deleted")? { + return Err(MetaError::NotFound("node in the trash")); + } + if row.try_get::, _>("purge_requested_ms")?.is_some() { + return Ok(()); + } + if flag(&row, "is_dir")? { + // What is in the trash below the folder goes with it, so it does not stay hidden and unmarked: up to the cap, + // shallowest first (so a marked node's folder is always marked too). Marking is bounded by the cap under + // the write lock. What is left is listed once the folder is marked. + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let cap = i64::try_from(self.trash_tree_cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; + sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} UPDATE nodes SET purge_requested_ms = $4 WHERE space_id = $1 AND id IN ( SELECT n.id FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $5)" + ))) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(bound) + .bind(now_ms) + .bind(cap) + .execute(&mut *w.tx) + .await?; + } + sqlx::query("UPDATE nodes SET purge_requested_ms = $1 WHERE space_id = $2 AND id = $3") + .bind(now_ms) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(()) + } +} diff --git a/crates/meta/src/tree.rs b/crates/meta/src/tree.rs index c5b30d05..47f0f9a6 100644 --- a/crates/meta/src/tree.rs +++ b/crates/meta/src/tree.rs @@ -77,7 +77,7 @@ fn node_state(row: &AnyRow) -> Result { impl Meta { /// Starts a read transaction that sees one consistent snapshot. On Postgres the isolation level has to be /// set as the transaction's first statement. SQLite read transactions are already snapshots. - async fn read_snapshot(&self) -> Result, MetaError> { + pub(crate) async fn read_snapshot(&self) -> Result, MetaError> { let mut tx = self.pool.begin().await?; if self.dialect == crate::Dialect::Postgres { sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY").execute(&mut *tx).await?; diff --git a/crates/meta/src/two_factor.rs b/crates/meta/src/two_factor.rs new file mode 100644 index 00000000..88664d0e --- /dev/null +++ b/crates/meta/src/two_factor.rs @@ -0,0 +1,967 @@ +//! Two-factor sign-in: the TOTP secret (stored encrypted, the server's `secretbox` makes and opens the ciphertext), the +//! recovery codes (stored as BLAKE3 hashes), the lock after wrong codes, and the challenges of a password sign-in that +//! still needs its second factor. +//! +//! - **One transaction contract for everything that issues a device or consumes a proof.** The account's row is locked +//! first (`users::lock_user`, the order every other transaction on the account uses), then what the caller checked +//! before is read again under the lock (the device still signed in, the password still the one that was verified, +//! the account's current two-factor state, the challenge), and only then is a challenge or a device made. A second +//! factor and the challenge or invite it opens are used up in the same transaction as the device they let in: the +//! proof is never spent by one commit and followed by another. +//! - **Proofs are single use.** A TOTP code is accepted for a time step greater than the last accepted one +//! (`last_accepted_step`, updated with `WHERE last_accepted_step < step`), so two requests with one code can't both +//! succeed, and a second confirmation in the same 30-second step needs the next code. Sign-in and the admin's +//! confirmations share that counter. A recovery code is a row updated once (`WHERE used_ms IS NULL`). +//! - **Wrong proofs are counted in the transaction that refused them, under the account's lock.** Every check runs before +//! anything is consumed, so the refusing transaction commits only the counters (the invite stays, the challenge stays, +//! no device), and the next request, which waited for the lock, sees the attempts and the lock this one made. After +//! [`LOCK_AFTER_FAILURES`] wrong proofs in a +//! row the second factor is locked for [`LOCK_BASE_MS`], and each wrong proof after that doubles the time up to +//! [`LOCK_MAX_MS`]. A lock is never for good. While it lasts every check answers [`MetaError::TwoFactorLocked`] +//! without counting. +//! - **Lock order** inside a transaction: the account's row, then its two-factor row, then challenges (the failed-proof +//! write takes the last two in that order and never the account's row, so it can't deadlock with them). +//! - **Turning it on or off, and a reset, end what an earlier check let through**: `users.auth_gen` goes up, every +//! open challenge, invite and pairing of the account is deleted, and the devices are signed out (all of them for a +//! disable made by someone else, a reset or the command line; all but the asking device for the account's own). + +use craft_cloud_proto::DeviceId; +use sqlx::Row; + +use crate::audit::{AuditAction, AuditDetails, AuditFailure, NewEvent, Step as AuditStep, record_in}; +use crate::codec::{uuid_bytes, uuid_from}; +use crate::users::{DeviceClient, account_blocked, check_device_name, delete_join_codes, insert_device, lock_user, revoke_devices}; +use crate::{Meta, MetaError, RevokeReason, UserId, WriteTx}; + +/// How long a set-up stays open: ten minutes. +pub const SETUP_TTL_MS: i64 = 10 * 60 * 1000; +/// Wrong codes one set-up takes before it is used up. +pub const MAX_SETUP_ATTEMPTS: i64 = 5; +/// How long a challenge works: five minutes. +pub const CHALLENGE_TTL_MS: i64 = 5 * 60 * 1000; +/// Wrong codes one challenge takes before it is deleted. +pub const MAX_CHALLENGE_ATTEMPTS: i64 = 5; +/// Open challenges one account may have; making another replaces the oldest. +pub const MAX_OPEN_CHALLENGES: i64 = 5; +/// Wrong proofs in a row that lock the second factor. +pub const LOCK_AFTER_FAILURES: i64 = 10; +/// The first lock: 15 minutes. Every wrong proof after a lock ran out doubles it… +pub const LOCK_BASE_MS: i64 = 15 * 60 * 1000; +/// …up to 24 hours. +pub const LOCK_MAX_MS: i64 = 24 * 60 * 60 * 1000; +/// Recovery codes made at once. +pub const RECOVERY_CODES: usize = 10; + +/// A stored secret that the caller's key could not open (the wrong key, a damaged value, another account's). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnreadableSecret; + +/// Checks a TOTP code against an account's stored secret (the ciphertext, which only the caller can open): the time step +/// of the code, if it is right for a step in the window, else `None`. Called with the transaction open, on the +/// ciphertext that is current under the account's lock. +pub type TotpVerifier<'a> = dyn Fn(UserId, &[u8]) -> Result, UnreadableSecret> + Send + Sync + 'a; + +/// Rewrites one stored ciphertext of an account: the one to store in its place, or `None` to leave it ([`Meta::two_factor_rewrite`]). +pub type Rewrite<'a> = dyn Fn(UserId, &[u8]) -> Result>, UnreadableSecret> + Send + Sync + 'a; + +/// A second factor a caller presents. +pub enum Proof<'a> { + /// A TOTP code, as the check that knows it. + Totp(&'a TotpVerifier<'a>), + /// The BLAKE3 hash of a normalised recovery code. + Recovery([u8; 32]), +} + +/// The signed-in device that asks for a change to its account's two-factor sign-in, as the handler knew it when it +/// verified the password: all of it is checked again under the account's lock. +#[derive(Clone, PartialEq, Eq)] +pub struct DeviceActor { + pub user: UserId, + /// The device the request came with; it must still be signed in. + pub device: DeviceId, + /// The stored password hash the password was verified against; it must still be the account's. + pub password_hash: String, +} + +impl std::fmt::Debug for DeviceActor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceActor").field("user", &self.user).field("device", &self.device).field("password_hash", &"").finish() + } +} + +/// What `GET /account/two-factor` tells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TwoFactorInfo { + pub enabled_ms: Option, + pub recovery_codes_left: u32, + /// A set-up is open (not expired, not used up). + pub pending: bool, +} + +/// A new challenge for a password sign-in that needs a second factor. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewChallenge { + /// The BLAKE3 hash of the challenge token: all the server keeps of it. + pub id_hash: [u8; 32], + /// The client address the challenge was made for. Only an additional binding: behind a proxy that hides addresses + /// every client has the same one, so the secret token is what binds the challenge. + pub client_key: String, +} + +/// The result of a password sign-in that checked out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SignIn { + /// The account has no second factor: the device is made. + Device(DeviceId), + /// The account needs its second factor: the challenge is made and expires at this time. No device. + Challenge { expires_ms: i64 }, +} + +/// What [`Meta::complete_second_factor`] made. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Completed { + pub user: UserId, + pub device: DeviceId, +} + +/// A stored secret for the key tools: the ciphertexts of one account. Its `Debug` output shows only their lengths. +#[derive(Clone, PartialEq, Eq)] +pub struct StoredSecrets { + pub user: UserId, + /// The ciphertext of the active secret, if two-factor sign-in is on. + pub active: Option>, + /// The ciphertext of an open set-up's secret, if there is one that is not expired. + pub pending: Option>, +} + +impl std::fmt::Debug for StoredSecrets { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StoredSecrets") + .field("user", &self.user) + .field("active", &self.active.as_ref().map(|sealed| format!("<{} bytes>", sealed.len()))) + .field("pending", &self.pending.as_ref().map(|sealed| format!("<{} bytes>", sealed.len()))) + .finish() + } +} + +/// The row of `user_two_factor`. +pub(crate) struct TfRow { + enabled_ms: Option, + secret: Option>, + pending: Option>, + pending_device: Option>, + pending_expires_ms: Option, + last_step: i64, + locked_until_ms: Option, +} + +impl TfRow { + /// Two-factor sign-in is on. + pub(crate) fn active(&self) -> bool { + self.enabled_ms.is_some() && self.secret.is_some() + } +} + +pub(crate) async fn load_row(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result, MetaError> { + let row = sqlx::query( + "SELECT enabled_ms, secret_ciphertext, pending_ciphertext, pending_device, pending_expires_ms, last_accepted_step, \ + locked_until_ms FROM user_two_factor WHERE user_id = $1", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut **tx) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some(TfRow { + enabled_ms: row.try_get("enabled_ms")?, + secret: row.try_get("secret_ciphertext")?, + pending: row.try_get("pending_ciphertext")?, + pending_device: row.try_get("pending_device")?, + pending_expires_ms: row.try_get("pending_expires_ms")?, + last_step: row.try_get("last_accepted_step")?, + locked_until_ms: row.try_get("locked_until_ms")?, + })) +} + +/// Whether two-factor sign-in is on for `user`, read in `tx`. +pub(crate) async fn is_enabled(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + Ok(load_row(tx, user).await?.is_some_and(|row| row.active())) +} + +/// How long the second factor stays locked after the `failures`-th wrong proof in a row (`None`: no lock yet). +fn lock_for(failures: i64) -> Option { + if failures < LOCK_AFTER_FAILURES { + return None; + } + let doublings = u32::try_from(failures.saturating_sub(LOCK_AFTER_FAILURES)).unwrap_or(u32::MAX).min(16); + Some(LOCK_BASE_MS.saturating_mul(1i64 << doublings).min(LOCK_MAX_MS)) +} + +/// Seconds a client should wait before trying again, for a `Retry-After` header: rounded up, at least one. +pub fn retry_after_secs(retry_after_ms: i64) -> u32 { + u32::try_from((retry_after_ms.max(0) + 999) / 1000).unwrap_or(u32::MAX).max(1) +} + +pub(crate) enum Checked { + Accepted, + Wrong, +} + +/// Checks `proof` against the account's two-factor row `row` and, when it is right, uses it up and clears the wrong-proof +/// counters, all in `tx` (the account's row is locked). A lock that is in force is [`MetaError::TwoFactorLocked`] and +/// nothing is checked or counted. A recovery code that is used up writes `auth.recovery_code_used` in `tx` (a code that +/// is not used up writes nothing); `device` is the device it was used from, where there is one. +pub(crate) async fn check_proof( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + device: Option, + web: bool, + row: &TfRow, + proof: &Proof<'_>, + now_ms: i64, +) -> Result { + if let Some(until) = row.locked_until_ms + && until > now_ms + { + return Err(MetaError::TwoFactorLocked { retry_after_ms: until - now_ms }); + } + match proof { + Proof::Totp(verify) => { + let secret = row.secret.as_deref().ok_or(MetaError::Corrupt("a two-factor row has no secret"))?; + let step = verify(user, secret).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))?; + // A step that was accepted before, or an older one, is a replay. + let Some(step) = step.filter(|step| *step > row.last_step) else { return Ok(Checked::Wrong) }; + let done = sqlx::query( + "UPDATE user_two_factor SET last_accepted_step = $1, failed_attempts = 0, locked_until_ms = NULL \ + WHERE user_id = $2 AND last_accepted_step < $1", + ) + .bind(step) + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + Ok(if done.rows_affected() == 1 { Checked::Accepted } else { Checked::Wrong }) + } + Proof::Recovery(hash) => { + let done = sqlx::query("UPDATE user_recovery_codes SET used_ms = $1 WHERE user_id = $2 AND code_hash = $3 AND used_ms IS NULL") + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .bind(hash.to_vec()) + .execute(&mut **tx) + .await?; + if done.rows_affected() != 1 { + return Ok(Checked::Wrong); + } + sqlx::query("UPDATE user_two_factor SET failed_attempts = 0, locked_until_ms = NULL WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + let event = NewEvent::new(AuditAction::AuthRecoveryCodeUsed).by(user, device).on_user(user); + let event = if web { event.with(AuditDetails::web()) } else { event }; + record_in(tx, now_ms, event).await?; + Ok(Checked::Accepted) + } + } +} + +/// Deletes the account's two-factor secret, recovery codes and open challenges, and raises `auth_gen`, in `tx`. Whether +/// there was anything to delete is the answer. +pub(crate) async fn remove_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + sqlx::query("DELETE FROM user_recovery_codes WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + let removed = sqlx::query("DELETE FROM user_two_factor WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("UPDATE users SET auth_gen = auth_gen + 1 WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(removed.rows_affected() > 0) +} + +/// The account's two-factor sign-in removed by someone else (the command line, an admin): the secret, the codes and the +/// challenges go, every device is signed out ([`RevokeReason::TwoFactor`]) and the invite and pairings are deleted. The +/// devices signed out are returned, and the answer is `None` when the account had none (nothing is changed then). +pub(crate) async fn reset_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, now_ms: i64) -> Result>, MetaError> { + if load_row(tx, user).await?.is_none() { + return Ok(None); + } + remove_in(tx, user).await?; + let revoked = revoke_devices(tx, user, None, RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(tx, user).await?; + Ok(Some(revoked)) +} + +/// Checks the device and the password a handler verified against what is stored now, under the account's lock. +async fn check_actor(w: &mut WriteTx, actor: &DeviceActor) -> Result<(), MetaError> { + if !lock_user(&mut w.tx, actor.user).await? { + return Err(MetaError::DeviceRevoked); + } + // The device must still be signed in; the no-op update locks its row too, so a revoke running now waits. + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(actor.device.as_uuid())) + .bind(uuid_bytes(&actor.user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, actor.user).await? { + return Err(MetaError::DeviceRevoked); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2") + .bind(uuid_bytes(&actor.user.0)) + .bind(actor.password_hash.as_str()) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Err(MetaError::Conflict("the password changed while this request was running; try again")); + } + Ok(()) +} + +impl Meta { + /// Whether `user` has two-factor sign-in on. + pub async fn two_factor_enabled(&self, user: UserId) -> Result { + let row = sqlx::query("SELECT 1 FROM user_two_factor WHERE user_id = $1 AND enabled_ms IS NOT NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + Ok(row.is_some()) + } + + /// How many milliseconds `user`'s second factor stays locked after `now_ms` (`None`: it is not locked, or is off). A plain + /// read, for a caller that wants to say "locked" rather than "too many tries". + pub async fn two_factor_locked_for(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let row = sqlx::query("SELECT locked_until_ms FROM user_two_factor WHERE user_id = $1 AND enabled_ms IS NOT NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + let until: Option = row.map(|row| row.try_get("locked_until_ms")).transpose()?.flatten(); + Ok(until.filter(|until| *until > now_ms).map(|until| until - now_ms)) + } + + /// What `GET /account/two-factor` tells about `user`. + pub async fn two_factor_info(&self, user: UserId, now_ms: i64) -> Result { + let row = sqlx::query("SELECT enabled_ms, pending_ciphertext, pending_expires_ms FROM user_two_factor WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(TwoFactorInfo { enabled_ms: None, recovery_codes_left: 0, pending: false }) }; + let enabled_ms: Option = row.try_get("enabled_ms")?; + let pending = row.try_get::>, _>("pending_ciphertext")?.is_some() + && row.try_get::, _>("pending_expires_ms")?.is_some_and(|expires| expires > now_ms); + let left = if enabled_ms.is_some() { + let n: i64 = sqlx::query("SELECT COUNT(*) AS n FROM user_recovery_codes WHERE user_id = $1 AND used_ms IS NULL") + .bind(uuid_bytes(&user.0)) + .fetch_one(&self.pool) + .await? + .try_get("n")?; + u32::try_from(n).unwrap_or(u32::MAX) + } else { + 0 + }; + Ok(TwoFactorInfo { enabled_ms, recovery_codes_left: left, pending }) + } + + /// Starts a set-up for `actor`'s account: `pending_ciphertext` (the new secret, sealed by the caller) is kept for + /// [`SETUP_TTL_MS`] and bound to the asking device, and its expiry is returned. Only while two-factor sign-in is off + /// (`Conflict` otherwise). A set-up that is open on another device is [`MetaError::TwoFactorPending`]; the same + /// device may start again, which replaces the secret and clears its wrong codes. A device that was signed out is + /// [`MetaError::DeviceRevoked`], a password that changed `Conflict`. + pub async fn two_factor_begin_setup(&self, actor: &DeviceActor, pending_ciphertext: &[u8], now_ms: i64) -> Result { + let expires_ms = now_ms.checked_add(SETUP_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + if let Some(row) = load_row(&mut w.tx, actor.user).await? { + if row.active() { + return Err(MetaError::Conflict("two-factor sign-in is on already; turn it off first")); + } + let open = row.pending.is_some() && row.pending_expires_ms.is_some_and(|expires| expires > now_ms); + if open && row.pending_device.as_deref() != Some(actor.device.as_uuid().as_bytes().as_slice()) { + return Err(MetaError::TwoFactorPending); + } + } + sqlx::query( + "INSERT INTO user_two_factor (user_id, pending_ciphertext, pending_device, pending_expires_ms, pending_attempts) \ + VALUES ($1, $2, $3, $4, 0) \ + ON CONFLICT (user_id) DO UPDATE SET pending_ciphertext = $2, pending_device = $3, pending_expires_ms = $4, pending_attempts = 0", + ) + .bind(uuid_bytes(&actor.user.0)) + .bind(pending_ciphertext.to_vec()) + .bind(uuid_bytes(actor.device.as_uuid())) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(expires_ms) + } + + /// Turns two-factor sign-in on with the first code of the set-up that `device` of `user` started: the code must be + /// right for the open set-up's secret (at most [`MAX_SETUP_ATTEMPTS`] wrong ones, then the set-up is used up). In one + /// transaction the pending secret becomes the active one, the account's recovery codes become `recovery_hashes` + /// (`BLAKE3` hashes of the codes the caller shows once), `auth_gen` goes up, every other device is signed out + /// ([`RevokeReason::TwoFactor`]) and the account's invite, pairings and challenges are deleted. Two confirmations + /// at once turn it on once. Returns the devices signed out. No open set-up, one that was started on another device + /// (the code is then not tried) or one that ran out is `Conflict`; a signed-out device [`MetaError::DeviceRevoked`]; + /// a wrong code [`MetaError::TwoFactorInvalid`]. + pub async fn two_factor_confirm( + &self, + user: UserId, + device: DeviceId, + verify: &TotpVerifier<'_>, + recovery_hashes: &[[u8; 32]], + now_ms: i64, + ) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let row = load_row(&mut w.tx, user).await?; + let Some(row) = row.filter(|row| !row.active()) else { + return Err(MetaError::Conflict("there is no two-factor set-up to confirm: start one first")); + }; + let open = row.pending.is_some() && row.pending_expires_ms.is_some_and(|expires| expires > now_ms); + if !open { + return Err(MetaError::Conflict("the two-factor set-up ran out: start it again")); + } + if row.pending_device.as_deref() != Some(device.as_uuid().as_bytes().as_slice()) { + return Err(MetaError::Conflict("this two-factor set-up was started on another device: confirm it there, or start it again here")); + } + let pending = row.pending.as_deref().ok_or(MetaError::Corrupt("an open set-up has no secret"))?; + let step = verify(user, pending).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))?; + let Some(step) = step else { + // Nothing was changed yet: the wrong code is counted and committed in this transaction, under the lock. + count_wrong_setup_in(&mut w.tx, user, device).await?; + w.commit().await?; + return Err(MetaError::TwoFactorInvalid); + }; + let done = sqlx::query( + "UPDATE user_two_factor SET secret_ciphertext = pending_ciphertext, enabled_ms = $1, pending_ciphertext = NULL, pending_device = NULL, \ + pending_expires_ms = NULL, pending_attempts = 0, last_accepted_step = $2, failed_attempts = 0, locked_until_ms = NULL \ + WHERE user_id = $3 AND enabled_ms IS NULL AND pending_ciphertext IS NOT NULL", + ) + .bind(now_ms) + .bind(step) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() != 1 { + return Err(MetaError::Conflict("two-factor sign-in changed while this request was running; try again")); + } + replace_recovery_codes(&mut w.tx, user, recovery_hashes).await?; + sqlx::query("UPDATE users SET auth_gen = auth_gen + 1 WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + let revoked = revoke_devices(&mut w.tx, user, Some(device), RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AccountTwoFactorEnabled).by(user, Some(device)).on_user(user)).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Turns `actor`'s two-factor sign-in off with `proof` (a code or a recovery code): in one transaction the secret, + /// the codes and the challenges go, `auth_gen` goes up, every other device is signed out ([`RevokeReason::TwoFactor`]) + /// and the invite and pairings are deleted. Returns the devices signed out. Off already is `Conflict`; a wrong + /// proof [`MetaError::TwoFactorInvalid`] (counted, and it can lock); a lock in force [`MetaError::TwoFactorLocked`]. + pub async fn two_factor_disable(&self, actor: &DeviceActor, proof: &Proof<'_>, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + let row = load_row(&mut w.tx, actor.user).await?.filter(TfRow::active); + let Some(row) = row else { return Err(MetaError::Conflict("two-factor sign-in is not on")) }; + if let Checked::Wrong = check_proof(&mut w.tx, actor.user, Some(actor.device), false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, actor.user, None, now_ms, false).await); + } + remove_in(&mut w.tx, actor.user).await?; + let revoked = revoke_devices(&mut w.tx, actor.user, Some(actor.device), RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(&mut w.tx, actor.user).await?; + let event = NewEvent::new(AuditAction::AccountTwoFactorDisabled).by(actor.user, Some(actor.device)).on_user(actor.user); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Replaces `actor`'s recovery codes with `recovery_hashes` (the old ones stop working at once), after `proof`. Off + /// is `Conflict`; the proof's errors are as for [`Meta::two_factor_disable`]. + pub async fn two_factor_new_recovery_codes( + &self, + actor: &DeviceActor, + proof: &Proof<'_>, + recovery_hashes: &[[u8; 32]], + now_ms: i64, + ) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + let row = load_row(&mut w.tx, actor.user).await?.filter(TfRow::active); + let Some(row) = row else { return Err(MetaError::Conflict("two-factor sign-in is not on")) }; + if let Checked::Wrong = check_proof(&mut w.tx, actor.user, Some(actor.device), false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, actor.user, None, now_ms, false).await); + } + replace_recovery_codes(&mut w.tx, actor.user, recovery_hashes).await?; + let event = NewEvent::new(AuditAction::AccountRecoveryCodesRegenerated).by(actor.user, Some(actor.device)).on_user(actor.user); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await + } + + /// The confirmation an admin gives for a change: when `user` has two-factor sign-in on, `proof` is checked and used up + /// (the time step or recovery code can't serve a second change), and the answer is `true`; when it is off, nothing is + /// checked and the answer is `false`. Whether it is on is decided under the account's lock. A missing proof for an + /// account that has it on is [`MetaError::TwoFactorInvalid`] (not counted: nothing was tried); the other errors are as + /// for [`Meta::two_factor_disable`]. + pub async fn confirm_second_factor(&self, user: UserId, proof: Option<&Proof<'_>>, now_ms: i64) -> Result { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let Some(row) = load_row(&mut w.tx, user).await?.filter(TfRow::active) else { return Ok(false) }; + let Some(proof) = proof else { return Err(MetaError::TwoFactorInvalid) }; + if let Checked::Wrong = check_proof(&mut w.tx, user, None, false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, user, None, now_ms, false).await); + } + // The proof is used up here, in a transaction of its own that commits before the change it confirms: a recovery + // code used for a change that then fails is still used, and its event stays. + self.commit_audited(w, now_ms).await?; + Ok(true) + } + + /// Takes away `user`'s two-factor sign-in on the command line: the secret, the codes and the challenges go, the lock is + /// gone with the row, every device is signed out ([`RevokeReason::TwoFactor`]) and the invite and pairings are + /// deleted. Returns the devices signed out. `NotFound` for a user that isn't there, `Conflict` for one without + /// two-factor sign-in (nothing is signed out then). + pub async fn reset_two_factor(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("user")); + } + let revoked = reset_in(&mut w.tx, user, now_ms).await?.ok_or(MetaError::Conflict("this account has no two-factor sign-in"))?; + let event = NewEvent::new(AuditAction::CliTwoFactorReset).on_user(user).with(AuditDetails::revoked(revoked.len())); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// A password sign-in that checked out, as [`Meta::sign_in_device`], for an account that may have two-factor sign-in: + /// under the account's lock the password hash and `auth_gen` the caller read before verifying the password are + /// compared again, then the account's **current** state decides. Without two-factor sign-in the device is made + /// ([`SignIn::Device`]); with it only `challenge` is made ([`SignIn::Challenge`], good for [`CHALLENGE_TTL_MS`]; the + /// account's expired challenges are deleted and the oldest goes when it would have more than + /// [`MAX_OPEN_CHALLENGES`]) and no device. `None`, a disabled account and the errors are as for `sign_in_device`. + #[allow(clippy::too_many_arguments)] + pub async fn sign_in( + &self, + user: UserId, + checked: (&str, i64), + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + challenge: &NewChallenge, + now_ms: i64, + ) -> Result, MetaError> { + check_device_name(name)?; + client.check()?; + let (hash, auth_gen) = checked; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Ok(None); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + if is_enabled(&mut w.tx, user).await? { + let expires_ms = now_ms.checked_add(CHALLENGE_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + sqlx::query("DELETE FROM login_challenges WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; + let open: i64 = sqlx::query("SELECT COUNT(*) AS n FROM login_challenges WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_one(&mut *w.tx) + .await? + .try_get("n")?; + if open >= MAX_OPEN_CHALLENGES { + sqlx::query( + "DELETE FROM login_challenges WHERE id_hash IN \ + (SELECT id_hash FROM login_challenges WHERE user_id = $1 ORDER BY expires_ms ASC, id_hash ASC LIMIT $2)", + ) + .bind(uuid_bytes(&user.0)) + .bind(open - MAX_OPEN_CHALLENGES + 1) + .execute(&mut *w.tx) + .await?; + } + sqlx::query( + "INSERT INTO login_challenges (id_hash, user_id, kind, client_key, device_name, client_kind, platform, app_version, attempts, expires_ms) \ + VALUES ($1, $2, 'login', $3, $4, $5, $6, $7, 0, $8)", + ) + .bind(challenge.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(challenge.client_key.as_str()) + .bind(name) + .bind(client.kind.as_str()) + .bind(client.platform.as_str()) + .bind(client.app_version.as_str()) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + return Ok(Some(SignIn::Challenge { expires_ms })); + } + let device = DeviceId::generate(); + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; + // A sign-in that made a device is a login; a challenge (above) is not: nobody is signed in yet. + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(SignIn::Device(device))) + } + + /// The account a challenge was made for and its username, by a plain read: only so the caller can count the try against + /// the account before the real check. `None` for a challenge that is unknown, expired or used. Only the challenges of a + /// device sign-in count: a web sign-in's ([`Meta::web_challenge_owner`]) can never make a device. + pub async fn challenge_owner(&self, id_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT c.user_id, u.username FROM login_challenges c JOIN users u ON u.id = c.user_id WHERE c.id_hash = $1 AND c.kind = 'login' AND c.expires_ms > $2", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some((UserId(uuid_from(&row.try_get::, _>("user_id")?)?), row.try_get("username")?))) + } + + /// Finishes a sign-in with the second factor: the account is found by a plain read of the challenge, its row is locked + /// **before** the challenge or the factor is touched, and under the lock the challenge (still there, not expired) and + /// the account's **current** state (two-factor sign-in still on, not disabled) are checked again. Only then are the + /// proof and the challenge used up and the device made, in this one transaction. A challenge that is unknown, expired, + /// used or no longer fit is [`MetaError::ChallengeGone`], the same for every case; a wrong proof is + /// [`MetaError::TwoFactorInvalid`] after its counters were committed on their own (the challenge's attempts, deleted at + /// the fifth; the account's wrong proofs and the lock); a lock in force is [`MetaError::TwoFactorLocked`]. + pub async fn complete_second_factor( + &self, + challenge: &[u8; 32], + proof: &Proof<'_>, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result { + let Some((user, _)) = self.challenge_owner(challenge, now_ms).await? else { return Err(MetaError::ChallengeGone) }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + let found = sqlx::query( + "SELECT device_name, client_kind, platform, app_version, attempts, expires_ms FROM login_challenges WHERE id_hash = $1 AND user_id = $2 AND kind = 'login'", + ) + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(found) = found.filter(|row| row.try_get::("expires_ms").is_ok_and(|expires| expires > now_ms)) else { + return Err(MetaError::ChallengeGone); + }; + let attempts: i64 = found.try_get("attempts")?; + let row = load_row(&mut w.tx, user).await?.filter(TfRow::active); + let Some(row) = row else { + // Turned off (or reset) since: the challenge is of no use any more. + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + return Err(MetaError::ChallengeGone); + }; + if attempts >= MAX_CHALLENGE_ATTEMPTS { + return Err(MetaError::ChallengeGone); + } + let device = DeviceId::generate(); + if let Checked::Wrong = check_proof(&mut w.tx, user, Some(device), false, &row, proof, now_ms).await? { + return Err(self.refuse_sign_in_proof(w, user, Some(challenge), proof, now_ms, false).await); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + let used = sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND user_id = $2") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if used.rows_affected() != 1 { + return Err(MetaError::ChallengeGone); + } + let client = + DeviceClient { kind: found.try_get("client_kind")?, platform: found.try_get("platform")?, app_version: found.try_get("app_version")? }; + let name: String = found.try_get("device_name")?; + insert_device(&mut w.tx, device, user, &name, &client, token_hash, now_ms).await?; + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(&client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Completed { user, device }) + } + + /// `w` refused a wrong proof and has changed nothing (every caller checks the proof before it consumes anything): the wrong + /// proof's counters are written and **committed in `w` itself**, with the account's row still locked, and the answer is + /// [`MetaError::TwoFactorInvalid`] (or the error that stopped the counters being written). Counting under the same lock as + /// the check means the next request sees the attempts and the lock this one made: no more than five wrong proofs per + /// challenge, and the lock set by exactly the tenth. + pub(crate) async fn refuse_proof(&self, mut w: WriteTx, user: UserId, challenge: Option<&[u8; 32]>, now_ms: i64, web: bool) -> MetaError { + let counted = async { + let locked = count_wrong_in(&mut w.tx, user, challenge, now_ms).await?; + w.commit().await?; + Ok::(locked) + } + .await; + match counted { + Ok(locked) => { + if locked { + // After the counters and the lock are committed, like a failed sign-in's event: an error in the log + // is reported and changes neither the answer nor what was counted (failing closed here would let a + // fault in the log undo the lock). + if let Err(e) = self.audit_lock_set(user, now_ms, web).await { + tracing::warn!(error = %e, "couldn't write an audit event"); + } + } + MetaError::TwoFactorInvalid + } + Err(e) => e, + } + } + + /// `auth.two_factor_locked`, in a transaction of its own. Nobody proved who they were: no actor. + async fn audit_lock_set(&self, user: UserId, now_ms: i64, web: bool) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + let event = NewEvent::new(AuditAction::AuthTwoFactorLocked).on_user(user); + let event = if web { event.with(AuditDetails::web()) } else { event }; + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await + } + + /// [`Meta::refuse_proof`] for a proof given at sign-in (the second step, or a join): after the counters are committed + /// and the answer is decided, the wrong proof is also counted in the audit log, `auth.second_factor_failed`, in a + /// transaction of its own and aggregated per account and minute. Writing it can fail without changing the answer. A + /// wrong proof on a signed-in route (turning it off, new recovery codes, an admin's confirmation) is not written one by + /// one: the lock they can cause is (`auth.two_factor_locked`). + pub(crate) async fn refuse_sign_in_proof( + &self, + w: WriteTx, + user: UserId, + challenge: Option<&[u8; 32]>, + proof: &Proof<'_>, + now_ms: i64, + web: bool, + ) -> MetaError { + let refused = self.refuse_proof(w, user, challenge, now_ms, web).await; + if matches!(refused, MetaError::TwoFactorInvalid) { + let step = match proof { + Proof::Totp(_) => AuditStep::Totp, + Proof::Recovery(_) => AuditStep::Recovery, + }; + let failure = AuditFailure { action: AuditAction::AuthSecondFactorFailed, target: Some(user), step }; + let result = if web { self.audit_record_web_failure(failure, now_ms).await } else { self.audit_record_failure(failure, now_ms).await }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } + } + refused + } + + /// The ciphertexts of every account's secrets (the active ones, and the open set-ups' that are not expired), for the + /// server to open them at start and for the key tools. + pub async fn two_factor_secrets(&self, now_ms: i64) -> Result, MetaError> { + let rows = sqlx::query( + "SELECT user_id, secret_ciphertext, pending_ciphertext, pending_expires_ms FROM user_two_factor \ + WHERE secret_ciphertext IS NOT NULL OR pending_ciphertext IS NOT NULL ORDER BY user_id", + ) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|row| { + let open = row.try_get::, _>("pending_expires_ms")?.is_some_and(|expires| expires > now_ms); + Ok(StoredSecrets { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + active: row.try_get("secret_ciphertext")?, + pending: if open { row.try_get("pending_ciphertext")? } else { None }, + }) + }) + .collect() + } + + /// Every account's ciphertexts, the open set-ups' and the expired ones too (a plain read, for the key tools). + pub async fn two_factor_all_secrets(&self) -> Result, MetaError> { + let rows = sqlx::query( + "SELECT user_id, secret_ciphertext, pending_ciphertext FROM user_two_factor \ + WHERE secret_ciphertext IS NOT NULL OR pending_ciphertext IS NOT NULL ORDER BY user_id", + ) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|row| { + Ok(StoredSecrets { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + active: row.try_get("secret_ciphertext")?, + pending: row.try_get("pending_ciphertext")?, + }) + }) + .collect() + } + + /// Rewrites `user`'s ciphertexts, the active secret and the set-up's both, **read fresh under the account's lock** (a + /// confirmation that moved a set-up's secret to the active one meanwhile is seen): `rewrite` gets each and answers the + /// ciphertext to store in its place, or `None` to leave it. Returns how many were rewritten and how many left. An account + /// that is gone, or has none, is `(0, 0)`; a ciphertext `rewrite` can't handle is an error and changes nothing. + pub async fn two_factor_rewrite(&self, user: UserId, rewrite: &Rewrite<'_>) -> Result<(usize, usize), MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok((0, 0)); + } + let row = sqlx::query("SELECT secret_ciphertext, pending_ciphertext FROM user_two_factor WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(row) = row else { return Ok((0, 0)) }; + let (mut rewritten, mut kept) = (0, 0); + for (column, update) in [ + ("secret_ciphertext", "UPDATE user_two_factor SET secret_ciphertext = $1 WHERE user_id = $2"), + ("pending_ciphertext", "UPDATE user_two_factor SET pending_ciphertext = $1 WHERE user_id = $2"), + ] { + let Some(sealed) = row.try_get::>, _>(column)? else { continue }; + match rewrite(user, &sealed).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))? { + Some(new) => { + sqlx::query(update).bind(new).bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + rewritten += 1; + } + None => kept += 1, + } + } + w.commit().await?; + Ok((rewritten, kept)) + } + + /// Replaces the ciphertext `old` of `user`'s active secret (`pending` false) or open set-up (`pending` true) by `new`, if + /// it still is `old`; `false` when it is not (changed meanwhile). For rewriting secrets under a new key. + pub async fn two_factor_replace_ciphertext(&self, user: UserId, pending: bool, old: &[u8], new: &[u8]) -> Result { + let sql = if pending { + "UPDATE user_two_factor SET pending_ciphertext = $1 WHERE user_id = $2 AND pending_ciphertext = $3" + } else { + "UPDATE user_two_factor SET secret_ciphertext = $1 WHERE user_id = $2 AND secret_ciphertext = $3" + }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(false); + } + let done = sqlx::query(sql).bind(new.to_vec()).bind(uuid_bytes(&user.0)).bind(old.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(done.rows_affected() == 1) + } +} + +/// Counts a wrong code against the open set-up of `device`, in `tx`; the fifth uses the set-up up. +async fn count_wrong_setup_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, device: DeviceId) -> Result<(), MetaError> { + sqlx::query( + "UPDATE user_two_factor SET pending_attempts = pending_attempts + 1 \ + WHERE user_id = $1 AND enabled_ms IS NULL AND pending_device = $2 AND pending_ciphertext IS NOT NULL", + ) + .bind(uuid_bytes(&user.0)) + .bind(uuid_bytes(device.as_uuid())) + .execute(&mut **tx) + .await?; + sqlx::query( + "UPDATE user_two_factor SET pending_ciphertext = NULL, pending_device = NULL, pending_expires_ms = NULL, pending_attempts = 0 \ + WHERE user_id = $1 AND enabled_ms IS NULL AND pending_attempts >= $2", + ) + .bind(uuid_bytes(&user.0)) + .bind(MAX_SETUP_ATTEMPTS) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Counts one wrong second factor of `user` in `tx` (the account's row is locked by it): the wrong proofs in a row go up, +/// and at [`LOCK_AFTER_FAILURES`] (and every one after) the lock is set or doubled; the challenge's attempts go up too, and +/// the challenge is deleted at [`MAX_CHALLENGE_ATTEMPTS`]. Lock order: the account's row, the two-factor row, the challenge. +/// Answers whether a lock was set or lengthened (for the caller to write its event once the counters are committed). +async fn count_wrong_in( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + challenge: Option<&[u8; 32]>, + now_ms: i64, +) -> Result { + let mut locked = false; + let counted = sqlx::query( + "UPDATE user_two_factor SET failed_attempts = failed_attempts + 1 WHERE user_id = $1 AND enabled_ms IS NOT NULL \ + RETURNING failed_attempts, locked_until_ms", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut **tx) + .await?; + if let Some(row) = counted + && let Some(duration) = lock_for(row.try_get("failed_attempts")?) + { + let until = now_ms.saturating_add(duration); + let until = row.try_get::, _>("locked_until_ms")?.map_or(until, |current| current.max(until)); + sqlx::query("UPDATE user_two_factor SET locked_until_ms = $1 WHERE user_id = $2") + .bind(until) + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + // Only here, where the lock is set or doubled, and never for a refusal while it is in force (`check_proof` stops + // those before anything is counted). + locked = true; + } + if let Some(challenge) = challenge { + sqlx::query("UPDATE login_challenges SET attempts = attempts + 1 WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND attempts >= $2") + .bind(challenge.to_vec()) + .bind(MAX_CHALLENGE_ATTEMPTS) + .execute(&mut **tx) + .await?; + } + Ok(locked) +} + +/// Makes `hashes` the account's recovery codes, all unused, in `tx` (the old ones, used or not, are gone). +async fn replace_recovery_codes(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, hashes: &[[u8; 32]]) -> Result<(), MetaError> { + sqlx::query("DELETE FROM user_recovery_codes WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + for hash in hashes { + sqlx::query("INSERT INTO user_recovery_codes (user_id, code_hash, used_ms) VALUES ($1, $2, NULL)") + .bind(uuid_bytes(&user.0)) + .bind(hash.to_vec()) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lock_starts_at_ten_and_doubles_up_to_a_day() { + assert_eq!(lock_for(0), None); + assert_eq!(lock_for(9), None); + assert_eq!(lock_for(10), Some(15 * 60 * 1000)); + assert_eq!(lock_for(11), Some(30 * 60 * 1000)); + assert_eq!(lock_for(12), Some(60 * 60 * 1000)); + assert_eq!(lock_for(15), Some(8 * 60 * 60 * 1000), "8 hours"); + assert_eq!(lock_for(16), Some(16 * 60 * 60 * 1000), "16 hours"); + assert_eq!(lock_for(17), Some(LOCK_MAX_MS), "32 hours is over a day"); + assert_eq!(lock_for(i64::MAX), Some(LOCK_MAX_MS)); + } + + #[test] + fn sealed_secrets_show_only_their_length() { + let secrets = StoredSecrets { user: UserId(uuid::Uuid::nil()), active: Some(vec![7, 8, 9]), pending: None }; + let shown = format!("{secrets:?}"); + assert!(shown.contains("<3 bytes>") && !shown.contains("[7"), "{shown}"); + let device = DeviceActor { user: UserId(uuid::Uuid::nil()), device: DeviceId::generate(), password_hash: "$argon2id$secret".to_owned() }; + assert!(!format!("{device:?}").contains("argon2id")); + } + + #[test] + fn retry_after_rounds_up_and_is_never_zero() { + assert_eq!(retry_after_secs(0), 1); + assert_eq!(retry_after_secs(1), 1); + assert_eq!(retry_after_secs(1000), 1); + assert_eq!(retry_after_secs(1001), 2); + assert_eq!(retry_after_secs(-5), 1); + } +} diff --git a/crates/meta/src/usage.rs b/crates/meta/src/usage.rs new file mode 100644 index 00000000..995b6851 --- /dev/null +++ b/crates/meta/src/usage.rs @@ -0,0 +1,207 @@ +//! Read-only storage estimates for one space. + +use craft_cloud_proto::SpaceId; +use sqlx::Row; + +use crate::blobs::effective_quota; +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::{Meta, MetaError, UserId}; + +const STORAGE_ESTIMATE_SQL: &str = include_str!("usage_estimate.sql"); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageEstimateReason { + IndexIncomplete, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StorageEstimate { + pub stored_bytes: u64, + pub quota_bytes: Option, + pub trash_bytes: Option, + pub history_bytes: Option, + pub reason: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageAccount { + pub id: UserId, + pub username: String, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageOverview { + pub accounts: u64, + pub stored_bytes: u64, + pub default_quota_bytes: Option, + pub accounts_with_override: u64, + pub accounts_near_quota: u64, + pub accounts_over_quota: u64, + pub top_accounts: Vec, + pub manifest_index: StorageIndexCounts, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StorageIndexCounts { + pub complete: u64, + pub building: u64, + pub not_indexed: u64, +} + +const STORAGE_OVERVIEW_SQL: &str = "WITH account_rows AS (\ + SELECT u.id, u.username, s.stored_bytes, s.quota_bytes, COALESCE(s.quota_bytes, $1) AS effective_quota \ + FROM users u JOIN spaces s ON s.owner = u.id WHERE u.removed_ms IS NULL\ +), account_totals AS (\ + SELECT COUNT(*) AS accounts, CAST(COALESCE(SUM(stored_bytes), 0) AS BIGINT) AS stored_bytes, \ + COALESCE(SUM(CASE WHEN quota_bytes IS NOT NULL THEN 1 ELSE 0 END), 0) AS accounts_with_override, \ + COALESCE(SUM(CASE WHEN effective_quota IS NOT NULL AND stored_bytes >= effective_quota - (effective_quota / 10) THEN 1 ELSE 0 END), 0) AS accounts_near_quota, \ + COALESCE(SUM(CASE WHEN effective_quota IS NOT NULL AND stored_bytes >= effective_quota THEN 1 ELSE 0 END), 0) AS accounts_over_quota \ + FROM account_rows\ +), index_counts AS (\ + SELECT COUNT(*) AS manifests, \ + COALESCE(SUM(CASE WHEN i.state = 1 THEN 1 ELSE 0 END), 0) AS complete, \ + COALESCE(SUM(CASE WHEN i.manifest IS NOT NULL AND i.state <> 1 THEN 1 ELSE 0 END), 0) AS building \ + FROM blobs b LEFT JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1\ +) SELECT account_totals.*, index_counts.manifests, index_counts.complete AS index_complete, index_counts.building AS index_building \ +FROM account_totals CROSS JOIN index_counts"; + +impl Meta { + /// Reads one consistent snapshot. Estimates are null while any manifest in the space has an incomplete index. + pub async fn storage_estimate(&self, space: SpaceId) -> Result { + let mut tx = self.read_snapshot().await?; + let row = sqlx::query(STORAGE_ESTIMATE_SQL) + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("space"))?; + let estimate = self.storage_estimate_from_row(&row)?; + tx.commit().await?; + Ok(estimate) + } + + /// Reads an account's estimate in one snapshot. Removed and unknown accounts are indistinguishable. + pub async fn admin_storage_estimate(&self, user: UserId) -> Result { + let mut tx = self.read_snapshot().await?; + let row = sqlx::query("SELECT s.id FROM users u JOIN spaces s ON s.owner = u.id WHERE u.id = $1 AND u.removed_ms IS NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("user"))?; + let space = uuid_from(&row.try_get::, _>("id")?)?; + let row = sqlx::query(STORAGE_ESTIMATE_SQL).bind(uuid_bytes(&space)).fetch_optional(&mut *tx).await?.ok_or(MetaError::NotFound("user"))?; + let estimate = self.storage_estimate_from_row(&row)?; + tx.commit().await?; + Ok(estimate) + } + + /// Aggregate counts and the ten largest current accounts from one consistent snapshot. + pub async fn admin_storage_overview(&self) -> Result { + let mut tx = self.read_snapshot().await?; + let cap = self.stored_bytes_cap().map(i64::try_from).transpose().map_err(|_| MetaError::Invalid("number too large"))?; + let row = sqlx::query(STORAGE_OVERVIEW_SQL).bind(cap).fetch_one(&mut *tx).await?; + let top_rows = sqlx::query( + "SELECT u.id, u.username, s.stored_bytes, COALESCE(s.quota_bytes, $1) AS effective_quota \ + FROM users u JOIN spaces s ON s.owner = u.id WHERE u.removed_ms IS NULL \ + ORDER BY s.stored_bytes DESC, u.username ASC, u.id ASC LIMIT 10", + ) + .bind(cap) + .fetch_all(&mut *tx) + .await?; + let top_accounts = top_rows + .iter() + .map(|row| { + let quota: Option = row.try_get("effective_quota")?; + Ok(StorageAccount { + id: UserId(uuid_from(&row.try_get::, _>("id")?)?), + username: row.try_get("username")?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + quota_bytes: quota.map(to_u64).transpose()?, + }) + }) + .collect::, MetaError>>()?; + let manifests: u64 = to_u64(row.try_get("manifests")?)?; + let complete: u64 = to_u64(row.try_get("index_complete")?)?; + let building: u64 = to_u64(row.try_get("index_building")?)?; + let overview = StorageOverview { + accounts: to_u64(row.try_get("accounts")?)?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + default_quota_bytes: self.stored_bytes_cap(), + accounts_with_override: to_u64(row.try_get("accounts_with_override")?)?, + accounts_near_quota: to_u64(row.try_get("accounts_near_quota")?)?, + accounts_over_quota: to_u64(row.try_get("accounts_over_quota")?)?, + top_accounts, + manifest_index: StorageIndexCounts { complete, building, not_indexed: manifests.saturating_sub(complete).saturating_sub(building) }, + }; + tx.commit().await?; + Ok(overview) + } + + fn storage_estimate_from_row(&self, row: &sqlx::any::AnyRow) -> Result { + let stored_bytes = to_u64(row.try_get("stored_bytes")?)?; + let stored_quota: Option = row.try_get("quota_bytes")?; + let quota_bytes = effective_quota(stored_quota.map(to_u64).transpose()?, self.stored_bytes_cap()); + let trash_bytes: Option = row.try_get("trash_bytes")?; + let history_bytes: Option = row.try_get("history_bytes")?; + let (trash_bytes, history_bytes, reason) = match (trash_bytes, history_bytes) { + (Some(trash), Some(history)) => (Some(to_u64(trash)?), Some(to_u64(history)?), None), + (None, None) => (None, None, Some(StorageEstimateReason::IndexIncomplete)), + _ => return Err(MetaError::Corrupt("storage estimates have inconsistent index state")), + }; + Ok(StorageEstimate { stored_bytes, quota_bytes, trash_bytes, history_bytes, reason }) + } +} + +#[cfg(test)] +mod tests { + use craft_cloud_proto::{ChunkId, ChunkRef, ChunkerParamsId, FileHash, Manifest}; + use sqlx::Row; + + use crate::{Meta, Role}; + + #[tokio::test] + async fn any_incomplete_space_manifest_makes_both_estimates_null() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let owner = meta.create_user("ana", "$argon2id$stub", Role::Member, 1).await.unwrap(); + let chunk = ChunkId::from_bytes([1; 32]); + meta.add_chunk(owner.space, &chunk, 6, 1).await.unwrap(); + let manifest = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([2; 32]), vec![ChunkRef { id: chunk, len: 5 }]) + .unwrap(); + meta.add_manifest(owner.space, &manifest, 100, 1).await.unwrap(); + sqlx::query("UPDATE manifest_index SET state = 0 WHERE manifest = $1") + .bind(manifest.id().as_bytes().to_vec()) + .execute(&meta.pool) + .await + .unwrap(); + + let estimate = meta.storage_estimate(owner.space).await.unwrap(); + assert_eq!((estimate.trash_bytes, estimate.history_bytes), (None, None)); + assert_eq!(estimate.reason, Some(super::StorageEstimateReason::IndexIncomplete)); + } + + #[tokio::test] + async fn storage_estimate_plan_uses_reference_and_version_indexes() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let owner = meta.create_user("ana", "$argon2id$stub", Role::Member, 1).await.unwrap(); + let plan_sql = format!("EXPLAIN QUERY PLAN {}", super::STORAGE_ESTIMATE_SQL); + let rows = + sqlx::query(sqlx::AssertSqlSafe(plan_sql)).bind(crate::codec::uuid_bytes(owner.space.as_uuid())).fetch_all(&meta.pool).await.unwrap(); + let plan = rows.iter().map(|row| row.try_get::("detail").unwrap()).collect::>().join(" | "); + assert!(plan.contains("versions_node_seq"), "missing existing version index in plan: {plan}"); + assert!(plan.contains("manifest_chunks_by_chunk"), "missing existing reverse-reference index in plan: {plan}"); + println!("storage estimate SQLite plan: {plan}"); + } + + #[tokio::test] + async fn storage_overview_plan_uses_bounded_aggregate_scans() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let plan_sql = format!("EXPLAIN QUERY PLAN {}", super::STORAGE_OVERVIEW_SQL); + let rows = sqlx::query(sqlx::AssertSqlSafe(plan_sql)).bind(Some(1_000_i64)).fetch_all(&meta.pool).await.unwrap(); + let plan = rows.iter().map(|row| row.try_get::("detail").unwrap()).collect::>().join(" | "); + assert!(plan.contains("SCAN u") || plan.contains("SEARCH u"), "missing account aggregate scan in plan: {plan}"); + assert!(plan.contains("b USING") && plan.contains("i USING"), "missing bounded manifest/index scans in plan: {plan}"); + println!("storage overview SQLite plan: {plan}"); + } +} diff --git a/crates/meta/src/usage_estimate.sql b/crates/meta/src/usage_estimate.sql new file mode 100644 index 00000000..80bc2682 --- /dev/null +++ b/crates/meta/src/usage_estimate.sql @@ -0,0 +1,50 @@ +WITH space_manifests AS ( + SELECT sb.id AS manifest FROM space_blobs sb WHERE sb.space_id = $1 AND sb.kind = 1 +), incomplete AS ( + SELECT 1 FROM space_manifests sm + LEFT JOIN manifest_index i ON i.manifest = sm.manifest + WHERE i.manifest IS NULL OR i.state <> 1 OR i.indexed_ordinals <> i.chunk_count + OR i.chunk_count <> (SELECT COUNT(*) FROM manifest_chunks mc WHERE mc.manifest = sm.manifest) + LIMIT 1 +), version_roles AS ( + SELECT v.manifest, + MAX(CASE WHEN n.deleted = 1 THEN 1 ELSE 0 END) AS has_deleted, + MAX(CASE WHEN n.deleted = 0 THEN 1 ELSE 0 END) AS has_live, + MAX(CASE WHEN n.deleted = 0 AND n.version = v.id THEN 1 ELSE 0 END) AS has_current, + MAX(CASE WHEN n.deleted = 0 AND n.version <> v.id THEN 1 ELSE 0 END) AS has_history, + MAX(CASE WHEN n.deleted = 0 AND v.save IS NOT NULL THEN 1 ELSE 0 END) AS has_saved + FROM nodes n JOIN versions v ON v.space_id = n.space_id AND v.node = n.id + WHERE n.space_id = $1 + GROUP BY v.manifest +), snapshot_manifests AS ( + SELECT snapshot AS manifest FROM live_docs WHERE space_id = $1 AND snapshot IS NOT NULL + UNION + SELECT import_snapshot AS manifest FROM live_docs WHERE space_id = $1 AND import_snapshot IS NOT NULL +), classified AS ( + SELECT sm.manifest, b.stored_size, + CASE WHEN vr.has_deleted = 1 AND vr.has_live = 0 AND snapshots.manifest IS NULL + THEN 1 ELSE 0 END AS trash_only, + CASE WHEN vr.has_history = 1 AND vr.has_deleted = 0 AND vr.has_current = 0 AND vr.has_saved = 0 AND snapshots.manifest IS NULL + THEN 1 ELSE 0 END AS history_only + FROM space_manifests sm JOIN blobs b ON b.kind = 1 AND b.id = sm.manifest + LEFT JOIN version_roles vr ON vr.manifest = sm.manifest + LEFT JOIN snapshot_manifests snapshots ON snapshots.manifest = sm.manifest +), eligible_chunks AS ( + SELECT sb.id, b.stored_size, + MIN(CASE WHEN c.trash_only = 1 THEN 1 ELSE 0 END) AS trash_only, + MIN(CASE WHEN c.history_only = 1 THEN 1 ELSE 0 END) AS history_only + FROM space_blobs sb JOIN blobs b ON b.kind = 0 AND b.id = sb.id + JOIN manifest_chunks mc ON mc.chunk = sb.id + JOIN space_manifests sm ON sm.manifest = mc.manifest + JOIN classified c ON c.manifest = sm.manifest + WHERE sb.space_id = $1 + GROUP BY sb.id, b.stored_size +) +SELECT s.stored_bytes, s.quota_bytes, + CASE WHEN EXISTS (SELECT 1 FROM incomplete) THEN NULL ELSE + CAST(COALESCE((SELECT SUM(stored_size) FROM classified WHERE trash_only = 1), 0) AS BIGINT) + + CAST(COALESCE((SELECT SUM(stored_size) FROM eligible_chunks WHERE trash_only = 1), 0) AS BIGINT) END AS trash_bytes, + CASE WHEN EXISTS (SELECT 1 FROM incomplete) THEN NULL ELSE + CAST(COALESCE((SELECT SUM(stored_size) FROM classified WHERE history_only = 1), 0) AS BIGINT) + + CAST(COALESCE((SELECT SUM(stored_size) FROM eligible_chunks WHERE history_only = 1), 0) AS BIGINT) END AS history_bytes +FROM spaces s WHERE s.id = $1 diff --git a/crates/meta/src/users.rs b/crates/meta/src/users.rs index 1d2b0255..42ac7c6f 100644 --- a/crates/meta/src/users.rs +++ b/crates/meta/src/users.rs @@ -3,11 +3,16 @@ //! Passwords are hashed by the caller (Argon2id, in the server); this module only stores the hash string. An account //! an invite created has an empty one until the invite is redeemed: no password matches it, since it is no hash. //! Device tokens are random 256-bit values; only their hash is stored, and a device is found by that hash. +//! +//! A device also records what client made it ([`DeviceClient`], shown in the device list and never a basis for a +//! permission), and, once revoked, when and why ([`RevokeReason`]). The `revoked` flag is the one that is checked: +//! the flag, the time and the reason are always set together, in one transaction, and only by the first revoke. use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId}; use sqlx::Row; use uuid::Uuid; +use crate::audit::{AuditAction, AuditDetails, NewEvent, Origin, record_in}; use crate::codec::{flag, uuid_bytes, uuid_from}; use crate::{Meta, MetaError}; @@ -21,14 +26,14 @@ pub enum Role { } impl Role { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Role::Admin => "admin", Role::Member => "member", } } - fn parse(s: &str) -> Result { + pub(crate) fn parse(s: &str) -> Result { match s { "admin" => Ok(Role::Admin), "member" => Ok(Role::Member), @@ -44,6 +49,9 @@ pub struct UserRecord { pub password_hash: String, pub role: Role, pub space: SpaceId, + /// Counts the changes that end what a sign-in checked before them (a disable, an enable, a removal, a password + /// change or reset). Read together with the password hash, and handed back to [`Meta::sign_in_device`]. + pub auth_gen: i64, } impl std::fmt::Debug for UserRecord { @@ -54,10 +62,114 @@ impl std::fmt::Debug for UserRecord { .field("password_hash", &"") .field("role", &self.role) .field("space", &self.space) + .field("auth_gen", &self.auth_gen) .finish() } } +/// What `GET /account` tells about a user. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountRecord { + pub id: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + /// When the password last changed through the server (`None`: never, the password is still the first one). + pub password_changed_ms: Option, + /// Whether signing in asks for a second factor. + pub two_factor_enabled: bool, +} + +/// The client kinds a device may name when it signs in. Empty means the client didn't say. +pub const CLIENT_KINDS: [&str; 3] = ["agent", "desktop", "cli"]; +/// The longest platform or app version, in bytes. +pub const MAX_CLIENT_FIELD_BYTES: usize = 32; + +/// What a device says about the client it signed in with. Display only: no permission ever depends on it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DeviceClient { + /// One of [`CLIENT_KINDS`], or empty. + pub kind: String, + pub platform: String, + pub app_version: String, +} + +impl DeviceClient { + /// The client a sign-in names; a field left out is empty. An unknown kind, a field over + /// [`MAX_CLIENT_FIELD_BYTES`] or a control character is `Invalid`. + pub fn new(kind: Option<&str>, platform: Option<&str>, app_version: Option<&str>) -> Result { + let client = Self { + kind: kind.unwrap_or_default().to_owned(), + platform: platform.unwrap_or_default().to_owned(), + app_version: app_version.unwrap_or_default().to_owned(), + }; + client.check()?; + Ok(client) + } + + pub(crate) fn check(&self) -> Result<(), MetaError> { + if !self.kind.is_empty() && !CLIENT_KINDS.contains(&self.kind.as_str()) { + return Err(MetaError::Invalid("client_kind is agent, desktop or cli")); + } + for field in [&self.platform, &self.app_version] { + if field.len() > MAX_CLIENT_FIELD_BYTES || field.chars().any(char::is_control) { + return Err(MetaError::Invalid("platform and app_version are at most 32 bytes without control characters")); + } + } + Ok(()) + } +} + +/// Why a device was revoked, as stored in `devices.revoke_reason`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RevokeReason { + /// The user revoked this device (`DELETE /devices/{id}`). + User, + /// The user signed out every other device. + Others, + /// The user changed the password. + Password, + /// An admin revoked it, on the command line or through the admin API. + Admin, + /// The account was disabled or marked for removal by an admin. + Disabled, + /// Two-factor sign-in was set up or changed (reserved). + TwoFactor, + /// The device sat idle too long (reserved). + Idle, + /// An admin reset the password (on the command line), or a reset code the admin made was redeemed. + Reset, +} + +impl RevokeReason { + pub fn as_str(self) -> &'static str { + match self { + RevokeReason::User => "user", + RevokeReason::Others => "others", + RevokeReason::Password => "password", + RevokeReason::Admin => "admin", + RevokeReason::Disabled => "disabled", + RevokeReason::TwoFactor => "two_factor", + RevokeReason::Idle => "idle", + RevokeReason::Reset => "reset", + } + } + + fn parse(s: &str) -> Result { + Ok(match s { + "user" => RevokeReason::User, + "others" => RevokeReason::Others, + "password" => RevokeReason::Password, + "admin" => RevokeReason::Admin, + "disabled" => RevokeReason::Disabled, + "two_factor" => RevokeReason::TwoFactor, + "idle" => RevokeReason::Idle, + "reset" => RevokeReason::Reset, + _ => return Err(MetaError::Corrupt("unknown revoke reason")), + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DeviceRecord { pub id: DeviceId, @@ -65,6 +177,11 @@ pub struct DeviceRecord { pub created_ms: i64, pub last_used_ms: i64, pub revoked: bool, + /// What the device said about its client when it signed in (empty for devices from before it was recorded). + pub client: DeviceClient, + /// When it was revoked, and why; both `None` while it isn't (and for devices revoked before they were recorded). + pub revoked_ms: Option, + pub revoke_reason: Option, } /// Who a device token belongs to. @@ -78,6 +195,14 @@ pub struct DeviceAuth { pub username: String, } +impl DeviceAuth { + /// The account's own (personal) space, for the routes that are about the account and never take a `craft-space` + /// header. A space-scoped handler takes its space from the server's `SpaceAccess` instead, never from here. + pub fn personal_space(&self) -> SpaceId { + self.space + } +} + /// Usernames: 1–64 bytes of ASCII letters, digits, `.`, `_`, `-`. They are stored in lowercase, so names that differ /// only in ASCII case are the same name. pub fn valid_username(name: &str) -> bool { @@ -129,7 +254,7 @@ impl Meta { pub async fn user_by_name(&self, username: &str) -> Result, MetaError> { let username = username.to_ascii_lowercase(); let row = sqlx::query( - "SELECT u.id, u.username, u.password_hash, u.role, s.id AS space FROM users u JOIN spaces s ON s.owner = u.id WHERE u.username = $1", + "SELECT u.id, u.username, u.password_hash, u.role, u.auth_gen, s.id AS space FROM users u JOIN spaces s ON s.owner = u.id WHERE u.username = $1", ) .bind(username.as_str()) .fetch_optional(&self.pool) @@ -141,38 +266,93 @@ impl Meta { password_hash: row.try_get("password_hash")?, role: Role::parse(&row.try_get::("role")?)?, space: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("space")?)?), + auth_gen: row.try_get("auth_gen")?, })) } - pub async fn set_password_hash(&self, user: UserId, password_hash: &str) -> Result<(), MetaError> { - let w = self.begin_write().await?; - let mut w = w; - let done = sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2") - .bind(password_hash) - .bind(uuid_bytes(&user.0)) - .execute(&mut *w.tx) - .await?; - if done.rows_affected() == 0 { - return Err(MetaError::NotFound("user")); - } - w.commit().await + /// Registers a device for `user` under the hash of its token, with no word on its client. + pub async fn create_device(&self, user: UserId, name: &str, token_hash: &[u8; 32], now_ms: i64) -> Result { + self.create_device_with(user, name, &DeviceClient::default(), token_hash, now_ms).await } - /// Registers a device for `user` under the hash of its token. - pub async fn create_device(&self, user: UserId, name: &str, token_hash: &[u8; 32], now_ms: i64) -> Result { + /// [`Meta::create_device`] for a device that says what client it is. + pub async fn create_device_with( + &self, + user: UserId, + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result { check_device_name(name)?; + client.check()?; let device = DeviceId::generate(); let mut w = self.begin_write().await?; - insert_device(&mut w.tx, device, user, name, token_hash, now_ms).await?; + // The account first, as every transaction that disables it or revokes its devices takes it: a device is + // never added to an account that was disabled or marked for removal meanwhile. + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("user")); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; w.commit().await?; Ok(device) } - /// The device holding this token hash, if any and not revoked. + /// Registers a device for `user` after a password sign-in, if the password it was checked against is still the + /// account's (an account with two-factor sign-in is [`MetaError::TwoFactorRequired`] here: it gets a device only from + /// [`Meta::complete_second_factor`], and [`Meta::sign_in`] is the call that knows both): `checked` is the stored hash and the generation ([`UserRecord::password_hash`], [`UserRecord::auth_gen`]) the caller read + /// before it verified the password. The account is locked first and both compared again in the same transaction, so a + /// password change or reset, or a disable followed by an enable (which leaves the hash as it was), that committed + /// while the (slow) check ran is not followed by a device that check let in. `None`: the user is gone or + /// something changed meanwhile; the caller answers as for a wrong password. An account that is disabled or marked for removal is + /// [`MetaError::AccountDisabled`], which the caller may only say after the password was right. + pub async fn sign_in_device( + &self, + user: UserId, + checked: (&str, i64), + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result, MetaError> { + check_device_name(name)?; + client.check()?; + let (hash, auth_gen) = checked; + let device = DeviceId::generate(); + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Ok(None); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + if crate::two_factor::is_enabled(&mut w.tx, user).await? { + return Err(MetaError::TwoFactorRequired); + } + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(device)) + } + + /// The device holding this token hash, if any, not revoked and of an account that isn't disabled. pub async fn device_by_token(&self, token_hash: &[u8; 32]) -> Result, MetaError> { let row = sqlx::query( "SELECT d.id, d.user_id, u.role, u.username, s.id AS space FROM devices d JOIN users u ON u.id = d.user_id JOIN spaces s ON s.owner = u.id \ - WHERE d.token_hash = $1 AND d.revoked = 0", + WHERE d.token_hash = $1 AND d.revoked = 0 AND u.disabled_ms IS NULL", ) .bind(token_hash.to_vec()) .fetch_optional(&self.pool) @@ -195,21 +375,34 @@ impl Meta { } pub async fn list_devices(&self, user: UserId) -> Result, MetaError> { - let rows = sqlx::query("SELECT id, name, created_ms, last_used_ms, revoked FROM devices WHERE user_id = $1 ORDER BY created_ms, id") - .bind(uuid_bytes(&user.0)) - .fetch_all(&self.pool) - .await?; - rows.iter() - .map(|row| { - Ok(DeviceRecord { - id: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), - name: row.try_get("name")?, - created_ms: row.try_get("created_ms")?, - last_used_ms: row.try_get("last_used_ms")?, - revoked: flag(row, "revoked")?, - }) - }) - .collect() + let rows = sqlx::query( + "SELECT id, name, created_ms, last_used_ms, revoked, client_kind, platform, app_version, revoked_ms, revoke_reason \ + FROM devices WHERE user_id = $1 ORDER BY created_ms, id", + ) + .bind(uuid_bytes(&user.0)) + .fetch_all(&self.pool) + .await?; + rows.iter().map(device_record).collect() + } + + /// What `GET /account` tells about `user`; `NotFound` for a user that isn't there. + pub async fn account(&self, user: UserId) -> Result { + let row = sqlx::query( + "SELECT username, role, created_ms, password_changed_ms, \ + (SELECT COUNT(*) FROM user_two_factor t WHERE t.user_id = users.id AND t.enabled_ms IS NOT NULL) AS two_factor FROM users WHERE id = $1", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await? + .ok_or(MetaError::NotFound("user"))?; + Ok(AccountRecord { + id: user, + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + password_changed_ms: row.try_get("password_changed_ms")?, + two_factor_enabled: row.try_get::("two_factor")? > 0, + }) } /// Whether `device` exists and isn't revoked (a long-lived connection asks again now and then: the admin CLI may @@ -220,23 +413,271 @@ impl Meta { Ok(row.is_some()) } - /// Revokes one of `user`'s devices, and deletes the pairing it made, if any (its add-a-computer code dies with - /// it). Revoking another user's device is `NotFound`. + /// Revokes one of `user`'s devices as the user's own act ([`RevokeReason::User`]), by the system clock. See + /// [`Meta::revoke_device_as`]. pub async fn revoke_device(&self, user: UserId, device: DeviceId) -> Result<(), MetaError> { + self.revoke_device_as(user, device, RevokeReason::User, wall_clock_ms()).await + } + + /// Revokes one of `user`'s devices, and deletes the pairing it made, if any (its add-a-computer code dies with + /// it). Revoking another user's device is `NotFound`. A device that is revoked already stays as it is, with its + /// first time and reason, and the call succeeds. The audit event is `device.revoked` for the account's own act + /// ([`Origin::Account`]); [`Meta::revoke_device_by`] says which device asked, or that the command line did. + pub async fn revoke_device_as(&self, user: UserId, device: DeviceId, reason: RevokeReason, now_ms: i64) -> Result<(), MetaError> { + self.revoke_device_by(user, device, reason, Origin::Account, now_ms).await + } + + /// [`Meta::revoke_device_as`] with who asked. A device that was revoked already is left as it is and writes no event: + /// nothing changed. + pub async fn revoke_device_by(&self, user: UserId, device: DeviceId, reason: RevokeReason, origin: Origin, now_ms: i64) -> Result<(), MetaError> { let mut w = self.begin_write().await?; - let done = sqlx::query("UPDATE devices SET revoked = 1 WHERE id = $1 AND user_id = $2") + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("device")); + } + let done = sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE id = $3 AND user_id = $4 AND revoked = 0") + .bind(now_ms) + .bind(reason.as_str()) .bind(uuid_bytes(device.as_uuid())) .bind(uuid_bytes(&user.0)) .execute(&mut *w.tx) .await?; if done.rows_affected() == 0 { - return Err(MetaError::NotFound("device")); + let owned = sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + if owned.is_none() { + return Err(MetaError::NotFound("device")); + } } sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; - w.commit().await + if done.rows_affected() > 0 { + let details = AuditDetails { reason: Some(reason), device: Some(device), ..AuditDetails::default() }; + let event = match origin { + Origin::Account => NewEvent::new(AuditAction::DeviceRevoked).by(user, None).on_device(device), + Origin::Device(by) => NewEvent::new(AuditAction::DeviceRevoked).by(user, Some(by)).on_device(device), + // The command line acts as no account. The event is about the account, whose own activity then shows it. + Origin::Cli => NewEvent::new(AuditAction::CliDeviceRevoked).on_user(user), + }; + record_in(&mut w.tx, now_ms, event.with(details)).await?; + } + self.commit_audited(w, now_ms).await + } + + /// Renames one of `user`'s devices that isn't revoked and returns it. Another user's, a revoked or a missing + /// device is `NotFound`, never a different answer, so nothing says which devices exist. + pub async fn rename_device(&self, user: UserId, device: DeviceId, name: &str) -> Result { + check_device_name(name)?; + let mut w = self.begin_write().await?; + let done = sqlx::query("UPDATE devices SET name = $1 WHERE id = $2 AND user_id = $3 AND revoked = 0") + .bind(name) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("device")); + } + let row = sqlx::query( + "SELECT id, name, created_ms, last_used_ms, revoked, client_kind, platform, app_version, revoked_ms, revoke_reason \ + FROM devices WHERE id = $1", + ) + .bind(uuid_bytes(device.as_uuid())) + .fetch_one(&mut *w.tx) + .await?; + let record = device_record(&row)?; + w.commit().await?; + Ok(record) + } + + /// Revokes every device of `user` but `keep` that isn't revoked (with `reason`), and deletes the pairings those + /// devices made, in one transaction. Returns the devices revoked, for the caller to end their live connections. + pub async fn revoke_other_devices(&self, user: UserId, keep: DeviceId, reason: RevokeReason, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(Vec::new()); + } + let revoked = revoke_devices(&mut w.tx, user, Some(keep), reason, now_ms).await?; + // One event with the count, however many devices went, and none when there was nobody to sign out. + if !revoked.is_empty() { + let details = + AuditDetails { reason: Some(reason), revoked: Some(u32::try_from(revoked.len()).unwrap_or(u32::MAX)), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::DeviceRevokedOthers).by(user, Some(keep)).on_user(user).with(details)).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// [`Meta::revoke_other_devices`] for every device of `user` (the admin's command line, which writes + /// `cli.devices_revoked_all` when there was a device to sign out). + pub async fn revoke_all_devices(&self, user: UserId, reason: RevokeReason, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(Vec::new()); + } + let revoked = revoke_devices(&mut w.tx, user, None, reason, now_ms).await?; + if !revoked.is_empty() { + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::CliDevicesRevokedAll).on_user(user).with(AuditDetails::revoked(revoked.len()))) + .await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Changes `user`'s password to `new_hash`, in one transaction: sets the hash and `password_changed_ms`, + /// revokes every device but `keep` ([`RevokeReason::Password`]), and deletes the account's open invite and all + /// its pairings (a code made before the change must not work after it). `current_hash` is the hash the caller + /// verified the current password against: if the password changed meanwhile, nothing changes and the answer is + /// `Conflict`; if `keep` was revoked meanwhile, it is `DeviceRevoked`. Returns the devices revoked. A failure at + /// any step changes nothing. + pub async fn change_password( + &self, + user: UserId, + current_hash: &str, + new_hash: &str, + keep: DeviceId, + now_ms: i64, + ) -> Result, MetaError> { + let mut w = self.begin_write().await?; + // The user's row first, as everywhere (see `lock_user`). + lock_user(&mut w.tx, user).await?; + // The caller's device must still be signed in: its Argon2 checks took a while, and a revoke since would + // otherwise leave a password changed by a device that is signed out. The no-op update locks the row on + // both databases, so a revoke running now waits for this transaction (and then finds the device kept). + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(keep.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let done = sqlx::query( + "UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3 AND password_hash = $4", + ) + .bind(new_hash) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .bind(current_hash) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::Conflict("the password changed while this request was running; try again")); + } + let revoked = revoke_devices(&mut w.tx, user, Some(keep), RevokeReason::Password, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + // The devices signed out with it are counted in this one event, not in events of their own. + let changed = + NewEvent::new(AuditAction::AccountPasswordChanged).by(user, Some(keep)).on_user(user).with(AuditDetails::revoked(revoked.len())); + record_in(&mut w.tx, now_ms, changed).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// The admin's password reset on the command line (`cli.password_reset`): sets `new_hash` and `password_changed_ms`, deletes the account's open invite and + /// pairings, and, unless `keep_devices`, revokes every device ([`RevokeReason::Reset`]), all in one transaction. + /// `NotFound` for a user that isn't there. Returns the devices revoked. + pub async fn reset_password(&self, user: UserId, new_hash: &str, keep_devices: bool, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + lock_user(&mut w.tx, user).await?; + let done = sqlx::query("UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3") + .bind(new_hash) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("user")); + } + let revoked = if keep_devices { Vec::new() } else { revoke_devices(&mut w.tx, user, None, RevokeReason::Reset, now_ms).await? }; + delete_join_codes(&mut w.tx, user).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::CliPasswordReset).on_user(user).with(AuditDetails::revoked(revoked.len()))).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) } } +/// Takes the row lock of `user` (a no-op update, so nothing changes), and says whether the user exists. Every +/// transaction that revokes devices or changes a password takes it first, then the rows of the devices, then those of +/// the pairings: one order for all of them, so two of them at once (a password change from one device, a sign-out of +/// the others from another, an admin reset) wait for each other on Postgres and never deadlock. SQLite writers are +/// serialised already. +pub(crate) async fn lock_user(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let done = sqlx::query("UPDATE users SET username = username WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(done.rows_affected() > 0) +} + +/// Whether `user` is disabled or marked for removal, read in `tx` after the user's row was locked. +pub(crate) async fn account_blocked(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let row = sqlx::query("SELECT disabled_ms, removed_ms FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut **tx).await?; + let Some(row) = row else { return Ok(false) }; + Ok(row.try_get::, _>("disabled_ms")?.is_some() || row.try_get::, _>("removed_ms")?.is_some()) +} + +fn device_record(row: &sqlx::any::AnyRow) -> Result { + Ok(DeviceRecord { + id: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + name: row.try_get("name")?, + created_ms: row.try_get("created_ms")?, + last_used_ms: row.try_get("last_used_ms")?, + revoked: flag(row, "revoked")?, + client: DeviceClient { kind: row.try_get("client_kind")?, platform: row.try_get("platform")?, app_version: row.try_get("app_version")? }, + revoked_ms: row.try_get("revoked_ms")?, + revoke_reason: row.try_get::, _>("revoke_reason")?.as_deref().map(RevokeReason::parse).transpose()?, + }) +} + +/// Revokes `user`'s devices that aren't revoked yet, except `keep`, and deletes the pairings of every revoked device +/// of theirs, in `tx`. Time and reason are set with the flag, and only here, so a device revoked before keeps its own. +pub(crate) async fn revoke_devices( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + keep: Option, + reason: RevokeReason, + now_ms: i64, +) -> Result, MetaError> { + let rows = + match keep { + Some(keep) => sqlx::query( + "UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE user_id = $3 AND revoked = 0 AND id <> $4 RETURNING id", + ) + .bind(now_ms) + .bind(reason.as_str()) + .bind(uuid_bytes(&user.0)) + .bind(uuid_bytes(keep.as_uuid())) + .fetch_all(&mut **tx) + .await?, + None => { + sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE user_id = $3 AND revoked = 0 RETURNING id") + .bind(now_ms) + .bind(reason.as_str()) + .bind(uuid_bytes(&user.0)) + .fetch_all(&mut **tx) + .await? + } + }; + sqlx::query("DELETE FROM pairings WHERE device_id IN (SELECT id FROM devices WHERE user_id = $1 AND revoked = 1)") + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + rows.iter().map(|row| Ok(DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?))).collect() +} + +/// Deletes `user`'s open invite, every pairing of theirs and every open sign-in challenge, in `tx`: a code or challenge +/// made before a password change, a reset or a disable must not work after it. +pub(crate) async fn delete_join_codes(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result<(), MetaError> { + sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM pairings WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(()) +} + +/// The system clock, for the call that takes no time of its own. +pub(crate) fn wall_clock_ms() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)).unwrap_or(0) +} + /// The device name rule: 1–128 bytes without control characters. pub(crate) fn check_device_name(name: &str) -> Result<(), MetaError> { if name.is_empty() || name.len() > 128 || name.chars().any(char::is_control) { @@ -269,11 +710,12 @@ pub(crate) async fn insert_user( sqlx::Error::Database(ref db) if db.is_unique_violation() => MetaError::UsernameTaken, other => MetaError::Db(other), })?; - sqlx::query("INSERT INTO spaces (id, owner, epoch, seq, oldest_seq, root) VALUES ($1, $2, $3, 0, 0, $4)") + sqlx::query("INSERT INTO spaces (id, owner, epoch, seq, oldest_seq, root, created_ms) VALUES ($1, $2, $3, 0, 0, $4, $5)") .bind(uuid_bytes(space.as_uuid())) .bind(uuid_bytes(&user.0)) .bind(uuid_bytes(Epoch::generate().as_uuid())) .bind(uuid_bytes(root.as_uuid())) + .bind(now_ms) .execute(&mut **tx) .await?; sqlx::query( @@ -284,7 +726,7 @@ pub(crate) async fn insert_user( .bind(uuid_bytes(root.as_uuid())) .execute(&mut **tx) .await?; - Ok(UserRecord { id: user, username: username.to_owned(), password_hash: password_hash.to_owned(), role, space }) + Ok(UserRecord { id: user, username: username.to_owned(), password_hash: password_hash.to_owned(), role, space, auth_gen: 0 }) } /// Inserts a device (its name already checked) in `tx`. @@ -293,16 +735,23 @@ pub(crate) async fn insert_device( device: DeviceId, user: UserId, name: &str, + client: &DeviceClient, token_hash: &[u8; 32], now_ms: i64, ) -> Result<(), MetaError> { - sqlx::query("INSERT INTO devices (id, user_id, name, token_hash, created_ms, last_used_ms, revoked) VALUES ($1, $2, $3, $4, $5, $5, 0)") - .bind(uuid_bytes(device.as_uuid())) - .bind(uuid_bytes(&user.0)) - .bind(name) - .bind(token_hash.to_vec()) - .bind(now_ms) - .execute(&mut **tx) - .await?; + sqlx::query( + "INSERT INTO devices (id, user_id, name, token_hash, created_ms, last_used_ms, revoked, client_kind, platform, app_version) \ + VALUES ($1, $2, $3, $4, $5, $5, 0, $6, $7, $8)", + ) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .bind(name) + .bind(token_hash.to_vec()) + .bind(now_ms) + .bind(client.kind.as_str()) + .bind(client.platform.as_str()) + .bind(client.app_version.as_str()) + .execute(&mut **tx) + .await?; Ok(()) } diff --git a/crates/meta/src/web.rs b/crates/meta/src/web.rs new file mode 100644 index 00000000..271ccc9a --- /dev/null +++ b/crates/meta/src/web.rs @@ -0,0 +1,347 @@ +//! Browser sessions of the web admin: the `web_sessions` table, signing in to one (with the second factor, when the +//! account has one), looking one up, logging out and pruning. +//! +//! - **Only a hash is stored.** The session token is 32 random bytes the server makes and puts in a cookie; the table keeps +//! its BLAKE3 hash (`id_hash`), like a device's token. A row holds no password, no token, no address and no user agent. +//! - **A session lives exactly as long as the account's `auth_gen` it was made under.** Every change that raises +//! `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off, an +//! admin's reset of it) ends the account's sessions with it, with no code of its own in those changes. A session is +//! also over when the account is disabled or marked for removal, when it is logged out (`revoked_ms`), when it was +//! made longer ago than the absolute lifetime, and when it was not seen for longer than the idle lifetime. +//! - **Only administrators sign in.** The check is made under the account's lock, together with the password's +//! generation, so a demotion or a password change that ran while the (slow) password check did cannot be followed by a +//! session. A refusal is `None` for every reason (wrong state, not an admin, blocked), which the caller answers like a +//! wrong password: nothing says whether an account exists, is a member, or is disabled. +//! - **The second factor is the device sign-in's, not a copy of it.** A web sign-in's challenge is a row of +//! `login_challenges` of its own kind (`web`), counted, locked and used up by the same code (`two_factor::check_proof`, +//! `refuse_sign_in_proof`), and can never be completed into a device (nor a device sign-in's into a session). +//! - **Pruning is bounded.** [`Meta::web_sessions_prune`] deletes at most a given number of expired or revoked rows at a +//! time, so the server can run it after a sign-in without stalling it, and the command line loops until it is done. + +use craft_cloud_proto::DeviceId; +use sqlx::Row; + +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::two_factor::{ + CHALLENGE_TTL_MS, Checked, MAX_CHALLENGE_ATTEMPTS, MAX_OPEN_CHALLENGES, NewChallenge, Proof, TfRow, check_proof, is_enabled, load_row, +}; +use crate::users::{account_blocked, lock_user}; +use crate::{Meta, MetaError, Role, UserId, WriteTx}; + +/// A session to make: the hash of its token and how long it may live in all. +#[derive(Clone, PartialEq, Eq)] +pub struct NewWebSession { + /// The BLAKE3 hash of the session token: all the server keeps of it. + pub id_hash: [u8; 32], + /// The most the session may live from now, whatever happens: its absolute lifetime. + pub absolute_ms: i64, +} + +impl std::fmt::Debug for NewWebSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NewWebSession").field("id_hash", &"").field("absolute_ms", &self.absolute_ms).finish() + } +} + +/// The result of a password sign-in to the web admin that checked out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WebSignIn { + /// The account has no second factor: the session is made. + Session, + /// The account needs its second factor: only the challenge is made, good until this time. + Challenge { expires_ms: i64 }, +} + +/// A session that is valid now, with the account it belongs to as the database has it now. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WebSession { + pub user: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + pub last_seen_ms: i64, + pub absolute_expires_ms: i64, + /// When a second factor was given at sign-in; `None` for an account without one. + pub mfa_verified_ms: Option, +} + +/// The SQL that says a session is over, for `$1` the time now and `$2` the oldest `last_seen_ms` that is still idle-valid. +const OVER: &str = "revoked_ms IS NOT NULL OR absolute_expires_ms <= $1 OR last_seen_ms <= $2"; + +/// Inserts the session of `user` in `tx` (the account's row locked), under the account's current generation. +async fn insert_session( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + session: &NewWebSession, + mfa_verified_ms: Option, + now_ms: i64, +) -> Result<(), MetaError> { + let absolute = now_ms.checked_add(session.absolute_ms).ok_or(MetaError::Invalid("the session's lifetime is too long"))?; + let generation: i64 = + sqlx::query("SELECT auth_gen FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_one(&mut **tx).await?.try_get("auth_gen")?; + sqlx::query( + "INSERT INTO web_sessions (id_hash, user_id, auth_gen, created_ms, last_seen_ms, absolute_expires_ms, revoked_ms, mfa_verified_ms) \ + VALUES ($1, $2, $3, $4, $4, $5, NULL, $6)", + ) + .bind(session.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(generation) + .bind(now_ms) + .bind(absolute) + .bind(mfa_verified_ms) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn is_admin(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let row = sqlx::query("SELECT role FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut **tx).await?; + match row { + Some(row) => Ok(Role::parse(&row.try_get::("role")?)? == Role::Admin), + None => Ok(false), + } +} + +impl Meta { + /// A password sign-in to the web admin that checked out: under the account's lock the password hash and `auth_gen` the + /// caller read before it verified the password are compared again, and the account must still be an administrator, not + /// disabled and not marked for removal. Then, without two-factor sign-in, the session is made and `auth.login` is + /// written (no device acts); with it only a challenge of kind `web` is made (the account's expired challenges are + /// deleted and the oldest goes when it would have more than [`MAX_OPEN_CHALLENGES`]) and no session. `None` for + /// everything that is not a sign-in, whatever the reason. + pub async fn web_sign_in( + &self, + user: UserId, + checked: (&str, i64), + challenge: &NewChallenge, + session: &NewWebSession, + now_ms: i64, + ) -> Result, MetaError> { + let (hash, auth_gen) = checked; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() || account_blocked(&mut w.tx, user).await? || !is_admin(&mut w.tx, user).await? { + return Ok(None); + } + if is_enabled(&mut w.tx, user).await? { + let expires_ms = now_ms.checked_add(CHALLENGE_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + sqlx::query("DELETE FROM login_challenges WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; + let open: i64 = sqlx::query("SELECT COUNT(*) AS n FROM login_challenges WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_one(&mut *w.tx) + .await? + .try_get("n")?; + if open >= MAX_OPEN_CHALLENGES { + sqlx::query( + "DELETE FROM login_challenges WHERE id_hash IN \ + (SELECT id_hash FROM login_challenges WHERE user_id = $1 ORDER BY expires_ms ASC, id_hash ASC LIMIT $2)", + ) + .bind(uuid_bytes(&user.0)) + .bind(open - MAX_OPEN_CHALLENGES + 1) + .execute(&mut *w.tx) + .await?; + } + sqlx::query( + "INSERT INTO login_challenges (id_hash, user_id, kind, client_key, device_name, attempts, expires_ms) \ + VALUES ($1, $2, 'web', $3, '', 0, $4)", + ) + .bind(challenge.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(challenge.client_key.as_str()) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + return Ok(Some(WebSignIn::Challenge { expires_ms })); + } + insert_session(&mut w.tx, user, session, None, now_ms).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AuthLogin).by(user, None::).with(AuditDetails::web())).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(WebSignIn::Session)) + } + + /// The account a web challenge was made for and its username, by a plain read, so the caller can count the try against + /// the account before the real check. `None` for a challenge that is unknown, expired, used, or a device sign-in's. + pub async fn web_challenge_owner(&self, id_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT c.user_id, u.username FROM login_challenges c JOIN users u ON u.id = c.user_id \ + WHERE c.id_hash = $1 AND c.kind = 'web' AND c.expires_ms > $2", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some((UserId(uuid_from(&row.try_get::, _>("user_id")?)?), row.try_get("username")?))) + } + + /// Finishes a web sign-in with the second factor, as [`Meta::complete_second_factor`] finishes a device's: the account's + /// row is locked before the challenge or the factor is touched, the challenge and the account's current state (still an + /// administrator, two-factor sign-in still on, not disabled) are checked again under the lock, and only then are the + /// proof and the challenge used up and the session made, in this one transaction. A challenge that is unknown, + /// expired, used or no longer fit is [`MetaError::ChallengeGone`], the same for every case, an account that is no longer + /// an administrator or is blocked too; a wrong proof is [`MetaError::TwoFactorInvalid`] after its counters were + /// committed on their own; a lock in force is [`MetaError::TwoFactorLocked`]. + pub async fn web_complete_second_factor( + &self, + challenge: &[u8; 32], + proof: &Proof<'_>, + session: &NewWebSession, + now_ms: i64, + ) -> Result { + let Some((user, _)) = self.web_challenge_owner(challenge, now_ms).await? else { return Err(MetaError::ChallengeGone) }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + let found = sqlx::query("SELECT attempts, expires_ms FROM login_challenges WHERE id_hash = $1 AND user_id = $2 AND kind = 'web'") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(found) = found.filter(|row| row.try_get::("expires_ms").is_ok_and(|expires| expires > now_ms)) else { + return Err(MetaError::ChallengeGone); + }; + let attempts: i64 = found.try_get("attempts")?; + let Some(row) = load_row(&mut w.tx, user).await?.filter(TfRow::active) else { + // Turned off (or reset) since: the challenge is of no use any more. + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + return Err(MetaError::ChallengeGone); + }; + if attempts >= MAX_CHALLENGE_ATTEMPTS || account_blocked(&mut w.tx, user).await? || !is_admin(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + if let Checked::Wrong = check_proof(&mut w.tx, user, None, true, &row, proof, now_ms).await? { + return Err(self.refuse_sign_in_proof(w, user, Some(challenge), proof, now_ms, true).await); + } + let used = sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND user_id = $2") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if used.rows_affected() != 1 { + return Err(MetaError::ChallengeGone); + } + insert_session(&mut w.tx, user, session, Some(now_ms), now_ms).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AuthLogin).by(user, None::).with(AuditDetails::web())).await?; + self.commit_audited(w, now_ms).await?; + Ok(user) + } + + /// The session under this token hash, if it is valid at `now_ms`: not logged out, younger than its absolute lifetime, + /// seen within `idle_ms`, made under the account's current `auth_gen`, and of an account that is neither disabled nor + /// marked for removal. The account's role is the one the database has now. + pub async fn web_session(&self, id_hash: &[u8; 32], now_ms: i64, idle_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT s.user_id, s.created_ms, s.last_seen_ms, s.absolute_expires_ms, s.mfa_verified_ms, u.username, u.role \ + FROM web_sessions s JOIN users u ON u.id = s.user_id \ + WHERE s.id_hash = $1 AND s.revoked_ms IS NULL AND s.absolute_expires_ms > $2 AND s.last_seen_ms > $3 \ + AND s.auth_gen = u.auth_gen AND u.disabled_ms IS NULL AND u.removed_ms IS NULL", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some(WebSession { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + last_seen_ms: row.try_get("last_seen_ms")?, + absolute_expires_ms: row.try_get("absolute_expires_ms")?, + mfa_verified_ms: row.try_get("mfa_verified_ms")?, + })) + } + + /// Records that the session was just used (its idle time starts again). Never moves the time back. + pub async fn web_session_seen(&self, id_hash: &[u8; 32], now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + sqlx::query("UPDATE web_sessions SET last_seen_ms = $1 WHERE id_hash = $2 AND last_seen_ms < $1 AND revoked_ms IS NULL") + .bind(now_ms) + .bind(id_hash.to_vec()) + .execute(&mut *w.tx) + .await?; + w.commit().await + } + + /// Logs the session out. One that is gone or over already is left as it is, and the call succeeds. + pub async fn web_logout(&self, id_hash: &[u8; 32], now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + sqlx::query("UPDATE web_sessions SET revoked_ms = $1 WHERE id_hash = $2 AND revoked_ms IS NULL") + .bind(now_ms) + .bind(id_hash.to_vec()) + .execute(&mut *w.tx) + .await?; + w.commit().await + } + + /// Deletes at most `max_rows` sessions that are over (logged out, past their absolute lifetime, or not seen for `idle_ms`) + /// and says how many went. Rows that are valid are never touched. The rows of sessions that an account change ended + /// (`auth_gen`) are deleted when their other lifetimes run out, which is at most the absolute lifetime. + pub async fn web_sessions_prune(&self, now_ms: i64, idle_ms: i64, max_rows: u32) -> Result { + let mut w = self.begin_write().await?; + let done = sqlx::query(sqlx::AssertSqlSafe(format!( + "DELETE FROM web_sessions WHERE id_hash IN (SELECT id_hash FROM web_sessions WHERE {OVER} LIMIT $3)" + ))) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .bind(i64::from(max_rows)) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(done.rows_affected()) + } + + /// How many sessions [`Meta::web_sessions_prune`] would delete now (the command line's dry run). + pub async fn web_sessions_count_over(&self, now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT COUNT(*) AS n FROM web_sessions WHERE {OVER}"))) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_one(&self.pool) + .await?; + to_u64(row.try_get("n")?) + } + + /// How many sessions the account has that are valid at `now_ms`, for tests and the overview. + pub async fn web_sessions_open(&self, user: UserId, now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) AS n FROM web_sessions s JOIN users u ON u.id = s.user_id \ + WHERE s.user_id = $1 AND s.revoked_ms IS NULL AND s.absolute_expires_ms > $2 AND s.last_seen_ms > $3 AND s.auth_gen = u.auth_gen", + ) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_one(&self.pool) + .await?; + to_u64(row.try_get("n")?) + } +} + +/// Whether the session `id_hash` of `user` is valid at `now_ms`, read inside a change (see `begin_admin`) under the account's +/// lock: not logged out, under the account's current generation, inside its absolute lifetime and used within `idle_ms`. A +/// request that began while the session was valid (a slow body, a wait for a lock or a password check) therefore cannot change +/// anything after the session ended. The account's disabled and removed marks need no look: they raise `auth_gen`. +pub(crate) async fn session_is_live(w: &mut WriteTx, user: UserId, id_hash: &[u8; 32], now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query( + "SELECT 1 FROM web_sessions s JOIN users u ON u.id = s.user_id WHERE s.id_hash = $1 AND s.user_id = $2 AND s.revoked_ms IS NULL AND s.auth_gen = u.auth_gen AND s.absolute_expires_ms > $3 AND s.last_seen_ms > $4", + ) + .bind(id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_optional(&mut *w.tx) + .await?; + Ok(row.is_some()) +} diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index d875fc30..b9bc2810 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -3,7 +3,10 @@ //! test creates and drops its own database. CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS`, which turns a missing //! `CRAFT_CLOUD_TEST_PG` into a failure instead of a skip. -use craft_cloud_meta::{BlobKind, InviteFor, Meta, MetaError, Redeemed, Role, UserRecord}; +use craft_cloud_meta::{ + AdminActor, AdminUser, AdminVia, BlobKind, DeviceActor, DeviceClient, DeviceRecord, InviteFor, MIN_QUOTA_BYTES, Meta, MetaError, NewChallenge, + NewWebSession, Proof, RECOVERY_CODES, RESET_TTL_MS, Redeemed, RevokeReason, Role, SignIn, UnreadableSecret, UserId, UserRecord, WebSignIn, +}; use craft_cloud_proto::api::PairingState; use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, PAIRING_CHARS, PAIRING_TTL_MS}; use craft_cloud_proto::limits::MAX_PAGE_NODES; @@ -90,6 +93,14 @@ on_every_backend!( a_late_save_record_is_appended_once, manifests_need_their_chunks_and_are_per_space, stored_bytes_are_counted_once_per_space_and_capped, + account_usage_estimates_history_and_trash_without_adding_them, + account_usage_excludes_chunks_shared_with_current_version, + account_usage_deduplicates_per_space_and_counts_pending_purge, + admin_storage_overview_is_aggregated_and_orders_ties, + admin_storage_overview_handles_an_empty_server, + admin_storage_estimates_hide_removed_and_unknown_accounts, + account_usage_excludes_live_document_snapshot_blobs, + account_usage_leaves_saved_versions_out_of_history, concurrent_chunk_batches_charge_a_shared_chunk_once, concurrent_signups_with_one_name_give_username_taken, an_offline_batch_creates_a_folder_and_files_in_it, @@ -104,6 +115,32 @@ on_every_backend!( ids_are_never_reused, the_change_feed_and_epochs, concurrent_commits_on_one_file_let_exactly_one_win, + history_pages_newest_first_for_own_files_only, + a_restore_is_a_new_version_after_the_head, + a_restore_refuses_what_it_can_not_restore, + concurrent_restores_let_exactly_one_win, + a_restore_and_a_late_save_record_both_apply, + deleting_stamps_the_time_once_and_coming_back_clears_it, + the_trash_pages_newest_first_for_its_own_space, + a_restore_brings_the_deleted_folders_back_with_their_ids, + a_restore_merges_into_live_folders_and_renames_around_taken_names, + a_restore_with_no_free_name_changes_nothing, + the_trash_lists_a_folder_without_what_is_below_it, + a_folder_restore_brings_back_everything_in_the_trash_below_it, + a_folder_restore_gives_children_of_one_name_a_conflict_name, + removing_a_folder_from_the_trash_takes_what_is_below_it, + a_folder_restore_takes_at_most_the_cap_below_it, + a_folder_restore_leaves_the_rest_listed_through_its_own_folder, + removing_a_folder_takes_at_most_the_cap_below_it, + only_nodes_in_the_trash_can_be_restored_or_removed, + removing_from_the_trash_frees_nothing, + a_trash_restore_and_a_commit_for_the_same_name_agree, + a_trash_restore_and_an_edit_of_the_file_agree, + a_restored_file_gets_a_new_version_with_its_old_content, + a_restore_does_not_exceed_the_depth_limit, + a_new_manifest_has_exact_edges_complete_in_its_own_transaction, + a_long_manifest_is_indexed_in_batches_of_the_budget, + the_index_stays_exact_through_every_write_path, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -139,6 +176,71 @@ on_every_backend!( concurrent_redeems_of_one_code_give_one_device, an_invite_to_an_account_with_a_password_needs_none, an_invite_to_a_new_account_needs_a_password, + a_device_records_its_client, + only_own_live_devices_are_renamed, + every_revoke_path_sets_flag_time_and_reason_together, + revoking_others_keeps_the_caller_and_ends_their_pairings, + a_password_change_is_one_step, + an_admin_reset_signs_out_every_device_unless_kept, + concurrent_password_changes_and_sign_outs_never_fail_the_database, + a_sign_in_checked_against_an_old_password_creates_no_device, + redeems_sign_ins_and_resets_at_the_same_moment_leave_no_live_device, + the_admin_list_pages_searches_and_counts, + a_sign_in_checked_before_a_disable_and_an_enable_creates_no_device, + an_admin_makes_an_account_with_a_role_and_an_invite, + an_admin_changes_a_role_and_a_quota, + the_acting_admins_state_is_checked_in_the_transaction, + disabling_signs_everything_out_and_enabling_brings_back_nothing, + issuing_access_while_an_account_is_disabled_leaves_no_usable_token, + invites_and_reset_codes_replace_each_other, + a_reset_code_sets_a_password_and_signs_everything_out, + a_quota_overrides_the_server_wide_cap, + an_admin_lists_and_revokes_a_users_devices, + two_demotions_at_once_keep_an_admin, + two_self_demotions_at_once_keep_an_admin, + two_disables_at_once_keep_an_admin, + a_demotion_and_a_disable_at_once_keep_an_admin, + a_removal_and_a_demotion_at_once_keep_an_admin, + a_self_demotion_and_a_disable_of_the_other_at_once_keep_an_admin, + every_operation_writes_exactly_its_events, + a_refused_change_writes_no_event, + failed_sign_ins_count_up_in_one_row_per_account_and_minute, + concurrent_failures_with_one_key_make_one_row_with_every_count, + audit_lists_filter_and_page_by_cursor, + pruning_works_in_batches_and_keeps_its_time_until_done, + a_removed_accounts_events_stay, + no_event_holds_a_name_or_a_secret, + set_up_confirmed_and_listed, + a_set_up_belongs_to_one_device_expires_and_takes_five_tries, + changes_of_the_settings_recheck_the_device_and_the_password, + two_confirmations_at_once_turn_it_on_once, + a_time_step_is_accepted_once_and_only_later_ones_after_it, + recovery_codes_work_once_and_new_ones_replace_them, + wrong_proofs_lock_for_a_while_double_and_reset_on_success, + wrong_proofs_at_the_same_moment_all_count, + a_challenge_is_made_counted_used_once_and_carries_the_device, + an_account_keeps_five_open_challenges_and_expired_ones_go, + a_challenge_dies_with_every_change_of_the_account, + two_completions_of_one_challenge_give_one_device, + completions_and_changes_of_the_account_at_the_same_moment_leave_no_stray_device, + turning_two_factor_on_and_off_at_the_same_moment_as_a_sign_in, + an_invite_to_a_protected_account_needs_the_second_factor_in_the_same_transaction, + a_reset_code_for_a_protected_account_needs_the_factor_unless_it_turns_it_off, + admins_two_factor_is_checked_in_the_change_and_resets_need_it, + the_command_line_resets_two_factor_and_the_account_shows_it, + secrets_are_listed_and_replaced_only_if_unchanged, + parallel_wrong_proofs_on_one_challenge_count_exactly_five, + parallel_wrong_proofs_lock_for_exactly_fifteen_minutes, + two_factor_changes_write_their_events, + two_factor_sign_in_writes_login_and_counts_failures, + the_lock_writes_one_event_when_it_is_set, + a_join_writes_its_two_factor_events, + resets_and_confirmations_write_their_events, + web_sessions_are_made_looked_up_expire_and_log_out, + a_web_session_ends_with_every_change_of_the_accounts_generation, + a_web_sign_in_with_two_factor_needs_its_own_challenge, + web_sessions_are_pruned_in_bounded_batches, + a_web_actor_changes_accounts_until_its_session_ends, ); // ---- helpers ---- @@ -291,7 +393,7 @@ async fn users_and_devices(meta: &Meta) { assert_eq!(meta.user_by_name("ana").await.unwrap(), Some(ana.clone())); assert_eq!(meta.user_by_name("bob").await.unwrap(), None); - meta.set_password_hash(ana.id, "$argon2id$new").await.unwrap(); + meta.reset_password(ana.id, "$argon2id$new", true, NOW + 1).await.unwrap(); assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); let token = [7u8; 32]; @@ -416,6 +518,242 @@ async fn stored_bytes_are_counted_once_per_space_and_capped(meta: &Meta) { assert!(!meta.has_manifest(ana.space, &m2.id()).await.unwrap()); } +async fn account_usage_estimates_history_and_trash_without_adding_them(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let empty = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((empty.stored_bytes, empty.trash_bytes, empty.history_bytes), (0, Some(0), Some(0))); + assert_eq!(empty.reason, None); + let _asset_manifest = manifest(meta, &ana, 3).await; + let assets = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((assets.trash_bytes, assets.history_bytes), (Some(0), Some(0)), "unrooted app assets are not freeable"); + + let (old, current) = (manifest(meta, &ana, 4).await, manifest(meta, &ana, 6).await); + let node = NodeId::generate(); + let first = version(old); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("file.txt"), version: first.clone() }], NOW) + .await + .unwrap(); + let live = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((live.trash_bytes, live.history_bytes), (Some(0), Some(0))); + + let latest = version(current); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: latest.clone() }], NOW).await.unwrap(); + let historical = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(historical.history_bytes, Some(105), "old chunk's 5 stored bytes plus its 100-byte manifest"); + assert_eq!(historical.trash_bytes, Some(0)); + assert_eq!(historical.stored_bytes, meta.stored_bytes(ana.space).await.unwrap()); + + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(latest.id) }], NOW).await.unwrap(); + let trashed = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(trashed.trash_bytes, Some(212), "both versions' chunks and manifests belong to trash"); + assert_eq!(trashed.history_bytes, Some(0), "trashed node history is not history estimate"); + assert_eq!(trashed.stored_bytes, historical.stored_bytes, "labeled estimates never add to stored bytes"); +} + +async fn admin_storage_overview_is_aggregated_and_orders_ties(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let mut accounts = Vec::new(); + for index in 0..13 { + accounts.push(user(meta, &format!("a{index:02}")).await); + } + let cap = 10_000_000; + let quota = 12_000_000; + meta.admin_update_user(&actor, accounts[0].id, None, Some(Some(quota)), NOW).await.unwrap(); + for (index, stored) in [quota, 8_999_999, 9_000_000, cap, 10, 10, 10, 10, 10, 10, 10, 10, 10].into_iter().enumerate() { + meta.add_chunk(accounts[index].space, &ChunkId::from_bytes([index as u8 + 1; 32]), stored, NOW).await.unwrap(); + } + + let report = meta.clone().with_stored_bytes_cap(Some(cap)).admin_storage_overview().await.unwrap(); + assert_eq!(report.accounts, 14); + assert_eq!(report.stored_bytes, 40_000_089); + assert_eq!(report.default_quota_bytes, Some(cap)); + assert_eq!(report.accounts_with_override, 1); + assert_eq!(report.accounts_near_quota, 3, "exactly 90% counts as near"); + assert_eq!(report.accounts_over_quota, 2, "exactly 100% counts as over"); + assert_eq!( + report.top_accounts.iter().map(|account| account.username.as_str()).collect::>(), + ["a00", "a03", "a02", "a01", "a04", "a05", "a06", "a07", "a08", "a09"] + ); + assert_eq!(report.top_accounts[0].quota_bytes, Some(quota), "account override beats cap"); + let capped = meta.clone().with_stored_bytes_cap(Some(cap)); + assert_eq!(capped.storage_estimate(accounts[0].space).await.unwrap().quota_bytes, Some(quota)); + assert_eq!(capped.storage_estimate(accounts[1].space).await.unwrap().quota_bytes, Some(cap)); + assert_eq!(meta.storage_estimate(accounts[1].space).await.unwrap().quota_bytes, None); + assert_eq!(report.manifest_index.complete, 0); + assert_eq!(report.manifest_index.building, 0); + assert_eq!(report.manifest_index.not_indexed, 0); +} + +async fn admin_storage_overview_handles_an_empty_server(meta: &Meta) { + let report = meta.admin_storage_overview().await.unwrap(); + assert_eq!(report.accounts, 0); + assert_eq!(report.stored_bytes, 0); + assert_eq!(report.accounts_with_override, 0); + assert_eq!(report.accounts_near_quota, 0); + assert_eq!(report.accounts_over_quota, 0); + assert!(report.top_accounts.is_empty()); + assert_eq!(report.manifest_index.complete, 0); + assert_eq!(report.manifest_index.building, 0); + assert_eq!(report.manifest_index.not_indexed, 0); +} + +async fn admin_storage_estimates_hide_removed_and_unknown_accounts(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let target = user(meta, "target").await; + let estimate = meta.admin_storage_estimate(target.id).await.unwrap(); + assert_eq!(estimate.quota_bytes, None); + assert_eq!(estimate.stored_bytes, 0); + assert!(matches!(meta.admin_storage_estimate(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound("user")))); + meta.admin_remove_user(&actor, target.id, NOW).await.unwrap(); + assert!(matches!(meta.admin_storage_estimate(target.id).await, Err(MetaError::NotFound("user")))); +} + +async fn account_usage_excludes_chunks_shared_with_current_version(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + let old = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([4; 32]), vec![ChunkRef { id: shared, len: 5 }]).unwrap(); + let current = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([5; 32]), vec![ChunkRef { id: shared, len: 5 }]).unwrap(); + meta.add_manifest(ana.space, &old, 100, NOW).await.unwrap(); + meta.add_manifest(ana.space, ¤t, 100, NOW).await.unwrap(); + let node = NodeId::generate(); + let first = version(old.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("shared.txt"), version: first.clone() }], NOW) + .await + .unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: version(current.id()) }], NOW).await.unwrap(); + + assert_eq!(meta.storage_estimate(ana.space).await.unwrap().history_bytes, Some(100), "the 6-byte chunk also belongs to the current version"); +} + +async fn account_usage_deduplicates_per_space_and_counts_pending_purge(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let (ana_device, bob_device) = (device(meta, &ana).await, device(meta, &bob).await); + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + meta.add_chunk(bob.space, &shared, 999, NOW).await.unwrap(); + + let make_manifest = |file_hash| { + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes(file_hash), vec![ChunkRef { id: shared, len: 5 }]).unwrap() + }; + let (trash_manifest, live_manifest, bob_manifest) = (make_manifest([1; 32]), make_manifest([2; 32]), make_manifest([3; 32])); + for (owner, manifest) in [(&ana, &trash_manifest), (&ana, &live_manifest), (&bob, &bob_manifest)] { + meta.add_manifest(owner.space, manifest, 100, NOW).await.unwrap(); + } + + let (ana_trash, ana_live) = (NodeId::generate(), NodeId::generate()); + let (trash_version, live_version) = (version(trash_manifest.id()), version(live_manifest.id())); + meta.commit( + ana.space, + ana_device, + &[ + Op::Create { node: ana_trash, parent: root(meta, &ana).await, name: name("trash.txt"), version: trash_version.clone() }, + Op::Create { node: ana_live, parent: root(meta, &ana).await, name: name("live.txt"), version: live_version }, + Op::Delete { node: ana_trash, expected: Some(trash_version.id) }, + ], + NOW, + ) + .await + .unwrap(); + let estimate = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(estimate.trash_bytes, Some(100), "shared chunk is also named by a live manifest"); + assert_eq!(estimate.history_bytes, Some(0)); + meta.request_purge(ana.space, ana_trash, NOW).await.unwrap(); + assert_eq!(meta.storage_estimate(ana.space).await.unwrap().trash_bytes, Some(100), "pending purge still stores trash blobs"); + + let bob_trash = NodeId::generate(); + let bob_version = version(bob_manifest.id()); + meta.commit( + bob.space, + bob_device, + &[ + Op::Create { node: bob_trash, parent: root(meta, &bob).await, name: name("shared.txt"), version: bob_version.clone() }, + Op::Delete { node: bob_trash, expected: Some(bob_version.id) }, + ], + NOW, + ) + .await + .unwrap(); + assert_eq!(meta.storage_estimate(bob.space).await.unwrap().trash_bytes, Some(106), "shared global chunk counts once in Bob's space"); +} + +async fn account_usage_excludes_live_document_snapshot_blobs(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let old = manifest(meta, &ana, 4).await; + let current = manifest(meta, &ana, 6).await; + let (node, first) = (NodeId::generate(), version(old)); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("document.kitdemo"), version: first.clone() }], NOW) + .await + .unwrap(); + let doc = craft_cloud_proto::live::node_doc(ana.space, node); + meta.create_live_document(ana.space, doc, "document:wordcraft", node, &old, NOW).await.unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: version(current) }], NOW).await.unwrap(); + + let estimate = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(estimate.history_bytes, Some(0), "old version's manifest and chunk remain named by the live-document snapshot"); +} + +async fn account_usage_leaves_saved_versions_out_of_history(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + let make_manifest = |file_hash| { + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes(file_hash), vec![ChunkRef { id: shared, len: 5 }]).unwrap() + }; + let (saved_old, unsaved_old) = (make_manifest([7; 32]), make_manifest([8; 32])); + for m in [&saved_old, &unsaved_old] { + meta.add_manifest(ana.space, m, 100, NOW).await.unwrap(); + } + let current = manifest(meta, &ana, 6).await; + + // v1 (saved) and v2 (not saved) name one chunk; v3 is current. The saved version's manifest and the chunk it shares + // with unsaved history stay out of the estimate: only v2's own 100-byte manifest is history. + let (node, v1) = (NodeId::generate(), version(saved_old.id())); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("saved.kitdemo"), version: v1.clone() }], NOW) + .await + .unwrap(); + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(ana.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(ana.space, node, v1.id, &record).await.unwrap(); + let v2 = version(unsaved_old.id()); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v1.id, version: v2.clone() }], NOW).await.unwrap(); + let v3 = version(current); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v2.id, version: v3.clone() }], NOW).await.unwrap(); + assert_eq!( + meta.storage_estimate(ana.space).await.unwrap().history_bytes, + Some(100), + "the saved version's manifest and the chunk it shares with unsaved history are kept" + ); + + // A saved version whose chunk nothing else names contributes nothing; an unsaved old version still counts in full. + let (exclusive_saved, exclusive_unsaved) = (manifest(meta, &ana, 4).await, manifest(meta, &ana, 4).await); + let (other, w1) = (NodeId::generate(), version(exclusive_saved)); + meta.commit(ana.space, dev, &[Op::Create { node: other, parent: root(meta, &ana).await, name: name("other.kitdemo"), version: w1.clone() }], NOW) + .await + .unwrap(); + let other_record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(ana.space, other), frontiers: vec![(1, 5)] }; + meta.attach_save(ana.space, other, w1.id, &other_record).await.unwrap(); + let w2 = version(exclusive_unsaved); + meta.commit(ana.space, dev, &[Op::Update { node: other, expected: w1.id, version: w2.clone() }], NOW).await.unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node: other, expected: w2.id, version: version(manifest(meta, &ana, 6).await) }], NOW).await.unwrap(); + assert_eq!( + meta.storage_estimate(ana.space).await.unwrap().history_bytes, + Some(205), + "unsaved 100 + 105 for the second file's unsaved old version" + ); + + // In the trash a saved version still counts: purging the node removes it with the rest. + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(v3.id) }], NOW).await.unwrap(); + let trashed = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(trashed.history_bytes, Some(105)); + assert_eq!(trashed.trash_bytes, Some(313), "three manifests, the shared 6-byte chunk and the current version's 7-byte chunk"); +} + /// Review finding 3 of M3c: two batches racing with an overlapping chunk charge it once (on Postgres both used to /// see it as new before inserting). SQLite runs one write at a time, so there it pins the result. async fn concurrent_chunk_batches_charge_a_shared_chunk_once(meta: &Meta) { @@ -1769,8 +2107,14 @@ async fn upgrading_counts_what_each_space_already_stores() { }; { // The database as it was before stored bytes were counted. - use sqlx::Connection; - let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let mut conn = raw(&url).await; + undo_team_spaces(&mut conn).await; + // Before the audit log. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE spaces DROP COLUMN stored_bytes").execute(&mut conn).await.unwrap(); // And before live documents of files. sqlx::query("ALTER TABLE spaces DROP COLUMN live_bytes").execute(&mut conn).await.unwrap(); @@ -1782,7 +2126,27 @@ async fn upgrading_counts_what_each_space_already_stores() { sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); // And before save records. + // And before the reference index. + sqlx::query("DROP TABLE manifest_chunks").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE manifest_index").execute(&mut conn).await.unwrap(); + // And before the trash. + sqlx::query("DROP INDEX nodes_trash").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN deleted_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN purge_requested_ms").execute(&mut conn).await.unwrap(); + // And before the history index. + sqlx::query("DROP INDEX versions_node_seq").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE versions DROP COLUMN save").execute(&mut conn).await.unwrap(); + // And before the admin API (its `invites` column went with the table). + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + // And before device sessions. + for column in ["client_kind", "platform", "app_version", "revoked_ms", "revoke_reason"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE devices DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + for column in ["password_changed_ms", "disabled_ms"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE users DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } sqlx::query("DELETE FROM schema_version WHERE version >= 2").execute(&mut conn).await.unwrap(); } let meta = Meta::open(&url).await.unwrap(); @@ -1807,5 +2171,5730 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 6); + assert_eq!(versions, 15); +} + +// ---- device sessions (migration 0007) ---- + +/// A device's record, read back from the list. +async fn record(meta: &Meta, user: &UserRecord, id: DeviceId) -> DeviceRecord { + meta.list_devices(user.id).await.unwrap().into_iter().find(|d| d.id == id).unwrap() +} + +/// The three columns of a revoke always agree: all set or none. +fn assert_revoke_columns_agree(devices: &[DeviceRecord]) { + for d in devices { + assert_eq!((d.revoked, d.revoked_ms.is_some(), d.revoke_reason.is_some()), (d.revoked, d.revoked, d.revoked), "{d:?}"); + } +} + +fn sorted(mut ids: Vec) -> Vec { + ids.sort_by_key(|id| *id.as_uuid()); + ids +} + +async fn a_device_records_its_client(meta: &Meta) { + let ana = user(meta, "ana").await; + let plain = device(meta, &ana).await; + let client = DeviceClient::new(Some("agent"), Some("windows x86_64"), Some("0.19.0")).unwrap(); + let agent = meta.create_device_with(ana.id, "studio", &client, &token(), NOW + 1).await.unwrap(); + assert_eq!(record(meta, &ana, agent).await.client, client); + let old = record(meta, &ana, plain).await; + assert_eq!((old.client, old.revoked_ms, old.revoke_reason), (DeviceClient::default(), None, None), "no word on the client: empty"); + + // What a sign-in may say: a known kind or none, short free text without control characters. + assert_eq!(DeviceClient::new(None, None, None).unwrap(), DeviceClient::default()); + assert!(DeviceClient::new(Some(""), Some(&"x".repeat(32)), Some(&"y".repeat(32))).is_ok()); + let long = "x".repeat(33); + for (kind, platform, version) in [ + (Some("browser"), None, None), + (Some("AGENT"), None, None), + (None, Some(long.as_str()), None), + (None, None, Some(long.as_str())), + (None, Some("a\nb"), None), + (None, None, Some("1.0\u{7}")), + ] { + assert!(matches!(DeviceClient::new(kind, platform, version), Err(MetaError::Invalid(_))), "{kind:?} {platform:?} {version:?}"); + } + // The store checks too, for a caller that built the value by hand: nothing is created. + let bad = DeviceClient { kind: "browser".into(), ..DeviceClient::default() }; + assert!(matches!(meta.create_device_with(ana.id, "x", &bad, &token(), NOW).await, Err(MetaError::Invalid(_)))); + assert_eq!(meta.list_devices(ana.id).await.unwrap().len(), 2); + + // A join says it too. + let (_, _, pairing, _) = paired(meta, "bob").await; + let desktop = DeviceClient::new(Some("desktop"), Some("macos"), Some("1.2")).unwrap(); + let joined = meta.redeem_with(&pairing.hash(), "mac", &desktop, &token(), None, NOW + 1).await.unwrap(); + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + assert_eq!(record(meta, &bob, joined.device).await.client, desktop); +} + +async fn only_own_live_devices_are_renamed(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let (mine, theirs) = (device(meta, &ana).await, device(meta, &bob).await); + let renamed = meta.rename_device(ana.id, mine, "studio pc").await.unwrap(); + assert_eq!((renamed.id, renamed.name.as_str()), (mine, "studio pc")); + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); + // Another account's device, and one that isn't there, are one answer. + assert!(matches!(meta.rename_device(ana.id, theirs, "mine now").await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.rename_device(ana.id, DeviceId::generate(), "x").await, Err(MetaError::NotFound(_)))); + assert_eq!(record(meta, &bob, theirs).await.name, "laptop"); + // The name rule holds. + for bad in ["", "tab\there", &"n".repeat(129)] { + assert!(matches!(meta.rename_device(ana.id, mine, bad).await, Err(MetaError::Invalid(_))), "{bad:?}"); + } + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); + // A revoked device keeps its name. + meta.revoke_device_as(ana.id, mine, RevokeReason::User, NOW + 1).await.unwrap(); + assert!(matches!(meta.rename_device(ana.id, mine, "again").await, Err(MetaError::NotFound(_)))); + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); +} + +async fn every_revoke_path_sets_flag_time_and_reason_together(meta: &Meta) { + // One device revoked by its user, and revoked again later: the first time and reason stay. + let ana = user(meta, "ana").await; + let (a1, a2, a3, a4) = (device(meta, &ana).await, device(meta, &ana).await, device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_device_as(ana.id, a1, RevokeReason::User, NOW + 10).await.unwrap(); + let first = record(meta, &ana, a1).await; + assert_eq!((first.revoked, first.revoked_ms, first.revoke_reason), (true, Some(NOW + 10), Some(RevokeReason::User))); + meta.revoke_device_as(ana.id, a1, RevokeReason::Admin, NOW + 20).await.unwrap(); + assert_eq!(record(meta, &ana, a1).await, first, "a second revoke changes nothing"); + assert!(matches!(meta.revoke_device_as(user(meta, "bob").await.id, a1, RevokeReason::User, NOW).await, Err(MetaError::NotFound(_)))); + // The others, but one: the device revoked before keeps its own reason, the kept one stays. + let revoked = meta.revoke_other_devices(ana.id, a2, RevokeReason::Others, NOW + 30).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![a3, a4]), "only the live others"); + for id in [a3, a4] { + let d = record(meta, &ana, id).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 30), Some(RevokeReason::Others))); + } + assert_eq!(record(meta, &ana, a1).await, first); + let kept = record(meta, &ana, a2).await; + assert_eq!((kept.revoked, kept.revoked_ms, kept.revoke_reason), (false, None, None)); + assert!(meta.revoke_other_devices(ana.id, a2, RevokeReason::Others, NOW + 40).await.unwrap().is_empty(), "nobody left to sign out"); + // Every device (the admin's command). + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + let (b1, b2) = (device(meta, &bob).await, device(meta, &bob).await); + let revoked = meta.revoke_all_devices(bob.id, RevokeReason::Admin, NOW + 50).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![b1, b2])); + assert!(meta.revoke_all_devices(bob.id, RevokeReason::Admin, NOW + 60).await.unwrap().is_empty()); + for id in [b1, b2] { + let d = record(meta, &bob, id).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 50), Some(RevokeReason::Admin))); + } + // A password change and a reset. + let cy = user(meta, "cy").await; + let (c1, c2) = (device(meta, &cy).await, device(meta, &cy).await); + meta.change_password(cy.id, "$argon2id$stub", "$argon2id$new", c1, NOW + 70).await.unwrap(); + let d = record(meta, &cy, c2).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 70), Some(RevokeReason::Password))); + let dee = user(meta, "dee").await; + let d1 = device(meta, &dee).await; + meta.reset_password(dee.id, "$argon2id$new", false, NOW + 80).await.unwrap(); + let d = record(meta, &dee, d1).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 80), Some(RevokeReason::Reset))); + // The plain call (no clock of its own) is the user's act, with a time. + let eve = user(meta, "eve").await; + let e1 = device(meta, &eve).await; + meta.revoke_device(eve.id, e1).await.unwrap(); + let d = record(meta, &eve, e1).await; + assert_eq!((d.revoked, d.revoke_reason), (true, Some(RevokeReason::User))); + assert!(d.revoked_ms.is_some_and(|ms| ms > NOW)); + for u in [&ana, &bob, &cy, &dee, &eve] { + assert_revoke_columns_agree(&meta.list_devices(u.id).await.unwrap()); + } +} + +async fn revoking_others_keeps_the_caller_and_ends_their_pairings(meta: &Meta) { + let (ana, keep, keep_pairing, keep_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_id = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + let revoked = meta.revoke_other_devices(ana.id, keep, RevokeReason::Others, NOW + 1).await.unwrap(); + assert_eq!(revoked, vec![other]); + assert!(meta.device_active(keep).await.unwrap() && !meta.device_active(other).await.unwrap()); + assert_eq!(meta.pairing_status(ana.id, keep_id, NOW + 2).await.unwrap(), Some((PairingState::Waiting, None)), "the caller's own stays"); + assert_eq!(meta.pairing_status(ana.id, other_id, NOW + 2).await.unwrap(), None, "a revoked device's pairing dies"); + redeem(meta, &keep_pairing, "pc", NOW + 3).await.unwrap(); + // Another account is never touched. + assert!(meta.device_active(bobs).await.unwrap()); + assert!(meta.pairing_status(bob.id, bobs_pairing, NOW + 2).await.unwrap().is_some()); +} + +async fn a_password_change_is_one_step(meta: &Meta) { + let (ana, keep, _, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_pairing = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + let bobs_invite = code(INVITE_CHARS); + meta.create_invite("bob", &bobs_invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + + let account = meta.account(ana.id).await.unwrap(); + assert_eq!((account.username.as_str(), account.role, account.created_ms, account.password_changed_ms), ("ana", Role::Member, NOW, None)); + assert!(matches!(meta.account(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound(_)))); + + // A password that is not the one the caller verified (it changed meanwhile): nothing at all changes. + let stale = meta.change_password(ana.id, "$argon2id$stale", "$argon2id$new", keep, NOW + 5).await; + assert!(matches!(stale, Err(MetaError::Conflict(_))), "{stale:?}"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + assert!(meta.device_active(keep).await.unwrap() && meta.device_active(other).await.unwrap()); + assert!(meta.pairing_status(ana.id, pairing_id, NOW + 6).await.unwrap().is_some()); + assert!(meta.pairing_status(ana.id, other_pairing, NOW + 6).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 6).await.unwrap(), Some(false)); + + // The change: new hash and time, the other device signed out, invite and pairings gone, the caller stays. + let revoked = meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 7).await.unwrap(); + assert_eq!(revoked, vec![other]); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 7)); + assert!(meta.device_active(keep).await.unwrap() && !meta.device_active(other).await.unwrap()); + let d = record(meta, &ana, other).await; + assert_eq!((d.revoked_ms, d.revoke_reason), (Some(NOW + 7), Some(RevokeReason::Password))); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 8).await.unwrap(), None); + assert_eq!(meta.pairing_status(ana.id, other_pairing, NOW + 8).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 8).await.unwrap(), None); + assert!(matches!(redeem(meta, &invite, "pc", NOW + 8).await, Err(MetaError::InvalidCode))); + // The old hash no longer matches, so the same change again (a replayed request) does nothing. + assert!(matches!(meta.change_password(ana.id, "$argon2id$stub", "$argon2id$third", keep, NOW + 9).await, Err(MetaError::Conflict(_)))); + // A caller that was signed out meanwhile changes nothing either. + assert!( + matches!(meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", other, NOW + 9).await, Err(MetaError::DeviceRevoked)), + "revoked" + ); + assert!( + matches!( + meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", DeviceId::generate(), NOW + 9).await, + Err(MetaError::DeviceRevoked) + ), + "unknown" + ); + let bobs_device = device(meta, &user(meta, "cy").await).await; + assert!( + matches!(meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", bobs_device, NOW + 9).await, Err(MetaError::DeviceRevoked)), + "another account's" + ); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); + // Another account is never touched. + assert_eq!(meta.user_by_name("bob").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert!(meta.device_active(bobs).await.unwrap()); + assert!(meta.pairing_status(bob.id, bobs_pairing, NOW + 8).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&bobs_invite.hash(), NOW + 8).await.unwrap(), Some(false)); +} + +async fn an_admin_reset_signs_out_every_device_unless_kept(meta: &Meta) { + let (ana, maker, _, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let revoked = meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![maker, other])); + assert!(!meta.device_active(maker).await.unwrap() && !meta.device_active(other).await.unwrap()); + assert_eq!(record(meta, &ana, other).await.revoke_reason, Some(RevokeReason::Reset)); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$reset"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 1)); + + // `--keep-devices`: the devices stay, the password changes, and codes made before it are still deleted. + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + assert!(meta.reset_password(bob.id, "$argon2id$reset", true, NOW + 3).await.unwrap().is_empty()); + assert!(meta.device_active(bobs).await.unwrap()); + assert_eq!(record(meta, &bob, bobs).await.revoke_reason, None); + assert_eq!(meta.pairing_status(bob.id, bobs_pairing, NOW + 4).await.unwrap(), None); + assert_eq!(meta.user_by_name("bob").await.unwrap().unwrap().password_hash, "$argon2id$reset"); + + // A user that isn't there changes nothing. + assert!(matches!(meta.reset_password(UserId(uuid::Uuid::now_v7()), "$argon2id$x", false, NOW).await, Err(MetaError::NotFound(_)))); +} + +/// A password change from one device, a sign-out of the others from another and an admin reset, at the same moment +/// and over and over: each ends as a success or as a conflict or a signed-out caller, never as a database error (on +/// Postgres, two transactions taking the same rows in different orders would deadlock and one would fail). +async fn concurrent_password_changes_and_sign_outs_never_fail_the_database(meta: &Meta) { + for round in 0..15 { + let ana = user(meta, &format!("ana{round}")).await; + let (a, b) = (device(meta, &ana).await, device(meta, &ana).await); + let mut tasks = Vec::new(); + for job in 0..6 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.change_password(ana.id, "$argon2id$stub", "$argon2id$one", a, NOW + 1).await.map(|_| ()), + 1 => meta.change_password(ana.id, "$argon2id$stub", "$argon2id$two", b, NOW + 1).await.map(|_| ()), + 2 => meta.revoke_other_devices(ana.id, b, RevokeReason::Others, NOW + 1).await.map(|_| ()), + 3 => meta.revoke_device_as(ana.id, a, RevokeReason::User, NOW + 1).await, + 4 => meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.map(|_| ()), + _ => meta.revoke_all_devices(ana.id, RevokeReason::Admin, NOW + 1).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!(matches!(result, Ok(()) | Err(MetaError::Conflict(_) | MetaError::DeviceRevoked)), "round {round}: {result:?}"); + } + assert_revoke_columns_agree(&meta.list_devices(ana.id).await.unwrap()); + } +} + +/// A device is created after a password sign-in only if the password that was checked is still the account's. +async fn a_sign_in_checked_against_an_old_password_creates_no_device(meta: &Meta) { + let ana = user(meta, "ana").await; + let client = DeviceClient::default(); + let first = meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "pc", &client, &token(), NOW).await.unwrap(); + assert!(first.is_some_and(|d| d != DeviceId::generate())); + assert!(meta.sign_in_device(ana.id, ("$argon2id$other", 0), "pc", &client, &token(), NOW).await.unwrap().is_none(), "not the stored hash"); + assert!( + meta.sign_in_device(UserId(uuid::Uuid::now_v7()), ("$argon2id$stub", 0), "pc", &client, &token(), NOW).await.unwrap().is_none(), + "no such user" + ); + assert!(matches!(meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "", &client, &token(), NOW).await, Err(MetaError::Invalid(_)))); + // The password was checked, then reset: the account's devices are signed out and the late sign-in adds none. + meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.unwrap(); + assert!(meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "late", &client, &token(), NOW + 2).await.unwrap().is_none()); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert_eq!(devices.len(), 1); + assert!(devices.iter().all(|d| d.revoked)); + // With the new hash it works. + assert!(meta.sign_in_device(ana.id, ("$argon2id$reset", gen_of(meta, "ana").await), "new", &client, &token(), NOW + 3).await.unwrap().is_some()); +} + +/// Redeeming an invite or a pairing, a password sign-in and a reset at the same moment, over and over: no database +/// error (on Postgres, the invite and the user taken in different orders would deadlock), and once the reset is +/// done no device of the account is signed in, whichever of them came first. +async fn redeems_sign_ins_and_resets_at_the_same_moment_leave_no_live_device(meta: &Meta) { + for round in 0..15 { + let (ana, maker, pairing, _) = paired(meta, &format!("ana{round}")).await; + let invite = code(INVITE_CHARS); + meta.create_invite(&format!("ana{round}"), &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..5 { + let (meta, invite, pairing) = (meta.clone(), invite.hash(), pairing.hash()); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.redeem(&invite, "invited", &token(), None, NOW + 1).await.map(|_| ()), + 1 => meta.redeem(&pairing, "paired", &token(), None, NOW + 1).await.map(|_| ()), + 2 => { + meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "signed in", &DeviceClient::default(), &token(), NOW + 1).await.map(|_| ()) + } + 3 => meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 1).await.map(|_| ()), + _ => meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 2).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!(matches!(result, Ok(()) | Err(MetaError::InvalidCode | MetaError::NotFound(_))), "round {round}: {result:?}"); + } + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| d.revoked), "round {round}: {devices:?}"); + assert_revoke_columns_agree(&devices); + } +} + +// ---- the migration from 0006 to 0007 ---- + +/// Drops what migration 0007 added and forgets it ran, which leaves the database as 0006 left it (with its rows). +async fn back_to_0006(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + // What migrations 0010, 0009 and 0008 added goes first: they run again with 0007 on the next open. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN purpose").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + for column in ["client_kind", "platform", "app_version", "revoked_ms", "revoke_reason"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE devices DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + for column in ["password_changed_ms", "disabled_ms"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE users DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + sqlx::query("DELETE FROM schema_version WHERE version >= 7").execute(&mut conn).await.unwrap(); +} + +async fn schema_version(url: &str) -> i64 { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap() +} + +/// Existing users and devices (live and revoked) survive the migration, read with empty client fields and no revoke +/// time or reason, still work (a live one is found by its token, a revoked one is not), and take the new columns; +/// a database newer than the program is refused. +async fn migrating_from_0006_keeps_the_rows(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, live, revoked, old_token) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + let live = meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + let revoked = device(&meta, &ana).await; + meta.revoke_device(ana.id, revoked).await.unwrap(); + (ana, live, revoked, old_token) + }; + back_to_0006(url).await; + assert_eq!(schema_version(url).await, 6); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15); + let old = record(&meta, &ana, live).await; + assert_eq!( + (old.name.as_str(), old.revoked, old.client, old.revoked_ms, old.revoke_reason), + ("old pc", false, DeviceClient::default(), None, None) + ); + let gone = record(&meta, &ana, revoked).await; + assert_eq!((gone.revoked, gone.client, gone.revoked_ms, gone.revoke_reason), (true, DeviceClient::default(), None, None)); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.device), Some(live)); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + // The new columns take values. + let client = DeviceClient::new(Some("cli"), Some("linux"), Some("1")).unwrap(); + let fresh = meta.create_device_with(ana.id, "new", &client, &token(), NOW + 1).await.unwrap(); + assert_eq!(record(&meta, &ana, fresh).await.client, client); + meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", fresh, NOW + 2).await.unwrap(); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 2)); + assert_eq!(record(&meta, &ana, live).await.revoke_reason, Some(RevokeReason::Password)); + + // A database from a newer Craft Cloud is still refused. + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (16)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0006_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0006_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0006_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0006_keeps_the_rows(&url).await; +} + +// ---- a failure in the middle of a transaction ---- + +/// A trigger that makes every delete from `pairings` fail, which is the step after the password and the devices +/// were already written in `change_password` and `reset_password`. Test only: the migrations carry no trigger. +async fn break_pairing_deletes(url: &str, postgres: bool) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + let statements: &[&str] = if postgres { + &[ + "CREATE FUNCTION cc_test_boom() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$ LANGUAGE plpgsql", + "CREATE TRIGGER cc_test_boom BEFORE DELETE ON pairings FOR EACH STATEMENT EXECUTE FUNCTION cc_test_boom()", + ] + } else { + &["CREATE TRIGGER cc_test_boom BEFORE DELETE ON pairings BEGIN SELECT RAISE(ABORT, 'boom'); END"] + }; + for statement in statements { + sqlx::query(sqlx::AssertSqlSafe((*statement).to_owned())).execute(&mut conn).await.unwrap(); + } +} + +async fn mend_pairing_deletes(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("DROP TRIGGER cc_test_boom ON pairings").execute(&mut conn).await.ok(); + sqlx::query("DROP TRIGGER cc_test_boom").execute(&mut conn).await.ok(); + sqlx::query("DROP FUNCTION cc_test_boom()").execute(&mut conn).await.ok(); +} + +/// A step that fails after others were written undoes them all: the hash, the time, the devices, the pairings and +/// the invite are as they were, for a password change and for an admin reset; once the step works, the same call +/// goes through. +async fn a_failure_in_the_middle_changes_nothing(url: &str, postgres: bool) { + let meta = Meta::open(url).await.unwrap(); + let (ana, keep, _, pairing_id) = paired(&meta, "ana").await; + let other = device(&meta, &ana).await; + let other_pairing = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + break_pairing_deletes(url, postgres).await; + + let invite_hash = invite.hash(); + let unchanged = || async { + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| !d.revoked && d.revoked_ms.is_none() && d.revoke_reason.is_none()), "{devices:?}"); + assert!(meta.pairing_status(ana.id, pairing_id, NOW + 1).await.unwrap().is_some()); + assert!(meta.pairing_status(ana.id, other_pairing, NOW + 1).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&invite_hash, NOW + 1).await.unwrap(), Some(false)); + }; + let failed = meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + let failed = meta.reset_password(ana.id, "$argon2id$new", false, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + let failed = meta.revoke_other_devices(ana.id, keep, RevokeReason::Others, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + + mend_pairing_deletes(url).await; + assert_eq!(meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 3).await.unwrap(), vec![other]); +} + +#[tokio::test] +async fn a_failure_in_the_middle_changes_nothing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_failure_in_the_middle_changes_nothing(&format!("sqlite:{}", dir.path().join("meta.db").display()), false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failure_in_the_middle_changes_nothing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_failure_in_the_middle_changes_nothing(&url, true).await; +} + +// ---- the admin API (migration 0008) ---- + +const ADMIN_HASH: &str = "$argon2id$admin"; + +/// An admin account with a device, and the actor an admin request on its behalf would carry. +async fn admin_actor(meta: &Meta, username: &str) -> (UserRecord, AdminActor) { + let u = meta.create_user(username, ADMIN_HASH, Role::Admin, NOW).await.unwrap(); + let device = device(meta, &u).await; + let actor = AdminActor { user: u.id, via: AdminVia::Device(device), password_hash: ADMIN_HASH.to_owned(), two_factor: false }; + (u, actor) +} + +/// The device a device-borne actor acts with. +fn device_of(actor: &AdminActor) -> DeviceId { + match actor.via { + AdminVia::Device(device) => device, + AdminVia::Web { .. } => panic!("a device actor was expected"), + } +} + +/// The generation a sign-in would read for `username` now. +async fn gen_of(meta: &Meta, username: &str) -> i64 { + meta.user_by_name(username).await.unwrap().unwrap().auth_gen +} + +async fn admin_view(meta: &Meta, user: UserId) -> AdminUser { + meta.admin_user(user, NOW).await.unwrap().unwrap() +} + +async fn users_page(meta: &Meta, after: Option<&str>, search: Option<&str>, limit: u32) -> (Vec, Option) { + let page = meta.admin_list_users(after, search, limit, NOW).await.unwrap(); + (page.users.into_iter().map(|u| u.username).collect(), page.next_after) +} + +async fn the_admin_list_pages_searches_and_counts(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for name in ["ana", "bob", "a_b", "axb", "carol"] { + user(meta, name).await; + } + // By username, in pages; the last page has no continuation. + let (first, next) = users_page(meta, None, None, 3).await; + assert_eq!(first.len(), 3); + let (second, next2) = users_page(meta, next.as_deref(), None, 3).await; + assert_eq!(second.len(), 3); + assert_eq!(next2, None); + let mut all: Vec<_> = first.into_iter().chain(second).collect(); + let mut sorted_all = all.clone(); + sorted_all.sort(); + assert_eq!(all, sorted_all, "pages follow one order"); + all.dedup(); + assert_eq!(all.len(), 6, "no account twice, none missing"); + assert_eq!(users_page(meta, None, None, 6).await.1, None, "exactly a full page has no continuation"); + // A search finds a substring, ignores case, and takes `%` and `_` literally. + assert_eq!(users_page(meta, None, Some("A_B"), 50).await.0, vec!["a_b"]); + assert_eq!(users_page(meta, None, Some("a"), 50).await.0, vec!["a_b", "ana", "axb", "carol"]); + assert_eq!(users_page(meta, None, Some("%"), 50).await.0, Vec::::new()); + assert_eq!(users_page(meta, None, Some("_"), 50).await.0, vec!["a_b"]); + assert_eq!(users_page(meta, None, Some("zzz"), 50).await.0, Vec::::new()); + // A search pages too. + let (one, next) = users_page(meta, None, Some("a"), 1).await; + assert_eq!(one, vec!["a_b"]); + assert_eq!(users_page(meta, next.as_deref(), Some("a"), 2).await.0, vec!["ana", "axb"]); + assert!(matches!(meta.admin_list_users(None, Some(&"x".repeat(65)), 10, NOW).await, Err(MetaError::Invalid(_)))); + meta.admin_list_users(None, Some(&"x".repeat(64)), 10, NOW).await.unwrap(); + + // What a row says. + let ana = meta.user_by_name("ana").await.unwrap().unwrap(); + let dev = device(meta, &ana).await; + meta.add_chunk(ana.space, &random_chunk(), 70, NOW).await.unwrap(); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let row = admin_view(meta, ana.id).await; + assert_eq!((row.role, row.devices_active, row.stored_bytes, row.quota_bytes, row.invite_pending), (Role::Member, 1, 70, None, true)); + meta.revoke_device(ana.id, dev).await.unwrap(); + assert_eq!(admin_view(meta, ana.id).await.devices_active, 0); + // An expired invite is not pending. + assert!(!meta.admin_user(ana.id, NOW + INVITE_TTL_MS).await.unwrap().unwrap().invite_pending); + // A reset code is not an invite. + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + meta.admin_issue_reset(&actor, bob.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + assert!(!admin_view(meta, bob.id).await.invite_pending); + + // The counters: accounts, live devices, bytes. + let counts = meta.admin_server_counts().await.unwrap(); + assert_eq!((counts.users, counts.devices_active, counts.stored_bytes, counts.live_bytes), (6, 1, 70, 0), "only root's device is live"); + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.admin_server_counts().await.unwrap().users, 5); + assert!(!users_page(meta, None, None, 50).await.0.contains(&"ana".to_owned()), "a removed account is not listed"); + assert!(meta.admin_user(ana.id, NOW).await.unwrap().is_some(), "but it can still be looked up"); +} + +async fn an_admin_makes_an_account_with_a_role_and_an_invite(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let invite = code(INVITE_CHARS); + let (made, issued) = meta.admin_create_user(&actor, "Newbie", Role::Admin, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + assert_eq!((made.username.as_str(), made.role, made.devices_active, made.invite_pending), ("newbie", Role::Admin, 0, true)); + assert_eq!(issued.expires_ms, NOW + INVITE_TTL_MS); + // A name that exists, in any case, any role, is taken, and nothing is changed. + for name in ["newbie", "NEWBIE", "root"] { + let again = meta.admin_create_user(&actor, name, Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await; + assert!(matches!(again, Err(MetaError::UsernameTaken)), "{name}: {again:?}"); + } + assert!(matches!( + meta.admin_create_user(&actor, "bad name", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::Invalid(_)) + )); + // It has no password: the invite asks for one, and sets it. The account keeps its role and has its own space. + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(true)); + assert!(matches!(meta.redeem(&invite.hash(), "pc", &token(), None, NOW + 1).await, Err(MetaError::PasswordRequired))); + let joined = meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$chosen"), NOW + 2).await.unwrap(); + assert_eq!(joined.user, made.id); + let record = meta.user_by_name("newbie").await.unwrap().unwrap(); + assert_eq!((record.role, record.password_hash.as_str()), (Role::Admin, "$argon2id$chosen")); + assert!(meta.space_info(record.space).await.is_ok()); + assert!(!admin_view(meta, made.id).await.invite_pending); +} + +async fn an_admin_changes_a_role_and_a_quota(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + // Nothing to change is invalid. + assert!(matches!(meta.admin_update_user(&actor, ana.id, None, None, NOW).await, Err(MetaError::Invalid(_)))); + let promoted = meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + assert_eq!((promoted.role, promoted.quota_bytes), (Role::Admin, None)); + // Role and quota together; a missing quota leaves it, `Some(None)` removes it. + let both = meta.admin_update_user(&actor, ana.id, Some(Role::Member), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert_eq!((both.role, both.quota_bytes), (Role::Member, Some(MIN_QUOTA_BYTES))); + let role_only = meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await.unwrap(); + assert_eq!(role_only.quota_bytes, Some(MIN_QUOTA_BYTES), "a missing quota leaves it"); + let cleared = meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + assert_eq!(cleared.quota_bytes, None); + // Bounds: a chunk up to i64::MAX. + for bad in [0, 1, MIN_QUOTA_BYTES - 1, i64::MAX as u64 + 1, u64::MAX] { + let result = meta.admin_update_user(&actor, ana.id, None, Some(Some(bad)), NOW).await; + assert!(matches!(result, Err(MetaError::Invalid(_))), "{bad}: {result:?}"); + } + for good in [MIN_QUOTA_BYTES, i64::MAX as u64] { + assert_eq!(meta.admin_update_user(&actor, ana.id, None, Some(Some(good)), NOW).await.unwrap().quota_bytes, Some(good)); + } + // Unknown users, and accounts marked for removal. + assert!(matches!(meta.admin_update_user(&actor, UserId(uuid::Uuid::now_v7()), Some(Role::Admin), None, NOW).await, Err(MetaError::NotFound(_)))); + // The last administrator can't be demoted, even by themselves; with another one they can. + let result = meta.admin_update_user(&actor, root.id, Some(Role::Member), None, NOW).await; + assert!(matches!(result, Err(MetaError::LastAdmin)), "{result:?}"); + assert_eq!(admin_view(meta, root.id).await.role, Role::Admin, "and nothing was changed"); + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + meta.admin_update_user(&actor, root.id, Some(Role::Member), None, NOW).await.unwrap(); + // The demoted admin's next change is refused: they are no admin any more. + let result = meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + let ana_actor = + AdminActor { user: ana.id, via: AdminVia::Device(device(meta, &ana).await), password_hash: "$argon2id$stub".to_owned(), two_factor: false }; + meta.admin_remove_user(&ana_actor, root.id, NOW).await.unwrap(); + let result = meta.admin_update_user(&ana_actor, root.id, Some(Role::Admin), None, NOW).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "{result:?}"); +} + +/// The acting admin's state is read again inside each change (the role and the password check the handler did can be +/// stale by then): a demotion, a sign-out, a password change or a disable since lets nothing through, and nothing is +/// changed. +async fn the_acting_admins_state_is_checked_in_the_transaction(meta: &Meta) { + let (_, boss) = admin_actor(meta, "boss").await; + let victim = user(meta, "victim").await; + let issue = |actor: AdminActor| { + let meta = meta.clone(); + async move { + meta.admin_create_user( + &actor, + &format!("made{}", uuid::Uuid::now_v7().simple()), + Role::Member, + &code(INVITE_CHARS).hash(), + NOW, + INVITE_TTL_MS, + ) + .await + .map(|_| ()) + } + }; + let users_before = meta.admin_server_counts().await.unwrap().users; + + // A member (a demoted admin) is forbidden. + let (demoted, demoted_actor) = admin_actor(meta, "demoted").await; + meta.admin_update_user(&boss, demoted.id, Some(Role::Member), None, NOW).await.unwrap(); + assert!(matches!(issue(demoted_actor).await, Err(MetaError::Forbidden(_)))); + // A signed-out device is unauthorized. + let (signed_out, signed_out_actor) = admin_actor(meta, "signedout").await; + meta.revoke_device(signed_out.id, device_of(&signed_out_actor)).await.unwrap(); + assert!(matches!(issue(signed_out_actor).await, Err(MetaError::DeviceRevoked))); + // A password changed since it was checked is a conflict. + let (changed, changed_actor) = admin_actor(meta, "changed").await; + meta.reset_password(changed.id, "$argon2id$other", true, NOW).await.unwrap(); + assert!(matches!(issue(changed_actor).await, Err(MetaError::Conflict(_)))); + // A disabled admin is unauthorized (their devices are revoked, and the check says it even for a device kept). + let (off, off_actor) = admin_actor(meta, "off").await; + meta.admin_disable_user(&boss, off.id, NOW).await.unwrap(); + assert!(matches!(issue(off_actor).await, Err(MetaError::DeviceRevoked))); + // Another account's device is not the actor's. + let borrowed = AdminActor { via: AdminVia::Device(device(meta, &victim).await), ..boss.clone() }; + assert!(matches!(issue(borrowed).await, Err(MetaError::DeviceRevoked))); + // A member who is no admin at all. + let member_actor = AdminActor { + user: victim.id, + via: AdminVia::Device(device(meta, &victim).await), + password_hash: "$argon2id$stub".to_owned(), + two_factor: false, + }; + assert!(matches!(meta.admin_disable_user(&member_actor, demoted.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert_eq!(meta.admin_server_counts().await.unwrap().users, users_before + 4, "only the four admins made here, no account from a refused change"); + // Every kind of change checks, not only the first: one of each against a stale actor. + let (stale_user, stale_actor) = admin_actor(meta, "stale").await; + meta.admin_update_user(&boss, stale_user.id, Some(Role::Member), None, NOW).await.unwrap(); + assert!(matches!(meta.admin_update_user(&stale_actor, victim.id, Some(Role::Admin), None, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_disable_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_enable_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_remove_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + let hash = code(INVITE_CHARS).hash(); + assert!(matches!(meta.admin_issue_invite(&stale_actor, victim.id, &hash, NOW, INVITE_TTL_MS).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_revoke_invite(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_issue_reset(&stale_actor, victim.id, &hash, false, NOW, RESET_TTL_MS).await, Err(MetaError::Forbidden(_)))); + let victims_device = device(meta, &victim).await; + assert!(matches!(meta.admin_revoke_device(&stale_actor, victim.id, victims_device, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(meta.device_active(victims_device).await.unwrap()); + assert_eq!(admin_view(meta, victim.id).await.role, Role::Member); +} + +/// A disabled account has nothing: its devices are signed out (with the reason), its codes gone, and no token, sign-in, +/// join or pairing works; enabling it brings back the account but never a device. +async fn disabling_signs_everything_out_and_enabling_brings_back_nothing(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (ana, maker, pairing, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_token = token(); + let third = meta.create_device(ana.id, "third", &other_token, NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + + // Not oneself. + assert!(matches!(meta.admin_disable_user(&actor, root.id, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.admin_remove_user(&actor, root.id, NOW).await, Err(MetaError::Invalid(_)))); + let revoked = meta.admin_disable_user(&actor, ana.id, NOW + 1).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![maker, other, third])); + let view = admin_view(meta, ana.id).await; + assert_eq!((view.disabled_ms, view.devices_active, view.invite_pending), (Some(NOW + 1), 0, false)); + assert_eq!(meta.device_by_token(&other_token).await.unwrap(), None); + for d in meta.list_devices(ana.id).await.unwrap() { + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 1), Some(RevokeReason::Disabled))); + } + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None); + // Disabling again is harmless. + assert!(meta.admin_disable_user(&actor, ana.id, NOW + 5).await.unwrap().is_empty()); + assert_eq!(admin_view(meta, ana.id).await.disabled_ms, Some(NOW + 1), "the first time stays"); + + // Nothing gets in: a sign-in that checked the right password, a device, a code the command line makes, a pairing. + let sign_in = meta.sign_in_device(ana.id, ("$argon2id$stub", gen_of(meta, "ana").await), "pc", &DeviceClient::default(), &token(), NOW + 3).await; + assert!(matches!(sign_in, Err(MetaError::AccountDisabled)), "{sign_in:?}"); + assert!(matches!(meta.create_device(ana.id, "pc", &token(), NOW + 3).await, Err(MetaError::AccountDisabled))); + let late_invite = code(INVITE_CHARS); + meta.create_invite("ana", &late_invite.hash(), NOW + 3, INVITE_TTL_MS).await.unwrap(); + assert!(matches!(redeem(meta, &late_invite, "pc", NOW + 4).await, Err(MetaError::AccountDisabled))); + assert!(matches!(redeem(meta, &pairing, "pc", NOW + 4).await, Err(MetaError::InvalidCode)), "its pairing is gone"); + assert!(meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 4).await.is_err()); + assert_eq!(meta.list_devices(ana.id).await.unwrap().len(), 3, "no device was made"); + + // Enabling: sign-in works again; what was signed out stays signed out. + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(admin_view(meta, ana.id).await.disabled_ms, None); + assert!(meta.list_devices(ana.id).await.unwrap().iter().all(|d| d.revoked)); + assert_eq!(meta.device_by_token(&other_token).await.unwrap(), None); + assert!( + meta.sign_in_device(ana.id, ("$argon2id$stub", gen_of(meta, "ana").await), "pc", &DeviceClient::default(), &token(), NOW + 6) + .await + .unwrap() + .is_some() + ); + // The code made while the account was disabled works now, as an invite made for an enabled account would. + redeem(meta, &late_invite, "pc2", NOW + 7).await.unwrap(); + + // Marked for removal: disabled, not listed, signed out, can't be enabled, and the name stays taken. + let (bob, bobs, _, _) = paired(meta, "bob").await; + assert_eq!(meta.admin_remove_user(&actor, bob.id, NOW + 8).await.unwrap(), vec![bobs]); + let removed = admin_view(meta, bob.id).await; + assert_eq!((removed.disabled_ms, removed.devices_active), (Some(NOW + 8), 0)); + assert!(matches!(meta.admin_enable_user(&actor, bob.id, NOW).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.create_user("bob", "x", Role::Member, NOW).await, Err(MetaError::UsernameTaken))); + assert!(matches!( + meta.admin_create_user(&actor, "bob", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::UsernameTaken) + )); + assert!(matches!( + meta.admin_issue_reset(&actor, bob.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await, + Err(MetaError::Conflict(_)) + )); + let sign_in = meta.sign_in_device(bob.id, ("$argon2id$stub", gen_of(meta, "bob").await), "pc", &DeviceClient::default(), &token(), NOW + 9).await; + assert!(matches!(sign_in, Err(MetaError::AccountDisabled)), "{sign_in:?}"); + assert!(meta.admin_remove_user(&actor, bob.id, NOW + 10).await.unwrap().is_empty(), "removing again is harmless"); + assert_eq!(meta.admin_server_counts().await.unwrap().users, 2); +} + +/// A sign-in, a join with an invite, a join with a pairing, a new pairing and a plain device, all at the moment an +/// admin disables the account: each either finishes before and is signed out by the disable, or finds the account +/// disabled. Either way no device that is signed in is left, and enabling afterwards brings none back. +async fn issuing_access_while_an_account_is_disabled_leaves_no_usable_token(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for round in 0..12 { + let name = format!("ana{round}"); + let (ana, maker, pairing, _) = paired(meta, &name).await; + let invite = code(INVITE_CHARS); + meta.create_invite(&name, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..6 { + let (meta, actor, invite, pairing) = (meta.clone(), actor.clone(), invite.hash(), pairing.hash()); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.redeem(&invite, "invited", &token(), None, NOW + 1).await.map(|_| ()), + 1 => meta.redeem(&pairing, "paired", &token(), None, NOW + 1).await.map(|_| ()), + 2 => { + meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "signed in", &DeviceClient::default(), &token(), NOW + 1).await.map(|_| ()) + } + 3 => meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 1).await.map(|_| ()), + 4 => meta.create_device(ana.id, "plain", &token(), NOW + 1).await.map(|_| ()), + _ => meta.admin_disable_user(&actor, ana.id, NOW + 2).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!( + matches!( + result, + Ok(()) | Err(MetaError::InvalidCode | MetaError::NotFound(_) | MetaError::AccountDisabled | MetaError::DeviceRevoked) + ), + "round {round}: {result:?}" + ); + } + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| d.revoked), "round {round}: {devices:?}"); + assert_revoke_columns_agree(&devices); + } +} + +/// A sign-in reads the hash and generation, verifies the password (slowly), and asks to create the device. A disable and +/// an enable in between leave the hash as it was but not the generation: the device is not created. So are a password +/// change and a removal. The interleaving is played out by calling the API in that order. +async fn a_sign_in_checked_before_a_disable_and_an_enable_creates_no_device(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let client = DeviceClient::default(); + for (name, id) in [("ana", ana.id), ("bob", bob.id)] { + let read = meta.user_by_name(name).await.unwrap().unwrap(); + if name == "ana" { + meta.admin_disable_user(&actor, id, NOW).await.unwrap(); + meta.admin_enable_user(&actor, id, NOW).await.unwrap(); + } else { + meta.admin_remove_user(&actor, id, NOW).await.unwrap(); + } + let late = meta.sign_in_device(id, (&read.password_hash, read.auth_gen), "pc", &client, &token(), NOW + 1).await; + match name { + "ana" => assert!(matches!(late, Ok(None)), "{late:?}"), + _ => assert!(matches!(late, Ok(None) | Err(MetaError::AccountDisabled)), "{late:?}"), + } + assert!(meta.list_devices(id).await.unwrap().is_empty(), "{name}: no device was made"); + } + // A sign-in that read the state after the enable goes through. + let fresh = meta.user_by_name("ana").await.unwrap().unwrap(); + assert!(fresh.auth_gen >= 2); + assert!(meta.sign_in_device(ana.id, (&fresh.password_hash, fresh.auth_gen), "pc", &client, &token(), NOW + 2).await.unwrap().is_some()); + // A password change moves the generation too. + let before = gen_of(meta, "ana").await; + meta.reset_password(ana.id, "$argon2id$new", true, NOW + 3).await.unwrap(); + assert!(gen_of(meta, "ana").await > before); +} + +async fn invites_and_reset_codes_replace_each_other(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (fresh, _) = meta.admin_create_user(&actor, "fresh", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let ana = user(meta, "ana").await; + let (first, second, reset) = (code(INVITE_CHARS), code(INVITE_CHARS), code(INVITE_CHARS)); + + // An invite only for an account without a password; it replaces the open one, and can be revoked. + meta.admin_issue_invite(&actor, fresh.id, &first.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.admin_issue_invite(&actor, fresh.id, &second.hash(), NOW + 1, INVITE_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&first.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), Some(true)); + let conflict = meta.admin_issue_invite(&actor, ana.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await; + assert!(matches!(conflict, Err(MetaError::Conflict(_))), "an account with a password gets none: {conflict:?}"); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), Some(true)); + meta.admin_revoke_invite(&actor, fresh.id, NOW).await.unwrap(); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), None); + assert!(matches!(meta.admin_revoke_invite(&actor, fresh.id, NOW).await, Err(MetaError::NotFound(_))), "nothing left to revoke"); + assert!(matches!(meta.admin_revoke_invite(&actor, UserId(uuid::Uuid::now_v7()), NOW).await, Err(MetaError::NotFound(_)))); + + // A reset code replaces an invite and the other way round, and the account has one row either way. + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW + 1, RESET_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 2).await.unwrap(), Some(true), "a reset always needs a password, though ana has one"); + let invite_again = code(INVITE_CHARS); + meta.create_invite("ana", &invite_again.hash(), NOW + 3, INVITE_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 4).await.unwrap(), None, "the command line's invite replaces a reset code"); + assert_eq!(meta.invite_needs_password(&invite_again.hash(), NOW + 4).await.unwrap(), Some(false)); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW + 5, RESET_TTL_MS).await.unwrap(); + meta.admin_revoke_invite(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 6).await.unwrap(), None, "a revoke ends a reset code too"); + + // Not oneself, and not another admin. + assert!(matches!(meta.admin_issue_reset(&actor, root.id, &reset.hash(), false, NOW, RESET_TTL_MS).await, Err(MetaError::Invalid(_)))); + let (boss, _) = admin_actor(meta, "boss").await; + let result = meta.admin_issue_reset(&actor, boss.id, &reset.hash(), false, NOW, RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 6).await.unwrap(), None); + // Not a disabled account. + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(matches!(meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await, Err(MetaError::Conflict(_)))); + assert!(matches!( + meta.admin_issue_reset(&actor, UserId(uuid::Uuid::now_v7()), &reset.hash(), false, NOW, RESET_TTL_MS).await, + Err(MetaError::NotFound(_)) + )); +} + +async fn a_reset_code_sets_a_password_and_signs_everything_out(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let (ana, maker, pairing, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + // Nothing changes until it is redeemed. + assert!(meta.device_active(maker).await.unwrap()); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + + // A reset without a new password is refused, and the code stays. + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), None, NOW + 1).await, Err(MetaError::PasswordRequired))); + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), Some(""), NOW + 1).await, Err(MetaError::PasswordRequired))); + assert!(meta.device_active(maker).await.unwrap()); + // With one: the password changes (though the account had one), every device is signed out, the pairings are gone, + // and the redeeming computer is the one device signed in. + let new_token = token(); + let joined = meta.redeem(&reset.hash(), "recovery pc", &new_token, Some("$argon2id$fresh"), NOW + 2).await.unwrap(); + assert_eq!(sorted(joined.revoked.clone()), sorted(vec![maker, other])); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$fresh"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 2)); + let devices = meta.list_devices(ana.id).await.unwrap(); + for d in &devices { + if d.id == joined.device { + assert!(!d.revoked); + } else { + assert_eq!((d.revoked, d.revoke_reason), (true, Some(RevokeReason::Reset))); + } + } + assert_revoke_columns_agree(&devices); + assert_eq!(meta.device_by_token(&new_token).await.unwrap().map(|a| a.device), Some(joined.device)); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 3).await.unwrap(), None); + assert!(matches!(redeem(meta, &pairing, "late", NOW + 3).await, Err(MetaError::InvalidCode))); + // Once only. + assert!(matches!(meta.redeem(&reset.hash(), "again", &token(), Some("$argon2id$x"), NOW + 4).await, Err(MetaError::InvalidCode))); + + // It lasts an hour. + let late = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &late.hash(), false, NOW + 10, RESET_TTL_MS).await.unwrap(); + let result = meta.redeem(&late.hash(), "late", &token(), Some("$argon2id$x"), NOW + 10 + RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::InvalidCode)), "{result:?}"); + assert_eq!(RESET_TTL_MS, 3_600_000); + meta.redeem(&late.hash(), "in time", &token(), Some("$argon2id$x"), NOW + 10 + RESET_TTL_MS - 1).await.unwrap(); + + // Disabling the account deletes the reset code it still held. + let disabled_code = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &disabled_code.hash(), false, NOW + 20, RESET_TTL_MS).await.unwrap(); + meta.admin_disable_user(&actor, ana.id, NOW + 21).await.unwrap(); + assert_eq!(meta.invite_needs_password(&disabled_code.hash(), NOW + 22).await.unwrap(), None, "disabling deletes it"); +} + +/// The quota of a space is its limit when set, with or without the server-wide cap, in the look before the bytes are +/// stored and in the count that follows. +async fn a_quota_overrides_the_server_wide_cap(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let quota = MIN_QUOTA_BYTES; + meta.admin_update_user(&actor, ana.id, None, Some(Some(quota)), NOW).await.unwrap(); + let (c1, c2, c3) = (random_chunk(), random_chunk(), random_chunk()); + + // No cap at all: the quota still holds, and a space without one is unlimited. + meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota).await.unwrap(); + let over = meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota + 1).await; + assert!(matches!(over, Err(MetaError::QuotaExceeded { used: 0, limit }) if limit == quota), "{over:?}"); + meta.check_room(bob.space, BlobKind::Chunk, c1.as_bytes(), quota * 10).await.unwrap(); + meta.add_chunk(ana.space, &c1, quota - 10, NOW).await.unwrap(); + let over = meta.add_chunk(ana.space, &c2, 11, NOW).await; + assert!(matches!(over, Err(MetaError::QuotaExceeded { limit, .. }) if limit == quota), "{over:?}"); + assert_eq!(meta.stored_bytes(ana.space).await.unwrap(), quota - 10, "nothing was recorded"); + assert!(matches!(meta.add_chunks(ana.space, &[(c2, 6), (c3, 6)], NOW).await, Err(MetaError::QuotaExceeded { .. }))); + meta.add_chunks(ana.space, &[(c2, 5), (c3, 5)], NOW).await.unwrap(); + assert_eq!(meta.stored_bytes(ana.space).await.unwrap(), quota); + meta.add_chunk(bob.space, &c1, quota * 3, NOW).await.unwrap(); + // What the space has already costs nothing to look at. + meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota * 5).await.unwrap(); + + // A cap larger than the quota: the quota wins. A cap smaller than the quota: the quota wins, too (it is the + // account's limit, whichever way it differs). + let big_cap = meta.clone().with_stored_bytes_cap(Some(quota * 100)); + assert!(matches!(big_cap.check_room(ana.space, BlobKind::Chunk, random_chunk().as_bytes(), 1).await, Err(MetaError::QuotaExceeded { .. }))); + assert!(matches!(big_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + let small_cap = meta.clone().with_stored_bytes_cap(Some(100)); + meta.admin_update_user(&actor, bob.id, None, Some(Some(quota * 10)), NOW).await.unwrap(); + small_cap.check_room(bob.space, BlobKind::Chunk, random_chunk().as_bytes(), quota).await.unwrap(); + small_cap.add_chunk(bob.space, &random_chunk(), quota, NOW).await.unwrap(); + // Removing the quota gives the cap back; a quota under the use deletes nothing and refuses new bytes. + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + big_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await.unwrap(); + assert!(matches!(small_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + meta.admin_update_user(&actor, bob.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert!(meta.stored_bytes(bob.space).await.unwrap() > MIN_QUOTA_BYTES); + assert!(matches!(meta.add_chunk(bob.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); +} + +async fn an_admin_lists_and_revokes_a_users_devices(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let (ana, maker, _, pairing_id) = paired(meta, "ana").await; + let bob = user(meta, "bob").await; + let bobs = device(meta, &bob).await; + let other = device(meta, &ana).await; + meta.revoke_device(ana.id, other).await.unwrap(); + // Every device, revoked ones with their reason. + let listed = meta.admin_devices(ana.id).await.unwrap(); + assert_eq!(listed.len(), 2); + assert_eq!(listed.iter().find(|d| d.id == other).unwrap().revoke_reason, Some(RevokeReason::User)); + assert!(matches!(meta.admin_devices(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound(_)))); + + meta.admin_revoke_device(&actor, ana.id, maker, NOW + 1).await.unwrap(); + let revoked = record(meta, &ana, maker).await; + assert_eq!((revoked.revoked, revoked.revoked_ms, revoked.revoke_reason), (true, Some(NOW + 1), Some(RevokeReason::Admin))); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None, "its pairing dies with it"); + // Again: the first time and reason stay. Another user's device, an unknown device or user: not found, nothing changed. + meta.admin_revoke_device(&actor, ana.id, maker, NOW + 5).await.unwrap(); + assert_eq!(record(meta, &ana, maker).await.revoked_ms, Some(NOW + 1)); + assert_eq!(record(meta, &ana, other).await.revoke_reason, Some(RevokeReason::User)); + assert!(matches!(meta.admin_revoke_device(&actor, ana.id, bobs, NOW).await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.admin_revoke_device(&actor, ana.id, DeviceId::generate(), NOW).await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.admin_revoke_device(&actor, UserId(uuid::Uuid::now_v7()), bobs, NOW).await, Err(MetaError::NotFound(_)))); + assert!(meta.device_active(bobs).await.unwrap()); +} + +// Two admins at the same moment, each taking the other (or oneself) away: whatever the order, one enabled admin is +// left, and no statement fails inside the database (on Postgres, rows taken in different orders would deadlock). + +#[derive(Clone, Copy, Debug)] +enum Act { + Demote(usize), + Disable(usize), + Remove(usize), +} + +/// Runs `acts` (who acts, what) at the same moment on a survivor and a new admin; exactly one enabled admin must be +/// left, and the second change of two must fail with a reason the rules give. Returns the admin left. +async fn race_two_admins(meta: &Meta, round: usize, survivor: (UserRecord, AdminActor), acts: [(usize, Act); 2]) -> (UserRecord, AdminActor) { + let fresh = admin_actor(meta, &format!("fresh{round}")).await; + let both = [survivor, fresh]; + let mut tasks = Vec::new(); + for (who, act) in acts { + let (meta, actor) = (meta.clone(), both[who].1.clone()); + let target = both[match act { + Act::Demote(t) | Act::Disable(t) | Act::Remove(t) => t, + }] + .0 + .id; + tasks.push(tokio::spawn(async move { + match act { + Act::Demote(_) => meta.admin_update_user(&actor, target, Some(Role::Member), None, NOW).await.map(|_| ()), + Act::Disable(_) => meta.admin_disable_user(&actor, target, NOW).await.map(|_| ()), + Act::Remove(_) => meta.admin_remove_user(&actor, target, NOW).await.map(|_| ()), + } + })); + } + for (task, (_, act)) in tasks.into_iter().zip(acts) { + let result = task.await.unwrap(); + assert!( + matches!(result, Ok(()) | Err(MetaError::LastAdmin | MetaError::Forbidden(_) | MetaError::DeviceRevoked | MetaError::Invalid(_))), + "round {round}, {act:?}: {result:?}" + ); + } + let page = meta.admin_list_users(None, None, 200, NOW).await.unwrap(); + let left: Vec<_> = page.users.iter().filter(|u| u.role == Role::Admin && u.disabled_ms.is_none()).collect(); + assert_eq!(left.len(), 1, "round {round}, {acts:?}: {left:?}"); + let left = left[0].id; + let [a, b] = both; + if a.0.id == left { a } else { b } +} + +async fn two_admins_racing_keep_one(meta: &Meta, combos: &[[(usize, Act); 2]]) { + let mut survivor = admin_actor(meta, "first").await; + for round in 0..8 { + for (i, combo) in combos.iter().enumerate() { + survivor = race_two_admins(meta, round * 10 + i, survivor, *combo).await; + } + } +} + +async fn two_demotions_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(1)), (1, Act::Demote(0))]]).await; +} + +/// Each admin steps down at the same moment: their rows are different, so only the `last_admin` lock keeps one of them. +async fn two_self_demotions_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(0)), (1, Act::Demote(1))]]).await; +} + +async fn two_disables_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Disable(1)), (1, Act::Disable(0))]]).await; +} + +async fn a_demotion_and_a_disable_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(1)), (1, Act::Disable(0))], [(0, Act::Disable(1)), (1, Act::Demote(0))]]).await; +} + +async fn a_removal_and_a_demotion_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Remove(1)), (1, Act::Demote(0))], [(0, Act::Demote(1)), (1, Act::Remove(0))]]).await; +} + +async fn a_self_demotion_and_a_disable_of_the_other_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(0)), (0, Act::Disable(1))], [(1, Act::Demote(1)), (1, Act::Disable(0))]]).await; +} + +// ---- the migration from 0007 to 0008 ---- + +/// An account, its invite and the audit log survive the two-factor migration (0010, which runs after the audit log's 0009). +async fn migrating_from_0009_keeps_the_rows(url: &str) { + use sqlx::{Connection, Row}; + sqlx::any::install_default_drivers(); + let (ana, invite, audit_rows) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + let events = all_events(&meta).await; + assert!(!events.is_empty(), "the audit log has rows to keep"); + (ana, invite, events.len()) + }; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 10").execute(&mut conn).await.unwrap(); + drop(conn); + assert_eq!(schema_version(url).await, 9, "the audit log's migration stays"); + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15); + assert_eq!(all_events(&meta).await.len(), audit_rows, "the audit rows are kept"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().id, ana.id); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(false)); + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + let row = sqlx::query("SELECT reset_two_factor FROM invites").fetch_one(&mut conn).await.unwrap(); + assert_eq!(row.try_get::("reset_two_factor").unwrap(), 0); +} + +#[tokio::test] +async fn migrating_from_0009_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0009_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0009_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0009_keeps_the_rows(&url).await; +} + +/// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. +async fn file_with_versions(meta: &Meta, u: &UserRecord, dev: DeviceId, n: usize) -> (NodeId, Vec) { + let r = root(meta, u).await; + let m = manifest(meta, u, 3).await; + let node = NodeId::generate(); + let mut versions = vec![version(m)]; + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name("history.txt"), version: versions[0].clone() }], NOW).await.unwrap(); + for i in 1..n { + let m = manifest(meta, u, 3 + i as u32).await; + let next = NewVersion { mtime_ms: NOW + i as i64, ..version(m) }; + let expected = versions[i - 1].id; + meta.commit(u.space, dev, &[Op::Update { node, expected, version: next.clone() }], NOW + i as i64).await.unwrap(); + versions.push(next); + } + (node, versions) +} + +async fn file_named(meta: &Meta, u: &UserRecord, dev: DeviceId, file_name: &str) -> (NodeId, NewVersion) { + let r = root(meta, u).await; + let node = NodeId::generate(); + let v = version(manifest(meta, u, 5).await); + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name(file_name), version: v.clone() }], NOW).await.unwrap(); + (node, v) +} + +fn refused_restore(result: Result) -> Vec<(usize, ConflictKind)> { + match result { + Err(MetaError::Rejected(list)) => list.into_iter().map(|c| (c.op, c.kind)).collect(), + other => panic!("expected a refused restore, got {other:?}"), + } +} + +/// The history comes newest first in pages that follow each other without gaps or repeats, also across versions that +/// share one sequence number (a batch of updates); only files of the caller's own space have one; a deleted file keeps +/// its history; `pruned` is false and `save` tells whether a record hangs on the version. +async fn history_pages_newest_first_for_own_files_only(meta: &Meta) { + use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, mut versions) = file_with_versions(meta, &u, dev, 4).await; + // Three more versions in one commit: they share a sequence number, so the order inside it is by ID. + let (mut ops, mut prev) = (Vec::new(), versions[3].id); + for i in 0..3 { + let next = version(manifest(meta, &u, 20 + i).await); + ops.push(Op::Update { node, expected: prev, version: next.clone() }); + prev = next.id; + versions.push(next); + } + meta.commit(u.space, dev, &ops, NOW + 100).await.unwrap(); + let head = prev; + let epoch = meta.space_info(u.space).await.unwrap().epoch; + + let all = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap(); + assert_eq!(all.next, None); + assert_eq!(all.versions.len(), 7); + let key = |v: &HistoryVersion| (v.seq, *v.id.as_uuid().as_bytes()); + assert!(all.versions.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (seq, id)"); + assert!(all.versions.iter().all(|v| !v.pruned && !v.save && v.device == dev && v.epoch == epoch)); + assert_eq!(all.versions.iter().filter(|v| v.seq == 5).count(), 3, "the batch shares one sequence number"); + let oldest = all.versions.last().unwrap(); + assert_eq!((oldest.id, oldest.parent_version, oldest.seq, oldest.size), (versions[0].id, None, 1, 3)); + let second = &all.versions[all.versions.len() - 2]; + assert_eq!((second.id, second.parent_version, second.mtime_ms, second.size), (versions[1].id, Some(versions[0].id), NOW + 1, 4)); + + for limit in [1u32, 2, 3] { + let (mut seen, mut after, mut pages) = (Vec::new(), None, 0); + loop { + let page = meta.history(u.space, node, after, limit).await.unwrap(); + assert!(page.versions.len() <= limit as usize); + seen.extend(page.versions.iter().cloned()); + pages += 1; + match page.next { + Some(next) => { + let last = page.versions.last().unwrap(); + assert_eq!(next, HistoryCursor { seq: last.seq, version: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.versions, "limit {limit}: every version once, in order"); + assert_eq!(pages, 7usize.div_ceil(limit as usize)); + } + + // The limit is 1 to 200; a cursor past the end gives an empty last page. + assert!(matches!(meta.history(u.space, node, None, 0).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT + 1).await, Err(MetaError::Invalid(_)))); + let past = meta.history(u.space, node, Some(HistoryCursor { seq: 0, version: VersionId::generate() }), 10).await.unwrap(); + assert_eq!((past.versions.len(), past.next), (0, None)); + + // A record hanging on a version shows as `save`. + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + let with_save: Vec<_> = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.into_iter().filter(|v| v.save).collect(); + assert_eq!(with_save.iter().map(|v| v.id).collect::>(), vec![versions[1].id]); + + // Folders have no history, another account's node is not found, and a deleted file keeps its history. + let r = root(meta, &u).await; + assert!(matches!(meta.history(u.space, r, None, 10).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, NodeId::generate(), None, 10).await, Err(MetaError::NotFound(_)))); + let bob = user(meta, "bob").await; + assert!(matches!(meta.history(bob.space, node, None, 10).await, Err(MetaError::NotFound(_)))); + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(head) }], NOW + 200).await.unwrap(); + assert_eq!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.len(), 7); +} + +/// A restore is an ordinary update after the head: a fresh version with the old content (manifest, size, mtime), the +/// head as its parent, the next sequence number, the same epoch and no save record; it shows in the change feed, and +/// the old version stays where it was. +async fn a_restore_is_a_new_version_after_the_head(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let other = meta.create_device(u.id, "phone", &[9u8; 32], NOW).await.unwrap(); + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[0].id, &record).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_version(u.space, other, node, versions[0].id, versions[2].id, NOW + 50).await.unwrap(); + assert_eq!((done.parent_version, done.epoch, done.seq), (versions[2].id, before.epoch, before.seq + 1)); + assert!(!versions.iter().any(|v| v.id == done.version), "a fresh version ID"); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch)); + + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!( + (f.version, f.manifest, f.mtime_ms, f.device, f.save.clone()), + (done.version, versions[0].manifest, versions[0].mtime_ms, other, None) + ); + + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 4); + assert_eq!(history[0].id, done.version); + assert_eq!((history[0].parent_version, history[0].size, history[0].save, history[0].seq), (Some(versions[2].id), 3, false, done.seq)); + assert!(history.iter().any(|v| v.id == versions[0].id && v.save), "the old version and its record stay"); + + // The restored content is the head now: a stale `expected` is refused and changes nothing. + let stale = meta.restore_version(u.space, dev, node, versions[1].id, versions[2].id, NOW + 60).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(done.version) })]); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, after.seq); + // Restoring the content that is already current is allowed: it is one more version. + let again = meta.restore_version(u.space, dev, node, done.version, done.version, NOW + 70).await.unwrap(); + assert_eq!(again.seq, after.seq + 1); +} + +/// What a restore turns down, each leaving the space as it was: a folder, a deleted file, a file with a live document, +/// a node or version that is not there, a version of another file, and anything of another account. +async fn a_restore_refuses_what_it_can_not_restore(meta: &Meta) { + let u = user(meta, "ana").await; + let (space, dev) = (u.space, device(meta, &u).await); + let (file, versions) = file_with_versions(meta, &u, dev, 2).await; + let (second, second_version) = file_named(meta, &u, dev, "second.txt").await; + let r = root(meta, &u).await; + let seq = meta.space_info(space).await.unwrap().seq; + let (old, head) = (versions[0].id, versions[1].id); + let restore = |node: NodeId, source: VersionId, expected: VersionId| { + let meta = meta.clone(); + async move { meta.restore_version(space, dev, node, source, expected, NOW + 9).await } + }; + + assert!(matches!(restore(r, old, head).await, Err(MetaError::Invalid(_))), "a folder"); + assert!(matches!(restore(NodeId::generate(), old, head).await, Err(MetaError::NotFound(_))), "no such node"); + assert!(matches!(restore(file, VersionId::generate(), head).await, Err(MetaError::NotFound(_))), "no such version"); + assert!(matches!(restore(file, second_version.id, head).await, Err(MetaError::NotFound(_))), "a version of another file"); + // The source is checked before the head: a missing source is not a conflict. + assert!(matches!(restore(file, VersionId::generate(), VersionId::generate()).await, Err(MetaError::NotFound(_)))); + assert_eq!(refused_restore(restore(file, old, VersionId::generate()).await).len(), 1, "a head that is not current"); + + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + assert!(matches!(meta.restore_version(bob.space, bob_device, file, old, head, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + + // A file with a live document. + let doc = craft_cloud_proto::live::node_doc(space, second); + meta.create_live_doc(space, doc, "document:kitdemo", Some(second), NOW).await.unwrap(); + assert!(matches!(restore(second, second_version.id, second_version.id).await, Err(MetaError::Invalid(_))), "a live document"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq, "nothing changed so far"); + + // A deleted file. + meta.commit(space, dev, &[Op::Delete { node: file, expected: Some(head) }], NOW + 5).await.unwrap(); + assert!(matches!(restore(file, old, head).await, Err(MetaError::Invalid(_))), "a deleted file"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq + 1, "only the delete"); + let feed = meta.changes(space, seq, None, MAX_PAGE_NODES).await.unwrap(); + assert!(feed.nodes.len() == 1 && find(&feed.nodes, file).deleted, "still deleted"); +} + +/// Several restores of different old versions against one head: exactly one applies, the rest are refused with a +/// version mismatch, and the space moves one step. +async fn concurrent_restores_let_exactly_one_win(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let head = versions[2].id; + let before = meta.space_info(u.space).await.unwrap().seq; + let mut tasks = Vec::new(); + for i in 0..8 { + let (meta, space, source) = (meta.clone(), u.space, versions[i % 2].id); + tasks.push(tokio::spawn(async move { meta.restore_version(space, dev, node, source, head, NOW + 20).await })); + } + let mut wins = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => wins += 1, + Err(MetaError::Rejected(list)) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + Err(other) => panic!("unexpected error {other:?}"), + } + } + assert_eq!(wins, 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before + 1); +} + +/// A restore and a late save record for the head it replaces run together: both apply under the space's one lock, the +/// record stays on the version it was meant for, and the restored version never gets a record. +async fn a_restore_and_a_late_save_record_both_apply(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let (head, old) = (versions[2].id, versions[0].id); + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let save = tokio::spawn(async move { m1.attach_save(space, node, head, &record).await }); + let restore = tokio::spawn(async move { m2.restore_version(space, dev, node, old, head, NOW + 30).await }); + save.await.unwrap().unwrap(); + let restored = restore.await.unwrap().unwrap(); + let history = meta.history(space, node, None, 50).await.unwrap().versions; + assert_eq!(history.len(), 4); + for v in &history { + assert_eq!(v.save, v.id == head, "only the old head has the record: {}", v.id); + } + assert_eq!(history[0].id, restored.version); + // The create, two updates, the record and the restore. + assert_eq!(meta.space_info(space).await.unwrap().seq, 5); +} + +/// The database at the newest version when the code knows it: the version table holds one row per migration, and a +/// database written by a newer build is refused rather than half-read. +#[tokio::test] +async fn the_schema_version_is_recorded_and_a_newer_database_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 15); + // Opening again changes nothing. + drop(Meta::open(&url).await.unwrap()); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(rows, 15); + sqlx::query("INSERT INTO schema_version (version) VALUES (16)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(&url).await, Err(MetaError::Corrupt(_))), "a newer database is refused"); +} + +/// A history page is read through the index on `(space_id, node, seq, id)`, not by scanning the versions table. +#[tokio::test] +async fn a_history_page_uses_the_node_index_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let plan = sqlx::query( + "EXPLAIN QUERY PLAN SELECT id FROM versions WHERE space_id = ?1 AND node = ?2 AND (seq, id) < (?3, ?4) ORDER BY seq DESC, id DESC LIMIT ?5", + ) + .bind(vec![0u8; 16]) + .bind(vec![0u8; 16]) + .bind(i64::MAX) + .bind(vec![0xffu8; 16]) + .bind(51i64) + .fetch_all(&mut conn) + .await + .unwrap(); + use sqlx::Row; + let detail: Vec = plan.iter().map(|r| r.get::("detail")).collect(); + assert!(detail.iter().any(|d| d.contains("versions_node_seq")), "{detail:?}"); + assert!(!detail.iter().any(|d| d.contains("TEMP B-TREE")), "no sort: {detail:?}"); +} + +// ---- the trash ---- + +/// A folder `name` in `parent`: its node. +async fn folder(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, folder_name: &str, now: i64) -> NodeId { + let node = NodeId::generate(); + meta.commit(u.space, dev, &[Op::Mkdir { node, parent, name: name(folder_name) }], now).await.unwrap(); + node +} + +/// A file `name` in `parent` with one version: its node and version. +async fn file_in(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, file_name: &str, now: i64) -> (NodeId, NewVersion) { + let node = NodeId::generate(); + let v = version(manifest(meta, u, 6).await); + meta.commit(u.space, dev, &[Op::Create { node, parent, name: name(file_name), version: v.clone() }], now).await.unwrap(); + (node, v) +} + +async fn delete_file(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, current: VersionId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(current) }], now).await.unwrap(); +} + +async fn delete_folder(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: None }], now).await.unwrap(); +} + +async fn trash_ids(meta: &Meta, u: &UserRecord) -> Vec { + meta.trash(u.space, None, 200, NOW).await.unwrap().entries.iter().map(|e| e.id).collect() +} + +/// The name a restore gives a node whose name is taken, in the minute of `NOW` (2025-10-09 08:53 UTC), from a device +/// called `laptop`. +fn conflict_of(original: &str, is_dir: bool, counter: u32) -> String { + let stamp = craft_cloud_proto::ConflictStamp { year: 2025, month: 10, day: 9, hour: 8, minute: 53, utc: true }; + craft_cloud_proto::conflict_name(&name(original), is_dir, "laptop", stamp, counter).unwrap().as_str().to_owned() +} + +async fn live_name(meta: &Meta, u: &UserRecord, node: NodeId) -> (Option, String, bool) { + // The feed from the start, which has deleted nodes too. + let feed = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + let n = find(&feed.nodes, node); + (n.parent, n.name.as_ref().map(|n| n.as_str().to_owned()).unwrap_or_default(), n.deleted) +} + +/// The time in the trash starts when a live node is deleted, not when a deleted one is deleted again; coming back +/// clears it, by an edit or by a restore from the trash, and the next deletion starts afresh. +async fn deleting_stamps_the_time_once_and_coming_back_clears_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (file, v) = file_in(meta, &u, dev, r, "stamp.txt", NOW).await; + assert!(trash_ids(meta, &u).await.is_empty()); + + delete_file(meta, &u, dev, file, v.id, NOW + 1_000).await; + let page = meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms, page.repaired), (1, NOW + 1_000, 0)); + // Deleting a deleted file again is no change: the time stays, and the file is not stamped twice. + delete_file(meta, &u, dev, file, v.id, NOW + 5_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 1_000); + + // An edit brings it back and leaves the trash; a later deletion starts the time again. + let edited = version(manifest(meta, &u, 7).await); + meta.commit(u.space, dev, &[Op::Update { node: file, expected: v.id, version: edited.clone() }], NOW + 6_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "an edit brings the file back"); + delete_file(meta, &u, dev, file, edited.id, NOW + 7_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 7_000); + + // A restore from the trash clears it too, and the next deletion starts afresh again. + let restored = meta.restore_from_trash(u.space, dev, file, NOW + 8_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + // The restore made a new current version: the next delete names it. + delete_file(meta, &u, dev, file, restored.version.unwrap(), NOW + 8_500).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 8_500); +} + +/// The trash lists a space's deleted files and folders, newest deletion first by `(deleted_ms, id)`, in pages without +/// gaps or repeats (also for nodes deleted in one instant), with sizes of files; it lists nothing of another space and +/// nothing that was removed from the trash. +async fn the_trash_pages_newest_first_for_its_own_space(meta: &Meta) { + use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut made = Vec::new(); + for i in 0..5 { + let (node, v) = file_in(meta, &u, dev, r, &format!("f{i}.txt"), NOW).await; + // The last three go in one instant. + delete_file(meta, &u, dev, node, v.id, NOW + 1_000 + i64::from(i.min(2)) * 10).await; + made.push(node); + } + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + delete_folder(meta, &u, dev, dir, NOW + 5_000).await; + + let all = meta.trash(u.space, None, TRASH_MAX_LIMIT, NOW).await.unwrap(); + assert_eq!((all.entries.len(), all.next), (6, None)); + assert_eq!(all.entries[0].id, dir); + assert!(all.entries[0].is_dir && all.entries[0].size.is_none() && all.entries[0].parent == r); + assert!(all.entries[1..].iter().all(|e| !e.is_dir && e.size == Some(6))); + let key = |e: &craft_cloud_meta::TrashEntry| (e.deleted_ms, *e.id.as_uuid().as_bytes()); + assert!(all.entries.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (deleted_ms, id)"); + for limit in [1u32, 2, 4] { + let (mut seen, mut after) = (Vec::new(), None); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize); + seen.extend(page.entries.iter().cloned()); + match page.next { + Some(next) => { + let last = page.entries.last().unwrap(); + assert_eq!(next, TrashCursor { deleted_ms: last.deleted_ms, node: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.entries, "limit {limit}"); + } + assert!(matches!(meta.trash(u.space, None, 0, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.trash(u.space, None, TRASH_MAX_LIMIT + 1, NOW).await, Err(MetaError::Invalid(_)))); + + // Another space sees none of it. A node removed from the trash is not listed any more. + let bob = user(meta, "bob").await; + assert!(meta.trash(bob.space, None, 10, NOW).await.unwrap().entries.is_empty()); + meta.request_purge(u.space, dir, NOW + 6_000).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await.len(), 5); + assert!(!trash_ids(meta, &u).await.contains(&dir)); +} + +/// A file in deleted folders comes back with the folders, every node keeping its ID, as one change of the space. +async fn a_restore_brings_the_deleted_folders_back_with_their_ids(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (f, v) = file_in(meta, &u, dev, b, "f.txt", NOW).await; + let (sibling, sv) = file_in(meta, &u, dev, b, "sibling.txt", NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 1).await; + delete_file(meta, &u, dev, sibling, sv.id, NOW + 2).await; + delete_folder(meta, &u, dev, b, NOW + 3).await; + delete_folder(meta, &u, dev, a, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed), (f, b, "f.txt", false)); + assert_eq!((done.seq, done.epoch), (before.seq + 1, before.epoch)); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch), "one change of the space"); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), false)); + assert_eq!(live_name(meta, &u, b).await, (Some(a), "b".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(b), "f.txt".to_owned(), false)); + assert_eq!(trash_ids(meta, &u).await, vec![sibling], "only the node that was asked for and its folders come back"); + + // The change feed reports exactly the nodes that came back, at the new sequence number, as live. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(a, false), (b, false), (f, false)]; + expected.sort(); + assert_eq!(changed, expected); + let NodeKind::File(state) = &find(&feed.nodes, f).kind else { panic!("a file") }; + assert_eq!(state.manifest, v.manifest, "the file's content is what it was"); + assert_eq!(Some(state.version), done.version, "as a new version after the last one"); + assert_ne!(state.version, v.id); +} + +/// A name taken by a file or by anything at the node itself gives a conflict name (never an overwrite); a deleted +/// folder whose name is held by a live folder is merged into it; every name tried is free, and a folder that is renamed +/// is a folder. +async fn a_restore_merges_into_live_folders_and_renames_around_taken_names(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + + // The node's own name is taken: a conflict name, and the live file is untouched. + let (f, fv) = file_in(meta, &u, dev, r, "plan.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (squatter, _) = file_in(meta, &u, dev, r, "PLAN.txt", NOW + 2).await; + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("plan.txt", false, 1))); + assert_eq!(live_name(meta, &u, squatter).await, (Some(r), "PLAN.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(r), conflict_of("plan.txt", false, 1), false)); + + // A second restore of the same name in the same minute takes the next counter. + let (g, gv) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 3).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 4).await; + let (_h, _) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 5).await; + let (_i, _) = file_in(meta, &u, dev, r, &conflict_of("plan2.txt", false, 1), NOW + 6).await; + let done = meta.restore_from_trash(u.space, dev, g, NOW).await.unwrap(); + assert_eq!(done.name, conflict_of("plan2.txt", false, 2)); + + // A deleted folder held by a live folder of the same name: the live folder takes what is restored. + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (x, xv) = file_in(meta, &u, dev, b, "x.txt", NOW).await; + delete_file(meta, &u, dev, x, xv.id, NOW + 10).await; + delete_folder(meta, &u, dev, b, NOW + 11).await; + delete_folder(meta, &u, dev, a, NOW + 12).await; + let live_a = folder(meta, &u, dev, r, "a", NOW + 13).await; + let done = meta.restore_from_trash(u.space, dev, x, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.as_str()), (false, "x.txt")); + assert_eq!(live_name(meta, &u, b).await, (Some(live_a), "b".to_owned(), false), "b moved into the live folder"); + assert_eq!(live_name(meta, &u, x).await, (Some(b), "x.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), true), "the deleted a stays deleted"); + assert_eq!(trash_ids(meta, &u).await, vec![a]); + + // A deleted folder held by a file: the folder comes back under a conflict name, with its content below it. + let c = folder(meta, &u, dev, r, "c", NOW).await; + let (y, yv) = file_in(meta, &u, dev, c, "y.txt", NOW).await; + delete_file(meta, &u, dev, y, yv.id, NOW + 20).await; + delete_folder(meta, &u, dev, c, NOW + 21).await; + let (_c_file, _) = file_in(meta, &u, dev, r, "c", NOW + 22).await; + let done = meta.restore_from_trash(u.space, dev, y, NOW).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (c, "y.txt")); + assert_eq!(live_name(meta, &u, c).await, (Some(r), conflict_of("c", true, 1), false)); + + // A deleted folder restored itself, its name held by a live folder: no merge for the node asked for. + let d = folder(meta, &u, dev, r, "d", NOW).await; + delete_folder(meta, &u, dev, d, NOW + 30).await; + let live_d = folder(meta, &u, dev, r, "d", NOW + 31).await; + let done = meta.restore_from_trash(u.space, dev, d, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("d", true, 1))); + assert_eq!(live_name(meta, &u, live_d).await.1, "d"); +} + +/// A folder in the trash is listed alone: what is in the trash below it is listed only through it, however long before +/// it was deleted, and pages never split or repeat the top items. +async fn the_trash_lists_a_folder_without_what_is_below_it(meta: &Meta) { + use craft_cloud_proto::api::TrashCursor; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut tops = Vec::new(); + for i in 0..7i64 { + let dir = folder(meta, &u, dev, r, &format!("d{i}"), NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub, "deep.txt", NOW).await; + // The child goes long before its folder, the deep file with the folders. + delete_file(meta, &u, dev, child, cv.id, NOW + 100 * i).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, sub, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, dir, NOW + 1_000 + 100 * i).await; + tops.push(dir); + } + for i in 0..3 { + let (f, v) = file_in(meta, &u, dev, r, &format!("loose{i}.txt"), NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 500 + i64::from(i)).await; + tops.push(f); + } + tops.sort(); + for limit in [1u32, 2, 3, 200] { + let (mut seen, mut after) = (Vec::new(), None::); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize && !page.entries.is_empty(), "limit {limit}"); + seen.extend(page.entries.iter().map(|e| e.id)); + match page.next { + Some(next) => { + // A page is full whenever another follows: the hidden nodes take no room in it. + assert_eq!(page.entries.len(), limit as usize, "limit {limit}"); + after = Some(next); + } + None => break, + } + } + seen.sort(); + assert_eq!(seen, tops, "every top item exactly once, limit {limit}"); + } +} + +/// Restoring a folder brings back all that is in the trash below it, whenever it was deleted, as one change of the space +/// with each node revived like a single restore; a node removed from the trash, and what hangs from it, stays deleted. +async fn a_folder_restore_brings_back_everything_in_the_trash_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir1", NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let (early, ev) = file_in(meta, &u, dev, dir, "early.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let sub2 = folder(meta, &u, dev, sub, "sub2", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub2, "deep.txt", NOW).await; + let gone = folder(meta, &u, dev, dir, "gone", NOW).await; + let (hung, hv) = file_in(meta, &u, dev, gone, "hung.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, dir, "kept.txt", NOW).await; + // `early` was deleted on its own long before; `gone` and `kept` are removed from the trash; the rest goes with + // the folder. + delete_file(meta, &u, dev, early, ev.id, NOW + 10).await; + delete_file(meta, &u, dev, hung, hv.id, NOW + 1_000).await; + delete_folder(meta, &u, dev, gone, NOW + 1_001).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1_002).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_003).await; + delete_folder(meta, &u, dev, sub2, NOW + 1_004).await; + delete_folder(meta, &u, dev, sub, NOW + 1_005).await; + delete_file(meta, &u, dev, child, cv.id, NOW + 1_006).await; + delete_folder(meta, &u, dev, dir, NOW + 1_007).await; + meta.request_purge(u.space, kept, NOW + 2_000).await.unwrap(); + meta.request_purge(u.space, gone, NOW + 2_001).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![dir], "the folder alone"); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, dir, NOW + 3_000).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed, done.version), (dir, r, "dir1", false, None)); + assert_eq!(done.seq, before.seq + 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq + 1, "one change of the space"); + assert!(trash_ids(meta, &u).await.is_empty(), "nothing is left in the trash: the removed nodes are no longer listed"); + for (node, parent, node_name) in + [(dir, r, "dir1"), (child, dir, "child.txt"), (early, dir, "early.txt"), (sub, dir, "sub"), (sub2, sub, "sub2"), (deep, sub2, "deep.txt")] + { + assert_eq!(live_name(meta, &u, node).await, (Some(parent), node_name.to_owned(), false), "{node_name}"); + } + for node in [gone, hung, kept] { + assert!(live_name(meta, &u, node).await.2, "a removed node stays deleted"); + } + + // The feed reports exactly the nodes that came back, each at the new sequence number; the files as new versions. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(dir, false), (child, false), (early, false), (sub, false), (sub2, false), (deep, false)]; + expected.sort(); + assert_eq!(changed, expected); + for (node, old) in [(child, &cv), (early, &ev), (deep, &dv)] { + let NodeKind::File(state) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!(state.manifest, old.manifest, "the content is what it was"); + assert_ne!(state.version, old.id, "as a new version"); + } +} + +/// Two deleted children of one name come back together: the newest deletion keeps the name, the other gets a conflict +/// name, and the folder itself is renamed only where its own name is taken. +async fn a_folder_restore_gives_children_of_one_name_a_conflict_name(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (old, ov) = file_in(meta, &u, dev, dir, "a.txt", NOW).await; + delete_file(meta, &u, dev, old, ov.id, NOW + 1).await; + let (new, nv) = file_in(meta, &u, dev, dir, "a.txt", NOW + 2).await; + delete_file(meta, &u, dev, new, nv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let (_squatter, _) = file_in(meta, &u, dev, r, "dir", NOW + 5).await; + + let done = meta.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("dir", true, 1))); + assert_eq!(live_name(meta, &u, new).await, (Some(dir), "a.txt".to_owned(), false), "the newest deletion keeps the name"); + assert_eq!(live_name(meta, &u, old).await, (Some(dir), conflict_of("a.txt", false, 1), false)); +} + +/// Removing a folder from the trash removes what is in the trash below it too, so nothing stays listed or restorable +/// alone (up to the cap, see below). +async fn removing_a_folder_from_the_trash_takes_what_is_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (c, cv) = file_in(meta, &u, dev, dir, "c.txt", NOW).await; + let s = folder(meta, &u, dev, dir, "s", NOW).await; + let (d, dv) = file_in(meta, &u, dev, s, "d.txt", NOW).await; + delete_file(meta, &u, dev, c, cv.id, NOW + 1).await; + delete_file(meta, &u, dev, d, dv.id, NOW + 2).await; + delete_folder(meta, &u, dev, s, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + meta.request_purge(u.space, dir, NOW + 5).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + for node in [dir, c, s, d] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "{node}"); + assert!(live_name(meta, &u, node).await.2); + } +} + +/// A folder `name` in the root with `count` files, the file `i` deleted at `base + 1 + i` and the folder at `base + 20`: +/// the folder and its files, the newest deletion last. +async fn folder_with_deleted_files(meta: &Meta, u: &UserRecord, dev: DeviceId, folder_name: &str, count: i64, base: i64) -> (NodeId, Vec) { + let r = root(meta, u).await; + let dir = folder(meta, u, dev, r, folder_name, base).await; + let mut files = Vec::new(); + for i in 0..count { + let (f, v) = file_in(meta, u, dev, dir, &format!("f{i}.txt"), base).await; + delete_file(meta, u, dev, f, v.id, base + 1 + i).await; + files.push(f); + } + delete_folder(meta, u, dev, dir, base + 20).await; + (dir, files) +} + +fn sorted_nodes(mut ids: Vec) -> Vec { + ids.sort(); + ids +} + +/// A folder brings back at most the cap of what is in the trash below it, the newest deletions first among equals in +/// depth; the rest stays in the trash, is listed on its own (its folder is live) and comes back by a further restore. A +/// folder with exactly the cap below it brings everything. +async fn a_folder_restore_takes_at_most_the_cap_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let (exact, exact_files) = folder_with_deleted_files(meta, &u, dev, "exact", 3, NOW + 100).await; + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![dir, exact])); + + let before = meta.space_info(u.space).await.unwrap(); + let done = small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.node, done.seq), (dir, before.seq + 1), "one change of the space"); + assert!(!live_name(meta, &u, dir).await.2); + for f in &files[2..] { + assert!(!live_name(meta, &u, *f).await.2, "the three newest deletions came back"); + } + for f in &files[..2] { + assert!(live_name(meta, &u, *f).await.2, "the oldest two stay in the trash"); + } + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0], files[1]]), "the rest is listed on its own"); + small.restore_from_trash(u.space, dev, files[1], NOW).await.unwrap(); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0]])); + + small.restore_from_trash(u.space, dev, exact, NOW).await.unwrap(); + for f in &exact_files { + assert!(!live_name(meta, &u, *f).await.2, "exactly the cap brings everything"); + } + assert_eq!(trash_ids(meta, &u).await, vec![files[0]]); +} + +/// Of the nodes below a folder the shallow ones come first, and what is left is listed only if its own folder is not in +/// the trash: a left-over folder is listed, what is in it is hidden behind it until it is restored. +async fn a_folder_restore_leaves_the_rest_listed_through_its_own_folder(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(1); + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let a = folder(meta, &u, dev, dir, "a", NOW).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW).await; + let (b, bv) = file_in(meta, &u, dev, dir, "b.txt", NOW).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 1).await; + delete_folder(meta, &u, dev, a, NOW + 2).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + assert_eq!(trash_ids(meta, &u).await, vec![dir]); + + small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert!(!live_name(meta, &u, dir).await.2 && !live_name(meta, &u, b).await.2, "the folder and its newest child"); + assert!(live_name(meta, &u, a).await.2 && live_name(meta, &u, a1).await.2); + assert_eq!(trash_ids(meta, &u).await, vec![a], "the folder is listed, what is in it is not"); + small.restore_from_trash(u.space, dev, a, NOW).await.unwrap(); + assert!(!live_name(meta, &u, a).await.2 && !live_name(meta, &u, a1).await.2); + assert!(trash_ids(meta, &u).await.is_empty()); +} + +/// Removing a folder marks it and at most the cap below it, shallowest first; the rest is listed on its own (its folder +/// is no longer in the trash) and goes by a further removal. +async fn removing_a_folder_takes_at_most_the_cap_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let before = meta.space_info(u.space).await.unwrap(); + small.request_purge(u.space, dir, NOW + 50).await.unwrap(); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![files[0], files[1]])); + for f in &files[2..] { + assert!(matches!(meta.restore_from_trash(u.space, dev, *f, NOW).await, Err(MetaError::NotFound(_))), "removed with the folder"); + } + small.request_purge(u.space, files[0], NOW + 51).await.unwrap(); + small.request_purge(u.space, files[1], NOW + 51).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + + // A left-over folder is listed and what is in it stays hidden behind it. + let tree = folder(meta, &u, dev, r, "tree", NOW + 100).await; + let a = folder(meta, &u, dev, tree, "a", NOW + 100).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW + 100).await; + let (b, bv) = file_in(meta, &u, dev, tree, "b.txt", NOW + 100).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 101).await; + delete_folder(meta, &u, dev, a, NOW + 102).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 103).await; + delete_folder(meta, &u, dev, tree, NOW + 104).await; + meta.clone().with_trash_tree_cap(1).request_purge(u.space, tree, NOW + 105).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![a], "only the left-over folder is listed"); + meta.clone().with_trash_tree_cap(1).request_purge(u.space, a, NOW + 106).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert!(matches!(meta.restore_from_trash(u.space, dev, a1, NOW).await, Err(MetaError::NotFound(_)))); +} + +/// When every conflict name is taken too, the restore is refused as a conflict and the space is exactly as it was: no +/// sequence number, no revived node, the node still in the trash. +async fn a_restore_with_no_free_name_changes_nothing(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "busy.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + // The name itself and its 100 conflict names, in one batch. + let m = manifest(meta, &u, 1).await; + let mut names = vec!["busy.txt".to_owned()]; + names.extend((1..=100).map(|n| conflict_of("busy.txt", false, n))); + let ops: Vec = names.iter().map(|n| Op::Create { node: NodeId::generate(), parent: r, name: name(n), version: version(m) }).collect(); + meta.commit(u.space, dev, &ops, NOW + 2).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let tree_before = full_tree(meta, u.space).await.nodes.len(); + + let refused = meta.restore_from_trash(u.space, dev, f, NOW).await; + let list = refused_restore(refused); + assert!(matches!(list[0].1, ConflictKind::NameTaken { .. }), "{list:?}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, vec![f]); + assert_eq!(full_tree(meta, u.space).await.nodes.len(), tree_before); + assert!(meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap().nodes.is_empty()); +} + +/// What is not in the trash can't be restored or removed from it: a live node, an unknown one, another space's, one +/// already removed from the trash. +async fn only_nodes_in_the_trash_can_be_restored_or_removed(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (live, _) = file_in(meta, &u, dev, r, "live.txt", NOW).await; + let (gone, gv) = file_in(meta, &u, dev, r, "gone.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, r, "kept.txt", NOW).await; + delete_file(meta, &u, dev, gone, gv.id, NOW + 1).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1).await; + meta.request_purge(u.space, gone, NOW + 2).await.unwrap(); + let seq = meta.space_info(u.space).await.unwrap().seq; + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + + for node in [live, NodeId::generate(), r, gone] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "restore {node}"); + } + for node in [live, NodeId::generate(), r] { + assert!(matches!(meta.request_purge(u.space, node, NOW).await, Err(MetaError::NotFound(_))), "remove {node}"); + } + assert!(matches!(meta.restore_from_trash(bob.space, bob_device, kept, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + assert!(matches!(meta.request_purge(bob.space, kept, NOW).await, Err(MetaError::NotFound(_)))); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, seq, "nothing changed"); + assert_eq!(trash_ids(meta, &u).await, vec![kept]); +} + +/// Removing from the trash only marks the node: no version, manifest or tombstone changes, the change feed and the +/// sequence number stay, the history is still readable; asking twice changes nothing; an edit that brings the node back +/// ends the mark, and the next deletion lists it again. +async fn removing_from_the_trash_frees_nothing(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + delete_file(meta, &u, dev, node, versions[2].id, NOW + 100).await; + let before = meta.space_info(u.space).await.unwrap(); + let feed_before = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + + meta.request_purge(u.space, node, NOW + 200).await.unwrap(); + meta.request_purge(u.space, node, NOW + 300).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "it left the trash"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(), feed_before, "the tombstone is as it was"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3, "every version is still there"); + for v in &versions { + assert!(meta.has_manifest(u.space, &v.manifest).await.unwrap(), "every manifest is still the space's"); + } + + // An edit still brings the file back (an edit beats a deletion), which ends the mark. + let edit = version(manifest(meta, &u, 9).await); + meta.commit(u.space, dev, &[Op::Update { node, expected: versions[2].id, version: edit.clone() }], NOW + 400).await.unwrap(); + assert!(!live_name(meta, &u, node).await.2); + delete_file(meta, &u, dev, node, edit.id, NOW + 500).await; + let page = meta.trash(u.space, None, 10, NOW + 600).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms), (1, NOW + 500)); +} + +/// A restore and a commit that wants the same name run together: both are applied in some order and the result is +/// consistent, never two live nodes with one name. +async fn a_trash_restore_and_a_commit_for_the_same_name_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + for round in 0..4 { + let file_name = format!("race{round}.txt"); + let (f, fv) = file_in(meta, &u, dev, r, &file_name, NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let rival = NodeId::generate(); + let rival_op = Op::Create { node: rival, parent: r, name: name(&file_name), version: version(manifest(meta, &u, 2).await) }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let create = tokio::spawn(async move { m2.commit(space, dev, &[rival_op], NOW + 2).await }); + let restored = restore.await.unwrap().unwrap(); + match create.await.unwrap() { + // The commit came first: it holds the name, and the restore went around it. + Ok(_) => { + assert!(restored.renamed, "round {round}"); + assert_eq!(live_name(meta, &u, rival).await.1, file_name); + } + // The restore came first: the commit finds the name taken. + Err(MetaError::Rejected(list)) => { + assert!(matches!(list[0].kind, ConflictKind::NameTaken { .. }), "round {round}"); + assert!(!restored.renamed); + assert_eq!(live_name(meta, &u, f).await.1, file_name); + } + Err(other) => panic!("unexpected error {other:?}"), + } + } + // No two live nodes share a name (the unique index would have refused it; this says what is seen). + let tree = full_tree(meta, u.space).await; + let mut names: Vec = tree.nodes.iter().filter_map(|n| n.name.as_ref().map(|n| n.fold_key().as_str().to_owned())).collect(); + let total = names.len(); + names.sort(); + names.dedup(); + assert_eq!(names.len(), total); +} + +/// A restore and an edit of the same deleted file run together: whichever is first, the file ends live with its versions +/// intact. If the edit is first it brings the file back and the restore finds nothing in the trash; if the restore is +/// first it makes a new current version, so the edit, made on the old one, is refused as a stale update. +async fn a_trash_restore_and_an_edit_of_the_file_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "both.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let edit = version(manifest(meta, &u, 4).await); + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let edit_op = Op::Update { node: f, expected: fv.id, version: edit.clone() }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let update = tokio::spawn(async move { m2.commit(space, dev, &[edit_op], NOW + 2).await }); + let restored = restore.await.unwrap(); + let updated = update.await.unwrap(); + match (&restored, &updated) { + // The edit came first and brought the file back: the restore finds nothing in the trash. + (Err(MetaError::NotFound(_)), Ok(_)) => {} + // The restore came first and made a new current version: the edit, made on the old one, is a stale update. + (Ok(_), Err(MetaError::Rejected(list))) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + other => panic!("one of them must win: {other:?}"), + } + assert_eq!(live_name(meta, &u, f).await, (Some(r), "both.txt".to_owned(), false)); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.history(u.space, f, None, 10).await.unwrap().versions.len(), 2); +} + +// ---- the trash: the migration and a damaged database ---- + +/// A raw connection to the database `url`, for what the library never does (reading and breaking columns). +async fn raw(url: &str) -> sqlx::AnyConnection { + use sqlx::Connection; + sqlx::any::install_default_drivers(); + sqlx::AnyConnection::connect(url).await.unwrap() +} + +/// Undoes migration 0015 (team spaces): the `spaces` table goes back to its shape in 0014, which older migration tests start +/// from. The version row is the caller's to delete. SQLite rebuilds the table, as the migration did. +async fn undo_team_spaces(conn: &mut sqlx::AnyConnection) { + let sqlite = sqlx::query("SELECT 1 FROM sqlite_master LIMIT 1").execute(&mut *conn).await.is_ok(); + let statements: &[&str] = if sqlite { + &[ + "PRAGMA foreign_keys = OFF", + "DROP TABLE space_members", + "CREATE TABLE spaces_old (id BLOB PRIMARY KEY, owner BLOB NOT NULL UNIQUE REFERENCES users (id), epoch BLOB NOT NULL, seq BIGINT NOT NULL, oldest_seq BIGINT NOT NULL, root BLOB NOT NULL, stored_bytes BIGINT NOT NULL DEFAULT 0, live_bytes BIGINT NOT NULL DEFAULT 0, quota_bytes BIGINT)", + "INSERT INTO spaces_old SELECT id, owner, epoch, seq, oldest_seq, root, stored_bytes, live_bytes, quota_bytes FROM spaces", + "DROP TABLE spaces", + "ALTER TABLE spaces_old RENAME TO spaces", + "PRAGMA foreign_keys = ON", + ] + } else { + &[ + "DROP TABLE space_members", + "DROP INDEX spaces_personal_owner", + "DROP INDEX spaces_team_name", + "ALTER TABLE spaces DROP CONSTRAINT spaces_owner_by_kind", + "ALTER TABLE spaces DROP CONSTRAINT spaces_name_by_kind", + "ALTER TABLE spaces DROP COLUMN kind", + "ALTER TABLE spaces DROP COLUMN name", + "ALTER TABLE spaces DROP COLUMN name_fold", + "ALTER TABLE spaces DROP COLUMN created_ms", + "ALTER TABLE spaces DROP COLUMN archived_ms", + "ALTER TABLE spaces ALTER COLUMN owner SET NOT NULL", + "ALTER TABLE spaces ADD CONSTRAINT spaces_owner_key UNIQUE (owner)", + ] + }; + for statement in statements.iter().copied() { + sqlx::query(statement).execute(&mut *conn).await.unwrap(); + } +} + +/// Removes the last three migrations and their version rows, so older migration tests can start from the version they name. +async fn drop_history_trash_and_manifest_index(conn: &mut sqlx::AnyConnection) { + undo_team_spaces(conn).await; + for statement in [ + "DROP TABLE manifest_chunks", + "DROP TABLE manifest_index", + "DROP INDEX nodes_trash", + "ALTER TABLE nodes DROP COLUMN deleted_ms", + "ALTER TABLE nodes DROP COLUMN purge_requested_ms", + "DROP INDEX versions_node_seq", + "DELETE FROM schema_version WHERE version >= 12", + ] { + sqlx::query(statement).execute(&mut *conn).await.unwrap(); + } +} + +/// Removes the trash and manifest-index migrations but retains the version-history migration. +async fn drop_trash_and_manifest_index(conn: &mut sqlx::AnyConnection) { + undo_team_spaces(conn).await; + for statement in [ + "DROP TABLE manifest_chunks", + "DROP TABLE manifest_index", + "DROP INDEX nodes_trash", + "ALTER TABLE nodes DROP COLUMN deleted_ms", + "ALTER TABLE nodes DROP COLUMN purge_requested_ms", + "DELETE FROM schema_version WHERE version >= 13", + ] { + sqlx::query(statement).execute(&mut *conn).await.unwrap(); + } +} + +/// Puts the database back to the state before the trash migration: no columns, no index, one version fewer. +async fn undo_the_trash_migration(url: &str) { + let mut conn = raw(url).await; + drop_trash_and_manifest_index(&mut conn).await; +} + +fn wall_clock_ms() -> i64 { + i64::try_from(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()).unwrap() +} + +/// Nodes deleted before the trash existed start their time in the trash at the migration, once: live nodes get no +/// time, and opening the database again does not stamp them again. +async fn the_trash_migration_dates_old_deletions_at_the_migration(url: &str) { + let (ana, file, folder_node) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "old.txt", NOW).await; + let (_kept, _) = file_in(&meta, &ana, dev, r, "kept.txt", NOW).await; + let dir = folder(&meta, &ana, dev, r, "olddir", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + delete_folder(&meta, &ana, dev, dir, NOW + 2).await; + (ana, file, dir) + }; + undo_the_trash_migration(url).await; + let before = wall_clock_ms(); + let meta = Meta::open(url).await.unwrap(); + let after = wall_clock_ms(); + let page = meta.trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(page.repaired, 0, "the migration stamped them, nothing was left to repair"); + let mut ids: Vec<_> = page.entries.iter().map(|e| e.id).collect(); + ids.sort(); + let mut expected = vec![file, folder_node]; + expected.sort(); + assert_eq!(ids, expected, "the live file is not in the trash"); + let stamps: Vec = page.entries.iter().map(|e| e.deleted_ms).collect(); + assert!(stamps.iter().all(|t| (before - 1_000..=after + 1_000).contains(t)), "{stamps:?} not in {before}..{after}"); + assert_eq!(stamps[0], stamps[1], "one baseline for all of them"); + // Another open changes nothing. + drop(meta); + let again = Meta::open(url).await.unwrap().trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(again.entries, page.entries); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 15); + let live_with_time: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM nodes WHERE deleted = 0 AND (deleted_ms IS NOT NULL OR purge_requested_ms IS NOT NULL)") + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(live_with_time, 0); +} + +#[tokio::test] +async fn the_trash_migration_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_trash_migration_dates_old_deletions_at_the_migration(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_trash_migration_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_trash_migration_dates_old_deletions_at_the_migration(&url).await; +} + +/// Drops what migration 0008 added and forgets it ran, which leaves the database as 0007 left it (with its rows). +async fn back_to_0007(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + // The audit log (migration 0009) and two-factor sign-in (migration 0010) go first: they run again with 0008 on the next open. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN purpose").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 8").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the admin API keeps its accounts, devices and invites: an old invite reads as an +/// invite and still works, no space has a quota and no user a removal mark, and the new calls work on it. +async fn migrating_from_0007_keeps_the_rows(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, old_token, invite) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + meta.add_chunk(ana.space, &random_chunk(), 42, NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + (ana, old_token, invite) + }; + back_to_0007(url).await; + assert_eq!(schema_version(url).await, 7); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(ana.id)); + let row = admin_view(&meta, ana.id).await; + assert_eq!((row.disabled_ms, row.quota_bytes, row.stored_bytes, row.devices_active, row.invite_pending), (None, None, 42, 1, true)); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(false), "an old invite is an invite"); + let (_, actor) = admin_actor(&meta, "root").await; + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.device_by_token(&old_token).await.unwrap(), None); + + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (16)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0007_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0007_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0007_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0007_keeps_the_rows(&url).await; +} + +/// A deleted node without a deletion time (it can't happen through this code) is stamped with the time of the read +/// and reported, never listed as expired; the stamp stays. +async fn a_deleted_node_without_a_time_is_stamped_when_read(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "damaged.txt", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + let mut conn = raw(url).await; + sqlx::query("UPDATE nodes SET deleted_ms = NULL WHERE deleted = 1").execute(&mut conn).await.unwrap(); + let page = meta.trash(ana.space, None, 10, NOW + 77).await.unwrap(); + assert_eq!((page.repaired, page.entries.len(), page.entries[0].deleted_ms), (1, 1, NOW + 77)); + let later = meta.trash(ana.space, None, 10, NOW + 99).await.unwrap(); + assert_eq!((later.repaired, later.entries[0].deleted_ms), (0, NOW + 77)); +} + +#[tokio::test] +async fn a_damaged_deletion_time_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_deleted_node_without_a_time_is_stamped_when_read(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_damaged_deletion_time_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_deleted_node_without_a_time_is_stamped_when_read(&url).await; +} + +// ---- the audit log (migration 0009) ---- + +use craft_cloud_meta::{ + ActionFilter, AuditAction, AuditCursor, AuditEvent, AuditFailure, AuditOutcome, AuditQuery, Origin, PRUNE_BATCH, PRUNE_EVERY_MS, Step, +}; + +/// Every event, newest first. +async fn all_events(meta: &Meta) -> Vec { + meta.audit_list(&AuditQuery { limit: 1000, ..AuditQuery::default() }).await.unwrap() +} + +/// Remembers which events a test has looked at, and hands out what was written since, oldest first. +struct Seen<'a> { + meta: &'a Meta, + ids: std::collections::HashSet, +} + +impl<'a> Seen<'a> { + async fn new(meta: &'a Meta) -> Self { + let mut seen = Self { meta, ids: Default::default() }; + seen.fresh().await; + seen + } + + async fn fresh(&mut self) -> Vec { + let Self { meta, ids } = self; + let mut fresh: Vec = all_events(meta).await.into_iter().filter(|e| ids.insert(e.id)).collect(); + fresh.reverse(); + fresh + } + + /// The names of the events written since the last look, sorted (two events of one change have no order to rely on). + async fn actions(&mut self) -> Vec<&'static str> { + let mut names: Vec<&'static str> = self.fresh().await.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + names + } + + /// The one event written since the last look. + async fn one(&mut self) -> AuditEvent { + let mut fresh = self.fresh().await; + assert_eq!(fresh.len(), 1, "{:?}", fresh.iter().map(|e| e.action.as_str()).collect::>()); + fresh.remove(0) + } +} + +fn details(e: &AuditEvent) -> serde_json::Value { + serde_json::from_str(&e.details).unwrap() +} + +/// An event of the day-to-day, shorthand for the matrix below. +fn json(text: &str) -> serde_json::Value { + serde_json::from_str(text).unwrap() +} + +/// Every operation of the matrix in `docs/design/sync.md` writes exactly the events it names, with the actor, target and +/// typed details it names; a repeat or a no-op writes none; a bulk sign-out writes one event with the count. Creating +/// an account or a device by the test seams (`create_user`, `create_device`) writes none. +async fn every_operation_writes_exactly_its_events(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let mut seen = Seen::new(meta).await; + assert!(all_events(meta).await.is_empty(), "the test seams write no events"); + + // A sign-in: the account, the new device, and the client reduced to the closed lists. + let client = DeviceClient::new(Some("agent"), Some("Linux"), Some("1.2.3")).unwrap(); + let dev = meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &client, &token(), NOW).await.unwrap().unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.count), + (AuditAction::AuthLogin, AuditOutcome::Success, Some(ana.id), Some(dev), 1) + ); + assert_eq!((e.target_kind.as_str(), e.target_id), ("device", Some(*dev.as_uuid()))); + assert_eq!(details(&e), json(r#"{"client_kind":"agent","platform":"linux"}"#)); + assert_eq!(e.at_ms, NOW); + // A sign-in that is refused writes nothing. + assert!(meta.sign_in_device(ana.id, ("$argon2id$other", 0), "pc", &client, &token(), NOW).await.unwrap().is_none()); + assert!(seen.actions().await.is_empty()); + + // Signing out one device: from a device, then the same again (nothing changed), then as the account without a device. + let (d2, d3) = (device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_device_by(ana.id, d2, RevokeReason::User, Origin::Device(dev), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::DeviceRevoked, Some(ana.id), Some(dev), Some(*d2.as_uuid()))); + assert_eq!(details(&e), json(r#"{"reason":"user"}"#), "the device is the event's target"); + meta.revoke_device_by(ana.id, d2, RevokeReason::User, Origin::Device(dev), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "a device that is out already: no change, no event"); + meta.revoke_device(ana.id, d3).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::DeviceRevoked, Some(ana.id), None)); + + // Signing out the others: one event with the count; with nobody to sign out, none. + let (_e1, _e2) = (device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_device, e.target_id), (AuditAction::DeviceRevokedOthers, Some(dev), Some(ana.id.0))); + assert_eq!(details(&e), json(r#"{"reason":"others","revoked":2}"#)); + meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // A password change: one event, the devices signed out with it are only in `revoked`. + let (_f1, _f2) = (device(meta, &ana).await, device(meta, &ana).await); + meta.change_password(ana.id, &ana.password_hash, "$argon2id$new", dev, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::AccountPasswordChanged, Some(ana.id), Some(dev))); + assert_eq!(details(&e), json(r#"{"revoked":2}"#)); + // A change that fails writes nothing: the password it checked is not the account's. + assert!(matches!(meta.change_password(ana.id, "$argon2id$wrong", "$argon2id$x", dev, NOW).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); + + // Accounts: created with a role; the invite that goes with it is not an event of its own. + let invite = code(INVITE_CHARS); + let (newbie, _) = meta.admin_create_user(&actor, "newbie", Role::Member, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AdminUserCreated, Some(root.id), Some(device_of(&actor)), Some(newbie.id.0)) + ); + assert_eq!(details(&e), json(r#"{"role_to":"member"}"#)); + // Invites: issued, revoked (a second revoke has nothing to revoke). + meta.admin_issue_invite(&actor, newbie.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.invite_issued"]); + meta.admin_revoke_invite(&actor, newbie.id, NOW).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.invite_revoked"]); + assert!(matches!(meta.admin_revoke_invite(&actor, newbie.id, NOW).await, Err(MetaError::NotFound(_)))); + assert!(seen.actions().await.is_empty()); + + // A role and a quota in one request: one event for each, in the same transaction. The same again changes nothing. + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + let both = seen.fresh().await; + let mut names: Vec<&str> = both.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["admin.quota_changed", "admin.role_changed"]); + for e in &both { + assert_eq!((e.actor_user, e.target_id), (Some(root.id), Some(ana.id.0))); + match e.action { + AuditAction::AdminRoleChanged => assert_eq!(details(e), json(r#"{"role_from":"member","role_to":"admin"}"#)), + _ => assert_eq!(details(e), json(&format!(r#"{{"quota_bytes":{MIN_QUOTA_BYTES}}}"#))), + } + } + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "the role and quota it has already: no change, no event"); + meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, details(&e)), (AuditAction::AdminRoleChanged, json(r#"{"role_from":"admin","role_to":"member"}"#))); + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, details(&e)), (AuditAction::AdminQuotaChanged, json(r#"{"quota_bytes":null}"#)), "removing the quota says so"); + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // A reset code, and its redemption: the devices it signs out are counted in the one event. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let active = meta.list_devices(ana.id).await.unwrap().iter().filter(|d| !d.revoked).count(); + let redeemed = meta.redeem_with(&reset.hash(), "recovery", &client, &token(), Some("$argon2id$reset"), NOW).await.unwrap(); + assert_eq!(redeemed.revoked.len(), active); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::JoinResetRedeemed, Some(ana.id), Some(redeemed.device), Some(*redeemed.device.as_uuid())) + ); + assert_eq!(details(&e), json(&format!(r#"{{"client_kind":"agent","platform":"linux","revoked":{active}}}"#))); + + // An invite redeemed with a password for the new account, and a pairing redeemed from its device. + meta.admin_issue_invite(&actor, newbie.id, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + seen.fresh().await; + let joined = meta.redeem(&invite.hash(), "newbie pc", &token(), Some("$argon2id$newbie"), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinInviteRedeemed, Some(newbie.id), Some(joined.device))); + assert_eq!(details(&e), json("{}"), "no client named: no client in the event"); + let pairing = code(PAIRING_CHARS); + meta.create_pairing(newbie.id, joined.device, &pairing.hash(), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "making a code is not an event"); + let paired = meta.redeem(&pairing.hash(), "second pc", &token(), None, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinPairingRedeemed, Some(newbie.id), Some(paired.device))); + // A code that does not work writes nothing. + assert!(matches!(meta.redeem(&pairing.hash(), "again", &token(), None, NOW).await, Err(MetaError::InvalidCode))); + assert!(seen.actions().await.is_empty()); + + // Disable (the devices go with it), a second disable, enable, a second enable. + let (_g1, _g2) = (device(meta, &ana).await, device(meta, &ana).await); + let signed_out = meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap().len(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.target_id), (AuditAction::AdminUserDisabled, Some(root.id), Some(ana.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{signed_out}}}"#))); + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "disabled already: no change, no event"); + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.user_enabled"]); + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // An admin signs out one device: about the account, naming the device; again, nothing. + let anas = device(meta, &ana).await; + meta.admin_revoke_device(&actor, ana.id, anas, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_kind.as_str(), e.target_id), + (AuditAction::AdminDeviceRevoked, Some(root.id), Some(device_of(&actor)), "user", Some(ana.id.0)) + ); + assert_eq!(details(&e), json(&format!(r#"{{"device":"{anas}","reason":"admin"}}"#))); + meta.admin_revoke_device(&actor, ana.id, anas, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // Marked for removal, once. + let _ = device(meta, &ana).await; + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.target_id), (AuditAction::AdminUserRemoved, Some(ana.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":1}"#)); + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // The command line acts as no account. + let cara = user(meta, "cara").await; + let (_h1, _h2) = (device(meta, &cara).await, device(meta, &cara).await); + meta.reset_password(cara.id, "$argon2id$cli", false, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliPasswordReset, None, None, Some(cara.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":2}"#)); + let one = device(meta, &cara).await; + meta.revoke_device_by(cara.id, one, RevokeReason::Admin, Origin::Cli, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliDeviceRevoked, None, None, Some(cara.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"device":"{one}","reason":"admin"}}"#))); + meta.revoke_device_by(cara.id, one, RevokeReason::Admin, Origin::Cli, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + let (_i1, _i2, _i3) = (device(meta, &cara).await, device(meta, &cara).await, device(meta, &cara).await); + meta.revoke_all_devices(cara.id, RevokeReason::Admin, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.target_id), (AuditAction::CliDevicesRevokedAll, None, Some(cara.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":3}"#)); + meta.revoke_all_devices(cara.id, RevokeReason::Admin, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + // A reset that keeps the devices counts none. + meta.reset_password(cara.id, "$argon2id$cli2", true, NOW).await.unwrap(); + assert_eq!(details(&seen.one().await), json(r#"{"revoked":0}"#)); +} + +/// A change that fails, or is refused, leaves no event behind, and the last administrator stays one. +async fn a_refused_change_writes_no_event(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let mut seen = Seen::new(meta).await; + // The only administrator cannot demote or disable themselves out of the job, and the role stays. + assert!(matches!(meta.admin_update_user(&actor, actor.user, Some(Role::Member), None, NOW).await, Err(MetaError::LastAdmin))); + assert!(matches!(meta.admin_disable_user(&actor, actor.user, NOW).await, Err(MetaError::Invalid(_)))); + // An actor whose role is gone, one whose password changed since, and a target that is not there. + let stale = AdminActor { password_hash: "$argon2id$old".to_owned(), ..actor.clone() }; + assert!(matches!(meta.admin_disable_user(&stale, ana.id, NOW).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.admin_update_user(&actor, UserId(uuid::Uuid::now_v7()), Some(Role::Admin), None, NOW).await, Err(MetaError::NotFound(_)))); + // A reset code without the new password, a name that is taken, a device of someone else. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + seen.fresh().await; + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), None, NOW).await, Err(MetaError::PasswordRequired))); + assert!(matches!( + meta.admin_create_user(&actor, "ana", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::UsernameTaken) + )); + assert!(matches!(meta.revoke_device_as(ana.id, DeviceId::generate(), RevokeReason::User, NOW).await, Err(MetaError::NotFound(_)))); + assert!(seen.actions().await.is_empty()); + assert_eq!(meta.account(actor.user).await.unwrap().role, Role::Admin); +} + +/// A sign-in step that failed is one row per account, action, client kind and minute, whose count goes up and whose identity, time and +/// details stay those of the first try; names no account has share one row per minute; an event that is not a failed step +/// cannot be aggregated. +async fn failed_sign_ins_count_up_in_one_row_per_account_and_minute(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let minute = NOW - NOW % 60_000; + let login = |target: Option| AuditFailure { action: AuditAction::AuthLoginFailed, target, step: Step::Password }; + meta.audit_record_failure(login(Some(ana.id)), minute + 1_000).await.unwrap(); + let first = all_events(meta).await; + assert_eq!(first.len(), 1); + let first = first[0].clone(); + assert_eq!( + (first.action, first.outcome, first.count, first.actor_user, first.target_id), + (AuditAction::AuthLoginFailed, AuditOutcome::Failure, 1, None, Some(ana.id.0)) + ); + assert_eq!((first.target_kind.as_str(), details(&first)), ("user", json(r#"{"step":"password"}"#))); + meta.audit_record_web_failure(login(Some(ana.id)), minute + 1_000).await.unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 2, "device and web failure aggregates stay separate"); + assert!(rows.iter().any(|event| details(event) == json(r#"{"client_kind":"web","step":"password"}"#))); + // The same minute: the count goes up, nothing else changes. + meta.audit_record_failure(login(Some(ana.id)), minute + 59_000).await.unwrap(); + meta.audit_record_failure(login(Some(ana.id)), minute + 59_999).await.unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 2); + let device_row = rows.iter().find(|event| event.id == first.id).unwrap(); + assert_eq!((device_row.id, device_row.at_ms, device_row.count, &device_row.details), (first.id, minute + 1_000, 3, &first.details)); + // The next minute is a new row; another account, another action, another row. + meta.audit_record_failure(login(Some(ana.id)), minute + 60_000).await.unwrap(); + meta.audit_record_failure(login(Some(bob.id)), minute + 1_000).await.unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthSecondFactorFailed, target: Some(ana.id), step: Step::Totp }, minute + 1_000) + .await + .unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 5); + let mut counts: Vec<(&str, u64)> = rows.iter().map(|e| (e.action.as_str(), e.count)).collect(); + counts.sort_unstable(); + assert_eq!( + counts, + [("auth.login_failed", 1), ("auth.login_failed", 1), ("auth.login_failed", 1), ("auth.login_failed", 3), ("auth.second_factor_failed", 1)] + ); + // Names nobody has: one row a minute whatever was typed, with no account in it. + for _ in 0..4 { + meta.audit_record_failure(login(None), minute + 2_000).await.unwrap(); + } + meta.audit_record_failure(login(None), minute + 62_000).await.unwrap(); + let unknown: Vec = all_events(meta).await.into_iter().filter(|e| e.target_id.is_none()).collect(); + assert_eq!(unknown.len(), 2); + assert!(unknown.iter().all(|e| e.actor_user.is_none() && e.target_kind.is_empty() && e.actor_device.is_none())); + let mut counts: Vec = unknown.iter().map(|e| e.count).collect(); + counts.sort_unstable(); + assert_eq!(counts, [1, 4]); + // Only the failed steps are aggregated. + for action in [AuditAction::AuthLogin, AuditAction::AdminUserCreated, AuditAction::CliPasswordReset] { + assert!(matches!( + meta.audit_record_failure(AuditFailure { action, target: None, step: Step::Password }, NOW).await, + Err(MetaError::Invalid(_)) + )); + } + // Other events have no key, so any number of them is allowed. + let (_, actor) = admin_actor(meta, "root").await; + for i in 0..3 { + meta.admin_update_user(&actor, ana.id, Some(if i % 2 == 0 { Role::Admin } else { Role::Member }), None, NOW).await.unwrap(); + } + assert_eq!(all_events(meta).await.iter().filter(|e| e.action == AuditAction::AdminRoleChanged).count(), 3); +} + +/// Many tries at one moment on one key: exactly one row, counting every one, and the first row's identity and time stay. +/// Run on both databases (on Postgres the collisions really are concurrent). +async fn concurrent_failures_with_one_key_make_one_row_with_every_count(meta: &Meta) { + let ana = user(meta, "ana").await; + let minute = NOW - NOW % 60_000; + const TRIES: u64 = 24; + for round in 0..3i64 { + let at = minute + round * 60_000; + let tries: Vec<_> = (0..TRIES) + .map(|i| { + let meta = meta.clone(); + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }; + tokio::spawn(async move { meta.audit_record_failure(failure, at + i as i64).await }) + }) + .collect(); + for t in tries { + t.await.unwrap().unwrap(); + } + } + let rows = all_events(meta).await; + assert_eq!(rows.len(), 3, "one row for each minute"); + assert!(rows.iter().all(|e| e.count == TRIES), "{:?}", rows.iter().map(|e| e.count).collect::>()); + // Each row kept the time of the try that made it: one of the first moment's tries, within its minute. + for (round, row) in rows.iter().rev().enumerate() { + let start = minute + round as i64 * 60_000; + assert!((start..start + TRIES as i64).contains(&row.at_ms), "{} not in {}..", row.at_ms, start); + } +} + +/// Lists: newest first in `(at_ms, id)` order, filters by account and by name or group of names, `before_ms` and an end +/// time, and a cursor that is exclusive so that events with one timestamp are neither repeated nor lost. +async fn audit_lists_filter_and_page_by_cursor(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + // Seven events at one moment. + for i in 0..7 { + meta.admin_update_user(&actor, ana.id, Some(if i % 2 == 0 { Role::Admin } else { Role::Member }), None, NOW).await.unwrap(); + } + meta.sign_in_device(bob.id, (&bob.password_hash, bob.auth_gen), "pc", &DeviceClient::default(), &token(), NOW + 5).await.unwrap().unwrap(); + let minute = NOW - NOW % 60_000 + 60_000; + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(bob.id), step: Step::Password }, minute) + .await + .unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }, minute + 60_000) + .await + .unwrap(); + + let all = all_events(meta).await; + assert_eq!(all.len(), 10); + let order: Vec<_> = all.iter().map(|e| (e.at_ms, e.id)).collect(); + assert!(order.windows(2).all(|w| w[0] > w[1]), "newest first, by (at_ms, id) descending"); + + // Pages of three through the cursor of the last event of each: all ten once, in order. + let mut paged = Vec::new(); + let mut after: Option = None; + for _ in 0..10 { + let page = meta.audit_list(&AuditQuery { after, limit: 3, ..AuditQuery::default() }).await.unwrap(); + let Some(last) = page.last() else { break }; + after = Some(last.cursor()); + paged.extend(page); + } + assert_eq!(paged, all); + // A cursor behind everything gives nothing; a cursor at an event never gives that event. + assert!( + meta.audit_list(&AuditQuery { after: Some(AuditCursor { at_ms: 0, id: uuid::Uuid::nil() }), limit: 10, ..AuditQuery::default() }) + .await + .unwrap() + .is_empty() + ); + let sevens: Vec<&AuditEvent> = all.iter().filter(|e| e.at_ms == NOW).collect(); + assert_eq!(sevens.len(), 7); + let page = meta.audit_list(&AuditQuery { after: Some(sevens[2].cursor()), limit: 100, ..AuditQuery::default() }).await.unwrap(); + assert_eq!(page.len(), 4, "the four behind the third of the seven that share a time"); + + // Times: before is exclusive, the end time inclusive. + let before = |ms| AuditQuery { before_ms: Some(ms), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&before(NOW)).await.unwrap().len(), 0); + assert_eq!(meta.audit_list(&before(NOW + 1)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&AuditQuery { until_ms: Some(NOW), limit: 100, ..AuditQuery::default() }).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&AuditQuery { until_ms: Some(NOW - 1), limit: 100, ..AuditQuery::default() }).await.unwrap().len(), 0); + + // Accounts: the actor or the target. + let by_user = |u: UserId| AuditQuery { user: Some(u), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&by_user(ana.id)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_user(root.id)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_user(bob.id)).await.unwrap().len(), 2, "his sign-in, and the guess about his password"); + assert_eq!(meta.audit_list(&by_user(UserId(uuid::Uuid::now_v7()))).await.unwrap().len(), 0); + + // Names: exact, or a group. + let by_action = |text: &str| AuditQuery { action: ActionFilter::parse(text), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&by_action("admin.role_changed")).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_action("auth.login")).await.unwrap().len(), 1, "exact: not auth.login_failed"); + assert_eq!(meta.audit_list(&by_action("auth.")).await.unwrap().len(), 3); + assert_eq!(meta.audit_list(&by_action("admin.")).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_action("cli.")).await.unwrap().len(), 0); + // Both at once, and with a page. + let both = AuditQuery { user: Some(bob.id), action: ActionFilter::parse("auth."), limit: 1, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&both).await.unwrap().len(), 1); +} + +/// Pruning deletes what is older than the limit and nothing else, a batch at a time; a pruning that leaves a backlog +/// does not count as done, so the next write goes on; one that finished holds off the next for an hour; an error is not +/// done either. +async fn pruning_works_in_batches_and_keeps_its_time_until_done(meta: &Meta) { + let ana = user(meta, "ana").await; + let day = 24 * 60 * 60 * 1000i64; + let old = NOW - 3 * day; + // More old events than one batch holds, each in a minute of its own. + let n = PRUNE_BATCH + 100; + let writers: Vec<_> = (0..8) + .map(|w| { + let meta = meta.clone(); + tokio::spawn(async move { + let mut i = w; + while i < n { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old + i * 60_000).await.unwrap(); + i += 8; + } + }) + }) + .collect(); + for w in writers { + w.await.unwrap(); + } + // Recent events, which pruning leaves. + for i in 0..3 { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }; + meta.audit_record_failure(failure, NOW - day / 2 + i * 60_000).await.unwrap(); + } + let limit = NOW - day; + assert_eq!(meta.audit_count_before(limit).await.unwrap(), n as u64); + assert_eq!(meta.audit_count_before(old).await.unwrap(), 0, "the limit is exclusive"); + + // Not configured: a write prunes nothing. + assert!(!meta.audit_prune_if_due(NOW).await.unwrap()); + // A second handle that prunes when asked. The writes below go through `meta`, which never starts a pruning of its own. + let pruner = meta.clone().with_audit_retention_days(1); + // One batch, then the rest: the first leaves a backlog and so does not hold off the next. + assert!(pruner.audit_prune_if_due_in(NOW, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 100, "one batch of {PRUNE_BATCH}"); + assert!(pruner.audit_prune_if_due_in(NOW + 1, 1).await.unwrap(), "a backlog is not done"); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 0); + assert_eq!(all_events(meta).await.len(), 3, "the recent events stay"); + // Done: the next hour is left alone, then it runs again. + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old).await.unwrap(); + assert!(!pruner.audit_prune_if_due_in(NOW + 2, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 1); + assert!(pruner.audit_prune_if_due_in(NOW + 1 + PRUNE_EVERY_MS, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 0); + + // `audit_prune` itself takes everything before the limit in batches, and reports the count. + for i in 0..(PRUNE_BATCH + 5) { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old + i * 60_000).await.unwrap(); + } + assert_eq!(meta.audit_prune(limit).await.unwrap(), (PRUNE_BATCH + 5) as u64); + assert_eq!(meta.audit_prune(limit).await.unwrap(), 0); + assert_eq!(all_events(meta).await.len(), 3); +} + +/// Events outlive the account they are about: marking it for removal deletes nothing, and the account's ID still finds them. +async fn a_removed_accounts_events_stay(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await.unwrap(); + meta.admin_remove_user(&actor, ana.id, NOW + 1).await.unwrap(); + let mine = meta.audit_list(&AuditQuery { user: Some(ana.id), limit: 100, ..AuditQuery::default() }).await.unwrap(); + let mut names: Vec<&str> = mine.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["admin.user_removed", "auth.login", "auth.login_failed"]); +} + +/// Nothing a client typed or a secret is in any event, whatever the account, the device or the client called itself. +async fn no_event_holds_a_name_or_a_secret(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let hash = "$argon2id$SECRETHASH"; + let zelda = meta.create_user("zelda", hash, Role::Member, NOW).await.unwrap(); + // A platform may be any printable text of 32 bytes: here an address, a file name, and 32 bytes of multi-byte characters. + let hostile = + ["10.0.0.1", "secret.psd", "fe80::1%eth0", "\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}", "windows 11"]; + let mut clock = NOW; + for platform in hostile { + let client = DeviceClient::new(Some("desktop"), Some(platform), Some("9.9.9-secretbuild")).unwrap(); + clock += 1; + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &client, &token(), clock).await.unwrap().unwrap(); + let e = all_events(meta).await.remove(0); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"other"}"#), "{platform:?}"); + } + let known = DeviceClient::new(Some("cli"), Some("MacOS"), None).unwrap(); + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &known, &token(), NOW + 100).await.unwrap().unwrap(); + assert_eq!(details(&all_events(meta).await.remove(0)), json(r#"{"client_kind":"cli","platform":"macos"}"#)); + let no_platform = DeviceClient::new(Some("agent"), None, None).unwrap(); + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &no_platform, &token(), NOW + 200).await.unwrap().unwrap(); + assert_eq!(details(&all_events(meta).await.remove(0)), json(r#"{"client_kind":"agent"}"#), "no platform: no field"); + // A few more changes, and the same scan over everything written. + meta.change_password(zelda.id, hash, "$argon2id$NEWSECRETHASH", meta.list_devices(zelda.id).await.unwrap()[0].id, NOW).await.unwrap(); + meta.admin_update_user(&actor, zelda.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }, NOW).await.unwrap(); + let events = all_events(meta).await; + assert!(events.len() > 8); + let shown = format!("{events:?}"); + for secret in ["zelda", "SECRETHASH", "secretbuild", "10.0.0.1", "secret.psd", "fe80", "127.0.0.1", "\u{20ac}", "argon2", "laptop", "ZELDA"] { + assert!(!shown.contains(secret), "{secret:?} in {shown}"); + } + // Every `details` is an object of the allowed words and numbers only. + for e in &events { + let value = details(e); + let object = value.as_object().unwrap(); + for key in object.keys() { + assert!( + ["client_kind", "platform", "reason", "step", "role_from", "role_to", "quota_bytes", "revoked", "device", "truncated"] + .contains(&key.as_str()), + "{key}" + ); + } + assert!(e.details.len() <= 2048); + } +} + +/// The log is append-only in the code: the sources of this crate update and delete in `audit_events` in exactly the two +/// places the design names (the counter of an aggregated failure, the pruning). +#[test] +fn the_sources_change_audit_rows_only_to_count_and_to_prune() { + fn sources(dir: &std::path::Path, out: &mut Vec<(std::path::PathBuf, String)>) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + sources(&path, out); + } else if path.extension().is_some_and(|e| e == "rs") { + let text = std::fs::read_to_string(&path).unwrap(); + out.push((path, text)); + } + } + } + let mut files = Vec::new(); + sources(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), &mut files); + assert!(files.len() > 5, "the sources were found"); + let count = |needle: &str| -> Vec { + files + .iter() + .flat_map(|(path, text)| { + // Comments may name statements, only code counts. + text.lines() + .filter(|l| !l.trim_start().starts_with("//") && l.contains(needle)) + .map(move |_| path.file_name().unwrap().to_string_lossy().into_owned()) + }) + .collect() + }; + assert!(count("UPDATE audit_events").is_empty(), "no statement updates the log"); + assert!(count("audit_events SET").is_empty()); + assert_eq!(count("DELETE FROM audit_events"), ["audit.rs"], "exactly one deletion, the pruning"); + assert_eq!(count("DO UPDATE SET agg_count = audit_events.agg_count + 1"), ["audit.rs"], "exactly one update, the counter"); + assert!(count("TRUNCATE").is_empty() && count("DROP TABLE audit").is_empty()); +} + +// ---- an error in the log ---- + +/// Makes every use of the log fail (the table is renamed away), which stands for any error of the database in the log +/// and needs no code in the product to cause it. +async fn break_audit(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("ALTER TABLE audit_events RENAME TO audit_events_away").execute(&mut conn).await.unwrap(); +} + +async fn repair_audit(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("ALTER TABLE audit_events_away RENAME TO audit_events").execute(&mut conn).await.unwrap(); +} + +/// A change whose event cannot be written is not made (fail closed), whichever it is; a failed sign-in step's event is +/// only reported to its caller, which ignores it; pruning that fails is not done, so the next write tries again. +async fn an_error_in_the_log_fails_every_change_and_changes_nothing(url: &str) { + sqlx::any::install_default_drivers(); + let meta = Meta::open(url).await.unwrap(); + let pruner = meta.clone().with_audit_retention_days(1); + let (_, actor) = admin_actor(&meta, "root").await; + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let others = [device(&meta, &ana).await, device(&meta, &ana).await]; + let invite = code(INVITE_CHARS); + let fresh = meta.admin_create_user(&actor, "fresh", Role::Member, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap().0; + let events_before = all_events(&meta).await.len(); + let active = |meta: &Meta, user: UserId| { + let meta = meta.clone(); + async move { meta.list_devices(user).await.unwrap().iter().filter(|d| !d.revoked).count() } + }; + assert_eq!(active(&meta, ana.id).await, 3); + + break_audit(url).await; + let is_db = |r: Result<(), MetaError>| assert!(matches!(r, Err(MetaError::Db(_))), "{r:?}"); + is_db(meta.sign_in_device(ana.id, (&ana.password_hash, 0), "new", &DeviceClient::default(), &token(), NOW).await.map(|_| ())); + is_db(meta.revoke_device_by(ana.id, others[0], RevokeReason::User, Origin::Device(dev), NOW).await); + is_db(meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.map(|_| ())); + is_db(meta.change_password(ana.id, &ana.password_hash, "$argon2id$new", dev, NOW).await.map(|_| ())); + is_db(meta.reset_password(ana.id, "$argon2id$cli", false, NOW).await.map(|_| ())); + is_db(meta.revoke_all_devices(ana.id, RevokeReason::Admin, NOW).await.map(|_| ())); + is_db(meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.map(|_| ())); + is_db(meta.admin_disable_user(&actor, ana.id, NOW).await.map(|_| ())); + is_db(meta.admin_remove_user(&actor, ana.id, NOW).await.map(|_| ())); + is_db(meta.admin_issue_reset(&actor, ana.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await.map(|_| ())); + is_db(meta.admin_revoke_device(&actor, ana.id, dev, NOW).await); + is_db(meta.admin_issue_invite(&actor, fresh.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.map(|_| ())); + is_db(meta.admin_revoke_invite(&actor, fresh.id, NOW).await); + is_db(meta.admin_create_user(&actor, "nobody", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.map(|_| ())); + is_db(meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$x"), NOW).await.map(|_| ())); + // A sign-in step that failed has no change to undo: the error is the caller's, and nothing else is touched. + is_db(meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await); + assert!(pruner.audit_prune_if_due(NOW).await.is_err(), "pruning with the log broken is an error"); + repair_audit(url).await; + + // Nothing was changed by any of it. + assert_eq!(all_events(&meta).await.len(), events_before, "no event, no change"); + assert_eq!(active(&meta, ana.id).await, 3, "no device was signed out or added"); + let row = meta.admin_user(ana.id, NOW).await.unwrap().unwrap(); + assert_eq!((row.role, row.disabled_ms, row.quota_bytes), (Role::Member, None, None)); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, ana.password_hash); + assert!(meta.user_by_name("nobody").await.unwrap().is_none(), "the account was not made"); + assert!(meta.admin_user(fresh.id, NOW).await.unwrap().unwrap().invite_pending, "the invite is still there"); + // And the same changes work once the log does, the invite included. + meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$x"), NOW).await.unwrap(); + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + // A pruning that failed left no time behind: the next write's goes ahead at once. + assert!(pruner.audit_prune_if_due(NOW + 1).await.unwrap()); +} + +#[tokio::test] +async fn an_error_in_the_log_fails_every_change_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_error_in_the_log_fails_every_change_and_changes_nothing(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_error_in_the_log_fails_every_change_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_error_in_the_log_fails_every_change_and_changes_nothing(&url).await; +} + +/// A write prunes old events after its commit, in the background, when the store is told how long to keep them. +#[tokio::test] +async fn a_write_prunes_old_events_after_its_commit() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + let plain = Meta::open(&url).await.unwrap(); + let ana = user(&plain, "ana").await; + let day = 24 * 60 * 60 * 1000i64; + for i in 0..4 { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + plain.audit_record_failure(failure, NOW - 40 * day + i * 60_000).await.unwrap(); + } + // Without a retention a write leaves them. + plain.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert_eq!(plain.audit_count_before(NOW - 30 * day).await.unwrap(), 4); + // With one, the next write starts the pruning. + let meta = plain.clone().with_audit_retention_days(30); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW + 1).await.unwrap().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while meta.audit_count_before(NOW - 30 * day).await.unwrap() > 0 { + assert!(std::time::Instant::now() < deadline, "the old events were not pruned"); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!(all_events(&meta).await.len(), 2, "the two sign-ins stay"); +} + +// ---- the migration from 0008 to 0009 (and on to 0010) ---- + +/// Drops what migrations 0009 (the audit log) and 0010 (two-factor sign-in) added and forgets they ran, which leaves the database as 0008 left it (with its rows). +async fn back_to_0008(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 9").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the audit log keeps its accounts and devices, gets an empty log that works, and a +/// database newer than the program is still refused. +async fn migrating_from_0008_adds_the_audit_log(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, old_token) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + assert_eq!(all_events(&meta).await.len(), 1); + (ana, old_token) + }; + back_to_0008(url).await; + assert_eq!(schema_version(url).await, 8); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15, "all migrations after the audit log run"); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(ana.id)); + assert!(all_events(&meta).await.is_empty(), "a new log is empty"); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await.unwrap(); + assert_eq!(all_events(&meta).await.len(), 2); + + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (16)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0008_adds_the_audit_log_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0008_adds_the_audit_log(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0008_adds_the_audit_log_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0008_adds_the_audit_log(&url).await; +} + +// ---- two-factor sign-in (migration 0010) ---- + +type Check = Box Result, UnreadableSecret> + Send + Sync>; + +/// A TOTP check that says every code is right for time step `step` (the crypto is the server's; here only the order of +/// operations and what the database does with the answer are under test). +fn right(step: i64) -> Check { + Box::new(move |_, _| Ok(Some(step))) +} + +/// A TOTP check that says every code is wrong. +fn wrong() -> Check { + Box::new(|_, _| Ok(None)) +} + +fn totp(check: &Check) -> Proof<'_> { + Proof::Totp(&**check) +} + +/// The admin's confirmation with a TOTP check that says `step` (`None`: a wrong code). +async fn confirm_with(meta: &Meta, user: UserId, step: Option, now: i64) -> Result { + let check = step.map_or_else(wrong, right); + meta.confirm_second_factor(user, Some(&totp(&check)), now).await +} + +async fn redeem_with_factor(meta: &Meta, hash: &[u8; 32], proof: Option<&Proof<'_>>, now: i64) -> Result { + meta.redeem_with_second_factor(hash, "new pc", &DeviceClient::default(), &token(), None, proof, now).await.map(|r| r.device) +} + +const FIRST_STEP: i64 = 1_000; +const SETUP_AT: i64 = NOW + 1_000; +const MINUTE: i64 = 60_000; + +/// An account with a password, two devices (`first` asked for the set-up and stays signed in, `second` is signed out by +/// turning it on), and two-factor sign-in on from [`SETUP_AT`] (its first code was step [`FIRST_STEP`]). +struct Protected { + user: UserRecord, + first: DeviceId, + second: DeviceId, + codes: Vec, +} + +impl Protected { + fn actor(&self) -> DeviceActor { + DeviceActor { user: self.user.id, device: self.first, password_hash: "$argon2id$stub".to_owned() } + } + + fn hashes(&self) -> Vec<[u8; 32]> { + self.codes.iter().map(Code::hash).collect() + } +} + +async fn protected(meta: &Meta, username: &str) -> Protected { + let user = user(meta, username).await; + let first = device(meta, &user).await; + let second = device(meta, &user).await; + let codes: Vec = (0..10).map(|_| code(INVITE_CHARS)).collect(); + let me = Protected { user, first, second, codes }; + meta.two_factor_begin_setup(&me.actor(), b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + let revoked = meta.two_factor_confirm(me.user.id, me.first, &*right(FIRST_STEP), &me.hashes(), SETUP_AT + 1).await.unwrap(); + assert_eq!(revoked, vec![me.second]); + me +} + +fn new_challenge() -> NewChallenge { + NewChallenge { id_hash: token(), client_key: "ip:203.0.113.7".to_owned() } +} + +async fn live_devices(meta: &Meta, user: &UserRecord) -> Vec { + meta.list_devices(user.id).await.unwrap().into_iter().filter(|d| !d.revoked).map(|d| d.id).collect() +} + +async fn set_up_confirmed_and_listed(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, second) = (device(meta, &ana).await, device(meta, &ana).await); + let info = meta.two_factor_info(ana.id, SETUP_AT).await.unwrap(); + assert_eq!((info.enabled_ms, info.recovery_codes_left, info.pending), (None, 0, false)); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let pairing = code(PAIRING_CHARS); + meta.create_pairing(ana.id, second, &pairing.hash(), NOW).await.unwrap(); + + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let expires = meta.two_factor_begin_setup(&actor, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + assert_eq!(expires, SETUP_AT + 10 * MINUTE); + let info = meta.two_factor_info(ana.id, SETUP_AT + 1).await.unwrap(); + assert_eq!((info.enabled_ms, info.pending), (None, true), "a set-up is not two-factor sign-in"); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + // A password sign-in still gives a device while it is only set up. + let generation = gen_of(meta, "ana").await; + let signed = + meta.sign_in(ana.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Device(_)))); + let extra = live_devices(meta, &ana).await.len(); + assert_eq!(extra, 3); + + // A wrong first code is refused and the set-up stays. + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 2).await, Err(MetaError::TwoFactorInvalid))); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let before = gen_of(meta, "ana").await; + let revoked = meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 3).await.unwrap(); + assert_eq!(revoked.len(), 2, "every other device is signed out"); + assert_eq!(live_devices(meta, &ana).await, vec![first]); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().filter(|d| d.revoked).all(|d| d.revoke_reason == Some(RevokeReason::TwoFactor))); + assert_revoke_columns_agree(&devices); + assert_eq!(gen_of(meta, "ana").await, before + 1, "what a sign-in checked before no longer counts"); + assert!(!meta.revoke_invite("ana").await.unwrap(), "the invite is gone"); + assert!(matches!(meta.redeem(&pairing.hash(), "x", &token(), None, SETUP_AT + 4).await, Err(MetaError::InvalidCode)), "pairings are deleted"); + let info = meta.two_factor_info(ana.id, SETUP_AT + 5).await.unwrap(); + assert_eq!((info.enabled_ms, info.recovery_codes_left, info.pending), (Some(SETUP_AT + 3), 10, false)); + assert!(meta.two_factor_enabled(ana.id).await.unwrap()); + assert!(meta.account(ana.id).await.unwrap().two_factor_enabled); + // It is on: a set-up is a conflict, and so is a second confirmation. + assert!(matches!(meta.two_factor_begin_setup(&actor, b"other", SETUP_AT + 6).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP + 5), &hashes, SETUP_AT + 7).await, Err(MetaError::Conflict(_)))); +} + +async fn a_set_up_belongs_to_one_device_expires_and_takes_five_tries(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, second) = (device(meta, &ana).await, device(meta, &ana).await); + let actor = |device| DeviceActor { user: ana.id, device, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor(first), b"one", SETUP_AT).await.unwrap(); + // Open on the first device: the second may not start, nor confirm (and its code is not tried at all). + assert!(matches!(meta.two_factor_begin_setup(&actor(second), b"two", SETUP_AT + 1).await, Err(MetaError::TwoFactorPending))); + assert!(matches!(meta.two_factor_confirm(ana.id, second, &*right(FIRST_STEP), &[], SETUP_AT + 1).await, Err(MetaError::Conflict(_)))); + // The first may start again; that replaces the secret and gives it five new tries. + for round in 0..2 { + meta.two_factor_begin_setup(&actor(first), b"again", SETUP_AT + 2).await.unwrap(); + for _ in 0..4 { + assert!( + matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 3).await, Err(MetaError::TwoFactorInvalid)), + "round {round}" + ); + } + assert!(meta.two_factor_info(ana.id, SETUP_AT + 3).await.unwrap().pending, "four wrong codes leave it open"); + } + // The fifth wrong code uses it up: even the right code then is a conflict, and the set-up has to start again. + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 3).await, Err(MetaError::TwoFactorInvalid))); + assert!(!meta.two_factor_info(ana.id, SETUP_AT + 3).await.unwrap().pending); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &[], SETUP_AT + 4).await, Err(MetaError::Conflict(_)))); + // After the ten minutes it ran out; another device may start one. + meta.two_factor_begin_setup(&actor(first), b"fresh", SETUP_AT + 5).await.unwrap(); + assert!(matches!( + meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &[], SETUP_AT + 5 + 10 * MINUTE).await, + Err(MetaError::Conflict(_)) + )); + assert!(!meta.two_factor_info(ana.id, SETUP_AT + 5 + 10 * MINUTE).await.unwrap().pending); + meta.two_factor_begin_setup(&actor(second), b"second's", SETUP_AT + 6 + 10 * MINUTE).await.unwrap(); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap(), "nothing was ever turned on"); +} + +/// Set-up, confirm, disable and new recovery codes check, under the account's lock, that the device is still signed in and +/// that the password that was verified is still the account's. +async fn changes_of_the_settings_recheck_the_device_and_the_password(meta: &Meta) { + let me = protected(meta, "ana").await; + let revoked_actor = DeviceActor { device: me.second, ..me.actor() }; + let stale_actor = DeviceActor { password_hash: "$argon2id$other".to_owned(), ..me.actor() }; + let hashes = me.hashes(); + for (name, actor, expect_gone) in [("revoked device", &revoked_actor, true), ("changed password", &stale_actor, false)] { + let results = [ + meta.two_factor_begin_setup(actor, b"x", SETUP_AT + 10).await.map(|_| ()), + meta.two_factor_disable(actor, &Proof::Recovery(hashes[0]), SETUP_AT + 10).await.map(|_| ()), + meta.two_factor_new_recovery_codes(actor, &totp(&right(FIRST_STEP + 50)), &hashes, SETUP_AT + 10).await, + ]; + for result in results { + if expect_gone { + assert!(matches!(result, Err(MetaError::DeviceRevoked)), "{name}: {result:?}"); + } else { + assert!(matches!(result, Err(MetaError::Conflict(_))), "{name}: {result:?}"); + } + } + } + // Confirming from a device that was signed out is refused too. + assert!(matches!(meta.two_factor_confirm(me.user.id, me.second, &*right(FIRST_STEP), &[], SETUP_AT + 11).await, Err(MetaError::DeviceRevoked))); + // Nothing changed: it is still on, with all ten codes, and no proof was spent. + let info = meta.two_factor_info(me.user.id, SETUP_AT + 12).await.unwrap(); + assert_eq!((info.enabled_ms.is_some(), info.recovery_codes_left), (true, 10)); + // A disabled account's device is gone as well. + meta.reset_password(me.user.id, "$argon2id$stub", false, SETUP_AT + 13).await.unwrap(); + assert!(matches!(meta.two_factor_begin_setup(&me.actor(), b"x", SETUP_AT + 14).await, Err(MetaError::DeviceRevoked))); +} + +async fn two_confirmations_at_once_turn_it_on_once(meta: &Meta) { + for round in 0..8 { + let ana = user(meta, &format!("ana{round}")).await; + let first = device(meta, &ana).await; + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"secret", SETUP_AT).await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..4 { + let (meta, id) = (meta.clone(), ana.id); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + tasks.push(tokio::spawn(async move { meta.two_factor_confirm(id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 1).await })); + } + let mut turned_on = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => turned_on += 1, + Err(MetaError::Conflict(_)) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(turned_on, 1, "round {round}"); + assert_eq!(meta.two_factor_info(ana.id, SETUP_AT + 2).await.unwrap().recovery_codes_left, 10, "one set of codes, round {round}"); + } +} + +/// The step of a code is accepted once, and only a later one after it. The admin's confirmation and the sign-in share it. +async fn a_time_step_is_accepted_once_and_only_later_ones_after_it(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let at = SETUP_AT + 100; + // The first code of the set-up was step 1000: it can't be used again, nor an earlier step. + for step in [FIRST_STEP, FIRST_STEP - 1, 0] { + assert!(matches!(confirm_with(meta, id, Some(step), at).await, Err(MetaError::TwoFactorInvalid)), "step {step}"); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), at).await, Ok(true))); + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), at).await, Err(MetaError::TwoFactorInvalid)), "the same step twice"); + assert!(matches!(confirm_with(meta, id, None, at).await, Err(MetaError::TwoFactorInvalid))); + // Eight requests with one code at once: exactly one gets through. + for round in 0..6 { + let step = FIRST_STEP + 10 + round; + let mut tasks = Vec::new(); + for _ in 0..8 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, Some(step), at).await })); + } + let mut accepted = 0; + for task in tasks { + match task.await.unwrap() { + Ok(true) => accepted += 1, + Err(MetaError::TwoFactorInvalid | MetaError::TwoFactorLocked { .. }) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(accepted, 1, "round {round}: one time step, one use"); + } + // An account without two-factor sign-in asks for nothing, and a missing proof for one with it is refused unspent. + let bob = user(meta, "bob").await; + assert!(matches!(meta.confirm_second_factor(bob.id, None, at).await, Ok(false))); + assert!(matches!(confirm_with(meta, bob.id, Some(1), at).await, Ok(false))); + assert!(matches!(meta.confirm_second_factor(id, None, at).await, Err(MetaError::TwoFactorInvalid))); +} + +async fn recovery_codes_work_once_and_new_ones_replace_them(meta: &Meta) { + let me = protected(meta, "ana").await; + let (id, hashes) = (me.user.id, me.hashes()); + let at = SETUP_AT + 100; + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[0])), at).await, Ok(true))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[0])), at).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(code(INVITE_CHARS).hash())), at).await, Err(MetaError::TwoFactorInvalid))); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 9); + // The same code at the same moment: one wins. + let mut tasks = Vec::new(); + for _ in 0..6 { + let (meta, hash) = (meta.clone(), hashes[1]); + tasks.push(tokio::spawn(async move { meta.confirm_second_factor(id, Some(&Proof::Recovery(hash)), at).await })); + } + let mut accepted = 0; + for task in tasks { + if matches!(task.await.unwrap(), Ok(true)) { + accepted += 1; + } + } + assert_eq!(accepted, 1); + // New codes: the old ones, used or not, are dead, and the new ones all live. + let new: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 1)), &new, at).await.unwrap(); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 10); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[5])), at).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(new[3])), at).await, Ok(true))); + // A repeated code can't be stored (the primary key), and then nothing changes: the old set is still there. + let again = vec![new[0]; 2]; + assert!(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 2)), &again, at).await.is_err()); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 9); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(new[7])), at).await, Ok(true))); + // A wrong proof for new codes changes none. + assert!(matches!(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&wrong()), &new, at).await, Err(MetaError::TwoFactorInvalid))); +} + +/// Ten wrong proofs in a row lock the second factor for 15 minutes; each wrong one after that doubles it, to 24 hours; a +/// right one resets all of it. While locked even a right proof is refused and nothing is counted. +async fn wrong_proofs_lock_for_a_while_double_and_reset_on_success(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let mut now = SETUP_AT + 100; + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}"); + } + // The tenth locks. + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1_000).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 15 * MINUTE - 1_000), "{locked:?}"); + // Every path refuses while it lasts: recovery codes, disable, new codes, a challenge. + let actor = me.actor(); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(me.hashes()[0])), now + 2).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(matches!(meta.two_factor_disable(&actor, &Proof::Recovery(me.hashes()[1]), now + 2).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(matches!( + meta.two_factor_new_recovery_codes(&actor, &totp(&right(5)), &me.hashes(), now + 2).await, + Err(MetaError::TwoFactorLocked { .. }) + )); + let generation = gen_of(meta, "ana").await; + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, now + 2).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[2]), &token(), now + 3).await, + Err(MetaError::TwoFactorLocked { .. }) + )); + assert_eq!(meta.two_factor_info(id, now + 2).await.unwrap().recovery_codes_left, 10, "a refused check uses nothing up"); + // Over, a wrong proof locks again for twice as long: 30 minutes, then 60. + now += 15 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 30 * MINUTE - 1), "{locked:?}"); + now += 30 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!( + confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await, + Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 60 * MINUTE - 1 + )); + // A right proof after it ran out resets the counters: nine more wrong ones do not lock. + now += 60 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), now).await, Ok(true))); + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "after the reset, wrong proof {n}"); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 2), now).await, Ok(true)), "nine are not ten"); + // The command line lifts any lock. + for _ in 0..10 { + let _ = confirm_with(meta, id, None, now).await; + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 3), now + 1).await, Err(MetaError::TwoFactorLocked { .. }))); + meta.reset_two_factor(id, now + 2).await.unwrap(); + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 3), now + 3).await, Ok(false)), "off, and nothing is locked"); +} + +/// The counters of wrong proofs are written outside the transaction that refused them, so requests at the same moment all +/// count: twenty wrong proofs lock it. +async fn wrong_proofs_at_the_same_moment_all_count(meta: &Meta) { + let me = protected(meta, "ana").await; + let (id, now) = (me.user.id, SETUP_AT + 100); + let mut tasks = Vec::new(); + for _ in 0..12 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, None, now).await })); + } + for task in tasks { + assert!(matches!(task.await.unwrap(), Err(MetaError::TwoFactorInvalid | MetaError::TwoFactorLocked { .. }))); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await, Err(MetaError::TwoFactorLocked { .. }))); +} + +async fn a_challenge_is_made_counted_used_once_and_carries_the_device(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let generation = gen_of(meta, "ana").await; + let client = DeviceClient::new(Some("desktop"), Some("linux"), Some("1.2")).unwrap(); + let chal = new_challenge(); + let made = meta.sign_in(id, ("$argon2id$stub", generation), "work laptop", &client, &token(), &chal, SETUP_AT + 10).await.unwrap(); + assert_eq!(made, Some(SignIn::Challenge { expires_ms: SETUP_AT + 10 + 5 * MINUTE })); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "no device without the second factor"); + // The old call, for a caller that doesn't know, refuses an account that has two-factor sign-in. + assert!(matches!( + meta.sign_in_device(id, ("$argon2id$stub", generation), "x", &client, &token(), SETUP_AT + 10).await, + Err(MetaError::TwoFactorRequired) + )); + assert_eq!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), Some((id, "ana".to_owned()))); + // Four wrong codes are counted, the challenge stays, and the right one then works (a fifth wrong would delete it). + for _ in 0..4 { + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), SETUP_AT + 12).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + } + let tok = token(); + let done = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &tok, SETUP_AT + 13).await.unwrap(); + assert_eq!(done.user, id); + let auth = meta.device_by_token(&tok).await.unwrap().unwrap(); + assert_eq!(auth.device, done.device); + let made = meta.list_devices(id).await.unwrap().into_iter().find(|d| d.id == done.device).unwrap(); + assert_eq!((made.name.as_str(), made.client), ("work laptop", client.clone()), "the device is what the first call said"); + // Once: the same challenge again is gone, for a right proof as for a wrong one. + for proof in [totp(&right(FIRST_STEP + 2)), totp(&wrong())] { + assert!(matches!(meta.complete_second_factor(&chal.id_hash, &proof, &token(), SETUP_AT + 14).await, Err(MetaError::ChallengeGone))); + } + assert!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 14).await.unwrap().is_none()); + + // Five wrong codes delete the challenge: the right one is too late. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 20).await.unwrap(); + for _ in 0..5 { + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), SETUP_AT + 21).await, + Err(MetaError::TwoFactorInvalid) + )); + } + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 22).await, + Err(MetaError::ChallengeGone) + )); + // One that ran out is as gone, and an unknown one. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 30).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 30 + 5 * MINUTE).await, + Err(MetaError::ChallengeGone) + )); + assert!(matches!( + meta.complete_second_factor(&token(), &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 30).await, + Err(MetaError::ChallengeGone) + )); + // The recovery code finishes a sign-in too, once. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 40).await.unwrap(); + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[4]), &token(), SETUP_AT + 41).await.unwrap(); + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 42).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[4]), &token(), SETUP_AT + 43).await, + Err(MetaError::TwoFactorInvalid) + )); +} + +/// Each account keeps at most five open challenges (the oldest goes), and expired ones are deleted by the next sign-in. +async fn an_account_keeps_five_open_challenges_and_expired_ones_go(meta: &Meta) { + let me = protected(meta, "ana").await; + let generation = gen_of(meta, "ana").await; + let mut made = Vec::new(); + for n in 0..7 { + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10 + n).await.unwrap(); + made.push(chal); + } + for (n, chal) in made.iter().enumerate() { + let open = meta.challenge_owner(&chal.id_hash, SETUP_AT + 20).await.unwrap().is_some(); + assert_eq!(open, n >= 2, "challenge {n}: the two oldest were replaced"); + } + // The next sign-in after they ran out deletes them (nothing else would). + let later = SETUP_AT + 10 + 5 * MINUTE + 100; + let fresh = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &fresh, later).await.unwrap(); + assert!(meta.challenge_owner(&fresh.id_hash, later).await.unwrap().is_some()); + // A stale password hash makes neither a challenge nor a device. + let stale = new_challenge(); + assert!( + meta.sign_in(me.user.id, ("$argon2id$old", generation), "pc", &DeviceClient::default(), &token(), &stale, later).await.unwrap().is_none() + ); + assert!( + meta.sign_in(me.user.id, ("$argon2id$stub", generation - 1), "pc", &DeviceClient::default(), &token(), &stale, later) + .await + .unwrap() + .is_none() + ); + assert!(meta.challenge_owner(&stale.id_hash, later).await.unwrap().is_none()); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first]); +} + +/// A challenge made before a password change, a reset, a turn-off, a reset of the second factor or a disable gives nothing after. +async fn a_challenge_dies_with_every_change_of_the_account(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for round in 0..6 { + let me = protected(meta, &format!("ana{round}")).await; + let generation = gen_of(meta, &format!("ana{round}")).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + let at = SETUP_AT + 11; + match round { + 0 => drop(meta.change_password(me.user.id, "$argon2id$stub", "$argon2id$new", me.first, at).await.unwrap()), + 1 => drop(meta.reset_password(me.user.id, "$argon2id$new", true, at).await.unwrap()), + 2 => drop(meta.two_factor_disable(&me.actor(), &Proof::Recovery(me.hashes()[0]), at).await.unwrap()), + 3 => drop(meta.reset_two_factor(me.user.id, at).await.unwrap()), + 4 => drop(meta.admin_disable_user(&actor, me.user.id, at).await.unwrap()), + _ => drop(meta.admin_reset_two_factor(&actor, me.user.id, at).await.unwrap()), + } + assert!(meta.challenge_owner(&chal.id_hash, at).await.unwrap().is_none(), "round {round}"); + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[1]), &token(), at + 1).await; + assert!(matches!(result, Err(MetaError::ChallengeGone)), "round {round}: {result:?}"); + assert!(live_devices(meta, &me.user).await.iter().all(|d| *d == me.first), "round {round}: no new device"); + } +} + +async fn two_completions_of_one_challenge_give_one_device(meta: &Meta) { + for round in 0..8 { + let me = protected(meta, &format!("ana{round}")).await; + let generation = gen_of(meta, &format!("ana{round}")).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..6 { + let (meta, hash, recovery) = (meta.clone(), chal.id_hash, me.hashes()[job % 2]); + tasks.push(tokio::spawn(async move { + // The same TOTP step for some, two recovery codes for the others: a code is no way past the challenge. + if job % 2 == 0 { + meta.complete_second_factor(&hash, &totp(&right(FIRST_STEP + 1)), &token(), SETUP_AT + 11).await + } else { + meta.complete_second_factor(&hash, &Proof::Recovery(recovery), &token(), SETUP_AT + 11).await + } + })); + } + let mut done = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => done += 1, + Err(MetaError::ChallengeGone | MetaError::TwoFactorInvalid) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(done, 1, "round {round}"); + assert_eq!(live_devices(meta, &me.user).await.len(), 2, "round {round}: the first device and one more"); + } +} + +/// Completing a sign-in and a password change, a reset, a turn-off or the command line's reset at the same moment, over and +/// over: no database error, and once the change is done no device of the account but the one that asked is signed in, +/// whichever came first. +async fn completions_and_changes_of_the_account_at_the_same_moment_leave_no_stray_device(meta: &Meta) { + for round in 0..12 { + let name = format!("ana{round}"); + let me = protected(meta, &name).await; + let generation = gen_of(meta, &name).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + // A password change keeps the device that asked; a reset of the second factor or of the password signs out all. + let all_gone = round % 3 != 0; + let (completing, changing) = { + let (m1, m2) = (meta.clone(), meta.clone()); + let (hash, id, first, recovery) = (chal.id_hash, me.user.id, me.first, me.hashes()); + let completing = + tokio::spawn( + async move { m1.complete_second_factor(&hash, &Proof::Recovery(recovery[0]), &token(), SETUP_AT + 11).await.map(|_| ()) }, + ); + let changing = tokio::spawn(async move { + match round % 3 { + 0 => m2.change_password(id, "$argon2id$stub", "$argon2id$new", first, SETUP_AT + 12).await.map(|_| ()), + 1 => m2.reset_two_factor(id, SETUP_AT + 12).await.map(|_| ()), + _ => m2.reset_password(id, "$argon2id$new", false, SETUP_AT + 12).await.map(|_| ()), + } + }); + (completing, changing) + }; + let (completed, changed) = (completing.await.unwrap(), changing.await.unwrap()); + assert!(matches!(completed, Ok(()) | Err(MetaError::ChallengeGone)), "round {round}: {completed:?}"); + changed.unwrap_or_else(|e| panic!("round {round}: {e:?}")); + let live = live_devices(meta, &me.user).await; + if all_gone { + assert!(live.is_empty(), "round {round}: {live:?}"); + } else { + assert!(live.iter().all(|d| *d == me.first), "round {round}: {live:?}"); + } + assert_revoke_columns_agree(&meta.list_devices(me.user.id).await.unwrap()); + } +} + +/// Turning two-factor sign-in on or off while a password sign-in is under way: a device made without a second factor +/// before it is signed out by the turn-on, and one checked before it makes no device at all. +async fn turning_two_factor_on_and_off_at_the_same_moment_as_a_sign_in(meta: &Meta) { + for round in 0..8 { + let name = format!("on{round}"); + let ana = user(meta, &name).await; + let first = device(meta, &ana).await; + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"cipher", SETUP_AT).await.unwrap(); + let generation = gen_of(meta, &name).await; + let (m1, m2) = (meta.clone(), meta.clone()); + let id = ana.id; + let signing = tokio::spawn(async move { + m1.sign_in(id, ("$argon2id$stub", generation), "late", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 5).await + }); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let enabling = tokio::spawn(async move { m2.two_factor_confirm(id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 5).await }); + let (signed, enabled) = (signing.await.unwrap(), enabling.await.unwrap()); + enabled.unwrap(); + assert!(matches!(signed, Ok(None | Some(SignIn::Device(_)))), "round {round}: a sign-in that checked before the turn-on makes no challenge"); + assert_eq!(live_devices(meta, &ana).await, vec![first], "round {round}: only the device that turned it on"); + + // And off: a challenge made under it works once, but a device it made is signed out by the turn-off. + let generation = gen_of(meta, &name).await; + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "x", &DeviceClient::default(), &token(), &chal, SETUP_AT + 6).await.unwrap(); + let (m1, m2) = (meta.clone(), meta.clone()); + let (hash, other_code) = (chal.id_hash, code(INVITE_CHARS).hash()); + let completing = + tokio::spawn(async move { m1.complete_second_factor(&hash, &Proof::Recovery(other_code), &token(), SETUP_AT + 7).await.map(|_| ()) }); + let actor = DeviceActor { user: id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let disabling = tokio::spawn(async move { m2.two_factor_disable(&actor, &totp(&right(FIRST_STEP + 1)), SETUP_AT + 7).await.map(|_| ()) }); + let completed = completing.await.unwrap(); + disabling.await.unwrap().unwrap(); + assert!(matches!(completed, Err(MetaError::TwoFactorInvalid | MetaError::ChallengeGone)), "round {round}: {completed:?}"); + assert_eq!(live_devices(meta, &ana).await, vec![first]); + } +} + +async fn an_invite_to_a_protected_account_needs_the_second_factor_in_the_same_transaction(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), SETUP_AT + 10, INVITE_TTL_MS).await.unwrap(); + // No second factor: refused and the invite is kept (not used up), and nothing is counted. + assert!(matches!(redeem_with_factor(meta, &invite.hash(), None, SETUP_AT + 11).await, Err(MetaError::TwoFactorRequired))); + let old_call = meta.redeem_with(&invite.hash(), "x", &DeviceClient::default(), &token(), None, SETUP_AT + 11).await; + assert!(matches!(old_call, Err(MetaError::TwoFactorRequired))); + // Wrong factors: refused, the invite stays, and nine of them leave the counters just short of the lock. + for n in 1..=9 { + let result = redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), SETUP_AT + 12).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "wrong factor {n}: {result:?}"); + } + assert_eq!(meta.invite_needs_password(&invite.hash(), SETUP_AT + 12).await.unwrap(), Some(false), "the invite is still there"); + // The tenth locks, with the invite still there, and a right one is refused while it lasts. + assert!(matches!(redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), SETUP_AT + 12).await, Err(MetaError::TwoFactorInvalid))); + let locked = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP + 1))), SETUP_AT + 13).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + // A step that was used is wrong, even after the lock. + let after = SETUP_AT + 12 + 15 * MINUTE + 1; + let replay = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP))), after).await; + assert!(matches!(replay, Err(MetaError::TwoFactorInvalid)), "the set-up's own step: {replay:?}"); + assert_eq!(meta.invite_needs_password(&invite.hash(), after).await.unwrap(), Some(false), "still there"); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "no device was made by any of it"); + let _ = id; + + // A fresh account: a new step is right, and the invite, the step and the device go together. + let bea = protected(meta, "bea").await; + let invite = code(INVITE_CHARS); + meta.create_invite("bea", &invite.hash(), SETUP_AT + 10, INVITE_TTL_MS).await.unwrap(); + let device = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP + 1))), SETUP_AT + 20).await.unwrap(); + assert_eq!(live_devices(meta, &bea.user).await.len(), 2); + assert!(live_devices(meta, &bea.user).await.contains(&device)); + assert_eq!(meta.invite_needs_password(&invite.hash(), SETUP_AT + 21).await.unwrap(), None, "used up with the device"); + // Nobody else can use that step now (sign-in and join share the counter). + assert!(matches!(confirm_with(meta, bea.user.id, Some(FIRST_STEP + 1), SETUP_AT + 22).await, Err(MetaError::TwoFactorInvalid))); + // A pairing needs no second factor: only a signed-in device can make one. + let pairing = code(PAIRING_CHARS); + meta.create_pairing(bea.user.id, bea.first, &pairing.hash(), SETUP_AT + 30).await.unwrap(); + meta.redeem_with(&pairing.hash(), "paired", &DeviceClient::default(), &token(), None, SETUP_AT + 31).await.unwrap(); +} + +async fn a_reset_code_for_a_protected_account_needs_the_factor_unless_it_turns_it_off(meta: &Meta) { + let (_, admin) = admin_actor(meta, "root").await; + let admin = AdminActor { two_factor: false, ..admin }; + let me = protected(meta, "ana").await; + let (id, stale) = (me.user.id, me.hashes()); + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), false, SETUP_AT + 10, RESET_TTL_MS).await.unwrap(); + assert!(matches!( + meta.redeem_with(&reset.hash(), "pc", &DeviceClient::default(), &token(), Some("$argon2id$fresh"), SETUP_AT + 11).await, + Err(MetaError::TwoFactorRequired) + )); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub", "nothing changed"); + assert!(matches!( + meta.redeem_with_second_factor( + &reset.hash(), + "pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$fresh"), + Some(&totp(&wrong())), + SETUP_AT + 11 + ) + .await, + Err(MetaError::TwoFactorInvalid) + )); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "the failed redeem signed nothing out"); + let first_done = meta + .redeem_with_second_factor( + &reset.hash(), + "pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$fresh"), + Some(&Proof::Recovery(stale[0])), + SETUP_AT + 12, + ) + .await + .unwrap(); + let done = &first_done; + assert_eq!(done.revoked, vec![me.first]); + assert!(meta.two_factor_enabled(id).await.unwrap(), "the code did not turn two-factor sign-in off"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$fresh"); + + // A code made to turn it off asks for no second factor, turns it off and signs everything out. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), true, SETUP_AT + 20, RESET_TTL_MS).await.unwrap(); + let done = meta.redeem_with(&reset.hash(), "pc2", &DeviceClient::default(), &token(), Some("$argon2id$newer"), SETUP_AT + 21).await.unwrap(); + assert_eq!(done.revoked, vec![first_done.device], "the computer of the first redeem is signed out with the rest"); + assert!(!meta.two_factor_enabled(id).await.unwrap()); + assert_eq!(meta.two_factor_info(id, SETUP_AT + 22).await.unwrap().recovery_codes_left, 0); + assert_eq!(live_devices(meta, &me.user).await, vec![done.device], "the redeeming computer is the only one signed in"); + // An account without two-factor sign-in redeems a reset code as it always did, whatever the flag. + let plain = user(meta, "plain").await; + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, plain.id, &reset.hash(), false, SETUP_AT + 30, RESET_TTL_MS).await.unwrap(); + meta.redeem_with(&reset.hash(), "pc", &DeviceClient::default(), &token(), Some("$argon2id$fresh"), SETUP_AT + 31).await.unwrap(); +} + +/// The admin's own two-factor state is compared again in every change, and resetting someone's two-factor sign-in, or another +/// admin's password, needs the acting admin's own. +async fn admins_two_factor_is_checked_in_the_change_and_resets_need_it(meta: &Meta) { + let (root, plain_actor) = admin_actor(meta, "root").await; + let (other, _) = admin_actor(meta, "other").await; + let member = protected(meta, "member").await; + let (boss_admin, boss_actor) = { + let (u, a) = admin_actor(meta, "boss").await; + (u, a) + }; + // Turn two-factor sign-in on for the acting admin "boss" (device `device_of(&boss_actor)`). + let actor_for = DeviceActor { user: boss_admin.id, device: device_of(&boss_actor), password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&actor_for, b"cipher", SETUP_AT).await.unwrap(); + meta.two_factor_confirm(boss_admin.id, device_of(&boss_actor), &*right(FIRST_STEP), &[code(INVITE_CHARS).hash()], SETUP_AT + 1).await.unwrap(); + // A confirmation that was given when it was off, and a change after it turned on, conflict (and the other way round). + let hash = code(INVITE_CHARS).hash(); + let result = meta.admin_issue_invite(&boss_actor, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "{result:?}"); + let with_factor = AdminActor { two_factor: true, ..boss_actor.clone() }; + let result = meta.admin_issue_invite(&with_factor, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "it has a password: {result:?}"); + let result = + meta.admin_issue_invite(&AdminActor { two_factor: true, ..plain_actor.clone() }, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "root has none: {result:?}"); + + // Resetting a member's two-factor sign-in: without the admin's own it works for a member, not for an admin. + let result = meta.admin_reset_two_factor(&plain_actor, other.id, SETUP_AT + 3).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "another admin, no two-factor of one's own: {result:?}"); + let result = meta.admin_issue_reset(&plain_actor, other.id, &hash, false, SETUP_AT + 3, RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, root.id, SETUP_AT + 3).await, Err(MetaError::Invalid(_))), "not one's own"); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, UserId(uuid::Uuid::now_v7()), SETUP_AT + 3).await, Err(MetaError::NotFound(_)))); + // An admin with it may reset another admin's password, and that admin's two-factor sign-in (once they have one). + meta.admin_issue_reset(&with_factor, other.id, &hash, false, SETUP_AT + 4, RESET_TTL_MS).await.unwrap(); + let result = meta.admin_reset_two_factor(&with_factor, other.id, SETUP_AT + 4).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "the other admin has none: {result:?}"); + // A member's: everything of it goes, every device is signed out, the lock goes with it. + let revoked = meta.admin_reset_two_factor(&plain_actor, member.user.id, SETUP_AT + 5).await.unwrap(); + assert_eq!(revoked, vec![member.first]); + assert!(!meta.two_factor_enabled(member.user.id).await.unwrap()); + assert!(live_devices(meta, &member.user).await.is_empty()); + let devices = meta.list_devices(member.user.id).await.unwrap(); + assert!(devices.iter().any(|d| d.revoke_reason == Some(RevokeReason::TwoFactor))); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, member.user.id, SETUP_AT + 6).await, Err(MetaError::Conflict(_))), "twice"); + let row = admin_view(meta, boss_admin.id).await; + assert!(row.two_factor_enabled); + assert!(!admin_view(meta, member.user.id).await.two_factor_enabled); +} + +async fn the_command_line_resets_two_factor_and_the_account_shows_it(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + assert!(meta.account(id).await.unwrap().two_factor_enabled); + assert!(admin_view(meta, id).await.two_factor_enabled); + let generation = gen_of(meta, "ana").await; + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 5).await.unwrap(); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), SETUP_AT + 5, INVITE_TTL_MS).await.unwrap(); + let revoked = meta.reset_two_factor(id, SETUP_AT + 6).await.unwrap(); + assert_eq!(revoked, vec![me.first]); + assert!(!meta.account(id).await.unwrap().two_factor_enabled); + assert!(!admin_view(meta, id).await.two_factor_enabled); + assert!(!meta.revoke_invite("ana").await.unwrap()); + assert!(matches!(meta.reset_two_factor(id, SETUP_AT + 7).await, Err(MetaError::Conflict(_))), "nothing to reset: nothing is signed out"); + assert!(matches!(meta.reset_two_factor(UserId(uuid::Uuid::now_v7()), SETUP_AT + 7).await, Err(MetaError::NotFound(_)))); + // After it a password sign-in gives a device again, and the account can set it up anew. + let generation = gen_of(meta, "ana").await; + let signed = + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 8).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Device(_)))); +} + +/// The key tools read the ciphertexts and replace one only if it is still the one they read. +async fn secrets_are_listed_and_replaced_only_if_unchanged(meta: &Meta) { + let me = protected(meta, "ana").await; + let other = user(meta, "bob").await; + let dev = device(meta, &other).await; + let actor = DeviceActor { user: other.id, device: dev, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"cipher-pending", SETUP_AT).await.unwrap(); + let listed = meta.two_factor_secrets(SETUP_AT + 1).await.unwrap(); + assert_eq!(listed.len(), 2); + let ana = listed.iter().find(|s| s.user == me.user.id).unwrap(); + assert_eq!((ana.active.as_deref(), ana.pending.as_deref()), (Some(b"cipher-of-the-secret".as_slice()), None)); + let bob = listed.iter().find(|s| s.user == other.id).unwrap(); + assert_eq!((bob.active.as_deref(), bob.pending.as_deref()), (None, Some(b"cipher-pending".as_slice()))); + let later = meta.two_factor_secrets(SETUP_AT + 11 * MINUTE).await.unwrap(); + assert!(later.iter().find(|s| s.user == other.id).is_some_and(|s| s.pending.is_none()), "an expired set-up is not listed"); + assert!(meta.two_factor_replace_ciphertext(me.user.id, false, b"cipher-of-the-secret", b"new-cipher").await.unwrap()); + assert!( + !meta.two_factor_replace_ciphertext(me.user.id, false, b"cipher-of-the-secret", b"newer").await.unwrap(), + "no longer the one that was read" + ); + assert!(meta.two_factor_replace_ciphertext(other.id, true, b"cipher-pending", b"new-pending").await.unwrap()); + let listed = meta.two_factor_secrets(SETUP_AT + 1).await.unwrap(); + assert_eq!(listed.iter().find(|s| s.user == me.user.id).unwrap().active.as_deref(), Some(b"new-cipher".as_slice())); + // A secret that isn't the account's is never touched. + assert!(!meta.two_factor_replace_ciphertext(UserId(uuid::Uuid::now_v7()), false, b"x", b"y").await.unwrap()); +} + +/// Wrong proofs are counted under the lock that checked them: of many at once on one challenge exactly five are counted (the +/// fifth deletes it, the rest find it gone), and they all count against the account's ten. +async fn parallel_wrong_proofs_on_one_challenge_count_exactly_five(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let generation = gen_of(meta, "ana").await; + for round in 0..4 { + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10 + round).await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..14 { + let (meta, hash) = (meta.clone(), chal.id_hash); + tasks.push(tokio::spawn(async move { meta.complete_second_factor(&hash, &totp(&wrong()), &token(), SETUP_AT + 20).await.map(|_| ()) })); + } + let (mut counted, mut gone) = (0, 0); + for task in tasks { + match task.await.unwrap() { + Err(MetaError::TwoFactorInvalid) => counted += 1, + Err(MetaError::ChallengeGone) => gone += 1, + Err(MetaError::TwoFactorLocked { .. }) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 21).await.unwrap().is_none(), "round {round}: five wrong proofs delete it"); + // Locked rounds stop counting; until then exactly five per challenge, never six. + if round == 0 { + assert_eq!((counted, gone), (5, 9), "round {round}"); + } + assert!(counted <= 5, "round {round}: {counted} proofs were checked against one challenge"); + if meta.two_factor_locked_for(id, SETUP_AT + 21).await.unwrap().is_some() { + break; + } + } +} + +/// Many wrong proofs at once, crossing ten: the lock is set by exactly the tenth, for exactly 15 minutes, and the ones after it +/// find it locked and count nothing (so it never jumps to 30). +async fn parallel_wrong_proofs_lock_for_exactly_fifteen_minutes(meta: &Meta) { + for round in 0..6 { + let me = protected(meta, &format!("ana{round}")).await; + let (id, now) = (me.user.id, SETUP_AT + 100); + let mut tasks = Vec::new(); + for _ in 0..24 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, None, now).await })); + } + let (mut wrong_ones, mut locked) = (0, 0); + for task in tasks { + match task.await.unwrap() { + Err(MetaError::TwoFactorInvalid) => wrong_ones += 1, + Err(MetaError::TwoFactorLocked { .. }) => locked += 1, + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!((wrong_ones, locked), (10, 14), "round {round}"); + assert_eq!(meta.two_factor_locked_for(id, now).await.unwrap(), Some(15 * MINUTE), "round {round}: 15 minutes, not 30"); + } +} + +// ---- the audit log of two-factor sign-in ---- + +/// The events of `action` in the log, oldest first (an aggregated row counts as one). +async fn events_of(meta: &Meta, action: AuditAction) -> Vec { + let mut events: Vec = all_events(meta).await.into_iter().filter(|e| e.action == action).collect(); + events.reverse(); + events +} + +/// Turning it on, new recovery codes and turning it off each write their event in their own transaction, as the account's own +/// act from the device that asked; a wrong proof on these routes writes nothing, and a recovery code that is used up writes +/// `auth.recovery_code_used` with the change it served. +async fn two_factor_changes_write_their_events(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, _second) = (device(meta, &ana).await, device(meta, &ana).await); + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let mut seen = Seen::new(meta).await; + let hashes: Vec<[u8; 32]> = (0..10).map(|_| code(INVITE_CHARS).hash()).collect(); + + meta.two_factor_begin_setup(&actor, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + assert!(seen.actions().await.is_empty(), "a set-up is not a change"); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &hashes, SETUP_AT + 1).await, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "a wrong first code writes nothing"); + meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 2).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.at_ms), + (AuditAction::AccountTwoFactorEnabled, AuditOutcome::Success, Some(ana.id), Some(first), Some(ana.id.0), SETUP_AT + 2) + ); + assert_eq!(e.target_kind.as_str(), "user"); + assert_eq!(details(&e), json("{}")); + + // New recovery codes: a wrong code writes nothing (the lock it can cause would), a right one the event. + let result = meta.two_factor_new_recovery_codes(&actor, &totp(&wrong()), &hashes, SETUP_AT + 3).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "a wrong proof on a signed-in route is not written one by one"); + meta.two_factor_new_recovery_codes(&actor, &totp(&right(FIRST_STEP + 1)), &hashes, SETUP_AT + 4).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AccountRecoveryCodesRegenerated, Some(ana.id), Some(first), Some(ana.id.0)) + ); + assert_eq!(details(&e), json("{}")); + + // Turning it off with a recovery code: the change and the code's use, from the same device. + meta.two_factor_disable(&actor, &Proof::Recovery(hashes[0]), SETUP_AT + 5).await.unwrap(); + assert_eq!(seen.actions().await, ["account.two_factor_disabled", "auth.recovery_code_used"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device, used.target_id, used.at_ms), (Some(ana.id), Some(first), Some(ana.id.0), SETUP_AT + 5)); + assert_eq!(details(used), json("{}"), "never the code"); + let off = &events_of(meta, AuditAction::AccountTwoFactorDisabled).await[0]; + assert_eq!((off.actor_user, off.actor_device), (Some(ana.id), Some(first))); + // Off: nothing to turn off, nothing written. + assert!(matches!(meta.two_factor_disable(&actor, &totp(&right(FIRST_STEP + 2)), SETUP_AT + 6).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); +} + +/// A sign-in that made a device is `auth.login` whichever way it went (no second factor, a code, a recovery code); the +/// challenge writes nothing; a wrong second step is one aggregated `auth.second_factor_failed` row per account and minute, +/// written after the answer is decided. +async fn two_factor_sign_in_writes_login_and_counts_failures(meta: &Meta) { + let me = protected(meta, "ana").await; + let plain = user(meta, "bob").await; + let mut seen = Seen::new(meta).await; + let client = DeviceClient::new(Some("desktop"), Some("Windows"), Some("1.0")).unwrap(); + let now = SETUP_AT + 100; + + // No two-factor sign-in: a login at once. + let generation = gen_of(meta, "bob").await; + let signed = meta.sign_in(plain.id, ("$argon2id$stub", generation), "pc", &client, &token(), &new_challenge(), now).await.unwrap(); + let Some(SignIn::Device(bobs)) = signed else { panic!("{signed:?}") }; + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthLogin, AuditOutcome::Success, Some(plain.id), Some(bobs), Some(*bobs.as_uuid())) + ); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"windows"}"#)); + + // Two-factor sign-in: the challenge is no login. + let generation = gen_of(meta, "ana").await; + let chal = new_challenge(); + let signed = meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Challenge { .. }))); + assert!(seen.actions().await.is_empty(), "no one is signed in yet"); + // A wrong code: one failure row, no login, the actor unknown (nobody proved who they were). + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), now + 1).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.count), + (AuditAction::AuthSecondFactorFailed, AuditOutcome::Failure, None, None, Some(me.user.id.0), 1) + ); + assert_eq!(details(&e), json(r#"{"step":"totp"}"#)); + // More in the same minute make the same row count up (a recovery code that is wrong too). + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(code(INVITE_CHARS).hash()), &token(), now + 2).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "the row was written already; only its count goes up"); + let rows = events_of(meta, AuditAction::AuthSecondFactorFailed).await; + assert_eq!((rows.len(), rows[0].count), (1, 2)); + // A right code: the login of the new device. + let done = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), now + 3).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthLogin, Some(me.user.id), Some(done.device), Some(*done.device.as_uuid())) + ); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"windows"}"#)); + + // A recovery code: the login and the code's use, the device the same. + let chal = new_challenge(); + let generation = gen_of(meta, "ana").await; + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now + 4).await.unwrap(); + let done = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[0]), &token(), now + 5).await.unwrap(); + assert_eq!(seen.actions().await, ["auth.login", "auth.recovery_code_used"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device, used.target_id), (Some(me.user.id), Some(done.device), Some(me.user.id.0))); + // The same code again is a wrong recovery code: a failure, not a use. + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now + 6).await.unwrap(); + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[0]), &token(), now + 7).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert_eq!(events_of(meta, AuditAction::AuthRecoveryCodeUsed).await.len(), 1); + assert_eq!(events_of(meta, AuditAction::AuthSecondFactorFailed).await[0].count, 3); +} + +/// The lock writes `auth.two_factor_locked` once, when it is set or lengthened, and never for a refusal while it holds; wrong +/// proofs on a signed-in route or an admin's confirmation count towards it without an event of their own. +async fn the_lock_writes_one_event_when_it_is_set(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let mut seen = Seen::new(meta).await; + let mut now = SETUP_AT + 100; + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}"); + } + assert!(seen.actions().await.is_empty(), "nine wrong proofs are not written one by one"); + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.at_ms), + (AuditAction::AuthTwoFactorLocked, AuditOutcome::Success, None, None, Some(id.0), now) + ); + assert_eq!(details(&e), json("{}")); + // Refused while it holds, by every path: no further event. + for offset in 1..4 { + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + offset).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + } + assert!(matches!(meta.two_factor_disable(&me.actor(), &Proof::Recovery(me.hashes()[1]), now + 5).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(seen.actions().await.is_empty(), "a refusal under the lock writes nothing"); + // Over, one more wrong proof sets a longer lock: a new event. + now += 15 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!((e.action, e.at_ms), (AuditAction::AuthTwoFactorLocked, now)); + assert_eq!(events_of(meta, AuditAction::AuthTwoFactorLocked).await.len(), 2); + assert!(events_of(meta, AuditAction::AuthSecondFactorFailed).await.is_empty(), "these are not sign-in steps"); +} + +/// The second factor at a join: a wrong one is one failure row and the invite stays; a recovery code used for it is written +/// with the redemption; a reset code made with `reset_two_factor` takes it away and the redemption says so, by the account that +/// redeemed it (no admin acts at that moment). +async fn a_join_writes_its_two_factor_events(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let (_, admin) = admin_actor(meta, "root").await; + let mut seen = Seen::new(meta).await; + let now = SETUP_AT + 100; + + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + assert!(matches!(redeem_with_factor(meta, &invite.hash(), None, now + 1).await, Err(MetaError::TwoFactorRequired))); + assert!(seen.actions().await.is_empty(), "asking for the factor writes nothing"); + assert!(matches!(redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), now + 2).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.target_id, e.count), + (AuditAction::AuthSecondFactorFailed, AuditOutcome::Failure, None, Some(id.0), 1) + ); + assert_eq!(details(&e), json(r#"{"step":"totp"}"#)); + // With a recovery code: the redemption and the code's use, by the new device. + let device = redeem_with_factor(meta, &invite.hash(), Some(&Proof::Recovery(me.hashes()[0])), now + 3).await.unwrap(); + assert_eq!(seen.actions().await, ["auth.recovery_code_used", "join.invite_redeemed"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device), (Some(id), Some(device))); + + // A reset code without the removal of two-factor: the right code is needed; the event has no such detail. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), false, now + 4, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let redeemed = meta + .redeem_with_second_factor( + &reset.hash(), + "new pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$new"), + Some(&totp(&right(FIRST_STEP + 1))), + now + 5, + ) + .await + .unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinResetRedeemed, Some(id), Some(redeemed.device))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{}}}"#, redeemed.revoked.len())), "two-factor sign-in is still on"); + assert!(meta.two_factor_enabled(id).await.unwrap()); + + // A reset code made with `reset_two_factor`: it takes the second factor away and the event says so. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), true, now + 6, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let redeemed = meta + .redeem_with_second_factor(&reset.hash(), "newer pc", &DeviceClient::default(), &token(), Some("$argon2id$newer"), None, now + 7) + .await + .unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::JoinResetRedeemed, Some(id), Some(redeemed.device), Some(*redeemed.device.as_uuid())) + ); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{},"two_factor_removed":true}}"#, redeemed.revoked.len()))); + assert!(!meta.two_factor_enabled(id).await.unwrap()); + assert!(events_of(meta, AuditAction::AdminTwoFactorReset).await.is_empty(), "no admin is made up for it"); +} + +/// An admin's reset of another account's two-factor sign-in and the command line's each write their event; the admin's +/// confirmation by recovery code writes the code's use, for the admin's account, even if the change it confirms fails. +async fn resets_and_confirmations_write_their_events(meta: &Meta) { + let (_, admin) = admin_actor(meta, "root").await; + let member = protected(meta, "member").await; + let other = protected(meta, "other").await; + let mut seen = Seen::new(meta).await; + let now = SETUP_AT + 100; + + let revoked = meta.admin_reset_two_factor(&admin, member.user.id, now).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AdminTwoFactorReset, AuditOutcome::Success, Some(admin.user), Some(device_of(&admin)), Some(member.user.id.0)) + ); + assert_eq!(details(&e), json("{}")); + assert_eq!(revoked.len(), 1); + assert!(matches!(meta.admin_reset_two_factor(&admin, member.user.id, now + 1).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty(), "nothing to reset: nothing written"); + + let revoked = meta.reset_two_factor(other.user.id, now + 2).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliTwoFactorReset, None, None, Some(other.user.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{}}}"#, revoked.len()))); + assert!(matches!(meta.reset_two_factor(other.user.id, now + 3).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); + + // An account's confirmation by recovery code, as an admin gives it: the code's use, no device. + let bea = protected(meta, "bea").await; + let mut seen = Seen::new(meta).await; + assert!(meta.confirm_second_factor(bea.user.id, Some(&Proof::Recovery(bea.hashes()[3])), now + 4).await.unwrap()); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthRecoveryCodeUsed, Some(bea.user.id), None, Some(bea.user.id.0)) + ); + // A wrong proof, or the same code again: nothing written. + assert!(matches!(confirm_with(meta, bea.user.id, None, now + 5).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!( + meta.confirm_second_factor(bea.user.id, Some(&Proof::Recovery(bea.hashes()[3])), now + 6).await, + Err(MetaError::TwoFactorInvalid) + )); + assert!(seen.actions().await.is_empty()); +} + +/// An error in the log fails every two-factor change and changes nothing; a wrong second step is still answered as wrong, since +/// its event is written after the answer is decided and only reported. +async fn an_error_in_the_log_fails_every_two_factor_change(url: &str) { + sqlx::any::install_default_drivers(); + let meta = Meta::open(url).await.unwrap(); + let me = protected(&meta, "ana").await; + let (_, admin) = admin_actor(&meta, "root").await; + let setup = user(&meta, "setup").await; + let setup_device = device(&meta, &setup).await; + let setup_actor = DeviceActor { user: setup.id, device: setup_device, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&setup_actor, b"cipher", SETUP_AT).await.unwrap(); + let plain = user(&meta, "plain").await; + let locker = protected(&meta, "locker").await; + let events_before = all_events(&meta).await.len(); + let now = SETUP_AT + 100; + let generation = gen_of(&meta, "ana").await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, now).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + + break_audit(url).await; + let is_db = |r: Result<(), MetaError>| assert!(matches!(r, Err(MetaError::Db(_))), "{r:?}"); + let hashes = me.hashes(); + is_db(meta.two_factor_confirm(setup.id, setup_device, &*right(FIRST_STEP), &hashes, now).await.map(|_| ())); + is_db(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 1)), &hashes, now).await); + is_db(meta.two_factor_disable(&me.actor(), &Proof::Recovery(hashes[0]), now).await.map(|_| ())); + is_db(meta.confirm_second_factor(me.user.id, Some(&Proof::Recovery(hashes[1])), now).await.map(|_| ())); + is_db(meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), now).await.map(|_| ())); + is_db(redeem_with_factor(&meta, &invite.hash(), Some(&Proof::Recovery(hashes[2])), now).await.map(|_| ())); + is_db(meta.sign_in(plain.id, ("$argon2id$stub", 0), "pc", &DeviceClient::default(), &token(), &new_challenge(), now).await.map(|_| ())); + is_db(meta.admin_reset_two_factor(&admin, me.user.id, now).await.map(|_| ())); + is_db(meta.reset_two_factor(me.user.id, now).await.map(|_| ())); + // A wrong second step: the answer is the same, whether or not the log works. + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), now).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + // The lock is set by the tenth wrong proof although its event can't be written: failing closed there would let a fault in + // the log undo the lock. The answer is the same and the counters and the lock are committed. + for n in 1..=10 { + let result = confirm_with(&meta, locker.user.id, None, now).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}: {result:?}"); + } + assert_eq!(meta.two_factor_locked_for(locker.user.id, now).await.unwrap(), Some(15 * MINUTE), "the lock holds"); + repair_audit(url).await; + + // Nothing was changed by any of it: still on, no code used, nobody signed out, the set-up still open. + assert_eq!(all_events(&meta).await.len(), events_before, "no event, no change (the lock's event was lost, only reported)"); + assert!(meta.two_factor_enabled(me.user.id).await.unwrap()); + assert_eq!(meta.two_factor_info(me.user.id, now).await.unwrap().recovery_codes_left, 10); + assert_eq!(live_devices(&meta, &me.user).await, vec![me.first]); + assert!(meta.two_factor_info(setup.id, now).await.unwrap().pending); + assert!(!meta.two_factor_enabled(setup.id).await.unwrap()); + assert_eq!(meta.invite_needs_password(&invite.hash(), now).await.unwrap(), Some(false), "the invite is still there"); + // And the same changes work once the log does. + meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 2)), &hashes, now + 1).await.unwrap(); +} + +#[tokio::test] +async fn an_error_in_the_log_fails_every_two_factor_change_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_error_in_the_log_fails_every_two_factor_change(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_error_in_the_log_fails_every_two_factor_change_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_error_in_the_log_fails_every_two_factor_change(&url).await; +} + +// ---- web sessions (migration 0011) ---- + +const WEB_IDLE_MS: i64 = 8 * 60 * MINUTE; +const WEB_ABSOLUTE_MS: i64 = 7 * 24 * 60 * MINUTE; + +fn new_session() -> NewWebSession { + NewWebSession { id_hash: token(), absolute_ms: WEB_ABSOLUTE_MS } +} + +/// An administrator with a password hash the web sign-ins below check against. +async fn web_admin(meta: &Meta, username: &str) -> UserRecord { + meta.create_user(username, ADMIN_HASH, Role::Admin, NOW).await.unwrap() +} + +/// A web sign-in of `user` as the handler would make it after verifying the password. +async fn web_sign_in(meta: &Meta, user: &UserRecord, session: &NewWebSession, now: i64) -> Option { + let generation = gen_of(meta, &user.username).await; + meta.web_sign_in(user.id, (&user.password_hash, generation), &new_challenge(), session, now).await.unwrap() +} + +async fn session_is_valid(meta: &Meta, session: &NewWebSession, now: i64) -> bool { + meta.web_session(&session.id_hash, now, WEB_IDLE_MS).await.unwrap().is_some() +} + +/// Only administrators get a session; it is found by its token hash, ends by idle time, by its absolute lifetime and by +/// logout, and the sign-in is written to the log without a device. +async fn web_sessions_are_made_looked_up_expire_and_log_out(meta: &Meta) { + let root = web_admin(meta, "root").await; + let member = user(meta, "ana").await; + // A member with the right password gets nothing, the same as a wrong generation or a wrong hash. + assert_eq!(web_sign_in(meta, &member, &new_session(), NOW).await, None); + let stale = meta.web_sign_in(root.id, (&root.password_hash, 99), &new_challenge(), &new_session(), NOW).await.unwrap(); + assert_eq!(stale, None, "the password was checked against another generation"); + let wrong_hash = meta.web_sign_in(root.id, ("$argon2id$other", root.auth_gen), &new_challenge(), &new_session(), NOW).await.unwrap(); + assert_eq!(wrong_hash, None); + assert_eq!(meta.web_sessions_open(member.id, NOW, WEB_IDLE_MS).await.unwrap(), 0); + + let session = new_session(); + assert_eq!(web_sign_in(meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let found = meta.web_session(&session.id_hash, NOW + 1, WEB_IDLE_MS).await.unwrap().unwrap(); + assert_eq!((found.user, found.username.as_str(), found.role, found.mfa_verified_ms), (root.id, "root", Role::Admin, None)); + assert_eq!((found.created_ms, found.last_seen_ms, found.absolute_expires_ms), (NOW, NOW, NOW + WEB_ABSOLUTE_MS)); + assert!(meta.web_session(&token(), NOW + 1, WEB_IDLE_MS).await.unwrap().is_none(), "an unknown token"); + let events = all_events(meta).await; + let login = events.iter().find(|e| e.action == AuditAction::AuthLogin).unwrap(); + assert_eq!((login.actor_user, login.actor_device), (Some(root.id), None)); + assert_eq!(details(login), json(r#"{"client_kind":"web"}"#)); + assert!(DeviceClient::new(Some("web"), None, None).is_err(), "devices cannot claim web"); + + // Idle: valid just inside the idle time, over at it, and a use moves it. + assert!(session_is_valid(meta, &session, NOW + WEB_IDLE_MS - 1).await); + assert!(!session_is_valid(meta, &session, NOW + WEB_IDLE_MS).await); + meta.web_session_seen(&session.id_hash, NOW + WEB_IDLE_MS - 1).await.unwrap(); + assert!(session_is_valid(meta, &session, NOW + 2 * WEB_IDLE_MS - 2).await, "a use starts the idle time again"); + // The time never moves back. + meta.web_session_seen(&session.id_hash, NOW + 5).await.unwrap(); + assert_eq!(meta.web_session(&session.id_hash, NOW + WEB_IDLE_MS, WEB_IDLE_MS).await.unwrap().unwrap().last_seen_ms, NOW + WEB_IDLE_MS - 1); + + // Absolute: keeping it busy does not help. + let busy = new_session(); + assert_eq!(web_sign_in(meta, &root, &busy, NOW).await, Some(WebSignIn::Session)); + let mut at = NOW; + while at + WEB_IDLE_MS / 2 < NOW + WEB_ABSOLUTE_MS { + at += WEB_IDLE_MS / 2; + meta.web_session_seen(&busy.id_hash, at).await.unwrap(); + assert!(session_is_valid(meta, &busy, at).await); + } + assert!(!session_is_valid(meta, &busy, NOW + WEB_ABSOLUTE_MS).await, "the absolute lifetime ends it"); + + // Logout ends it at once, and a second logout, or one of an unknown session, is no error. + let out = new_session(); + assert_eq!(web_sign_in(meta, &root, &out, NOW + 10).await, Some(WebSignIn::Session)); + assert!(session_is_valid(meta, &out, NOW + 11).await); + meta.web_logout(&out.id_hash, NOW + 12).await.unwrap(); + meta.web_logout(&out.id_hash, NOW + 13).await.unwrap(); + meta.web_logout(&token(), NOW + 13).await.unwrap(); + assert!(!session_is_valid(meta, &out, NOW + 14).await); + meta.web_session_seen(&out.id_hash, NOW + 15).await.unwrap(); + assert!(!session_is_valid(meta, &out, NOW + 16).await, "a use does not revive a logged-out session"); +} + +/// An administrator with two-factor sign-in on, and a session made with it (the sign-in's second step). `step` is the time +/// step of the code that made the session. +async fn web_admin_with_factor(meta: &Meta, username: &str, step: i64, now: i64) -> (UserRecord, DeviceId, NewWebSession) { + let me = web_admin(meta, username).await; + let first = device(meta, &me).await; + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: me.id, device: first, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", now).await.unwrap(); + meta.two_factor_confirm(me.id, first, &*right(step), &hashes, now + 1).await.unwrap(); + let (chal, session) = (new_challenge(), new_session()); + let generation = gen_of(meta, username).await; + let made = meta.web_sign_in(me.id, (ADMIN_HASH, generation), &chal, &session, now + 2).await.unwrap(); + assert!(matches!(made, Some(WebSignIn::Challenge { .. }))); + meta.web_complete_second_factor(&chal.id_hash, &totp(&right(step + 1)), &session, now + 3).await.unwrap(); + assert!(session_is_valid(meta, &session, now + 4).await); + (me, first, session) +} + +/// Every change that raises `users.auth_gen` ends the account's sessions: a password change and reset, a disable, an enable +/// after it, a removal, and two-factor sign-in turned on, off, or reset by an admin or the command line. +async fn a_web_session_ends_with_every_change_of_the_accounts_generation(meta: &Meta) { + // The acting admin has two-factor sign-in, which resetting another administrator's needs. + let (boss, plain) = admin_actor(meta, "boss").await; + let boss_device = device_of(&plain); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: boss.id, device: boss_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", NOW).await.unwrap(); + meta.two_factor_confirm(boss.id, boss_device, &*right(FIRST_STEP), &hashes, NOW + 1).await.unwrap(); + let actor = AdminActor { two_factor: true, ..plain }; + let mut now = NOW + 10; + + // A password reset by the command line. + let root = web_admin(meta, "root").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &root, &s, now).await, Some(WebSignIn::Session)); + meta.reset_password(root.id, "$argon2id$new", true, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a password reset ends the session"); + now += 10; + + // A password change by a device. + let (ana, ana_actor) = admin_actor(meta, "ana").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &ana, &s, now).await, Some(WebSignIn::Session)); + meta.change_password(ana.id, ADMIN_HASH, "$argon2id$changed", device_of(&ana_actor), now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a password change ends the session"); + now += 10; + + // Disable ends it, and an enable afterwards does not bring it back. + let target = web_admin(meta, "target").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &target, &s, now).await, Some(WebSignIn::Session)); + meta.admin_disable_user(&actor, target.id, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a disabled account's session is over"); + meta.admin_enable_user(&actor, target.id, now + 3).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 4).await, "enabling does not revive it"); + now += 10; + + // A removal. + let doomed = web_admin(meta, "doomed").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &doomed, &s, now).await, Some(WebSignIn::Session)); + meta.admin_remove_user(&actor, doomed.id, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a removed account's session is over"); + now += 10; + + // Two-factor sign-in turned on: a session from before it is over (a set-up that is not confirmed changes nothing). + let guarded = web_admin(meta, "guarded").await; + let guarded_device = device(meta, &guarded).await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &guarded, &s, now).await, Some(WebSignIn::Session)); + let setup = DeviceActor { user: guarded.id, device: guarded_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", now + 1).await.unwrap(); + assert!(session_is_valid(meta, &s, now + 2).await, "an open set-up is no change"); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + meta.two_factor_confirm(guarded.id, guarded_device, &*right(FIRST_STEP), &hashes, now + 3).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 4).await, "turning two-factor sign-in on ends the session"); + now += 10; + + // Turned off by the account itself. + let (own, own_device, s) = web_admin_with_factor(meta, "own", FIRST_STEP, now).await; + let setup = DeviceActor { user: own.id, device: own_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_disable(&setup, &totp(&right(FIRST_STEP + 5)), now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "turning two-factor sign-in off ends the session"); + now += 20; + + // Reset by an admin, and by the command line. + let (by_admin, _, s) = web_admin_with_factor(meta, "byadmin", FIRST_STEP, now).await; + meta.admin_reset_two_factor(&actor, by_admin.id, now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "an admin's reset ends the session"); + now += 20; + let (by_cli, _, s) = web_admin_with_factor(meta, "bycli", FIRST_STEP, now).await; + meta.reset_two_factor(by_cli.id, now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "the command line's reset ends the session"); + + // Another account's session is untouched by all of that. + let bystander = web_admin(meta, "bystander").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &bystander, &s, now + 30).await, Some(WebSignIn::Session)); + meta.reset_password(root.id, "$argon2id$again", true, now + 31).await.unwrap(); + assert!(session_is_valid(meta, &s, now + 32).await); +} + +/// An admin with two-factor sign-in gets a challenge of kind `web`, which no device sign-in can use and which makes no +/// device, and completes it with a code, once. The second factor's counters and lock are the device sign-in's. +async fn a_web_sign_in_with_two_factor_needs_its_own_challenge(meta: &Meta) { + let me = web_admin(meta, "root").await; + let first = device(meta, &me).await; + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: me.id, device: first, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + meta.two_factor_confirm(me.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 1).await.unwrap(); + + let generation = gen_of(meta, "root").await; + let chal = new_challenge(); + let session = new_session(); + let made = meta.web_sign_in(me.id, (ADMIN_HASH, generation), &chal, &session, SETUP_AT + 10).await.unwrap(); + assert_eq!(made, Some(WebSignIn::Challenge { expires_ms: SETUP_AT + 10 + 5 * MINUTE })); + assert!(!session_is_valid(meta, &session, SETUP_AT + 11).await, "no session without the second factor"); + assert_eq!(live_devices(meta, &me).await, vec![first], "and no device"); + // The challenge is seen as a web one only. + assert_eq!(meta.web_challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), Some((me.id, "root".to_owned()))); + assert_eq!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), None, "a device sign-in does not see it"); + let as_device = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), SETUP_AT + 12).await; + assert!(matches!(as_device, Err(MetaError::ChallengeGone)), "{as_device:?}"); + assert_eq!(live_devices(meta, &me).await, vec![first], "still no new device, and the code was not used"); + // And the other way round. + let device_chal = new_challenge(); + let made = meta.sign_in(me.id, (ADMIN_HASH, generation), "pc", &DeviceClient::default(), &token(), &device_chal, SETUP_AT + 13).await.unwrap(); + assert!(matches!(made, Some(SignIn::Challenge { .. }))); + let as_web = meta.web_complete_second_factor(&device_chal.id_hash, &totp(&right(FIRST_STEP + 1)), &session, SETUP_AT + 14).await; + assert!(matches!(as_web, Err(MetaError::ChallengeGone)), "{as_web:?}"); + assert!(!session_is_valid(meta, &session, SETUP_AT + 15).await); + + // Wrong codes are counted (four, and the fifth would delete the challenge), then the right one makes the session, once. + for _ in 0..4 { + let result = meta.web_complete_second_factor(&chal.id_hash, &totp(&wrong()), &session, SETUP_AT + 20).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + } + let done = meta.web_complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &session, SETUP_AT + 21).await.unwrap(); + assert_eq!(done, me.id); + let found = meta.web_session(&session.id_hash, SETUP_AT + 22, WEB_IDLE_MS).await.unwrap().unwrap(); + assert_eq!(found.mfa_verified_ms, Some(SETUP_AT + 21), "the session says when the second factor was given"); + assert_eq!(live_devices(meta, &me).await, vec![first], "a web sign-in makes no device"); + let login = all_events(meta).await.into_iter().find(|event| event.action == AuditAction::AuthLogin).unwrap(); + assert_eq!(details(&login), json(r#"{"client_kind":"web"}"#)); + let failed = all_events(meta).await.into_iter().find(|event| event.action == AuditAction::AuthSecondFactorFailed).unwrap(); + assert_eq!(details(&failed), json(r#"{"client_kind":"web","step":"totp"}"#)); + let again = meta.web_complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 2)), &new_session(), SETUP_AT + 23).await; + assert!(matches!(again, Err(MetaError::ChallengeGone)), "once: {again:?}"); + + // The same time step is not accepted again (the counter is the device sign-in's and the admin confirmation's). + let generation = gen_of(meta, "root").await; + let replay = new_challenge(); + meta.web_sign_in(me.id, (ADMIN_HASH, generation), &replay, &new_session(), SETUP_AT + 24).await.unwrap(); + let replayed = meta.web_complete_second_factor(&replay.id_hash, &totp(&right(FIRST_STEP + 1)), &new_session(), SETUP_AT + 25).await; + assert!(matches!(replayed, Err(MetaError::TwoFactorInvalid)), "{replayed:?}"); + // Wrong proofs lock the second factor for every way of using it: a device sign-in's step too. + for n in 0..12 { + let _ = confirm_with(meta, me.id, None, SETUP_AT + 30 + n).await; + } + let locked = meta.web_complete_second_factor(&replay.id_hash, &totp(&right(FIRST_STEP + 9)), &new_session(), SETUP_AT + 50).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + let locked = meta.complete_second_factor(&device_chal.id_hash, &totp(&right(FIRST_STEP + 9)), &token(), SETUP_AT + 51).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); +} + +/// Pruning deletes sessions that are over (logged out, expired, idle), never a valid one, and at most the batch asked for. +async fn web_sessions_are_pruned_in_bounded_batches(meta: &Meta) { + let root = web_admin(meta, "root").await; + let late = NOW + WEB_ABSOLUTE_MS - 2 * WEB_IDLE_MS; + let now = NOW + WEB_ABSOLUTE_MS - WEB_IDLE_MS; + let keep = new_session(); + assert_eq!(web_sign_in(meta, &root, &keep, now - MINUTE).await, Some(WebSignIn::Session)); + for n in 0..5 { + assert_eq!(web_sign_in(meta, &root, &new_session(), NOW + n).await, Some(WebSignIn::Session)); + } + let out = new_session(); + assert_eq!(web_sign_in(meta, &root, &out, late).await, Some(WebSignIn::Session)); + meta.web_logout(&out.id_hash, late + 1).await.unwrap(); + + assert_eq!(meta.web_sessions_count_over(now, WEB_IDLE_MS).await.unwrap(), 6, "five idle ones and the logged-out one"); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 4, "a batch is bounded"); + assert_eq!(meta.web_sessions_count_over(now, WEB_IDLE_MS).await.unwrap(), 2); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 2); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 0, "nothing is left to delete"); + assert!(session_is_valid(meta, &keep, now).await, "a valid session is never pruned"); + // Past the absolute lifetime the last one goes too. + let end = NOW + 2 * WEB_ABSOLUTE_MS; + assert_eq!(meta.web_sessions_prune(end, WEB_IDLE_MS, 100).await.unwrap(), 1); + assert_eq!(meta.web_sessions_count_over(end, WEB_IDLE_MS).await.unwrap(), 0); +} + +/// An admin change made on a browser session is audited without a device, and a session that was logged out, or whose account +/// changed, can no longer make one. +async fn a_web_actor_changes_accounts_until_its_session_ends(meta: &Meta) { + // The store judges the session by its own clock, which follows the time each call below is made at. + let clock = std::sync::Arc::new(std::sync::atomic::AtomicI64::new(NOW)); + let ticking = clock.clone(); + let meta = &meta.clone().with_clock(std::sync::Arc::new(move || ticking.load(std::sync::atomic::Ordering::SeqCst))); + let at_ms = |t: i64| { + clock.store(t, std::sync::atomic::Ordering::SeqCst); + t + }; + let root = web_admin(meta, "root").await; + let session = new_session(); + assert_eq!(web_sign_in(meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let actor = AdminActor { + user: root.id, + via: AdminVia::Web { id_hash: session.id_hash, idle_ms: WEB_IDLE_MS }, + password_hash: ADMIN_HASH.to_owned(), + two_factor: false, + }; + assert!(!format!("{actor:?}").contains(&format!("{:?}", session.id_hash)), "the session's hash is not in Debug"); + let ana = user(meta, "ana").await; + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 1)).await.unwrap(); + let events = all_events(meta).await; + let changed = events.iter().find(|e| e.action == AuditAction::AdminQuotaChanged).unwrap(); + assert_eq!((changed.actor_user, changed.actor_device), (Some(root.id), None), "no device acted"); + // Another account's session is not this admin's. + let other = web_admin(meta, "other").await; + let theirs = new_session(); + assert_eq!(web_sign_in(meta, &other, &theirs, NOW).await, Some(WebSignIn::Session)); + let borrowed = AdminActor { via: AdminVia::Web { id_hash: theirs.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + assert!(matches!(meta.admin_update_user(&borrowed, ana.id, None, Some(None), at_ms(NOW + 2)).await, Err(MetaError::DeviceRevoked))); + // A session that does not exist. + let unknown = AdminActor { via: AdminVia::Web { id_hash: token(), idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + assert!(matches!(meta.admin_update_user(&unknown, ana.id, None, Some(None), at_ms(NOW + 3)).await, Err(MetaError::DeviceRevoked))); + // Logged out: refused, and nothing changed. + meta.web_logout(&session.id_hash, NOW + 4).await.unwrap(); + assert!(matches!(meta.admin_update_user(&actor, ana.id, None, Some(None), at_ms(NOW + 5)).await, Err(MetaError::DeviceRevoked))); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, Some(MIN_QUOTA_BYTES)); + // A session that expired between the request's start and the change (the handler's check was passed, the transaction runs + // later): refused by the idle time and by the absolute lifetime, and nothing changed. + let slow = new_session(); + assert_eq!(web_sign_in(meta, &root, &slow, NOW + 20).await, Some(WebSignIn::Session)); + let slow_actor = AdminActor { via: AdminVia::Web { id_hash: slow.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + let idle = meta.admin_update_user(&slow_actor, ana.id, None, Some(None), at_ms(NOW + 20 + WEB_IDLE_MS)).await; + assert!(matches!(idle, Err(MetaError::DeviceRevoked)), "idle: {idle:?}"); + meta.admin_update_user(&slow_actor, ana.id, None, Some(None), at_ms(NOW + 20 + WEB_IDLE_MS - 1)).await.unwrap(); + let busy = new_session(); + assert_eq!(web_sign_in(meta, &root, &busy, NOW + 30).await, Some(WebSignIn::Session)); + let busy_actor = AdminActor { via: AdminVia::Web { id_hash: busy.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + let mut at = NOW + 30; + while at + WEB_IDLE_MS / 2 < NOW + 30 + WEB_ABSOLUTE_MS { + at += WEB_IDLE_MS / 2; + meta.web_session_seen(&busy.id_hash, at).await.unwrap(); + } + meta.admin_update_user(&busy_actor, ana.id, None, Some(None), at_ms(at)).await.unwrap(); + let absolute = meta.admin_update_user(&busy_actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 30 + WEB_ABSOLUTE_MS)).await; + assert!(matches!(absolute, Err(MetaError::DeviceRevoked)), "absolute: {absolute:?}"); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, None); + // A session of a changed account: the generation moved on. + let late = new_session(); + assert_eq!(web_sign_in(meta, &root, &late, NOW + 6).await, Some(WebSignIn::Session)); + let late_actor = AdminActor { via: AdminVia::Web { id_hash: late.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + meta.admin_update_user(&late_actor, ana.id, None, Some(None), at_ms(NOW + 7)).await.unwrap(); + meta.reset_password(root.id, "$argon2id$new", true, NOW + 8).await.unwrap(); + let after = meta.admin_update_user(&late_actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 9)).await; + assert!(matches!(after, Err(MetaError::DeviceRevoked)), "{after:?}"); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, None); +} + +// ---- the migration from 0010 to 0011 ---- + +/// Drops what migration 0011 added and forgets it ran, which leaves the database as 0010 left it (with its rows). +async fn back_to_0010(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 11").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the web admin keeps its accounts, devices and audit log, gets an empty session table +/// that works (also after the migration is rolled back and run again), and a database newer than the program is still refused. +async fn migrating_from_0010_adds_the_web_sessions(url: &str) { + sqlx::any::install_default_drivers(); + let (root, old_token, events) = { + let meta = Meta::open(url).await.unwrap(); + let root = web_admin(&meta, "root").await; + let old_token = token(); + meta.create_device(root.id, "old pc", &old_token, NOW).await.unwrap(); + meta.sign_in_device(root.id, (&root.password_hash, root.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + let events = all_events(&meta).await.len(); + (root, old_token, events) + }; + back_to_0010(url).await; + assert_eq!(schema_version(url).await, 10); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15, "the current schema includes history, trash and the reference index after web sessions"); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(root.id)); + assert_eq!(all_events(&meta).await.len(), events, "the audit rows are kept"); + let session = new_session(); + assert_eq!(web_sign_in(&meta, &root, &session, NOW + 5).await, Some(WebSignIn::Session)); + assert!(session_is_valid(&meta, &session, NOW + 6).await); + + // The rollback: the table goes, the version is forgotten, and opening again makes the table again. + drop(meta); + back_to_0010(url).await; + assert_eq!(schema_version(url).await, 10); + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 15); + assert!(!session_is_valid(&meta, &session, NOW + 6).await, "the table is new and empty"); + drop(meta); + + // A database from a newer Craft Cloud is still refused. + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (16)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0010_adds_the_web_sessions_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0010_adds_the_web_sessions(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0010_adds_the_web_sessions_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0010_adds_the_web_sessions(&url).await; +} + +// ---- a change that waits for its locks while its session ends ---- + +/// The time a change judges its session by is read after the change has its locks: one that waited while the session went idle +/// is refused, and changes nothing. +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(url: &str) { + use sqlx::Connection; + use std::sync::Arc; + use std::sync::atomic::{AtomicI64, Ordering}; + sqlx::any::install_default_drivers(); + let now = Arc::new(AtomicI64::new(NOW)); + let ticking = now.clone(); + let meta = Meta::open(url).await.unwrap().with_clock(Arc::new(move || ticking.load(Ordering::SeqCst))); + let root = web_admin(&meta, "root").await; + let ana = user(&meta, "ana").await; + let session = new_session(); + assert_eq!(web_sign_in(&meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let actor = AdminActor { + user: root.id, + via: AdminVia::Web { id_hash: session.id_hash, idle_ms: WEB_IDLE_MS }, + password_hash: ADMIN_HASH.to_owned(), + two_factor: false, + }; + // Another connection holds the write lock (SQLite) or the admin's row (Postgres) until the test lets go. + let mut holder = sqlx::AnyConnection::connect(url).await.unwrap(); + if url.starts_with("sqlite:") { + sqlx::query("BEGIN IMMEDIATE").execute(&mut holder).await.unwrap(); + } else { + sqlx::query("BEGIN").execute(&mut holder).await.unwrap(); + sqlx::query("UPDATE users SET username = username WHERE username = 'root'").execute(&mut holder).await.unwrap(); + } + let change = { + let (meta, actor, ana) = (meta.clone(), actor.clone(), ana.id); + tokio::spawn(async move { meta.admin_update_user(&actor, ana, None, Some(Some(MIN_QUOTA_BYTES)), NOW + 1).await }) + }; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert!(!change.is_finished(), "the change is waiting for the lock"); + now.store(NOW + WEB_IDLE_MS + 5, Ordering::SeqCst); + sqlx::query("COMMIT").execute(&mut holder).await.unwrap(); + let result = change.await.unwrap(); + assert!(matches!(result, Err(MetaError::DeviceRevoked)), "{result:?}"); + assert_eq!(admin_view(&meta, ana.id).await.quota_bytes, None, "nothing changed"); + // The same change while the session is valid goes through. + now.store(NOW + 2, Ordering::SeqCst); + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW + 2).await.unwrap(); +} + +#[tokio::test] +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(&url).await; +} + +/// A file that comes back from the trash gets a new current version with its last content, after the last one and +/// without a save record, so a delete that was sent against the old version (and whose answer a machine never heard) +/// is refused as a version mismatch and can't delete the restored file again; a folder has no version. +async fn a_restored_file_gets_a_new_version_with_its_old_content(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 2).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + delete_file(meta, &u, dev, node, versions[1].id, NOW + 10).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, node, NOW + 20).await.unwrap(); + let new = done.version.unwrap(); + assert!(!versions.iter().any(|v| v.id == new)); + assert_eq!(done.seq, before.seq + 1, "one change of the space"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3); + assert_eq!( + (history[0].id, history[0].parent_version, history[0].size, history[0].save, history[0].seq), + (new, Some(versions[1].id), 4, false, done.seq) + ); + assert!(history.iter().any(|v| v.id == versions[1].id && v.save), "the old version keeps its record"); + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!((f.version, f.manifest, f.mtime_ms, f.save.clone()), (new, versions[1].manifest, versions[1].mtime_ms, None)); + + // The delete a machine sent before the restore, with the old version, is refused and the file stays. + let stale = meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(versions[1].id) }], NOW + 30).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(new) })]); + assert!(!live_name(meta, &u, node).await.2); + + // A folder has no version. + let dir = folder(meta, &u, dev, r, "plain-dir", NOW + 40).await; + delete_folder(meta, &u, dev, dir, NOW + 41).await; + assert_eq!(meta.restore_from_trash(u.space, dev, dir, NOW + 42).await.unwrap().version, None); +} + +/// A restore never takes the tree past its depth limit: a folder that comes back below a live folder that has moved +/// deep meanwhile is refused as too deep, for the folder itself and for a file whose folder would come back, and +/// the refusal changes nothing; a file in a folder at the limit is fine. +async fn a_restore_does_not_exceed_the_depth_limit(meta: &Meta) { + use craft_cloud_proto::limits::MAX_TREE_DEPTH; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let shallow = folder(meta, &u, dev, r, "shallow", NOW).await; + let c = folder(meta, &u, dev, shallow, "c", NOW).await; + let (f, fv) = file_in(meta, &u, dev, c, "f.txt", NOW).await; + let spare = folder(meta, &u, dev, shallow, "spare", NOW).await; + let (g, gv) = file_in(meta, &u, dev, spare, "g.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + delete_folder(meta, &u, dev, c, NOW + 2).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 3).await; + // A chain of folders down to depth MAX_TREE_DEPTH - 1, and the live folder moved to its end: depth MAX_TREE_DEPTH. + let mut ops = Vec::new(); + let mut parent = r; + for i in 0..MAX_TREE_DEPTH - 1 { + let id = NodeId::generate(); + ops.push(Op::Mkdir { node: id, parent, name: name(&format!("d{i}")) }); + parent = id; + } + meta.commit(u.space, dev, &ops, NOW + 4).await.unwrap(); + meta.commit(u.space, dev, &[Op::Move { node: shallow, from_parent: r, from_name: name("shallow"), parent, name: name("shallow") }], NOW + 5) + .await + .unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let trash_before = trash_ids(meta, &u).await; + + // The deleted folder would sit one level too deep, and so would the folder of the file: refused, nothing changes. + for node in [c, f] { + let refused = meta.restore_from_trash(u.space, dev, node, NOW + 6).await; + assert_eq!(refused_restore(refused), vec![(0, ConflictKind::TooDeep)], "restoring {node}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, trash_before); + } + // A file may sit in the deepest folder: it comes back, in `spare`, which is live (it was never deleted). + let done = meta.restore_from_trash(u.space, dev, g, NOW + 7).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (spare, "g.txt")); +} + +// ---- the reference index ---- + +/// A manifest over `chunks` (each one byte), the chunks recorded as `u`'s; the manifest itself is not recorded. +async fn manifest_over(meta: &Meta, u: &UserRecord, chunks: &[ChunkId]) -> Manifest { + let unique: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(u.space, &unique, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), chunks.len() as u64, FileHash::from_bytes([chunks.len() as u8; 32]), refs).unwrap() +} + +fn fresh_chunks(n: usize) -> Vec { + (0..n).map(|_| random_chunk()).collect() +} + +fn edge_list(chunks: &[ChunkId]) -> Vec<(u64, ChunkId)> { + chunks.iter().enumerate().map(|(i, c)| (i as u64, *c)).collect() +} + +/// The index of a manifest is written in the transaction that records it, exactly: in order, repeated chunks at each +/// of their places; recording it again (another space, the same bytes) changes nothing; a manifest the quota refuses +/// leaves no index behind; an empty file's manifest is complete with no edges. +async fn a_new_manifest_has_exact_edges_complete_in_its_own_transaction(meta: &Meta) { + use craft_cloud_meta::{EdgeCheck, IndexState}; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let (a, b, c) = (random_chunk(), random_chunk(), random_chunk()); + let chunks = [a, b, a, c]; + let m = manifest_over(meta, &ana, &chunks).await; + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert_eq!(meta.index_check(&m).await.unwrap(), EdgeCheck::Exact); + + // The same manifest from another space: nothing is written twice. + let edges = meta.index_edge_count().await.unwrap(); + let again = manifest_over(meta, &bob, &chunks).await; + meta.add_manifest(bob.space, &again, 100, NOW).await.unwrap(); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + + // An empty file. + let empty = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 0, FileHash::from_bytes([0; 32]), vec![]).unwrap(); + meta.add_manifest(ana.space, &empty, 10, NOW).await.unwrap(); + assert_eq!(meta.index_state(&empty.id()).await.unwrap(), IndexState::Complete { total: 0 }); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + + // A manifest the stored-bytes cap refuses leaves no index: its transaction rolls back whole. + let capped = meta.clone().with_stored_bytes_cap(Some(50)); + let carol = user(&capped, "carol").await; + let chunks = fresh_chunks(3); + let refused = manifest_over(&capped, &carol, &chunks).await; + assert!(matches!(capped.add_manifest(carol.space, &refused, 1_000, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + assert_eq!(capped.index_state(&refused.id()).await.unwrap(), IndexState::Absent); + assert_eq!(capped.index_edge_count().await.unwrap(), edges); + // A manifest whose chunks the space lacks is refused before anything is written. + let lacking = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 1, FileHash::from_bytes([7; 32]), vec![ChunkRef { id: random_chunk(), len: 1 }]) + .unwrap(); + assert!(matches!(meta.add_manifest(ana.space, &lacking, 100, NOW).await, Err(MetaError::MissingChunks(_)))); + assert_eq!(meta.index_state(&lacking.id()).await.unwrap(), IndexState::Absent); +} + +/// A manifest longer than the budget is indexed in batches of at most that many edges, each its own transaction, and is +/// complete only after the comparison with its manifest. +async fn a_long_manifest_is_indexed_in_batches_of_the_budget(meta: &Meta) { + use craft_cloud_meta::{IndexState, Validation}; + let meta = meta.clone().with_index_batch_rows(5); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + let before = meta.index_edge_count().await.unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 5, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 5, "no more than the budget in the recording transaction"); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + + for (expected_done, expected_rows) in [(10u64, 10u64), (12, 12)] { + let state = meta.index_stage(&m).await.unwrap(); + assert_eq!(state, IndexState::Building { done: expected_done, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, expected_rows, "at most the budget more per stage"); + } + // Every edge is written, and still it is not complete: only the comparison makes it so. + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 12, total: 12 }); + assert_eq!(meta.index_validate(&m).await.unwrap(), Validation::Complete); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + // Once complete, more stages change nothing. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 12); + + // `index_complete` does all of it, and again is a no-op. + let more = fresh_chunks(23); + let long = manifest_over(&meta, &ana, &more).await; + meta.add_manifest(ana.space, &long, 100, NOW).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + assert_eq!(meta.index_state(&long.id()).await.unwrap(), IndexState::Complete { total: 23 }); + assert_eq!(meta.index_edges(&long.id()).await.unwrap(), edge_list(&more)); +} + +/// The index is built from what a manifest says, not from what is there: wrong, swapped, missing and extra edges, a +/// repeated chunk collapsed or changed, and a wrong edge written before the build started are each found by the +/// comparison, the index does not become complete, and a build that repeats the manifest's own list makes it exact. +async fn an_index_is_complete_only_when_every_edge_is_right_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let (a, b, c, d, e, f) = (random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk()); + let spare = random_chunk(); + // `(what, chunks, damage SQL, expected first problem)`: `$1` is a chunk ID and `$2` the manifest. + type Case<'a> = (&'a str, Vec, &'a str, (u64, EdgeProblemKind)); + let cases: Vec = vec![ + ( + "wrong chunk", + vec![a, b, c, d, e, f], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a missing edge", + vec![a, b, c, d, e, f], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 4 AND chunk <> $1", + (4, EdgeProblemKind::MissingEdge), + ), + ( + "an extra edge", + vec![a, b, c, d, e, f], + "INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($2, 6, $1)", + (6, EdgeProblemKind::ExtraEdge), + ), + ( + "a repeated chunk changed", + vec![a, b, a, c], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a repeated chunk collapsed", + vec![a, b, a, c], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 2 AND chunk = $1", + (2, EdgeProblemKind::MissingEdge), + ), + ]; + for (what, chunks, damage, (ordinal, kind)) in cases { + // A distinct manifest per case: its chunks differ in size by the case. + let chunks: Vec = + chunks.iter().map(|c| ChunkId::from_bytes(*blake3::hash(&[c.as_bytes().as_slice(), what.as_bytes()].concat()).as_bytes())).collect(); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + assert!(matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { .. }), "{what}: all edges written, not yet compared"); + // Damage it with every edge written, before the comparison. + let wrong = chunks.first().map(|c| *c.as_bytes()).unwrap_or(*spare.as_bytes()); + let other = if damage.starts_with("UPDATE") { *spare.as_bytes() } else { wrong }; + sqlx::query(sqlx::AssertSqlSafe(damage.to_owned())).bind(other.to_vec()).bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let outcome = meta.index_validate(&m).await.unwrap(); + match outcome { + Validation::Mismatch(problem) => assert_eq!((problem.ordinal, problem.kind), (ordinal, kind), "{what}"), + other => panic!("{what}: expected a mismatch, got {other:?}"), + } + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{what}: not complete"); + // A build repeats the manifest's own list and ends exact. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }, "{what}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{what}"); + } + + // A wrong edge that was there before the build began is not taken for the right one. + let chunks = fresh_chunks(4); + let m = manifest_over(&meta, &ana, &chunks).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "recorded with a wrong edge present"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + + // A complete index that was damaged afterwards is found by the checks that read it, not trusted. + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 3 }); + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 0") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert!(matches!(meta.index_check(&m).await.unwrap(), craft_cloud_meta::EdgeCheck::Mismatch(p) if p.ordinal == 0)); + // The builder leaves a complete row alone (it is the checker's to report), and a mismatch there is an error, not a silent rewrite. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 3 }); +} + +/// A build stopped part way resumes where it was, after the database was closed and opened again, and a manifest +/// recorded before the index existed (no row at all) is found and indexed. +async fn an_index_build_resumes_after_a_restart_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let (ana, m) = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + (ana, m) + }; + // The "restart": a new handle on the same database. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + assert_eq!(meta.index_edge_count().await.unwrap(), 8); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + + // A manifest from before the index: recorded, no row, no edges. + let old_chunks = fresh_chunks(6); + let old = manifest_over(&meta, &ana, &old_chunks).await; + meta.add_manifest(ana.space, &old, 100, NOW).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + assert_eq!(meta.index_state(&old.id()).await.unwrap(), IndexState::Absent); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![old.id()]); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent), (2, 1, 1)); + meta.index_complete(&old).await.unwrap(); + assert_eq!(meta.index_edges(&old.id()).await.unwrap(), edge_list(&old_chunks)); + assert_eq!(meta.index_audit(5).await.unwrap().complete, 2); +} + +/// The migration that adds the index does not pretend: manifests recorded before it have no row and are reported as not +/// indexed until a build indexes them. +async fn the_index_migration_leaves_old_manifests_unindexed_on(url: &str) { + use craft_cloud_meta::IndexState; + let (m, ana) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &fresh_chunks(3)).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + (m, ana) + }; + let _ = ana; + { + let mut conn = raw(url).await; + undo_team_spaces(&mut conn).await; + for statement in ["DROP TABLE manifest_chunks", "DROP TABLE manifest_index", "DELETE FROM schema_version WHERE version >= 14"] { + sqlx::query(statement).execute(&mut conn).await.unwrap(); + } + } + let meta = Meta::open(url).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent, "recorded before the index: not indexed"); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent, audit.building), (1, 0, 1, 0)); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 15); +} + +/// What the database alone can say is wrong, in seeded damage: orphan edges, an index row without its manifest, an edge +/// to a chunk that is not recorded, a root naming a manifest that is not recorded; and the notices: manifests no version +/// or snapshot names (the live-document asset case) and chunks no manifest names. +async fn the_audit_names_what_is_wrong_on(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + + // A healthy file, an asset no version names, and a chunk no manifest names. + let chunks = fresh_chunks(3); + let file_manifest = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &file_manifest, 100, NOW).await.unwrap(); + let node = NodeId::generate(); + let v = version(file_manifest.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("f.txt"), version: v }], NOW).await.unwrap(); + let asset = manifest_over(&meta, &ana, &fresh_chunks(2)).await; + meta.add_manifest(ana.space, &asset, 100, NOW).await.unwrap(); + let stray = random_chunk(); + meta.add_chunk(ana.space, &stray, 10, NOW).await.unwrap(); + let healthy = meta.index_audit(5).await.unwrap(); + assert_eq!((healthy.manifests, healthy.complete, healthy.building, healthy.absent), (2, 2, 0, 0)); + assert_eq!(healthy.unrooted_manifests.count, 1, "the asset is named by no version"); + assert_eq!(healthy.unrooted_manifests.examples, vec![asset.id().to_string()]); + assert_eq!(healthy.unreferenced_chunks.count, 1); + for finding in [ + &healthy.index_without_manifest, + &healthy.edges_without_index, + &healthy.complete_with_wrong_count, + &healthy.unknown_dependencies, + &healthy.roots_without_manifest, + &healthy.roots_not_indexed, + &healthy.space_blobs_without_blob, + ] { + assert_eq!(finding.count, 0, "{finding:?}"); + } + + // Damage: orphan edges, an index row without a manifest, an unknown dependency, a root without its manifest. + let ghost = ManifestId::from_bytes([0xaa; 32]); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 0, $2)") + .bind(ghost.as_bytes().to_vec()) + .bind(stray.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + let ghost_row = ManifestId::from_bytes([0xbb; 32]); + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, 0, 0, 1)") + .bind(ghost_row.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 0 AND id = $1").bind(chunks[1].as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 1 AND id = $1").bind(file_manifest.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let damaged = meta.index_audit(5).await.unwrap(); + assert_eq!(damaged.edges_without_index.examples, vec![ghost.to_string()]); + assert_eq!(damaged.index_without_manifest.count, 2, "the made-up row and the manifest whose record was removed"); + assert!(damaged.index_without_manifest.examples.contains(&ghost_row.to_string())); + assert_eq!(damaged.unknown_dependencies.count, 1); + assert_eq!(damaged.unknown_dependencies.examples, vec![format!("{}/{}", file_manifest.id(), chunks[1])]); + assert_eq!(damaged.roots_without_manifest.examples, vec![file_manifest.id().to_string()], "the version names a manifest that is gone"); + assert_eq!(damaged.space_blobs_without_blob.count, 2, "the memberships of the removed blobs"); + // A complete index whose edge count is not its chunk count. + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal = 0") + .bind(asset.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(meta.index_audit(5).await.unwrap().complete_with_wrong_count.examples, vec![asset.id().to_string()]); +} + +/// Every path that records or uses a manifest leaves the index exactly as the manifests made it: commit, update, +/// restore of a version, restore from the trash, a late save record, a live document's import and compaction. Their +/// roots are always indexed, and none of them writes an edge. +async fn the_index_stays_exact_through_every_write_path(meta: &Meta) { + use craft_cloud_proto::live::node_doc; + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let r = root(meta, &ana).await; + let clean = |label: &'static str, edges: u64| async move { + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.complete, audit.manifests, "{label}: every manifest is complete"); + assert_eq!(audit.roots_not_indexed.count + audit.roots_without_manifest.count + audit.unknown_dependencies.count, 0, "{label}"); + assert_eq!(audit.edges_without_index.count + audit.index_without_manifest.count + audit.complete_with_wrong_count.count, 0, "{label}"); + assert_eq!(meta.index_edge_count().await.unwrap(), edges, "{label}: the path wrote no edge"); + }; + let (c1, c2, c3) = (fresh_chunks(3), fresh_chunks(2), fresh_chunks(4)); + let (m1, m2, m3) = (manifest_over(meta, &ana, &c1).await, manifest_over(meta, &ana, &c2).await, manifest_over(meta, &ana, &c3).await); + for m in [&m1, &m2, &m3] { + meta.add_manifest(ana.space, m, 100, NOW).await.unwrap(); + } + let edges = 3 + 2 + 4; + clean("uploads", edges).await; + + let node = NodeId::generate(); + let v1 = version(m1.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("path.txt"), version: v1.clone() }], NOW).await.unwrap(); + clean("create", edges).await; + let v2 = version(m2.id()); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v1.id, version: v2.clone() }], NOW + 1).await.unwrap(); + clean("update", edges).await; + let restored = meta.restore_version(ana.space, dev, node, v1.id, v2.id, NOW + 2).await.unwrap(); + clean("restore of a version", edges).await; + let record = craft_cloud_proto::SaveRecord { doc: node_doc(ana.space, node), frontiers: vec![(1, 2)] }; + meta.attach_save(ana.space, node, v2.id, &record).await.unwrap(); + clean("a late save record", edges).await; + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(restored.version) }], NOW + 3).await.unwrap(); + clean("delete", edges).await; + meta.restore_from_trash(ana.space, dev, node, NOW + 4).await.unwrap(); + clean("restore from the trash", edges).await; + + // A live document of a file: its import snapshot and a compaction's snapshot are roots too. + let doc_node = file(meta, &ana, "doc.txt").await; + let doc = node_doc(ana.space, doc_node); + let first = snapshot(meta, &ana, 100).await; + let edges = meta.index_edge_count().await.unwrap(); + meta.create_live_document(ana.space, doc, "document:wordcraft", doc_node, &first.manifest, NOW).await.unwrap(); + clean("a live document's import", edges).await; + meta.append_live_update(ana.space, doc, dev, 1, &[1; 10], NOW).await.unwrap(); + let compacted = snapshot(meta, &ana, 40).await; + let edges = meta.index_edge_count().await.unwrap(); + assert!(meta.compact_live_doc(ana.space, doc, 1, &compacted.manifest).await.unwrap()); + clean("a compaction", edges).await; + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.unrooted_manifests.count, 1, "only m3 is named by nothing"); +} + +/// A build runs beside commits and uploads: manifests recorded before the index (no rows) are indexed while new files +/// are committed and new manifests recorded, and the result is exact for all of them. +async fn a_build_runs_beside_commits_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + let mut old = Vec::new(); + for i in 0..6 { + let chunks = fresh_chunks(7 + i); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + old.push((m, chunks)); + } + let builder = { + let (meta, old) = (meta.clone(), old.clone()); + tokio::spawn(async move { + for (m, _) in &old { + meta.index_complete(m).await.unwrap(); + } + }) + }; + let writer = { + let (meta, space) = (meta.clone(), ana.space); + tokio::spawn(async move { + let mut made = Vec::new(); + for i in 0..8 { + let chunks = fresh_chunks(5); + let ids: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(space, &ids, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + let m = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([i; 32]), refs).unwrap(); + meta.add_manifest(space, &m, 100, NOW).await.unwrap(); + // As the server's upload handler does, beside the builder: the part the recording left. + meta.index_complete(&m).await.unwrap(); + let node = NodeId::generate(); + let v = NewVersion { id: VersionId::generate(), manifest: m.id(), mtime_ms: NOW, exec: false, save: None }; + meta.commit(space, dev, &[Op::Create { node, parent: r, name: name(&format!("w{i}.txt")), version: v }], NOW).await.unwrap(); + made.push((m, chunks)); + } + made + }) + }; + builder.await.unwrap(); + let made = writer.await.unwrap(); + for (m, chunks) in old.iter().chain(made.iter()) { + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(chunks)); + } + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete), (14, 14)); + assert_eq!(audit.roots_not_indexed.count, 0); +} + +#[tokio::test] +async fn an_index_is_complete_only_when_every_edge_is_right_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_is_complete_only_when_every_edge_is_right_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_is_complete_only_when_every_edge_is_right_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_is_complete_only_when_every_edge_is_right_on(&url).await; +} + +#[tokio::test] +async fn an_index_build_resumes_after_a_restart_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_build_resumes_after_a_restart_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_build_resumes_after_a_restart_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_build_resumes_after_a_restart_on(&url).await; +} + +#[tokio::test] +async fn the_index_migration_leaves_old_manifests_unindexed_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_index_migration_leaves_old_manifests_unindexed_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_index_migration_leaves_old_manifests_unindexed_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_index_migration_leaves_old_manifests_unindexed_on(&url).await; +} + +#[tokio::test] +async fn the_audit_names_what_is_wrong_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_audit_names_what_is_wrong_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_audit_names_what_is_wrong_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_audit_names_what_is_wrong_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_runs_beside_commits_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_runs_beside_commits_on(&url).await; +} + +/// An edge at a negative ordinal, which pagination starting at -1 once skipped, is found by the comparison: the index +/// of such a manifest is not marked complete, however it got there, and a build throws the edge away. +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let spare = random_chunk(); + for ordinal in [-1i64, -7, i64::MIN] { + let chunks = fresh_chunks(5); + let m = manifest_over(&meta, &ana, &chunks).await; + // The stray edge is there before the manifest is recorded, and stays after all its own edges were written. + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(ordinal) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + match meta.index_check(&m).await.unwrap() { + craft_cloud_meta::EdgeCheck::Mismatch(p) => { + assert_eq!((p.ordinal, p.kind), (ordinal.unsigned_abs(), EdgeProblemKind::NegativeOrdinal), "{ordinal}") + } + other => panic!("{ordinal}: expected a mismatch, got {other:?}"), + } + assert!(matches!(meta.index_validate(&m).await.unwrap(), Validation::Mismatch(p) if p.kind == EdgeProblemKind::NegativeOrdinal), "{ordinal}"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{ordinal}: not complete"); + // The build rebuilds from the manifest, and the stray edge is gone. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 5 }, "{ordinal}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{ordinal}"); + } +} + +/// Throwing away the edges of an index that starts again takes batches of the index budget, each its own transaction, +/// and a run that stops between two of them leaves a marker the next one resumes from; nothing is written or marked +/// complete while edges of the old list are left. +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(12); + let spare = random_chunk(); + let m = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + m + }; + let mut conn = raw(url).await; + // A wrong edge, and one at a negative ordinal; the index was not marked complete. + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, -1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // The reset starts (what a build does on a mismatch) and the process stops right after: only the marker is there. + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // A new handle carries on, one batch of 3 rows per transaction, and writes nothing meanwhile. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + assert_eq!(meta.index_edge_count().await.unwrap(), 13); + let mut left = 13u64; + while left > 0 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }, "{left} edges left"); + let now = meta.index_edge_count().await.unwrap(); + assert_eq!(left - now, left.min(3), "a batch deletes at most the budget"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. })); + left = now; + } + // The last batch (fewer rows than the budget) put the row back to building, so the next stage writes edges. + assert_eq!(meta.index_edge_count().await.unwrap(), 0); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// A build that meets a wrong edge resets the index across several batches by itself, and ends exact. +async fn a_build_resets_a_wrong_index_in_batches_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + let mut conn = raw(url).await; + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 7") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 10 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// An extra edge at the highest possible ordinal, `i64::MAX`, after valid ones is reported as a missing edge at the first +/// ordinal that is not there, and the report is stable. +/// +/// This does not reach the `raw.checked_add(1)` that guards the page cursor in `compare`. An edge at `i64::MAX` is +/// returned as `MissingEdge` as soon as it is seen, because its ordinal is above the next expected one, and no +/// manifest has the 2^63 chunks that would make it the expected one, so the cursor step at `i64::MAX` cannot be +/// reached through the public meta API. The test pins that a high-ordinal edge is rejected without an error or an +/// overflow, not the checked addition. +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(url: &str) { + use craft_cloud_meta::{EdgeCheck, EdgeProblemKind}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(i64::MAX) + .bind(random_chunk().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + let expected = EdgeCheck::Mismatch(craft_cloud_meta::EdgeProblem { ordinal: 3, kind: EdgeProblemKind::MissingEdge }); + assert_eq!(meta.index_check(&m).await.unwrap(), expected); + assert_eq!(meta.index_check(&m).await.unwrap(), expected, "the report is stable"); + let edges = meta.index_edges(&m.id()).await.unwrap(); + assert_eq!(edges.len(), 4); + assert_eq!(edges.last().unwrap().0, i64::MAX as u64, "the stored edge at the maximum ordinal is still there"); +} + +/// A reset of 12 edges at budget 3 takes four full batches and one empty batch before returning to building state 0. +async fn an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let before = meta.index_edge_count().await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + for batch in 1..=4 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap(), before - batch * 3); + } + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 2, "four full batches leave reset state 2 until an empty batch proves completion"); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 0); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }, "the next call writes instead of looping"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// Two independent Postgres handles finish one damaged manifest reset without duplicate or missing ordinals. +async fn concurrent_index_completes_reset_one_manifest_on(url: &str) { + use craft_cloud_meta::IndexState; + let left = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let right = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&left, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&left, &ana, &chunks).await; + left.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + left.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + // Force the overlap: a third connection holds the row lock of the manifest's index row, both builders start and are + // proven to wait on it, and only then is it released, so both reach the reset together. + let mut holder = raw(url).await; + sqlx::query("BEGIN").execute(&mut holder).await.unwrap(); + sqlx::query("SELECT 1 FROM manifest_index WHERE manifest = $1 FOR UPDATE").bind(m.id().as_bytes().to_vec()).execute(&mut holder).await.unwrap(); + + let left_manifest = m.clone(); + let right_manifest = m.clone(); + let left_builder = left.clone(); + let mut first = tokio::spawn(async move { left_builder.index_complete(&left_manifest).await }); + let mut second = tokio::spawn(async move { right.index_complete(&right_manifest).await }); + let mut waiting = 0i64; + for _ in 0..300 { + waiting = sqlx::query_scalar( + "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock' AND query LIKE '%manifest_index%FOR UPDATE%'", + ) + .fetch_one(&mut conn) + .await + .unwrap(); + if waiting >= 2 || first.is_finished() || second.is_finished() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + let overlapped = waiting == 2 && !first.is_finished() && !second.is_finished(); + sqlx::query("ROLLBACK").execute(&mut holder).await.unwrap(); + if !overlapped { + first.abort(); + second.abort(); + } + assert!(overlapped, "both builders must be blocked on the manifest's index row at once (waiting backends: {waiting})"); + let joined = tokio::time::timeout(std::time::Duration::from_secs(30), async { + (&mut first).await.unwrap().unwrap(); + (&mut second).await.unwrap().unwrap(); + }) + .await; + if joined.is_err() { + first.abort(); + second.abort(); + } + assert!(joined.is_ok(), "concurrent index completion must finish within 30 seconds"); + + assert_eq!(left.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(left.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + let (edges, ordinals): (i64, i64) = sqlx::query_as("SELECT COUNT(*), COUNT(DISTINCT ordinal) FROM manifest_chunks WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!((edges, ordinals), (12, 12)); +} + +#[tokio::test] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&url).await; +} + +#[tokio::test] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_reset_of_an_index_goes_in_batches_and_resumes_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_reset_of_an_index_goes_in_batches_and_resumes_on(&url).await; +} + +#[tokio::test] +async fn a_build_resets_a_wrong_index_in_batches_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_resets_a_wrong_index_in_batches_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_resets_a_wrong_index_in_batches_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_resets_a_wrong_index_in_batches_on(&url).await; +} + +#[tokio::test] +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&url).await; +} + +#[tokio::test] +async fn an_exact_multiple_index_reset_finishes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_exact_multiple_index_reset_finishes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_index_completes_reset_one_manifest_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + concurrent_index_completes_reset_one_manifest_on(&url).await; } diff --git a/crates/meta/tests/team_spaces.rs b/crates/meta/tests/team_spaces.rs new file mode 100644 index 00000000..27bd8261 --- /dev/null +++ b/crates/meta/tests/team_spaces.rs @@ -0,0 +1,1003 @@ +//! Team spaces (migration 0015): the migration from a filled 0014 database, the table constraints, who has which role +//! where, the member changes with their audit events, and the last-owner rule under concurrent changes. SQLite always +//! runs; Postgres runs when `CRAFT_CLOUD_TEST_PG` holds an admin URL (`CRAFT_CLOUD_REQUIRE_BACKENDS` makes a missing one a +//! failure), as in `behaviour.rs`. + +use craft_cloud_meta::{ + ActionFilter, AdminActor, AdminVia, AuditEvent, AuditQuery, Meta, MetaError, Role, SpaceActor, UserId, UserRecord, WriteGuard, +}; +use craft_cloud_proto::api::SpaceRole; +use craft_cloud_proto::{DeviceId, SpaceId, SpaceName}; +use sqlx::{AnyPool, Row}; + +const NOW: i64 = 1_700_000_000_000; +const ADMIN_HASH: &str = "$argon2id$admin"; + +struct Db { + meta: Meta, + url: String, + postgres: bool, + _guard: Guard, +} + +enum Guard { + Sqlite(#[allow(dead_code)] tempfile::TempDir), + Postgres { admin_url: String, name: String }, +} + +impl Drop for Guard { + fn drop(&mut self) { + if let Guard::Postgres { admin_url, name } = self { + let (admin_url, name) = (admin_url.clone(), name.clone()); + // Drop the test database from a separate runtime; the test's runtime may be gone. + let _ = std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + rt.block_on(async { + use sqlx::Connection; + if let Ok(mut conn) = sqlx::PgConnection::connect(&admin_url).await { + let _ = sqlx::query(sqlx::AssertSqlSafe(format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)"))).execute(&mut conn).await; + } + }); + }) + .join(); + } + } +} + +/// A fresh, empty SQLite database: its URL and the guard that removes it. +fn sqlite_database() -> (String, Guard) { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}?mode=rwc", dir.path().join("meta.db").display()); + (url, Guard::Sqlite(dir)) +} + +/// A fresh, empty Postgres database: its URL, and the guard that drops it. +async fn postgres_database() -> Option<(String, Guard)> { + let Ok(admin_url) = std::env::var("CRAFT_CLOUD_TEST_PG") else { + assert!(std::env::var_os("CRAFT_CLOUD_REQUIRE_BACKENDS").is_none(), "CRAFT_CLOUD_REQUIRE_BACKENDS is set but CRAFT_CLOUD_TEST_PG is not"); + return None; + }; + use sqlx::Connection; + let name = format!("cc_test_{}", uuid::Uuid::now_v7().simple()); + let mut conn = sqlx::PgConnection::connect(&admin_url).await.unwrap(); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))).execute(&mut conn).await.unwrap(); + let (base, _) = admin_url.rsplit_once('/').unwrap(); + Some((format!("{base}/{name}"), Guard::Postgres { admin_url, name })) +} + +async fn sqlite() -> Db { + let (url, guard) = sqlite_database(); + Db { meta: Meta::open(&url).await.unwrap(), url, postgres: false, _guard: guard } +} + +async fn postgres() -> Option { + let (url, guard) = postgres_database().await?; + Some(Db { meta: Meta::open(&url).await.unwrap(), url, postgres: true, _guard: guard }) +} + +/// A second, raw connection pool on the database, for what the store's API does not offer: setting columns, breaking +/// rules on purpose. +async fn raw(url: &str) -> AnyPool { + sqlx::any::install_default_drivers(); + sqlx::any::AnyPoolOptions::new().max_connections(4).connect(url).await.unwrap() +} + +async fn run(pool: &AnyPool, sql: String) -> Result<(), sqlx::Error> { + sqlx::query(sqlx::AssertSqlSafe(sql)).execute(pool).await.map(|_| ()) +} + +macro_rules! on_every_backend { + ($($scenario:ident),* $(,)?) => { + mod on_sqlite { + $( #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn $scenario() { let db = super::sqlite().await; super::$scenario(&db).await; } )* + } + mod on_postgres { + $( #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn $scenario() { + let Some(db) = super::postgres().await else { eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); return }; + super::$scenario(&db).await; + } )* + } + }; +} + +on_every_backend!( + the_table_constraints_hold, + the_access_matrix, + members_come_and_go_with_their_events, + the_command_line_and_admins_create_and_manage_spaces, + the_last_owner_stays, + review_fixes_of_the_routes_hold, + an_owner_who_cannot_sign_in_is_not_counted, + concurrent_changes_never_leave_a_space_without_an_owner, + guarded_writes_check_the_role_in_their_own_transaction, + concurrent_guarded_uploads_do_not_deadlock, +); + +// ---- helpers ---- + +/// The review of the routes (P1-3): a role too low inside the transaction is `ReadOnly`, an invitation to an archived space +/// is no invitation, the command line accepts a pending invitation, and the administrator's search is not a name. +async fn review_fixes_of_the_routes_hold(db: &Db) { + let meta = &db.meta; + let (ana, ben, cy) = (user(meta, "ana").await, user(meta, "ben").await, user(meta, "cy").await); + let space = team(meta, "Studio", &ana).await; + join(meta, space, &ana, &ben, SpaceRole::Editor).await; + + // A role that is too low: an editor and an owner of an archived space (capped at viewer) are `ReadOnly`, not + // `Forbidden`; an account with no access at all still learns nothing. + assert!(matches!(meta.rename_team_space(&by(&ben), space, &team_name("Other"), NOW + 5).await, Err(MetaError::ReadOnly))); + meta.set_space_archived(&by(&ana), space, true, NOW + 6).await.unwrap(); + assert!(matches!(meta.rename_team_space(&by(&ana), space, &team_name("Other"), NOW + 7).await, Err(MetaError::ReadOnly))); + assert!(matches!(meta.rename_team_space(&by(&cy), space, &team_name("Other"), NOW + 7).await, Err(MetaError::NotFound("space")))); + meta.set_space_archived(&by(&ana), space, false, NOW + 8).await.unwrap(); + + // An invitation to an archived space can be neither accepted nor declined (it is not listed either): no such space. + meta.invite_member(&by(&ana), space, cy.id, SpaceRole::Viewer, NOW + 9).await.unwrap(); + meta.set_space_archived(&by(&ana), space, true, NOW + 10).await.unwrap(); + assert!(matches!(meta.accept_invitation(cy.id, None, space, NOW + 11).await, Err(MetaError::NotFound("space")))); + assert!(matches!(meta.decline_invitation(cy.id, None, space, NOW + 11).await, Err(MetaError::NotFound("space")))); + assert_eq!(meta.space_access(cy.id, space).await.unwrap(), None, "nothing was accepted"); + meta.set_space_archived(&by(&ana), space, false, NOW + 12).await.unwrap(); + + // The command line sets a role and accepts a pending invitation in one step (whoever runs it has the files). + let pending = meta.list_members(space).await.unwrap().into_iter().find(|m| m.user == cy.id).unwrap(); + assert!(!pending.accepted()); + let done = meta.change_member_role(&SpaceActor::Cli, space, cy.id, SpaceRole::Viewer, NOW + 13).await.unwrap(); + assert!(done.accepted()); + assert_eq!(meta.space_access(cy.id, space).await.unwrap(), Some(SpaceRole::Viewer)); + // Again: nothing to change, nothing written. + let before = space_events(meta).await.len(); + meta.change_member_role(&SpaceActor::Cli, space, cy.id, SpaceRole::Viewer, NOW + 14).await.unwrap(); + assert_eq!(space_events(meta).await.len(), before); + + // The administrator's search is folded text, not a name: one letter and padded text work. + assert_eq!(meta.admin_list_spaces(None, Some(" STU "), 10).await.unwrap().spaces.len(), 1); + assert_eq!(meta.admin_list_spaces(None, Some("u"), 10).await.unwrap().spaces.len(), 1); + assert!(meta.admin_list_spaces(None, Some("0194a2d0-1c3e-7b5a-9d41-2f6e8a7b3c10"), 10).await.unwrap().spaces.is_empty()); + assert!(matches!(meta.admin_list_spaces(None, Some(&"x".repeat(65)), 10).await, Err(MetaError::Invalid(_)))); + // A name shaped like a space ID is refused, so a space is never named like another. + assert!(SpaceName::new("0194a2d0-1c3e-7b5a-9d41-2f6e8a7b3c10").is_err()); +} + +/// An owner whose account is disabled or marked for removal has no access, so the administrator's list does not count it +/// (the space may have nobody left to manage it), and the owners of a page come in one query with their account's state. +async fn an_owner_who_cannot_sign_in_is_not_counted(db: &Db) { + let meta = &db.meta; + let (boss, actor) = admin_actor(meta, "boss").await; + let _ = boss; + let (ana, ben, cy) = (user(meta, "ana").await, user(meta, "ben").await, user(meta, "cy").await); + let (solo, pair, empty) = (team(meta, "Solo", &ana).await, team(meta, "Pair", &ben).await, team(meta, "Empty", &cy).await); + meta.set_owner_accepted(&actor, pair, ana.id, NOW + 1).await.unwrap(); + let owners = |page: Vec| page.into_iter().map(|s| (s.name, s.owners)).collect::>(); + let all = || async { owners(meta.admin_list_spaces(None, None, 10).await.unwrap().spaces) }; + assert_eq!(all().await, [("Empty".to_owned(), 1), ("Pair".to_owned(), 2), ("Solo".to_owned(), 1)]); + + meta.admin_disable_user(&actor, ana.id, NOW + 2).await.unwrap(); + meta.admin_remove_user(&actor, cy.id, NOW + 3).await.unwrap(); + assert_eq!(all().await, [("Empty".to_owned(), 0), ("Pair".to_owned(), 1), ("Solo".to_owned(), 0)]); + assert_eq!(meta.admin_space(solo).await.unwrap().owners, 0); + assert_eq!(meta.admin_space(pair).await.unwrap().members, 2, "the member is still a member"); + + // The owners of the listed spaces: one query, every accepted owner with the state of the account, by space and name. + let found = meta.space_owners(&[solo, pair, empty]).await.unwrap(); + let mut seen: Vec<_> = found.iter().map(|o| (o.space, o.username.as_str(), o.active())).collect(); + seen.sort_by_key(|(space, name, _)| (space.to_string(), *name)); + let mut expected = vec![(solo, "ana", false), (pair, "ana", false), (pair, "ben", true), (empty, "cy", false)]; + expected.sort_by_key(|(space, name, _)| (space.to_string(), *name)); + assert_eq!(seen, expected); + assert!(found.iter().any(|o| o.username == "ana" && o.disabled_ms.is_some() && o.removed_ms.is_none())); + assert!(found.iter().any(|o| o.username == "cy" && o.removed_ms.is_some())); + assert!(meta.space_owners(&[]).await.unwrap().is_empty()); + assert!(meta.space_owners(&[pair]).await.unwrap().iter().all(|o| o.space == pair)); + // An invited owner is no owner yet. + meta.invite_member(&by(&ben), pair, user(meta, "dee").await.id, SpaceRole::Owner, NOW + 4).await.unwrap(); + assert_eq!(meta.space_owners(&[pair]).await.unwrap().len(), 2); + + // Naming a new owner brings the space back. + meta.set_owner_accepted(&actor, solo, ben.id, NOW + 5).await.unwrap(); + assert_eq!(meta.admin_space(solo).await.unwrap().owners, 1); +} + +async fn user(meta: &Meta, name: &str) -> UserRecord { + meta.create_user(name, "$argon2id$stub", Role::Member, NOW).await.unwrap() +} + +async fn device(meta: &Meta, u: &UserRecord) -> DeviceId { + let token = *blake3::hash(uuid::Uuid::now_v7().as_bytes()).as_bytes(); + meta.create_device(u.id, "laptop", &token, NOW).await.unwrap() +} + +async fn admin_actor(meta: &Meta, name: &str) -> (UserRecord, AdminActor) { + let u = meta.create_user(name, ADMIN_HASH, Role::Admin, NOW).await.unwrap(); + let device = device(meta, &u).await; + (u.clone(), AdminActor { user: u.id, via: AdminVia::Device(device), password_hash: ADMIN_HASH.to_owned(), two_factor: false }) +} + +fn team_name(name: &str) -> SpaceName { + SpaceName::new(name).unwrap() +} + +fn by(u: &UserRecord) -> SpaceActor { + SpaceActor::User { user: u.id, device: None } +} + +/// A team space made on the command line with `owner` as its first owner. +async fn team(meta: &Meta, name: &str, owner: &UserRecord) -> SpaceId { + meta.create_team_space(&SpaceActor::Cli, &team_name(name), owner.id, None, NOW).await.unwrap() +} + +/// `u` invited by the owner `by_owner` with `role`, and accepting. +async fn join(meta: &Meta, space: SpaceId, by_owner: &UserRecord, u: &UserRecord, role: SpaceRole) { + meta.invite_member(&by(by_owner), space, u.id, role, NOW + 1).await.unwrap(); + assert_eq!(meta.accept_invitation(u.id, None, space, NOW + 2).await.unwrap(), role); +} + +async fn space_events(meta: &Meta) -> Vec { + let query = AuditQuery { action: ActionFilter::parse("space."), limit: 1000, ..AuditQuery::default() }; + let mut events = meta.audit_list(&query).await.unwrap(); + let cli = AuditQuery { action: ActionFilter::parse("cli."), limit: 1000, ..AuditQuery::default() }; + events.extend(meta.audit_list(&cli).await.unwrap().into_iter().filter(|e| e.action.as_str().starts_with("cli.space_"))); + events.sort_by_key(|e| (e.at_ms, e.id)); + events +} + +fn names(events: &[AuditEvent]) -> Vec<&'static str> { + events.iter().map(|e| e.action.as_str()).collect() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A blob literal of `len` bytes made of `tag` and `n`, in the backend's syntax. +fn blob(pg: bool, tag: u8, n: u8, len: usize) -> String { + let mut bytes = vec![0u8; len]; + bytes[0] = tag; + bytes[1] = n; + if pg { format!("'\\x{}'::bytea", hex(&bytes)) } else { format!("X'{}'", hex(&bytes)) } +} + +// ---- the migration ---- + +const SQLITE_0014: [&str; 14] = [ + include_str!("../migrations/sqlite/0001_init.sql"), + include_str!("../migrations/sqlite/0002_stored_bytes.sql"), + include_str!("../migrations/sqlite/0003_live.sql"), + include_str!("../migrations/sqlite/0004_join.sql"), + include_str!("../migrations/sqlite/0005_live_documents.sql"), + include_str!("../migrations/sqlite/0006_save_records.sql"), + include_str!("../migrations/sqlite/0007_devices_v2.sql"), + include_str!("../migrations/sqlite/0008_admin_resets.sql"), + include_str!("../migrations/sqlite/0009_audit_events.sql"), + include_str!("../migrations/sqlite/0010_two_factor.sql"), + include_str!("../migrations/sqlite/0011_web_sessions.sql"), + include_str!("../migrations/sqlite/0012_versions_node_seq.sql"), + include_str!("../migrations/sqlite/0013_trash.sql"), + include_str!("../migrations/sqlite/0014_manifest_index.sql"), +]; + +const POSTGRES_0014: [&str; 14] = [ + include_str!("../migrations/postgres/0001_init.sql"), + include_str!("../migrations/postgres/0002_stored_bytes.sql"), + include_str!("../migrations/postgres/0003_live.sql"), + include_str!("../migrations/postgres/0004_join.sql"), + include_str!("../migrations/postgres/0005_live_documents.sql"), + include_str!("../migrations/postgres/0006_save_records.sql"), + include_str!("../migrations/postgres/0007_devices_v2.sql"), + include_str!("../migrations/postgres/0008_admin_resets.sql"), + include_str!("../migrations/postgres/0009_audit_events.sql"), + include_str!("../migrations/postgres/0010_two_factor.sql"), + include_str!("../migrations/postgres/0011_web_sessions.sql"), + include_str!("../migrations/postgres/0012_versions_node_seq.sql"), + include_str!("../migrations/postgres/0013_trash.sql"), + include_str!("../migrations/postgres/0014_manifest_index.sql"), +]; + +/// A database at schema version 14 with rows in every kind of table that holds user data: two accounts with devices, +/// their spaces with a folder, a file with a version, a deleted file, blobs, live documents with updates, an audit event, +/// an invite and the manifest index. +async fn database_at_0014(url: &str, pg: bool) -> AnyPool { + let pool = raw(url).await; + run(&pool, "CREATE TABLE IF NOT EXISTS schema_version (version BIGINT NOT NULL)".to_owned()).await.unwrap(); + for (index, sql) in (if pg { POSTGRES_0014 } else { SQLITE_0014 }).iter().enumerate() { + for statement in sql.split(';').map(str::trim).filter(|s| !s.is_empty()) { + run(&pool, statement.to_owned()).await.unwrap(); + } + run(&pool, format!("INSERT INTO schema_version (version) VALUES ({})", index + 1)).await.unwrap(); + } + let id = |tag: u8, n: u8| blob(pg, tag, n, 16); + let hash = |tag: u8, n: u8| blob(pg, tag, n, 32); + let (ana, ben) = (id(1, 1), id(1, 2)); + let (space_a, space_b) = (id(2, 1), id(2, 2)); + let sql = vec![ + format!( + "INSERT INTO users (id, username, password_hash, role, created_ms) VALUES ({ana}, 'ana', 'h1', 'admin', 111), ({ben}, 'ben', 'h2', 'member', 222)" + ), + format!( + "INSERT INTO spaces (id, owner, epoch, seq, oldest_seq, root, stored_bytes, live_bytes, quota_bytes) VALUES \ + ({space_a}, {ana}, {e1}, 7, 2, {r1}, 12345, 77, 99999999), ({space_b}, {ben}, {e2}, 0, 0, {r2}, 0, 0, NULL)", + e1 = id(3, 1), + r1 = id(4, 1), + e2 = id(3, 2), + r2 = id(4, 2) + ), + format!( + "INSERT INTO devices (id, user_id, name, token_hash, created_ms, last_used_ms, revoked) VALUES ({d1}, {ana}, 'laptop', {t1}, 1, 2, 0), ({d2}, {ben}, 'pc', {t2}, 3, 4, 1)", + d1 = id(5, 1), + t1 = hash(6, 1), + d2 = id(5, 2), + t2 = hash(6, 2) + ), + format!( + "INSERT INTO nodes (space_id, id, parent, name, fold_key, is_dir, version, deleted, changed_seq, deleted_ms) VALUES \ + ({space_a}, {root}, NULL, '', '', 1, NULL, 0, 0, NULL), \ + ({space_a}, {dir}, {root}, 'Docs', 'docs', 1, NULL, 0, 3, NULL), \ + ({space_a}, {file}, {dir}, 'a.txt', 'a.txt', 0, {v1}, 0, 5, NULL), \ + ({space_a}, {gone}, {root}, 'old.txt', 'old.txt', 0, {v2}, 1, 6, 1234), \ + ({space_b}, {root_b}, NULL, '', '', 1, NULL, 0, 0, NULL)", + root = id(4, 1), + dir = id(7, 1), + file = id(7, 2), + gone = id(7, 3), + v1 = id(8, 1), + v2 = id(8, 2), + root_b = id(4, 2) + ), + format!( + "INSERT INTO versions (space_id, id, node, parent_version, epoch, manifest, size, mtime_ms, exec, device, created_ms, seq) VALUES \ + ({space_a}, {v1}, {file}, NULL, {e1}, {m1}, 10, 100, 0, {d1}, 5, 5), \ + ({space_a}, {v2}, {gone}, NULL, {e1}, {m2}, 20, 200, 1, {d1}, 6, 6)", + v1 = id(8, 1), + v2 = id(8, 2), + file = id(7, 2), + gone = id(7, 3), + e1 = id(3, 1), + m1 = hash(9, 1), + m2 = hash(9, 2), + d1 = id(5, 1) + ), + format!( + "INSERT INTO blobs (kind, id, stored_size, file_size, created_ms) VALUES (0, {c1}, 100, 90, 1), (1, {m1}, 50, NULL, 2)", + c1 = hash(10, 1), + m1 = hash(9, 1) + ), + format!( + "INSERT INTO space_blobs (space_id, kind, id) VALUES ({space_a}, 0, {c1}), ({space_a}, 1, {m1}), ({space_b}, 0, {c1})", + c1 = hash(10, 1), + m1 = hash(9, 1) + ), + format!( + "INSERT INTO live_docs (space_id, id, kind, node, created_ms, head_seq, bytes) VALUES ({space_a}, {doc}, 'document', {file}, 9, 2, 33)", + doc = id(11, 1), + file = id(7, 2) + ), + format!( + "INSERT INTO live_updates (space_id, doc, seq, bytes, device, client_seq, created_ms) VALUES ({space_a}, {doc}, 1, {u1}, {d1}, 1, 10), ({space_a}, {doc}, 2, {u2}, {d1}, 2, 11)", + doc = id(11, 1), + u1 = id(12, 1), + u2 = id(12, 2), + d1 = id(5, 1) + ), + format!("INSERT INTO live_spaces (space_id, generation) VALUES ({space_a}, 4)"), + format!("INSERT INTO audit_events (id, at_ms, action, outcome) VALUES ({a1}, 5, 'auth.login', 'success')", a1 = id(13, 1)), + format!("INSERT INTO invites (user_id, code_hash, expires_ms) VALUES ({ben}, {c}, 99)", c = hash(14, 1)), + format!("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ({m1}, 0, {c1})", m1 = hash(9, 1), c1 = hash(10, 1)), + format!("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ({m1}, 1, 1, 2)", m1 = hash(9, 1)), + ]; + for statement in sql { + run(&pool, statement.clone()).await.unwrap_or_else(|e| panic!("{e}: {statement}")); + } + pool +} + +/// Every column of a row as text, `NULL` for none. +fn cells(row: &sqlx::any::AnyRow) -> String { + use sqlx::Row; + (0..row.columns().len()) + .map(|i| { + if let Ok(v) = row.try_get::, _>(i) { + return v.map_or("NULL".to_owned(), |v| v.to_string()); + } + if let Ok(v) = row.try_get::, _>(i) { + return v.map_or("NULL".to_owned(), |v| v.to_string()); + } + if let Ok(v) = row.try_get::, _>(i) { + return v.map_or("NULL".to_owned(), |v| v.to_string()); + } + if let Ok(v) = row.try_get::, _>(i) { + return v.map_or("NULL".to_owned(), |v| format!("{v:?}")); + } + if let Ok(v) = row.try_get::>, _>(i) { + return v.map_or("NULL".to_owned(), |v| hex(&v)); + } + panic!("a column of a type the test cannot read"); + }) + .collect::>() + .join("|") +} + +/// Every row of every table the migration must not touch, as text, rows sorted: equal before and after means nothing +/// else changed. +async fn fingerprint(pool: &AnyPool, pg: bool) -> Vec<(String, Vec)> { + let list = if pg { + "SELECT CAST(table_name AS TEXT) AS name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'" + } else { + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + }; + let mut tables: Vec = sqlx::query(list).fetch_all(pool).await.unwrap().iter().map(|r| r.try_get::("name").unwrap()).collect(); + tables.retain(|t| !["spaces", "schema_version", "space_members"].contains(&t.as_str())); + tables.sort(); + let mut out = Vec::new(); + for table in tables { + let rows = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT * FROM {table}"))).fetch_all(pool).await.unwrap(); + let mut rows: Vec = rows.iter().map(cells).collect(); + rows.sort(); + out.push((table, rows)); + } + out +} + +async fn old_space_columns(pool: &AnyPool) -> Vec { + let rows = sqlx::query("SELECT id, owner, epoch, seq, oldest_seq, root, stored_bytes, live_bytes, quota_bytes FROM spaces ORDER BY id") + .fetch_all(pool) + .await + .unwrap(); + rows.iter().map(cells).collect() +} + +async fn count(pool: &AnyPool, sql: &str) -> i64 { + sqlx::query(sqlx::AssertSqlSafe(sql.to_owned())).fetch_one(pool).await.unwrap().try_get::(0).unwrap() +} + +async fn the_migration_from_a_filled_0014_database(url: &str, pg: bool) { + let pool = database_at_0014(url, pg).await; + let fingerprint_before = fingerprint(&pool, pg).await; + assert!(fingerprint_before.iter().any(|(t, rows)| t == "versions" && rows.len() == 2)); + for filled in [ + "users", + "devices", + "nodes", + "versions", + "blobs", + "space_blobs", + "live_docs", + "live_updates", + "live_spaces", + "audit_events", + "invites", + "manifest_chunks", + ] { + assert!(fingerprint_before.iter().any(|(t, rows)| t == filled && !rows.is_empty()), "{filled} is empty"); + } + let spaces_before = old_space_columns(&pool).await; + assert_eq!(spaces_before.len(), 2); + pool.close().await; + + let meta = Meta::open(url).await.unwrap(); + let pool = raw(url).await; + assert_eq!(count(&pool, "SELECT MAX(version) FROM schema_version").await, 15); + assert_eq!(fingerprint(&pool, pg).await, fingerprint_before, "a table other than spaces changed"); + assert_eq!(old_space_columns(&pool).await, spaces_before, "a column of spaces that existed before changed"); + + // Every space is a personal one with its owner and its owner's creation time; the new columns hold nothing else. + assert_eq!( + count( + &pool, + "SELECT COUNT(*) FROM spaces WHERE kind = 'personal' AND owner IS NOT NULL AND name IS NULL AND name_fold IS NULL AND archived_ms IS NULL" + ) + .await, + 2 + ); + assert_eq!(count(&pool, "SELECT COUNT(*) FROM spaces s JOIN users u ON u.id = s.owner WHERE s.created_ms = u.created_ms").await, 2); + assert_eq!(count(&pool, "SELECT COUNT(*) FROM space_members").await, 0); + + // The foreign keys are intact. + if !pg { + let broken = sqlx::query("PRAGMA foreign_key_check").fetch_all(&pool).await.unwrap(); + assert!(broken.is_empty(), "foreign_key_check found {} rows", broken.len()); + } + for table in ["nodes", "versions", "space_blobs", "live_docs", "live_spaces"] { + assert_eq!(count(&pool, &format!("SELECT COUNT(*) FROM {table} WHERE space_id NOT IN (SELECT id FROM spaces)")).await, 0, "{table}"); + } + // ... and enforced: a node of a space that does not exist is refused. + let orphan = run(&pool, format!("INSERT INTO nodes (space_id, id, parent, name, fold_key, is_dir, version, deleted, changed_seq) VALUES ({}, {}, NULL, '', '', 1, NULL, 0, 0)", blob(pg, 99, 99, 16), blob(pg, 98, 98, 16))).await; + assert!(orphan.is_err(), "an orphan node was accepted"); + + // The old accounts and spaces work through the store as before. + let ana = meta.user_by_name("ana").await.unwrap().unwrap(); + assert_eq!(meta.space_access(ana.id, ana.space).await.unwrap(), Some(SpaceRole::Owner)); + assert_eq!(meta.space_info(ana.space).await.unwrap().seq, 7); + let ben = meta.user_by_name("ben").await.unwrap().unwrap(); + assert_eq!(meta.space_access(ben.id, ana.space).await.unwrap(), None); + // ... and a team space can be made beside them. + let space = meta.create_team_space(&SpaceActor::Cli, &team_name("Studio"), ana.id, None, NOW).await.unwrap(); + assert_eq!(meta.space_access(ana.id, space).await.unwrap(), Some(SpaceRole::Owner)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_migration_from_a_filled_0014_database_on_sqlite() { + let (url, _guard) = sqlite_database(); + the_migration_from_a_filled_0014_database(&url, false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_migration_from_a_filled_0014_database_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_migration_from_a_filled_0014_database(&url, true).await; +} + +// ---- constraints ---- + +/// The table's own rules, tried with raw SQL: what the checks and indexes of 0015 refuse and what they allow. +async fn the_table_constraints_hold(db: &Db) { + let pg = db.postgres; + let meta = &db.meta; + let pool = raw(&db.url).await; + let ana = user(meta, "ana").await; + let id = |n: u8| blob(pg, 50, n, 16); + let space = |n: u8, owner: &str, kind: &str, name: &str, fold: &str| { + format!( + "INSERT INTO spaces (id, owner, epoch, seq, oldest_seq, root, kind, name, name_fold) VALUES ({}, {owner}, {}, 0, 0, {}, '{kind}', {name}, {fold})", + id(n), + id(n + 100), + id(n + 50) + ) + }; + let owner = |u: &UserRecord| blob_of(pg, &u.id.0); + // A personal space needs its owner, and a team space must not have one. + assert!(run(&pool, space(1, "NULL", "personal", "NULL", "NULL")).await.is_err(), "a personal space without an owner"); + assert!(run(&pool, space(2, &owner(&ana), "team", "'T'", "'t'")).await.is_err(), "a team space with an owner"); + // A team space needs both names, and a personal one has none. + assert!(run(&pool, space(3, "NULL", "team", "NULL", "NULL")).await.is_err(), "a team space without a name"); + assert!(run(&pool, space(4, "NULL", "team", "'T'", "NULL")).await.is_err(), "a team space without a folded name"); + // The same rules on a space that exists: ana's personal space cannot gain a name, lose its owner or become a team space. + let hers = blob_of(pg, ana.space.as_uuid()); + assert!(run(&pool, format!("UPDATE spaces SET name = 'T', name_fold = 't' WHERE id = {hers}")).await.is_err(), "a personal space with a name"); + assert!(run(&pool, format!("UPDATE spaces SET owner = NULL WHERE id = {hers}")).await.is_err(), "a personal space losing its owner"); + assert!(run(&pool, format!("UPDATE spaces SET kind = 'team' WHERE id = {hers}")).await.is_err(), "a personal space turned into a team space"); + assert!(run(&pool, space(6, "NULL", "shared", "'T'", "'t'")).await.is_err(), "an unknown kind"); + // A second personal space for one account is refused (ana has hers from sign-up). + assert!(run(&pool, space(7, &owner(&ana), "personal", "NULL", "NULL")).await.is_err(), "a second personal space"); + // Team spaces: two with one folded name are refused, different ones are fine, and they have no owner column value. + run(&pool, space(8, "NULL", "team", "'Studio'", "'studio'")).await.unwrap(); + assert!(run(&pool, space(9, "NULL", "team", "'STUDIO'", "'studio'")).await.is_err(), "a second team space of one folded name"); + run(&pool, space(10, "NULL", "team", "'Other'", "'other'")).await.unwrap(); + // `space_members`: the role is a closed list, a member appears once, and both ends must exist. + let member = |space: u8, user: &str, role: &str| { + format!("INSERT INTO space_members (space_id, user_id, role, invited_ms) VALUES ({}, {user}, '{role}', 1)", id(space)) + }; + assert!(run(&pool, member(8, &owner(&ana), "admin")).await.is_err(), "an unknown role"); + run(&pool, member(8, &owner(&ana), "editor")).await.unwrap(); + assert!(run(&pool, member(8, &owner(&ana), "viewer")).await.is_err(), "a second row for one member"); + assert!(run(&pool, member(11, &owner(&ana), "viewer")).await.is_err(), "a member of a space that does not exist"); + assert!(run(&pool, member(8, &blob(pg, 77, 77, 16), "viewer")).await.is_err(), "a member that is no account"); + // A space with members cannot be deleted from under them. + assert!(run(&pool, format!("DELETE FROM spaces WHERE id = {}", id(8))).await.is_err(), "a space with members was deleted"); +} + +fn blob_of(pg: bool, id: &uuid::Uuid) -> String { + let hex = hex(id.as_bytes()); + if pg { format!("'\\x{hex}'::bytea") } else { format!("X'{hex}'") } +} + +// ---- access ---- + +/// `space_access` for every kind of account in every kind of space. +async fn the_access_matrix(db: &Db) { + let meta = &db.meta; + let pool = raw(&db.url).await; + let pg = db.postgres; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let cy = user(meta, "cy").await; + let dee = user(meta, "dee").await; + let eve = user(meta, "eve").await; + let fay = user(meta, "fay").await; + let space = team(meta, "Studio", &ana).await; + join(meta, space, &ana, &ben, SpaceRole::Viewer).await; + join(meta, space, &ana, &cy, SpaceRole::Editor).await; + join(meta, space, &ana, &dee, SpaceRole::Owner).await; + // fay is invited and has not answered. + meta.invite_member(&by(&ana), space, fay.id, SpaceRole::Editor, NOW + 3).await.unwrap(); + let access = |u: &UserRecord, s: SpaceId| { + let meta = meta.clone(); + let id = u.id; + async move { meta.space_access(id, s).await.unwrap() } + }; + + // The team space, by role. + assert_eq!(access(&ana, space).await, Some(SpaceRole::Owner)); + assert_eq!(access(&ben, space).await, Some(SpaceRole::Viewer)); + assert_eq!(access(&cy, space).await, Some(SpaceRole::Editor)); + assert_eq!(access(&dee, space).await, Some(SpaceRole::Owner)); + assert_eq!(access(&fay, space).await, None, "a pending invitation gives no access"); + assert_eq!(access(&eve, space).await, None, "a stranger"); + + // Personal spaces belong to their owners and nobody else, whatever team spaces exist. + assert_eq!(access(&ana, ana.space).await, Some(SpaceRole::Owner)); + assert_eq!(access(&ben, ana.space).await, None); + assert_eq!(access(&eve, ben.space).await, None); + assert_eq!(access(&ben, ben.space).await, Some(SpaceRole::Owner)); + + // A space that does not exist is the same answer as no access, and so is an account that does not exist. + assert_eq!(meta.space_access(ana.id, SpaceId::generate()).await.unwrap(), None); + assert_eq!(meta.space_access(UserId(uuid::Uuid::now_v7()), space).await.unwrap(), None); + + // An archived team space gives at most the viewer role; a personal space is never archived. + run(&pool, format!("UPDATE spaces SET archived_ms = 5 WHERE id = {}", blob_of(pg, space.as_uuid()))).await.unwrap(); + assert_eq!(access(&ana, space).await, Some(SpaceRole::Viewer)); + assert_eq!(access(&ben, space).await, Some(SpaceRole::Viewer)); + assert_eq!(access(&cy, space).await, Some(SpaceRole::Viewer)); + assert_eq!(access(&fay, space).await, None); + assert_eq!(access(&eve, space).await, None); + assert_eq!(access(&ana, ana.space).await, Some(SpaceRole::Owner)); + run(&pool, "UPDATE spaces SET archived_ms = NULL".to_owned()).await.unwrap(); + assert_eq!(access(&cy, space).await, Some(SpaceRole::Editor)); + + // A disabled account has no access anywhere, a removed one neither; the member rows stay, so enabling restores them. + run(&pool, format!("UPDATE users SET disabled_ms = 9 WHERE id = {}", blob_of(pg, &cy.id.0))).await.unwrap(); + assert_eq!(access(&cy, space).await, None); + assert_eq!(access(&cy, cy.space).await, None, "its personal space too"); + run(&pool, format!("UPDATE users SET disabled_ms = NULL, removed_ms = 9 WHERE id = {}", blob_of(pg, &cy.id.0))).await.unwrap(); + assert_eq!(access(&cy, space).await, None); + run(&pool, format!("UPDATE users SET removed_ms = NULL WHERE id = {}", blob_of(pg, &cy.id.0))).await.unwrap(); + assert_eq!(access(&cy, space).await, Some(SpaceRole::Editor)); + assert_eq!(meta.list_members(space).await.unwrap().len(), 5); + + // Roles follow the changes at once. + meta.change_member_role(&by(&ana), space, cy.id, SpaceRole::Viewer, NOW + 10).await.unwrap(); + assert_eq!(access(&cy, space).await, Some(SpaceRole::Viewer)); + meta.remove_member(&by(&ana), space, cy.id, NOW + 11).await.unwrap(); + assert_eq!(access(&cy, space).await, None); + // The ordering of the roles is the ordering of permissions. + assert!(SpaceRole::Viewer < SpaceRole::Editor && SpaceRole::Editor < SpaceRole::Owner); +} + +// ---- members and their events ---- + +async fn members_come_and_go_with_their_events(db: &Db) { + let meta = &db.meta; + let pool = raw(&db.url).await; + let pg = db.postgres; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let cy = user(meta, "cy").await; + let gone = user(meta, "gone").await; + let off = user(meta, "off").await; + let eve = user(meta, "eve").await; + let device_ana = device(meta, &ana).await; + let space = team(meta, "Studio", &ana).await; + let owner = SpaceActor::User { user: ana.id, device: Some(device_ana) }; + + // Whoever is not an owner may not manage; a stranger learns nothing (the answer is the one for no such space). + meta.invite_member(&owner, space, ben.id, SpaceRole::Editor, NOW + 1).await.unwrap(); + assert!(matches!(meta.invite_member(&by(&eve), space, cy.id, SpaceRole::Viewer, NOW + 2).await, Err(MetaError::NotFound("space")))); + // ben is only invited so far: still not a member who could invite. + assert!(matches!(meta.invite_member(&by(&ben), space, cy.id, SpaceRole::Viewer, NOW + 2).await, Err(MetaError::NotFound("space")))); + // One answer for an account that does not exist, one that is disabled and one that is marked for removal. + run(&pool, format!("UPDATE users SET disabled_ms = 1 WHERE id = {}", blob_of(pg, &off.id.0))).await.unwrap(); + run(&pool, format!("UPDATE users SET disabled_ms = 1, removed_ms = 1 WHERE id = {}", blob_of(pg, &gone.id.0))).await.unwrap(); + for target in [UserId(uuid::Uuid::now_v7()), off.id, gone.id] { + assert!(matches!(meta.invite_member(&owner, space, target, SpaceRole::Viewer, NOW + 3).await, Err(MetaError::NotFound("user")))); + } + // A second invitation, or an invitation of a member, is a conflict. + assert!(matches!(meta.invite_member(&owner, space, ben.id, SpaceRole::Viewer, NOW + 3).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.invite_member(&owner, space, ana.id, SpaceRole::Viewer, NOW + 3).await, Err(MetaError::Conflict(_)))); + // The pending invitation shows in the list, not as a member. + let list = meta.list_members(space).await.unwrap(); + assert_eq!( + list.iter().map(|m| (m.username.as_str(), m.role, m.accepted())).collect::>(), + [("ana", SpaceRole::Owner, true), ("ben", SpaceRole::Editor, false)] + ); + assert_eq!(list[1].invited_by, Some(ana.id)); + assert_eq!(list[0].invited_by, None, "made on the command line"); + + // Accepting turns it into access; again, or without an invitation, finds none and writes nothing. + let before = space_events(meta).await.len(); + // An account with no row at all gets the answer for a space that does not exist, so it learns nothing about the space. + let nowhere = craft_cloud_proto::SpaceId::generate(); + for space in [space, nowhere] { + assert!(matches!(meta.accept_invitation(cy.id, None, space, NOW + 4).await, Err(MetaError::NotFound("space")))); + assert!(matches!(meta.decline_invitation(cy.id, None, space, NOW + 4).await, Err(MetaError::NotFound("space")))); + assert!(matches!(meta.leave_space(cy.id, None, space, NOW + 4).await, Err(MetaError::NotFound("space")))); + } + assert_eq!(meta.accept_invitation(ben.id, None, space, NOW + 5).await.unwrap(), SpaceRole::Editor); + assert!(matches!(meta.accept_invitation(ben.id, None, space, NOW + 6).await, Err(MetaError::NotFound("invitation")))); + assert!(matches!(meta.decline_invitation(ben.id, None, space, NOW + 6).await, Err(MetaError::NotFound("invitation")))); + assert_eq!(space_events(meta).await.len(), before + 1); + + // An editor may not manage the members: a member whose role is too low is `ReadOnly` (403 `read_only`). + assert!(matches!(meta.invite_member(&by(&ben), space, cy.id, SpaceRole::Viewer, NOW + 7).await, Err(MetaError::ReadOnly))); + assert!(matches!(meta.change_member_role(&by(&ben), space, ben.id, SpaceRole::Owner, NOW + 7).await, Err(MetaError::ReadOnly))); + assert!(matches!(meta.remove_member(&by(&ben), space, ana.id, NOW + 7).await, Err(MetaError::ReadOnly))); + + // Decline deletes the invitation, and the account can be invited again. + meta.invite_member(&owner, space, cy.id, SpaceRole::Viewer, NOW + 8).await.unwrap(); + meta.decline_invitation(cy.id, None, space, NOW + 9).await.unwrap(); + assert_eq!(meta.space_access(cy.id, space).await.unwrap(), None); + meta.invite_member(&owner, space, cy.id, SpaceRole::Viewer, NOW + 10).await.unwrap(); + // Withdrawing a pending invitation is a removal; an account that is only invited cannot "leave". + assert!(matches!(meta.leave_space(cy.id, None, space, NOW + 11).await, Err(MetaError::NotFound("member")))); + meta.remove_member(&owner, space, cy.id, NOW + 12).await.unwrap(); + assert!(matches!(meta.remove_member(&owner, space, cy.id, NOW + 13).await, Err(MetaError::NotFound("member")))); + + // A role change returns the row; the role the member has already is no change and writes nothing. + let changed = meta.change_member_role(&owner, space, ben.id, SpaceRole::Viewer, NOW + 14).await.unwrap(); + assert_eq!((changed.role, changed.accepted()), (SpaceRole::Viewer, true)); + let events = space_events(meta).await.len(); + assert_eq!(meta.change_member_role(&owner, space, ben.id, SpaceRole::Viewer, NOW + 15).await.unwrap().role, SpaceRole::Viewer); + assert_eq!(space_events(meta).await.len(), events); + assert!(matches!(meta.change_member_role(&owner, space, eve.id, SpaceRole::Viewer, NOW + 15).await, Err(MetaError::NotFound("member")))); + + // Leaving: the member's own row goes, and the files of the space are untouched. + meta.leave_space(ben.id, None, space, NOW + 16).await.unwrap(); + assert_eq!(meta.space_access(ben.id, space).await.unwrap(), None); + assert!(matches!(meta.leave_space(ben.id, None, space, NOW + 17).await, Err(MetaError::NotFound("space")))); + + // A device that was signed out meanwhile no longer acts. + meta.revoke_device(ana.id, device_ana).await.unwrap(); + assert!(matches!(meta.invite_member(&owner, space, eve.id, SpaceRole::Viewer, NOW + 18).await, Err(MetaError::DeviceRevoked))); + + // The events: every change once, by the right actor, about the space, with typed details and no name in the row. + let events = space_events(meta).await; + assert_eq!( + names(&events), + [ + "cli.space_created", + "space.member_invited", + "space.member_joined", + "space.member_invited", + "space.member_declined", + "space.member_invited", + "space.member_removed", + "space.member_role_changed", + "space.member_left", + ] + ); + assert!(events.iter().all(|e| e.target_kind == "space" && e.target_id == Some(*space.as_uuid()))); + assert_eq!(events[0].actor_user, None); + assert_eq!(events[0].details, format!(r#"{{"quota_bytes":null,"user":"{}"}}"#, ana.id.0)); + assert_eq!((events[1].actor_user, events[1].actor_device), (Some(ana.id), Some(device_ana))); + assert_eq!(events[1].details, format!(r#"{{"role_to":"editor","user":"{}"}}"#, ben.id.0)); + assert_eq!(events[2].actor_user, Some(ben.id)); + assert_eq!(events[7].details, format!(r#"{{"role_from":"editor","role_to":"viewer","user":"{}"}}"#, ben.id.0)); + assert_eq!(events[8].actor_user, Some(ben.id)); + assert!(events.iter().all(|e| !e.details.contains("Studio"))); +} + +async fn the_command_line_and_admins_create_and_manage_spaces(db: &Db) { + let meta = &db.meta; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let (boss, admin) = admin_actor(meta, "boss").await; + + // An account does not create team spaces; a bad quota and an owner that is no account are refused. + assert!(matches!(meta.create_team_space(&by(&ana), &team_name("Mine"), ana.id, None, NOW).await, Err(MetaError::Forbidden(_)))); + let admin_actor = SpaceActor::Admin(admin.clone()); + assert!(matches!(meta.create_team_space(&admin_actor, &team_name("Small"), ana.id, Some(5), NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!( + meta.create_team_space(&admin_actor, &team_name("Nobody"), UserId(uuid::Uuid::now_v7()), None, NOW).await, + Err(MetaError::NotFound("user")) + )); + + // An administrator creates one with its first owner already a member. + let space = meta.create_team_space(&admin_actor, &team_name("Straße"), ana.id, Some(8 * 1024 * 1024), NOW + 1).await.unwrap(); + assert_eq!(meta.space_access(ana.id, space).await.unwrap(), Some(SpaceRole::Owner)); + // The administrator has no access of their own to the content. + assert_eq!(meta.space_access(boss.id, space).await.unwrap(), None); + // The name is unique in any case; the failed attempts left nothing behind. + for clash in ["STRASSE", "straße", "Strasse"] { + assert!( + matches!(meta.create_team_space(&admin_actor, &team_name(clash), ben.id, None, NOW + 2).await, Err(MetaError::Conflict(_))), + "{clash}" + ); + } + assert_eq!(meta.all_spaces().await.unwrap().len(), 4, "three personal spaces and the one team space"); + // It has a root like every space and a quota of its own. + let info = meta.space_info(space).await.unwrap(); + assert_eq!((info.seq, info.oldest_seq), (0, 0)); + assert_eq!(meta.tree(space, None, 10).await.unwrap().nodes.len(), 1); + + // An administrator invites; the invitation needs the account's acceptance, and a stale admin is refused. + let member = meta.invite_member(&admin_actor, space, ben.id, SpaceRole::Editor, NOW + 3).await.unwrap(); + assert!(!member.accepted()); + assert_eq!(meta.space_access(ben.id, space).await.unwrap(), None); + let stale = AdminActor { password_hash: "$argon2id$changed".to_owned(), ..admin.clone() }; + assert!(matches!(meta.invite_member(&SpaceActor::Admin(stale), space, ana.id, SpaceRole::Viewer, NOW + 4).await, Err(MetaError::Conflict(_)))); + let not_admin = AdminActor { user: ana.id, ..admin.clone() }; + assert!(meta.invite_member(&SpaceActor::Admin(not_admin), space, ana.id, SpaceRole::Viewer, NOW + 4).await.is_err()); + + // The command line adds a member who is accepted at once, and changes roles without being a member. + let cy = user(meta, "cy").await; + let added = meta.invite_member(&SpaceActor::Cli, space, cy.id, SpaceRole::Viewer, NOW + 5).await.unwrap(); + assert!(added.accepted() && added.invited_by.is_none()); + assert_eq!(meta.space_access(cy.id, space).await.unwrap(), Some(SpaceRole::Viewer)); + meta.change_member_role(&SpaceActor::Cli, space, cy.id, SpaceRole::Editor, NOW + 6).await.unwrap(); + meta.remove_member(&SpaceActor::Cli, space, cy.id, NOW + 7).await.unwrap(); + + // A personal space has no members to manage. + assert!(matches!(meta.invite_member(&SpaceActor::Cli, ana.space, ben.id, SpaceRole::Viewer, NOW + 8).await, Err(MetaError::NotFound("space")))); + assert!(matches!(meta.list_members(ana.space).await, Err(MetaError::NotFound("space")))); + + let events = space_events(meta).await; + assert_eq!( + names(&events), + ["space.created", "space.member_invited", "cli.space_member_changed", "cli.space_member_changed", "cli.space_member_changed"] + ); + assert_eq!(events[0].actor_user, Some(boss.id)); + assert_eq!(events[0].details, format!(r#"{{"quota_bytes":8388608,"user":"{}"}}"#, ana.id.0)); + assert!(events[2..].iter().all(|e| e.actor_user.is_none() && e.actor_device.is_none())); + assert_eq!(events[3].details, format!(r#"{{"role_from":"viewer","role_to":"editor","user":"{}"}}"#, cy.id.0)); +} + +// ---- the last owner ---- + +async fn the_last_owner_stays(db: &Db) { + let meta = &db.meta; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let space = team(meta, "Studio", &ana).await; + let owner = by(&ana); + + // The only owner cannot be lowered, removed or leave, by themselves, by the command line or by anyone. + assert!(matches!(meta.change_member_role(&owner, space, ana.id, SpaceRole::Editor, NOW + 1).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.change_member_role(&SpaceActor::Cli, space, ana.id, SpaceRole::Editor, NOW + 1).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.remove_member(&SpaceActor::Cli, space, ana.id, NOW + 1).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.remove_member(&owner, space, ana.id, NOW + 1).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.leave_space(ana.id, None, space, NOW + 1).await, Err(MetaError::LastOwner))); + assert_eq!(meta.space_access(ana.id, space).await.unwrap(), Some(SpaceRole::Owner)); + + // An invited owner does not count until they accept. + meta.invite_member(&owner, space, ben.id, SpaceRole::Owner, NOW + 2).await.unwrap(); + assert!(matches!(meta.change_member_role(&owner, space, ana.id, SpaceRole::Editor, NOW + 3).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.leave_space(ana.id, None, space, NOW + 3).await, Err(MetaError::LastOwner))); + // ... and a pending invitation can be changed and withdrawn without the rule. + meta.change_member_role(&owner, space, ben.id, SpaceRole::Viewer, NOW + 3).await.unwrap(); + meta.change_member_role(&owner, space, ben.id, SpaceRole::Owner, NOW + 3).await.unwrap(); + meta.accept_invitation(ben.id, None, space, NOW + 4).await.unwrap(); + + // With a second owner the first may be lowered; then the second is the last. + meta.change_member_role(&owner, space, ana.id, SpaceRole::Editor, NOW + 5).await.unwrap(); + assert!(matches!(meta.change_member_role(&by(&ben), space, ben.id, SpaceRole::Viewer, NOW + 6).await, Err(MetaError::LastOwner))); + assert!(matches!(meta.leave_space(ben.id, None, space, NOW + 6).await, Err(MetaError::LastOwner))); + // ana is an editor now and can leave freely. + meta.leave_space(ana.id, None, space, NOW + 7).await.unwrap(); + assert!(meta.list_members(space).await.unwrap().iter().all(|m| m.user == ben.id)); +} + +/// Two owners lower each other, or leave, at the same time, many times over: never both succeed, and the space always keeps +/// an accepted owner. On Postgres the space's row lock orders them (and, for the commands where no user lock separates +/// the two, nothing else does: the rounds that use the command line twice or two leaves test exactly that lock; the +/// locks of the two accounts already order an owner lowering or removing the other). On SQLite the process's single writer +/// orders them by design, so a mutation that removes the row lock cannot fail there. Both tasks wait at a barrier so +/// that they start their transactions together. +async fn concurrent_changes_never_leave_a_space_without_an_owner(db: &Db) { + let meta = &db.meta; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + // Mostly the cases that depend on the space's row lock (0: the command line twice, 3: both leave). + const HOW: [u8; 6] = [0, 3, 0, 3, 1, 2]; + for round in 0..48 { + let space = team(meta, &format!("Studio {round}"), &ana).await; + join(meta, space, &ana, &ben, SpaceRole::Owner).await; + let (a, b) = (ana.clone(), ben.clone()); + let (m1, m2) = (meta.clone(), meta.clone()); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2)); + let attempt = |meta: Meta, actor: SpaceActor, target: UserId, how: u8, barrier: std::sync::Arc| async move { + barrier.wait().await; + match how { + // The command line lowers the owner it is told to; only the rule can stop both. + 0 => meta.change_member_role(&actor, space, target, SpaceRole::Editor, NOW + 20).await.map(|_| ()), + // An owner lowers the other one; the loser is also no longer an owner when its turn comes. + 1 => meta.change_member_role(&actor, space, target, SpaceRole::Viewer, NOW + 20).await.map(|_| ()), + 2 => meta.remove_member(&actor, space, target, NOW + 20).await, + _ => match actor { + SpaceActor::User { user, device } => meta.leave_space(user, device, space, NOW + 20).await, + _ => Ok(()), + }, + } + }; + let how = HOW[round % HOW.len()]; + let (actor1, actor2) = if how == 0 { (SpaceActor::Cli, SpaceActor::Cli) } else { (by(&a), by(&b)) }; + let t1 = tokio::spawn(attempt(m1, actor1, b.id, how, barrier.clone())); + let t2 = tokio::spawn(attempt(m2, actor2, a.id, how, barrier)); + let (r1, r2) = (t1.await.unwrap(), t2.await.unwrap()); + let ok = [&r1, &r2].iter().filter(|r| r.is_ok()).count(); + let refused = |r: &Result<(), MetaError>| { + matches!(r, Err(MetaError::LastOwner | MetaError::Forbidden(_) | MetaError::ReadOnly | MetaError::NotFound(_))) + }; + assert!(ok == 1 && (refused(&r1) || refused(&r2)), "round {round} ({how}): {r1:?} / {r2:?}"); + let owners = meta.list_members(space).await.unwrap().into_iter().filter(|m| m.role == SpaceRole::Owner && m.accepted()).count(); + assert_eq!(owners, 1, "round {round} ({how}): {r1:?} / {r2:?}"); + } +} + +// ---- guarded writes ---- + +/// A write through a handle from `Meta::guarded` reads the caller's role again inside its own transaction, so a role lowered, +/// a member removed or a space archived after the server's access check (and before the write) is respected: the write is +/// refused and changes nothing. The same write through the plain handle goes through, which shows it is the guard that refuses. +async fn guarded_writes_check_the_role_in_their_own_transaction(db: &Db) { + let meta = &db.meta; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let space = team(meta, "Studio", &ana).await; + join(meta, space, &ana, &ben, SpaceRole::Editor).await; + let dev = device(meta, &ben).await; + let root = meta.space_info(space).await.unwrap().root; + let mkdir = |name: &str| { + vec![craft_cloud_proto::Op::Mkdir { + node: craft_cloud_proto::NodeId::generate(), + parent: root, + name: craft_cloud_proto::Name::new(name).unwrap(), + }] + }; + let chunk = craft_cloud_proto::ChunkId::from_bytes([3; 32]); + let editor = meta.guarded(WriteGuard { user: ben.id, min: SpaceRole::Editor }); + + // As an editor every write goes through. + editor.commit(space, dev, &mkdir("one"), NOW + 3).await.unwrap(); + editor.add_chunk(space, &chunk, 10, NOW + 3).await.unwrap(); + let seq = meta.space_info(space).await.unwrap().seq; + + // The role is lowered after the handle was made: every write is refused, and nothing changed. + meta.change_member_role(&by(&ana), space, ben.id, SpaceRole::Viewer, NOW + 4).await.unwrap(); + assert!(matches!(editor.commit(space, dev, &mkdir("two"), NOW + 5).await, Err(MetaError::ReadOnly))); + assert!(matches!(editor.add_chunk(space, &craft_cloud_proto::ChunkId::from_bytes([4; 32]), 10, NOW + 5).await, Err(MetaError::ReadOnly))); + assert!(matches!(editor.add_chunks(space, &[(craft_cloud_proto::ChunkId::from_bytes([5; 32]), 10)], NOW + 5).await, Err(MetaError::ReadOnly))); + assert!(matches!(editor.restore_from_trash(space, dev, craft_cloud_proto::NodeId::generate(), NOW + 5).await, Err(MetaError::ReadOnly))); + assert!(matches!(editor.request_purge(space, craft_cloud_proto::NodeId::generate(), NOW + 5).await, Err(MetaError::ReadOnly))); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq, "a refused write takes no sequence number"); + assert_eq!(meta.missing_chunks(space, &[craft_cloud_proto::ChunkId::from_bytes([4; 32])]).await.unwrap().len(), 1, "nothing was recorded"); + // A viewer's handle for a read-level minimum still works, and the plain handle is not guarded. + meta.guarded(WriteGuard { user: ben.id, min: SpaceRole::Viewer }).commit(space, dev, &mkdir("three"), NOW + 6).await.unwrap(); + meta.commit(space, dev, &mkdir("four"), NOW + 6).await.unwrap(); + + // Removed: no access at all, which is the answer for a space that is not there. + meta.remove_member(&SpaceActor::Cli, space, ben.id, NOW + 7).await.unwrap(); + assert!(matches!(editor.commit(space, dev, &mkdir("five"), NOW + 8).await, Err(MetaError::NotFound("space")))); + + // Archived: at most a viewer, so even the owner's write is refused. + let owner = meta.guarded(WriteGuard { user: ana.id, min: SpaceRole::Editor }); + owner.commit(space, dev, &mkdir("six"), NOW + 9).await.unwrap(); + run(&raw(&db.url).await, format!("UPDATE spaces SET archived_ms = 1 WHERE id = {}", blob_of(db.postgres, space.as_uuid()))).await.unwrap(); + assert!(matches!(owner.commit(space, dev, &mkdir("seven"), NOW + 10).await, Err(MetaError::ReadOnly))); + meta.guarded(WriteGuard { user: ana.id, min: SpaceRole::Viewer }).commit(space, dev, &mkdir("eight"), NOW + 10).await.unwrap(); +} + +/// Uploads by editors of one team space, started together: each records a blob and charges the space's byte count, which +/// updates the space's row. A guard that took a weaker lock on that row first would make two of them deadlock on +/// Postgres (one answer is a database error); all of them must succeed. +async fn concurrent_guarded_uploads_do_not_deadlock(db: &Db) { + let meta = &db.meta; + let ana = user(meta, "ana").await; + let ben = user(meta, "ben").await; + let cy = user(meta, "cy").await; + let space = team(meta, "Studio", &ana).await; + join(meta, space, &ana, &ben, SpaceRole::Editor).await; + join(meta, space, &ana, &cy, SpaceRole::Editor).await; + for round in 0..20u8 { + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2)); + let upload = |who: UserId, tag: u8, barrier: std::sync::Arc| { + let meta = meta.guarded(WriteGuard { user: who, min: SpaceRole::Editor }); + async move { + barrier.wait().await; + meta.add_chunk( + space, + &craft_cloud_proto::ChunkId::from_bytes([ + tag, round, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]), + 10, + NOW + 5, + ) + .await + } + }; + let (a, b) = (tokio::spawn(upload(ben.id, 1, barrier.clone())), tokio::spawn(upload(cy.id, 2, barrier))); + let (a, b) = (a.await.unwrap(), b.await.unwrap()); + assert!(a.is_ok() && b.is_ok(), "round {round}: {a:?} / {b:?}"); + } +} diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 6ff15c5d..43a2d209 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -5,15 +5,23 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::hash::ChunkId; -use crate::ids::{DeviceId, Epoch, NodeId, SpaceId}; +use crate::ids::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; use crate::ops::{Conflict, Op}; /// What apps see of these bodies lives in `craft-cloud-api` (no postcard), re-exported here. -pub use craft_cloud_api::api::{DeviceInfo, ErrorAnswer, PairingState, PairingStatus, codes}; +pub use craft_cloud_api::api::{ + AUDIT_ACTIONS, AccountInfo, AccountUsage, AddSpaceMember, AdminCode, AdminConfirm, AdminCreateSpace, AdminCreateUser, AdminCreatedUser, + AdminDevices, AdminManifestIndex, AdminPasswordReset, AdminServerInfo, AdminSetSpaceOwner, AdminSpaceInfo, AdminSpaceList, AdminStorageAccount, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditDetails, AuditEvent, AuditExportEnd, AuditPage, ChangeSpaceMember, + DeviceInfo, ErrorAnswer, PairingState, PairingStatus, RecoveryCodes, RevokedOthers, SpaceConfirm, SpaceList, SpaceListItem, SpaceMember, + SpaceMembers, SpaceRole, SpaceUsage, TwoFactorChallenge, TwoFactorSetupResponse, TwoFactorStatus, UpdateSpace, UsageEstimateReason, + UsageEstimates, codes, valid_audit_action, +}; -/// Protocol versions this build speaks. A client and server must share at least one. +/// Protocol versions this build speaks. A client and server must share at least one. Version 2 adds the `craft-space` +/// header and its echo ([`SPACE_HEADER`]) and the team-space routes; version 1 clients keep working unchanged. pub const PROTOCOL_MIN: u32 = 1; -pub const PROTOCOL_MAX: u32 = 1; +pub const PROTOCOL_MAX: u32 = 2; /// Most chunk IDs in one `POST /chunks/missing`. pub const MAX_MISSING_IDS: usize = 10_000; @@ -44,15 +52,32 @@ pub struct SpaceResponse { pub seq: u64, /// The account's username, in lowercase. pub username: String, + /// `personal` or `team` (appended: an older server leaves it out). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kind: Option, + /// The team space's name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The caller's role in the space. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, } -/// `POST /api/v1/auth/login`. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +/// `POST /api/v1/auth/login`. `client_kind`, `platform` and `app_version` are optional and only shown in the device +/// list, never a basis for a permission: `client_kind` is `agent`, `desktop` or `cli`, the others are free text of at +/// most 32 bytes (all without control characters). A body without them works as before. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct LoginRequest { pub username: String, pub password: String, /// Shown in the device list, e.g. the computer's name. pub device_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, } impl std::fmt::Debug for LoginRequest { @@ -61,6 +86,9 @@ impl std::fmt::Debug for LoginRequest { .field("username", &self.username) .field("password", &"") .field("device_name", &self.device_name) + .field("client_kind", &self.client_kind) + .field("platform", &self.platform) + .field("app_version", &self.app_version) .finish() } } @@ -97,14 +125,27 @@ impl std::fmt::Debug for PairingCreated { /// `POST /api/v1/join` (no auth): redeem an invite or an add-a-computer code (normalised or as typed) for a new /// device; answered with a [`LoginResponse`]. `password` is required, and only taken, for an invite to an account -/// that has none yet. Its `Debug` output hides the code and the password. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +/// that has none yet. `client_kind`, `platform` and `app_version` are as in [`LoginRequest`]. `totp_code` or +/// `recovery_code` (one of them) is the second factor of an account that has two-factor sign-in: the server answers +/// `two_factor_required` without using the code up, and the same body is sent again with it. Its `Debug` output hides +/// the code, the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct JoinRequest { pub code: String, /// Shown in the device list, e.g. the computer's name. pub device_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, } impl std::fmt::Debug for JoinRequest { @@ -113,6 +154,109 @@ impl std::fmt::Debug for JoinRequest { .field("code", &"") .field("device_name", &self.device_name) .field("password", &self.password.as_ref().map(|_| "")) + .field("client_kind", &self.client_kind) + .field("platform", &self.platform) + .field("app_version", &self.app_version) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /api/v1/account/password` (device auth): changes the password, answered `204`. Every other device of the +/// account is signed out, and its open invite and pairings are deleted. A wrong `current_password` is `403 +/// wrong_password` (never 401), a `new_password` that breaks the rules `400 weak_password`. Its `Debug` output hides +/// both passwords. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangePasswordRequest { + pub current_password: String, + pub new_password: String, +} + +impl std::fmt::Debug for ChangePasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChangePasswordRequest").field("current_password", &"").field("new_password", &"").finish() + } +} + +/// `PATCH /api/v1/devices/{id}` (device auth): renames one of the account's devices that isn't revoked; answered with +/// its [`DeviceInfo`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenameDeviceRequest { + pub name: String, +} + +// ---- two-factor sign-in (`/api/v1/account/two-factor/…`, `/api/v1/auth/login/second-factor`) ---- +// +// The second factor is a TOTP code (`totp_code`, six digits) or a recovery code (`recovery_code`, 16 letters and +// digits, with dashes or without). A wrong one is `403 two_factor_invalid` on a signed-in route and `401` at sign-in and +// join (which are not signed in); too many wrong ones lock the account's second factor for a while (`429 +// two_factor_locked` with `Retry-After`). Every type here that holds a secret hides it in `Debug`. + +/// `POST /account/two-factor/setup`: the account's password starts setting two-factor sign-in up. Its `Debug` output +/// hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorSetupRequest { + pub password: String, +} + +impl std::fmt::Debug for TwoFactorSetupRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorSetupRequest").field("password", &"").finish() + } +} + +/// `POST /account/two-factor/confirm`: the first code from the authenticator app turns two-factor sign-in on. Its +/// `Debug` output hides the code. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorConfirmRequest { + pub totp_code: String, +} + +impl std::fmt::Debug for TwoFactorConfirmRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorConfirmRequest").field("totp_code", &"").finish() + } +} + +/// `POST /account/two-factor/disable` and `POST /account/two-factor/recovery-codes`: the password and one second +/// factor (`recovery-codes` takes `totp_code` only). Its `Debug` output hides all three. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorProofRequest { + pub password: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for TwoFactorProofRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorProofRequest") + .field("password", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /auth/login/second-factor`: the challenge `two_factor_required` gave, and one second factor. Answered with a +/// [`LoginResponse`]. Its `Debug` output hides all of it. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SecondFactorRequest { + pub challenge: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for SecondFactorRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorRequest") + .field("challenge", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) .finish() } } @@ -145,6 +289,166 @@ pub struct SaveAttach { pub save: crate::SaveRecord, } +/// Versions in one `GET /api/v1/nodes/{id}/history` page when `limit` is not given. +pub const HISTORY_DEFAULT_LIMIT: u32 = 50; +/// Most versions in one history page; a larger `limit` is `invalid`. +pub const HISTORY_MAX_LIMIT: u32 = 200; + +/// Where the next history page starts: after this `(seq, version)` in the newest-first order. On the wire it is an +/// opaque string; clients pass back the `next` they were given and nothing else. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HistoryCursor { + pub seq: u64, + pub version: VersionId, +} + +impl std::fmt::Display for HistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.seq, self.version) + } +} + +/// A history cursor that does not read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BadHistoryCursor; + +impl std::fmt::Display for BadHistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("not a history cursor") + } +} + +impl std::error::Error for BadHistoryCursor {} + +impl std::str::FromStr for HistoryCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted, so one cursor has one spelling. + fn from_str(s: &str) -> Result { + let (seq, version) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = HistoryCursor { seq: seq.parse().map_err(|_| BadHistoryCursor)?, version: version.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One version of a file in its history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryVersion { + pub id: VersionId, + pub parent_version: Option, + pub epoch: Epoch, + /// The space's change sequence number of the commit that made this version. + pub seq: u64, + pub size: u64, + pub mtime_ms: i64, + /// The device that committed it. + pub device: DeviceId, + pub created_ms: i64, + /// Whether a save record (a Craft app's live-document save) hangs on this version. + pub save: bool, + /// Thinned out of the history: its row stays, but it can't be restored. Always `false` until thinning exists. + pub pruned: bool, +} + +/// `GET /api/v1/nodes/{id}/history?cursor&limit`: the versions of a file, newest first by `(seq, id)`. `next` is the +/// `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryResponse { + pub versions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/nodes/{id}/restore`: make `version` the file's content again as a new version. `expected_version` +/// must still be the file's current version, else the answer is `conflict`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreRequest { + pub version: VersionId, + pub expected_version: VersionId, +} + +/// A restore that applied: the new version, which follows `parent_version` (the one that was current). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreResponse { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// Items in one `GET /api/v1/trash` page when `limit` is not given. +pub const TRASH_DEFAULT_LIMIT: u32 = 50; +/// Most items in one trash page; a larger `limit` is `invalid`. +pub const TRASH_MAX_LIMIT: u32 = 200; + +/// Where the next trash page starts: after this `(deleted_ms, node)` in the newest-deleted-first order. Opaque on the +/// wire, like [`HistoryCursor`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TrashCursor { + pub deleted_ms: i64, + pub node: NodeId, +} + +impl std::fmt::Display for TrashCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.deleted_ms, self.node) + } +} + +impl std::str::FromStr for TrashCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted. + fn from_str(s: &str) -> Result { + let (deleted_ms, node) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = TrashCursor { deleted_ms: deleted_ms.parse().map_err(|_| BadHistoryCursor)?, node: node.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashItem { + pub id: NodeId, + /// The folder it was in when it was deleted (itself deleted, perhaps). + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + pub deleted_ms: i64, + /// `deleted_ms` plus the server's `retention.trash_days`: when the server may remove the node for good. Nothing is + /// removed yet, so this is a promise of the earliest time, not an event. + pub purge_after_ms: i64, +} + +/// `GET /api/v1/trash?cursor&limit`: the deleted nodes of your space that were not removed from the trash, newest +/// deletion first by `(deleted_ms, id)`. `next` is the `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashResponse { + pub items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/trash/{id}/restore`: the node is back. `parent` and `name` say where, which differs from where it was +/// when a folder of its path was merged into a live folder of the same name, or when `renamed` is set: the name was +/// taken, so the node got a conflict name like the sync engine makes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashRestoreResponse { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + /// For a file, its new current version: a copy of its last one (the same content), so a delete sent against the old + /// version, whose answer a machine never heard, can't delete it again. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub seq: u64, + pub epoch: Epoch, +} + /// A commit that applied. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitResponse { @@ -160,7 +464,8 @@ pub struct QuotaUse { } /// Every error answer. `error` is a stable code, one of [`codes`]; `message` is for people. `conflicts` is -/// set for `conflict`, `missing` for `missing_chunks`, `quota` for `quota_exceeded`. Clients ignore codes and JSON +/// set for `conflict`, `missing` for `missing_chunks`, `quota` for `quota_exceeded`, `two_factor` for +/// `two_factor_required`. Clients ignore codes and JSON /// fields they don't know (the server may add more later), so match on the codes you handle and treat the rest by /// HTTP status. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -173,26 +478,133 @@ pub struct ErrorBody { pub missing: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub quota: Option, + /// Set for `two_factor_required`: the challenge to answer at `POST /auth/login/second-factor`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor: Option, } /// Response header carrying the space's epoch on every signed-in request. pub const EPOCH_HEADER: &str = "craft-epoch"; +/// Request header naming the space a space-scoped request works on (a space UUID; absent: the device's personal space). +/// Every answer of a space-scoped route echoes it with the space the server worked on, also when the request had none. +pub const SPACE_HEADER: &str = "craft-space"; + #[cfg(test)] mod tests { use super::*; #[test] fn secrets_stay_out_of_debug_output() { - let req = LoginRequest { username: "ana".into(), password: "hunter2-secret".into(), device_name: "pc".into() }; + let req = LoginRequest { username: "ana".into(), password: "hunter2-secret".into(), device_name: "pc".into(), ..LoginRequest::default() }; assert!(!format!("{req:?}").contains("hunter2-secret")); let res = LoginResponse { device: DeviceId::generate(), token: "tok-secret".into() }; assert!(!format!("{res:?}").contains("tok-secret")); - let join = JoinRequest { code: "K7M4Q9TX".into(), device_name: "pc".into(), password: Some("hunter2-secret".into()) }; + let join = + JoinRequest { code: "K7M4Q9TX".into(), device_name: "pc".into(), password: Some("hunter2-secret".into()), ..JoinRequest::default() }; let shown = format!("{join:?}"); assert!(!shown.contains("K7M4Q9TX") && !shown.contains("hunter2-secret"), "{shown}"); let created = PairingCreated { id: Uuid::nil(), code: "K7M4Q9TX".into(), expires_ms: 1 }; assert!(!format!("{created:?}").contains("K7M4Q9TX")); + let change = ChangePasswordRequest { current_password: "old-secret-pw".into(), new_password: "new-secret-pw".into() }; + let shown = format!("{change:?}"); + assert!(!shown.contains("old-secret-pw") && !shown.contains("new-secret-pw"), "{shown}"); + assert_eq!(shown.matches("").count(), 2); + let confirm = AdminConfirm { + password: Some("admin-secret-pw".into()), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERYSECRET00".into()), + }; + let shown = format!("{confirm:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && !shown.contains("RECOVERYSECRET"), "{shown}"); + let reset = + AdminPasswordReset { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), ..AdminPasswordReset::default() }; + let shown = format!("{reset:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + assert!(!format!("{create:?}").contains("admin-secret-pw")); + let update = AdminUpdateUser { password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + assert!(!format!("{update:?}").contains("admin-secret-pw")); + let code = AdminCode { code: "ABCDEFGHJKMNPQRS".into(), join_url: Some("cloud.example.org/ABCD-EFGH-JKMN-PQRS".into()), expires_ms: 1 }; + let shown = format!("{code:?} {:?}", AdminCreatedUser { user: sample_user(), invite: code.clone() }); + assert!(!shown.contains("ABCDEFGH") && !shown.contains("cloud.example.org"), "{shown}"); + } + + #[test] + fn two_factor_secrets_stay_out_of_debug_output() { + let setup = TwoFactorSetupResponse { + secret: "JBSWY3DPEHPK3PXP".into(), + otpauth_uri: "otpauth://totp/Craft%20Cloud:ana?secret=JBSWY3DPEHPK3PXP".into(), + expires_ms: 5, + }; + let shown = format!("{setup:?}"); + assert!(!shown.contains("JBSWY3DP") && !shown.contains("otpauth:"), "{shown}"); + let codes = RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into(), "WXYZ-0123-4567-89AB".into()] }; + let shown = format!("{codes:?}"); + assert!(!shown.contains("ABCD") && !shown.contains("WXYZ"), "{shown}"); + let second = + SecondFactorRequest { challenge: "challenge-secret".into(), totp_code: Some("123456".into()), recovery_code: Some("RC-SECRET".into()) }; + let shown = format!("{second:?}"); + assert!(!shown.contains("challenge-secret") && !shown.contains("123456") && !shown.contains("RC-SECRET"), "{shown}"); + let proof = TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: Some("654321".into()), recovery_code: None }; + let shown = format!("{proof:?}"); + assert!(!shown.contains("pw-secret-1") && !shown.contains("654321"), "{shown}"); + assert!(!format!("{:?}", TwoFactorSetupRequest { password: "pw-secret-1".into() }).contains("pw-secret-1")); + assert!(!format!("{:?}", TwoFactorConfirmRequest { totp_code: "654321".into() }).contains("654321")); + let join = JoinRequest { + code: "K7M4Q9TX".into(), + totp_code: Some("111222".into()), + recovery_code: Some("RC-SECRET".into()), + ..JoinRequest::default() + }; + let shown = format!("{join:?}"); + assert!(!shown.contains("111222") && !shown.contains("RC-SECRET"), "{shown}"); + let body = ErrorBody { + error: codes::TWO_FACTOR_REQUIRED.into(), + message: "m".into(), + two_factor: Some(TwoFactorChallenge { challenge: "challenge-secret".into(), expires_ms: 1 }), + ..ErrorBody::default() + }; + assert!(!format!("{body:?}").contains("challenge-secret")); + // The challenge travels in the error body, and an older client that doesn't know the field ignores it. + let json = serde_json::to_value(&body).unwrap(); + assert_eq!(json["two_factor"], serde_json::json!({ "challenge": "challenge-secret", "expires_ms": 1 })); + assert_eq!(serde_json::from_value::(json).unwrap(), body); + } + + fn sample_user() -> AdminUserInfo { + AdminUserInfo { + id: Uuid::nil(), + username: "ana".into(), + role: "member".into(), + created_ms: 1, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 0, + stored_bytes: 0, + quota_bytes: None, + invite_pending: false, + } + } + + #[test] + fn an_update_tells_a_missing_quota_from_a_null_one() { + let missing: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "role": "admin", "password": "p" })).unwrap(); + assert_eq!(missing.quota_bytes, None); + let null: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": null })).unwrap(); + assert_eq!(null.quota_bytes, Some(None)); + let set: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": 5_000_000 })).unwrap(); + assert_eq!(set.quota_bytes, Some(Some(5_000_000))); + assert!(serde_json::from_value::(serde_json::json!({ "quota_bytes": -1 })).is_err()); + assert_eq!(serde_json::to_value(&set).unwrap(), serde_json::json!({ "quota_bytes": 5_000_000 })); + } + + #[test] + fn an_admin_body_ignores_fields_it_does_not_know() { + let confirm: AdminConfirm = serde_json::from_value(serde_json::json!({ "password": "p", "totp_code": "123456", "later": 1 })).unwrap(); + assert_eq!((confirm.password.as_deref(), confirm.totp_code.as_deref(), confirm.recovery_code), (Some("p"), Some("123456"), None)); + assert_eq!(serde_json::from_value::(serde_json::json!({})).unwrap().password, None); } #[test] @@ -204,11 +616,42 @@ mod tests { assert_eq!(waiting, PairingStatus { state: PairingState::Waiting, device_name: None }); let join: JoinRequest = serde_json::from_value(serde_json::json!({ "code": "c", "device_name": "d" })).unwrap(); assert_eq!(join.password, None); + assert_eq!((join.client_kind, join.platform, join.app_version), (None, None, None)); + let login = LoginRequest { username: "ana".into(), password: "pw".into(), device_name: "pc".into(), ..LoginRequest::default() }; + assert_eq!(serde_json::to_value(&login).unwrap(), serde_json::json!({ "username": "ana", "password": "pw", "device_name": "pc" })); assert_eq!(codes::INVALID_CODE, "invalid_code"); assert_eq!(codes::PASSWORD_REQUIRED, "password_required"); assert_eq!(codes::WEAK_PASSWORD, "weak_password"); } + #[test] + fn audit_bodies_have_a_plain_json_form() { + let details: AuditDetails = serde_json::from_str(r#"{"revoked":2,"a_later_field":"x","quota_bytes":null}"#).unwrap(); + assert_eq!(details.revoked, Some(2)); + assert_eq!(details.quota_bytes, Some(None)); + assert_eq!(serde_json::to_string(&details).unwrap(), r#"{"quota_bytes":null,"revoked":2}"#); + assert_eq!(serde_json::from_str::("{}").unwrap(), AuditDetails::default()); + assert_eq!(serde_json::to_string(&AuditDetails::default()).unwrap(), "{}"); + let end = AuditExportEnd { complete: Some(true), truncated: None, exported: 3 }; + assert_eq!(serde_json::to_string(&end).unwrap(), r#"{"complete":true,"exported":3}"#); + let event = AuditEvent { + id: Uuid::nil(), + at_ms: 5, + actor_user: None, + actor_device: None, + action: "auth.login_failed".into(), + target_kind: String::new(), + target_id: None, + outcome: "failure".into(), + details: AuditDetails { step: Some("password".into()), ..AuditDetails::default() }, + count: 4, + }; + let json = serde_json::to_value(&event).unwrap(); + assert_eq!(json["actor_user"], serde_json::Value::Null); + assert_eq!(json["count"], 4); + assert_eq!(serde_json::from_value::(json).unwrap(), event); + } + #[test] fn error_bodies_omit_empty_extras() { let body = ErrorBody { error: "not_found".into(), message: "no such chunk".into(), ..ErrorBody::default() }; @@ -217,6 +660,36 @@ mod tests { assert_eq!(serde_json::from_value::(json).unwrap(), body); } + #[test] + fn account_usage_round_trips_and_ignores_unknown_fields() { + let response = AccountUsage { + stored_bytes: 17, + quota_bytes: Some(99), + trash_bytes: None, + history_bytes: None, + estimates: UsageEstimates { index_complete: false, reason: Some(UsageEstimateReason::IndexIncomplete) }, + }; + let value = serde_json::to_value(&response).unwrap(); + assert_eq!(serde_json::from_value::(value.clone()).unwrap(), response); + let mut value = value; + value.as_object_mut().unwrap().insert("future_field".into(), serde_json::json!(1)); + assert_eq!(serde_json::from_value::(value).unwrap(), response); + } + + #[test] + fn account_usage_decodes_complete_and_incomplete_estimates() { + let complete: AccountUsage = + serde_json::from_str(r#"{"stored_bytes":11,"quota_bytes":null,"trash_bytes":0,"history_bytes":0,"estimates":{"index_complete":true}}"#) + .unwrap(); + assert_eq!(complete.estimates, UsageEstimates { index_complete: true, reason: None }); + let incomplete: AccountUsage = serde_json::from_str( + r#"{"stored_bytes":11,"quota_bytes":null,"trash_bytes":null,"history_bytes":null,"estimates":{"index_complete":false,"reason":"index_incomplete"}}"#, + ).unwrap(); + assert_eq!(incomplete.trash_bytes, None); + assert_eq!(incomplete.history_bytes, None); + assert_eq!(incomplete.estimates.reason, Some(UsageEstimateReason::IndexIncomplete)); + } + #[test] fn a_quota_answer_says_how_much_is_used() { let body = ErrorBody { @@ -237,6 +710,42 @@ mod tests { assert_eq!(body.error, "something_new"); } + #[test] + fn history_cursors_have_one_spelling() { + let cursor = HistoryCursor { seq: 7, version: VersionId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in + ["", "7", "7.", ".x", "x.y", "+7.00000000-0000-0000-0000-000000000000", &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] + { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + assert_eq!("-1.00000000-0000-0000-0000-000000000000".parse::(), Err(BadHistoryCursor)); + } + + #[test] + fn trash_cursors_have_one_spelling() { + let cursor = TrashCursor { deleted_ms: 1_760_000_000_000, node: NodeId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in ["", "7", "7.", ".x", "x.y", &format!("+{text}"), &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + } + + #[test] + fn history_and_restore_bodies_have_a_plain_json_form() { + let restore: RestoreRequest = serde_json::from_value(serde_json::json!({ + "version": "01a12b0b-9fca-73db-b82d-c7c33683601e", + "expected_version": "01a12b0b-9fca-73db-b82d-c7c33683601f", + "unknown": 1 + })) + .unwrap(); + assert_ne!(restore.version, restore.expected_version); + let last = HistoryResponse { versions: vec![], next: None }; + assert_eq!(serde_json::to_value(&last).unwrap(), serde_json::json!({ "versions": [] })); + } + #[test] fn error_codes_are_stable() { assert_eq!(codes::MISSING_CHUNKS, "missing_chunks"); diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index ae043989..7ec87d85 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -20,5 +20,5 @@ pub use conflict::{ConflictStamp, conflict_name}; pub use hash::{BlobKind, ChunkId, FileHash, HexIdError, ManifestId}; pub use ids::{DeviceId, DocId, Epoch, NodeId, SpaceId, VersionId}; pub use manifest::{ChunkRef, ChunkerParamsId, Manifest, ManifestError}; -pub use name::{FoldKey, Name, NameError, UNICODE_VERSION}; +pub use name::{FoldKey, MAX_SPACE_NAME_CHARS, Name, NameError, SpaceName, SpaceNameError, UNICODE_VERSION, fold_search}; pub use ops::{ChangeSet, Conflict, ConflictKind, FileState, NewVersion, NodeKind, NodeState, Op, PageCursor, SaveRecord}; diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 4f219f67..6054bbdd 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -12,6 +12,7 @@ async-trait.workspace = true blake3.workspace = true bytes.workspace = true craft-cloud-proto.workspace = true +futures-util.workspace = true object_store.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/store/src/disk.rs b/crates/store/src/disk.rs index cea80de5..9c3436fc 100644 --- a/crates/store/src/disk.rs +++ b/crates/store/src/disk.rs @@ -5,8 +5,10 @@ //! their parents as well, so a crash can't lose the entry that points at a new shard directory. //! A crash leaves the old object or the new one. -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use async_trait::async_trait; use bytes::Bytes; @@ -15,6 +17,8 @@ use crate::{Backend, StoreError}; pub struct DiskBackend { root: PathBuf, + /// How many directories listings have opened, for tests that check a page doesn't walk finished directories again. + dirs_read: Arc, } impl DiskBackend { @@ -22,12 +26,34 @@ impl DiskBackend { pub fn new(root: impl Into) -> Result { let root = root.into(); std::fs::create_dir_all(&root)?; - Ok(Self { root }) + Ok(Self { root, dirs_read: Arc::new(AtomicUsize::new(0)) }) } fn full_path(&self, path: &str) -> PathBuf { self.root.join(path) } + + /// The number of directories listings have read so far. Only meant for tests of the listing's cost. + #[doc(hidden)] + pub fn directories_read(&self) -> usize { + self.dirs_read.load(Ordering::Relaxed) + } +} + +/// The name of a write's temporary file: `.tmp-` and a hyphenated lowercase UUIDv7. +fn temp_name(id: uuid::Uuid) -> String { + format!("{TEMP_PREFIX}{}", id.hyphenated()) +} + +const TEMP_PREFIX: &str = ".tmp-"; + +/// Whether `name` is exactly what [`temp_name`] writes. Any other `.tmp-` name is somebody else's file and +/// is listed as foreign. +fn is_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + uuid::Uuid::try_parse(rest).is_ok_and(|id| id.get_version() == Some(uuid::Version::SortRand) && id.hyphenated().to_string() == rest) } fn write_atomic(target: &Path, data: &[u8]) -> io::Result<()> { @@ -35,7 +61,7 @@ fn write_atomic(target: &Path, data: &[u8]) -> io::Result<()> { // Directories this write will create, deepest first. Their entries in the parents must be flushed too. let created: Vec<&Path> = dir.ancestors().take_while(|p| !p.as_os_str().is_empty() && !p.exists()).collect(); std::fs::create_dir_all(dir)?; - let tmp = dir.join(format!(".tmp-{}", uuid::Uuid::now_v7())); + let tmp = dir.join(temp_name(uuid::Uuid::now_v7())); let result = (|| { let mut file = std::fs::File::create(&tmp)?; file.write_all(data)?; @@ -73,15 +99,38 @@ async fn blocking(f: impl FnOnce() -> io::Result + Send + #[async_trait] impl Backend for DiskBackend { + fn kind(&self) -> &'static str { + "disk" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { let target = self.full_path(path); blocking(move || write_atomic(&target, &data)).await } async fn get_raw(&self, path: &str) -> Result, StoreError> { + self.get_raw_limited(path, usize::MAX).await + } + + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + let target = self.full_path(path); + blocking(move || match std::fs::File::open(&target) { + Ok(file) => { + let mut bytes = Vec::new(); + file.take(u64::try_from(max_bytes.saturating_add(1)).unwrap_or(u64::MAX)).read_to_end(&mut bytes)?; + Ok(Some(Bytes::from(bytes))) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + }) + .await + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { let target = self.full_path(path); - blocking(move || match std::fs::read(&target) { - Ok(bytes) => Ok(Some(Bytes::from(bytes))), + blocking(move || match std::fs::symlink_metadata(&target) { + Ok(meta) if meta.file_type().is_file() => Ok(Some(meta.len())), + Ok(_) => Ok(None), Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), Err(e) => Err(e), }) @@ -97,4 +146,108 @@ impl Backend for DiskBackend { }) .await } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + if Path::new(prefix).is_absolute() || Path::new(prefix).components().any(|part| !matches!(part, Component::Normal(_))) { + return Err(StoreError::Backend("listing prefix must be a relative path".into())); + } + let root = self.root.clone(); + let prefix = prefix.to_owned(); + let after = after.map(str::to_owned); + let dirs_read = Arc::clone(&self.dirs_read); + blocking(move || { + let Some(start) = list_prefix_dir(&root, &prefix)? else { + return Ok(Vec::new()); + }; + let mut out = Vec::new(); + walk_sorted(&start, &prefix, after.as_deref(), limit, &dirs_read, &mut out)?; + Ok(out) + }) + .await + } +} + +fn list_prefix_dir(root: &Path, prefix: &str) -> io::Result> { + let mut current = root.to_path_buf(); + for component in Path::new(prefix).components() { + let Component::Normal(name) = component else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "listing prefix must be a relative path")); + }; + current.push(name); + let metadata = match std::fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Ok(None); + } + } + Ok(Some(current)) +} + +/// Collects up to `limit` file paths below `dir` that sort strictly after `after`, in byte order of the +/// relative path (`/` separators), the order memory and S3 use. `relative` is `dir`'s path relative to the root. +/// Returns `true` once `out` is full. +/// +/// One directory's entries are read and sorted at a time, so memory is bounded by the largest single +/// directory listing (a shard directory holds at most 256 entries) plus `limit` paths. A subtree whose paths +/// all sort at or before `after` is skipped without being opened, and the walk stops as soon as `limit` paths +/// are found, so a page costs the directories on the way to `after` and to the end of the page, not the +/// store. A directory sorts as `name/`, so `a.b` comes before everything in `a/`. +fn walk_sorted(dir: &Path, relative: &str, after: Option<&str>, limit: usize, dirs_read: &AtomicUsize, out: &mut Vec) -> io::Result { + if out.len() >= limit { + return Ok(true); + } + let metadata = match std::fs::symlink_metadata(dir) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Ok(false); + } + let reader = match std::fs::read_dir(dir) { + Ok(reader) => reader, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + dirs_read.fetch_add(1, Ordering::Relaxed); + // (sort key, name, is a directory) + let mut entries: Vec<(String, String, bool)> = Vec::new(); + for entry in reader { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "store path is not valid UTF-8"))?.to_owned(); + let file_type = entry.file_type()?; + if file_type.is_symlink() { + continue; + } + if file_type.is_dir() { + entries.push((format!("{name}/"), name, true)); + } else if file_type.is_file() && !is_temp_name(&name) { + entries.push((name.clone(), name, false)); + } + } + entries.sort_unstable(); + for (_, name, is_dir) in entries { + let path = if relative.is_empty() { name.clone() } else { format!("{relative}/{name}") }; + if is_dir { + let subtree = format!("{path}/"); + // Everything below sorts after `subtree` and starts with it; it is all at or before `after` + // exactly when `after` is greater and is not itself below it. + if after.is_some_and(|cursor| cursor > subtree.as_str() && !cursor.starts_with(&subtree)) { + continue; + } + if walk_sorted(&dir.join(&name), &path, after, limit, dirs_read, out)? { + return Ok(true); + } + } else if after.is_none_or(|cursor| path.as_str() > cursor) { + out.push(path); + if out.len() >= limit { + return Ok(true); + } + } + } + Ok(false) } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 43f37475..b4ff4eb8 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -6,6 +6,15 @@ //! //! Each stored object is one format byte followed by the blob. Format 0 is plain bytes; other values are //! reserved for compression or encryption and are refused today. +//! +//! [`Store::list`] returns bounded, ordered pages scoped to one blob namespace. Foreign object paths remain +//! visible in the page. S3-compatible backends may return incomplete or stale listings, so listing is not a +//! proof that the namespace contains every object. Disk listing reports an I/O error for non-UTF-8 names, +//! which the string path API cannot represent. Disk and memory list in byte order of the relative path and +//! resume from the cursor without visiting what lies before it; a disk page reads one directory at a time. On +//! S3 a page reads the listing only up to its limit, and a key that `object_store` cannot represent (an empty +//! path segment, as in `chunks//a`) ends the listing with an error instead of appearing as foreign; a key +//! with a trailing slash is reported without the slash. Cursors longer than any real path are refused. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod disk; @@ -89,14 +98,106 @@ pub enum StoreError { /// Moves raw bytes. Implementations don't check anything; [`Store`] does. #[async_trait] pub trait Backend: Send + Sync + 'static { + /// What kind of storage this is, for the admin's server overview (`disk`, `s3` or `memory`). A backend that + /// doesn't say is `other`. Never a path, an address or a setting. + fn kind(&self) -> &'static str { + "other" + } /// Writes the object atomically: a reader sees the old object, the new one, or none, never a mix. async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError>; /// `None` when there is no such object. async fn get_raw(&self, path: &str) -> Result, StoreError>; + /// Reads at most `max_bytes` plus one byte. Backends should override this for bounded I/O. + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + Ok(self.get_raw(path).await?.map(|bytes| bytes.slice(..bytes.len().min(max_bytes.saturating_add(1))))) + } + /// Stored object size without reading object contents. + async fn size_raw(&self, _path: &str) -> Result, StoreError> { + Err(StoreError::Backend("backend does not support metadata-only object sizes".into())) + } /// Removing a missing object succeeds. async fn delete_raw(&self, path: &str) -> Result<(), StoreError>; + /// Lists paths below `prefix`, in ascending order, strictly after `after`. + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError>; +} + +/// The longest object path a listing cursor can carry. S3 keys are at most 1024 bytes and Linux paths at most +/// 4096, so no object the backends can hold needs more. +const MAX_CURSOR_PATH_BYTES: usize = 4096; +/// The longest valid token: the `1:c:` header plus two hex digits per path byte. +const MAX_CURSOR_TOKEN_BYTES: usize = 4 + 2 * MAX_CURSOR_PATH_BYTES; + +/// Opaque continuation token returned by [`Store::list`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListCursor(String); + +impl ListCursor { + /// Reconstructs a cursor token saved from a previous page. + pub fn from_token(token: impl Into) -> Self { + Self(token.into()) + } + + /// Returns this cursor's opaque token for storage or transport. + pub fn as_token(&self) -> &str { + &self.0 + } + + fn new(kind: BlobKind, path: &str) -> Result { + if path.len() > MAX_CURSOR_PATH_BYTES { + return Err(StoreError::Backend("object path is too long for a listing cursor".into())); + } + let kind = match kind { + BlobKind::Chunk => "c", + BlobKind::Manifest => "m", + }; + let encoded = path.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect::(); + Ok(Self(format!("1:{kind}:{encoded}"))) + } + + fn decode(&self, kind: BlobKind) -> Result { + let invalid = || StoreError::Backend("invalid listing cursor".into()); + // Checked before anything is split, allocated or decoded: a token is untrusted input. + if self.0.len() > MAX_CURSOR_TOKEN_BYTES { + return Err(invalid()); + } + let mut parts = self.0.splitn(3, ':'); + let version = parts.next().ok_or_else(invalid)?; + let cursor_kind = parts.next().ok_or_else(invalid)?; + let encoded = parts.next().ok_or_else(invalid)?; + let expected = match kind { + BlobKind::Chunk => "c", + BlobKind::Manifest => "m", + }; + if version != "1" || cursor_kind != expected || encoded.is_empty() || encoded.len() % 2 != 0 { + return Err(invalid()); + } + let mut bytes = Vec::with_capacity(encoded.len() / 2); + let mut index = 0; + while index < encoded.len() { + let end = index.checked_add(2).ok_or_else(invalid)?; + let pair = encoded.as_bytes().get(index..end).ok_or_else(invalid)?; + let hex = std::str::from_utf8(pair).map_err(|_| invalid())?; + bytes.push(u8::from_str_radix(hex, 16).map_err(|_| invalid())?); + index = end; + } + let path = String::from_utf8(bytes).map_err(|_| invalid())?; + if !path.starts_with(&format!("{}/", dir(kind))) { + return Err(invalid()); + } + Ok(path) + } } +/// One bounded page. Invalid object names appear in `foreign`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListPage { + pub blobs: Vec, + pub foreign: Vec, + pub next_cursor: Option, +} + +pub const MAX_LIST_PAGE_SIZE: usize = 1000; + /// The blob store the server uses. Cheap to clone. #[derive(Clone)] pub struct Store { @@ -108,6 +209,11 @@ impl Store { Self { backend: Arc::new(backend) } } + /// The backend's kind (`disk`, `s3`, `memory` or `other`). + pub fn kind(&self) -> &'static str { + self.backend.kind() + } + /// Stores `data` under `key` after checking its size and hash. Storing a blob that is already there /// succeeds and changes nothing. pub async fn put(&self, key: BlobKey, data: &[u8]) -> Result<(), StoreError> { @@ -126,13 +232,21 @@ impl Store { /// Reads a blob and checks it. A blob whose bytes no longer match its ID is an error, never served. pub async fn get(&self, key: BlobKey) -> Result { - let object = self.backend.get_raw(&key.path()).await?.ok_or(StoreError::NotFound(key))?; + self.get_sized(key).await.map(|(data, _)| data) + } + + /// Like [`Store::get`], and also says how many bytes the object is stored as (the format byte included), which is + /// what the metadata records as the blob's stored size. + pub async fn get_sized(&self, key: BlobKey) -> Result<(Bytes, u64), StoreError> { + let max_raw = max_len(key.kind).saturating_add(1); + let object = self.backend.get_raw_limited(&key.path(), max_raw).await?.ok_or(StoreError::NotFound(key))?; let Some((&format, _)) = object.split_first() else { return Err(StoreError::Corrupt { key, reason: "empty object" }); }; if format != FORMAT_PLAIN { return Err(StoreError::Corrupt { key, reason: "unknown format byte" }); } + let stored = u64::try_from(object.len()).unwrap_or(u64::MAX); let data = object.slice(1..); if data.len() > max_len(key.kind) { return Err(StoreError::Corrupt { key, reason: "larger than the limit" }); @@ -140,12 +254,75 @@ impl Store { if *blake3::hash(&data).as_bytes() != key.id { return Err(StoreError::Corrupt { key, reason: "hash mismatch" }); } - Ok(data) + Ok((data, stored)) + } + + /// Returns stored object size without materializing the object. + pub async fn stored_size(&self, key: BlobKey) -> Result { + self.backend.size_raw(&key.path()).await?.ok_or(StoreError::NotFound(key)) } pub async fn delete(&self, key: BlobKey) -> Result<(), StoreError> { self.backend.delete_raw(&key.path()).await } + + /// Lists one kind beneath its namespace. S3-compatible listings can be incomplete or stale. + pub async fn list(&self, kind: BlobKind, cursor: Option, limit: usize) -> Result { + if limit == 0 { + return Err(StoreError::Backend("listing limit must be greater than zero".into())); + } + let limit = limit.min(MAX_LIST_PAGE_SIZE); + let after = cursor.as_ref().map(|cursor| cursor.decode(kind)).transpose()?; + if let Some(path) = &after + && !path.starts_with(&format!("{}/", dir(kind))) + { + return Err(StoreError::Backend("invalid listing cursor".into())); + } + let request_limit = limit.checked_add(1).ok_or_else(|| StoreError::Backend("listing limit is too large".into()))?; + let mut paths = self.backend.list_raw(dir(kind), after.as_deref(), request_limit).await?; + paths.sort(); + paths.dedup(); + if paths.iter().any(|path| !path.starts_with(&format!("{}/", dir(kind)))) { + return Err(StoreError::Backend("backend returned object outside listing prefix".into())); + } + let has_more = paths.len() > limit; + paths.truncate(limit); + let next_cursor = if has_more { paths.last().map(|path| ListCursor::new(kind, path)).transpose()? } else { None }; + let mut blobs = Vec::new(); + let mut foreign = Vec::new(); + for path in paths { + match parse_key(kind, &path) { + Some(key) => blobs.push(key), + None => foreign.push(path), + } + } + Ok(ListPage { blobs, foreign, next_cursor }) + } +} + +fn parse_key(kind: BlobKind, path: &str) -> Option { + let mut parts = path.split('/'); + if parts.next()? != dir(kind) { + return None; + } + let first = parts.next()?; + let second = parts.next()?; + let hex = parts.next()?; + if parts.next().is_some() + || first.len() != 2 + || second.len() != 2 + || hex.len() != 64 + || !hex.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + || &hex[..2] != first + || &hex[2..4] != second + { + return None; + } + let mut id = [0u8; 32]; + for (i, byte) in id.iter_mut().enumerate() { + *byte = u8::from_str_radix(hex.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(BlobKey { kind, id }) } #[cfg(test)] @@ -161,4 +338,16 @@ mod tests { assert_eq!(key.path(), format!("chunks/ab/cd/abcd{}", "00".repeat(30))); assert!(BlobKey { kind: BlobKind::Manifest, id }.path().starts_with("manifests/ab/cd/")); } + + #[tokio::test] + async fn disk_read_rejects_huge_object_without_reading_it_all() { + let root = tempfile::tempdir().unwrap(); + let key = BlobKey { kind: BlobKind::Chunk, id: [7; 32] }; + let path = root.path().join(key.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let file = std::fs::File::create(&path).unwrap(); + file.set_len(u64::from(MAX_CHUNK_BYTES) + 1_000_000_000).unwrap(); + let store = Store::new(disk::DiskBackend::new(root.path()).unwrap()); + assert!(matches!(store.get_sized(key).await, Err(StoreError::Corrupt { reason: "larger than the limit", .. }))); + } } diff --git a/crates/store/src/memory.rs b/crates/store/src/memory.rs index 47a007ce..ca41d901 100644 --- a/crates/store/src/memory.rs +++ b/crates/store/src/memory.rs @@ -1,6 +1,6 @@ //! An in-memory backend for tests. -use std::collections::HashMap; +use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use async_trait::async_trait; @@ -11,7 +11,7 @@ use crate::{Backend, StoreError}; /// Clones share the same objects, so a test can keep a handle to the backend it gave a `Store`. #[derive(Default, Clone)] pub struct MemoryBackend { - objects: Arc>>, + objects: Arc>>, } impl MemoryBackend { @@ -19,7 +19,7 @@ impl MemoryBackend { Self::default() } - fn lock(&self) -> Result>, StoreError> { + fn lock(&self) -> Result>, StoreError> { self.objects.lock().map_err(|_| StoreError::Backend("memory store lock poisoned".into())) } @@ -32,6 +32,10 @@ impl MemoryBackend { #[async_trait] impl Backend for MemoryBackend { + fn kind(&self) -> &'static str { + "memory" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { self.lock()?.insert(path.to_owned(), data); Ok(()) @@ -41,8 +45,34 @@ impl Backend for MemoryBackend { Ok(self.lock()?.get(path).cloned()) } + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + Ok(self.lock()?.get(path).map(|bytes| bytes.slice(..bytes.len().min(max_bytes.saturating_add(1))))) + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + Ok(self.lock()?.get(path).map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX))) + } + async fn delete_raw(&self, path: &str) -> Result<(), StoreError> { self.lock()?.remove(path); Ok(()) } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + let objects = self.lock()?; + let boundary = if prefix.is_empty() { String::new() } else { format!("{prefix}/") }; + // Start at the later of the prefix boundary and `after`, and stop at the first key outside the + // boundary: keys of later namespaces are never visited, and the mutex is held for at most `limit` steps. + let start = match after { + Some(path) if path >= boundary.as_str() => std::ops::Bound::Excluded(path.to_owned()), + _ => std::ops::Bound::Included(boundary.clone()), + }; + Ok(objects + .range((start, std::ops::Bound::Unbounded)) + .map(|(path, _)| path) + .take_while(|path| path.starts_with(&boundary)) + .take(limit) + .cloned() + .collect()) + } } diff --git a/crates/store/src/s3.rs b/crates/store/src/s3.rs index 0557eba2..a9cbb5ba 100644 --- a/crates/store/src/s3.rs +++ b/crates/store/src/s3.rs @@ -3,9 +3,10 @@ use async_trait::async_trait; use bytes::Bytes; +use futures_util::TryStreamExt; use object_store::aws::{AmazonS3, AmazonS3Builder}; use object_store::path::Path; -use object_store::{ObjectStoreExt, PutPayload}; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; use crate::{Backend, StoreError}; @@ -67,16 +68,65 @@ fn backend_error(e: object_store::Error) -> StoreError { StoreError::Backend(e.to_string()) } +/// The path of a listed key relative to the configured prefix, or `None` when the key is outside the +/// listing prefix (`boundary`, which ends in `/`) or outside the backend's root (`root_boundary`). +fn relative_path(full: &str, boundary: &str, root_boundary: &str) -> Option { + if !full.starts_with(boundary) { + return None; + } + full.strip_prefix(root_boundary).map(str::to_owned) +} + +/// `object_store` parses every listed key as a `Path`, and one key it cannot represent (an empty segment, +/// as in `chunks//a`) fails the whole ListObjectsV2 response page it arrived in and ends the stream. Such a +/// key cannot be reported as foreign from here, and the objects of that page are not visible either, so the +/// listing stops with an error that says why. A key with a trailing slash (`chunks/a/`) is not an error: +/// `object_store` strips the slash and the object appears as `chunks/a`. +fn list_error(error: object_store::Error) -> StoreError { + match error { + object_store::Error::InvalidPath { source } => { + StoreError::Backend(format!("S3 listing contains a key that object_store cannot represent, so the listing cannot continue: {source}")) + } + other => backend_error(other), + } +} + #[async_trait] impl Backend for S3Backend { + fn kind(&self) -> &'static str { + "s3" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { self.store.put(&self.object_path(path), PutPayload::from(data)).await.map_err(backend_error)?; Ok(()) } async fn get_raw(&self, path: &str) -> Result, StoreError> { + self.get_raw_limited(path, usize::MAX).await + } + + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { match self.store.get(&self.object_path(path)).await { - Ok(result) => Ok(Some(result.bytes().await.map_err(backend_error)?)), + Ok(result) => { + let limit = max_bytes.saturating_add(1); + let mut body = result.into_stream(); + let mut bytes = Vec::with_capacity(limit.min(1024 * 1024)); + while bytes.len() < limit { + let Some(part) = body.try_next().await.map_err(backend_error)? else { break }; + let take = part.len().min(limit.saturating_sub(bytes.len())); + bytes.extend_from_slice(&part[..take]); + } + Ok(Some(Bytes::from(bytes))) + } + Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(backend_error(e)), + } + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + match self.store.head(&self.object_path(path)).await { + Ok(meta) => Ok(Some(meta.size)), Err(object_store::Error::NotFound { .. }) => Ok(None), Err(e) => Err(backend_error(e)), } @@ -88,11 +138,90 @@ impl Backend for S3Backend { Err(e) => Err(backend_error(e)), } } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + let object_prefix = self.object_path(prefix); + let normalized_prefix = object_prefix.as_ref().to_owned(); + let boundary = if normalized_prefix.is_empty() { String::new() } else { format!("{normalized_prefix}/") }; + let normalized_root = if self.prefix.is_empty() { String::new() } else { Path::from(self.prefix.as_str()).as_ref().to_owned() }; + let root_boundary = if normalized_root.is_empty() { String::new() } else { format!("{normalized_root}/") }; + let offset = after + .map(|path| { + let full_path = if normalized_root.is_empty() { path.to_owned() } else { format!("{normalized_root}/{path}") }; + Path::parse(full_path).map_err(|error| StoreError::Backend(error.to_string())) + }) + .transpose()?; + let mut stream = if let Some(offset) = &offset { + self.store.list_with_offset(Some(&object_prefix), offset) + } else { + self.store.list(Some(&object_prefix)) + }; + // ListObjectsV2 returns keys in ascending UTF-8 binary order, which is the order `list_raw` promises. + // The stream is read only until `limit` results are collected, then dropped, so a page costs at most + // the S3 requests that cover `limit` keys. A key that is not strictly greater than the one before it + // (a service that lists out of order, such as S3 Express directory buckets) is an error: with an + // unsorted listing a bounded page cannot be correct, and draining the namespace is not an option. + let mut paths: Vec = Vec::new(); + while paths.len() < limit { + let meta = match stream.try_next().await { + Ok(Some(meta)) => meta, + Ok(None) => break, + Err(error) => return Err(list_error(error)), + }; + let Some(relative) = relative_path(meta.location.as_ref(), &boundary, &root_boundary) else { + continue; + }; + if after.is_some_and(|cursor| relative.as_str() <= cursor) { + continue; + } + if paths.last().is_some_and(|previous| relative.as_str() <= previous.as_str()) { + return Err(StoreError::Backend("S3 listing is not in ascending key order".into())); + } + paths.push(relative); + } + Ok(paths) + } } #[cfg(test)] mod debug_tests { - use super::S3Config; + use super::{S3Backend, S3Config, list_error, relative_path}; + use crate::StoreError; + + #[test] + fn relative_path_keeps_the_prefix_boundary_exact() { + assert_eq!(relative_path("root/chunks/ab", "root/chunks/", "root/").as_deref(), Some("chunks/ab")); + assert_eq!(relative_path("root/chunksx/ab", "root/chunks/", "root/"), None); + assert_eq!(relative_path("chunks/ab", "chunks/", "").as_deref(), Some("chunks/ab")); + assert_eq!(relative_path("other/chunks/ab", "root/chunks/", "root/"), None); + } + + #[test] + fn keys_object_store_cannot_parse_give_a_specific_error() { + let source = object_store::path::Path::parse("chunks//a").unwrap_err(); + let message = match list_error(object_store::Error::InvalidPath { source }) { + StoreError::Backend(message) => message, + other => panic!("unexpected {other:?}"), + }; + assert!(message.contains("cannot represent"), "{message}"); + // A trailing slash is normalised away by object_store, before this crate sees the key. + assert_eq!(object_store::path::Path::parse("chunks/a/").unwrap().as_ref(), "chunks/a"); + } + + #[test] + fn object_prefix_uses_object_store_path_normalization() { + let config = S3Config { + bucket: "b".into(), + prefix: "a//b%bar".into(), + region: "us-east-1".into(), + endpoint: None, + access_key_id: "key".into(), + secret_access_key: "secret".into(), + allow_http: false, + }; + let backend = S3Backend::new(&config).unwrap(); + assert_eq!(backend.object_path("chunks").as_ref(), "a/b%25bar/chunks"); + } #[test] fn debug_output_hides_the_secret_key() { diff --git a/crates/store/tests/conformance.rs b/crates/store/tests/conformance.rs index 84150342..ae8ba483 100644 --- a/crates/store/tests/conformance.rs +++ b/crates/store/tests/conformance.rs @@ -10,13 +10,37 @@ use craft_cloud_proto::{ChunkId, ManifestId}; use craft_cloud_store::disk::DiskBackend; use craft_cloud_store::memory::MemoryBackend; use craft_cloud_store::s3::{S3Backend, S3Config}; -use craft_cloud_store::{BlobKey, Store, StoreError}; +use craft_cloud_store::{Backend, BlobKey, ListCursor, MAX_LIST_PAGE_SIZE, Store, StoreError}; fn chunk_key(data: &[u8]) -> BlobKey { BlobKey::chunk(&ChunkId::from_bytes(*blake3::hash(data).as_bytes())) } async fn conformance(store: Store) { + let empty = store.list(craft_cloud_store::BlobKind::Chunk, None, 3).await.unwrap(); + assert!(empty.blobs.is_empty() && empty.foreign.is_empty() && empty.next_cursor.is_none()); + let mut expected = Vec::new(); + for n in 0..8 { + let data = format!("listing conformance {n} {}", uuid::Uuid::now_v7()).into_bytes(); + let key = chunk_key(&data); + store.put(key, &data).await.unwrap(); + expected.push(key); + } + expected.sort_by_key(BlobKey::path); + let mut seen = Vec::new(); + let mut cursor = None; + loop { + let page = store.list(craft_cloud_store::BlobKind::Chunk, cursor, 3).await.unwrap(); + assert!(page.foreign.is_empty()); + assert!(page.blobs.len() <= 3); + seen.extend(page.blobs); + cursor = page.next_cursor.map(|cursor| ListCursor::from_token(cursor.as_token().to_owned())); + if cursor.is_none() { + break; + } + } + assert_eq!(seen, expected); + // Round trip, both kinds. A unique payload per run keeps S3 runs independent. let data = format!("chunk bytes {}", uuid::Uuid::now_v7()).into_bytes(); let key = chunk_key(&data); @@ -69,14 +93,27 @@ async fn s3_backend() { let var = |name: &str| std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set with CRAFT_CLOUD_TEST_S3_ENDPOINT")); let config = S3Config { bucket: var("CRAFT_CLOUD_TEST_S3_BUCKET"), - prefix: format!("test-{}", uuid::Uuid::now_v7()), + prefix: format!("test-{}/a//b%bar", uuid::Uuid::now_v7()), region: "us-east-1".into(), endpoint: Some(endpoint), access_key_id: var("CRAFT_CLOUD_TEST_S3_ACCESS_KEY"), secret_access_key: var("CRAFT_CLOUD_TEST_S3_SECRET_KEY"), allow_http: true, }; - conformance(Store::new(S3Backend::new(&config).unwrap())).await; + let store = Store::new(S3Backend::new(&config).unwrap()); + conformance(store.clone()).await; + let raw = S3Backend::new(&config).unwrap(); + raw.put_raw("chunks/foreign-object", Bytes::new()).await.unwrap(); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); + let sibling_config = S3Config { prefix: format!("{}-sibling", config.prefix), ..config.clone() }; + let sibling = Store::new(S3Backend::new(&sibling_config).unwrap()); + let data = b"sibling prefix object"; + let key = chunk_key(data); + sibling.put(key, data).await.unwrap(); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(!page.blobs.contains(&key)); + assert!(!page.foreign.iter().any(|path| path.starts_with("chunks-sibling/"))); } #[tokio::test] @@ -101,6 +138,126 @@ async fn damaged_objects_are_errors_not_data() { assert!(matches!(store.get(key).await, Err(StoreError::Corrupt { reason: "empty object", .. }))); } +#[tokio::test] +async fn memory_list_is_prefix_scoped_and_paginated() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + let key = BlobKey::chunk(&ChunkId::from_bytes([0x11; 32])); + backend.overwrite_raw(&key.path(), Bytes::new()).unwrap(); + backend.overwrite_raw("chunks//a", Bytes::new()).unwrap(); + backend.overwrite_raw("chunksx/sibling-object", Bytes::new()).unwrap(); + + let first = store.list(craft_cloud_store::BlobKind::Chunk, None, 1).await.unwrap(); + assert!(first.blobs.is_empty()); + assert_eq!(first.foreign, vec!["chunks//a"]); + let second = store.list(craft_cloud_store::BlobKind::Chunk, first.next_cursor, 1).await.unwrap(); + assert_eq!(second.blobs, vec![key]); + assert!(second.foreign.is_empty()); + assert!(second.next_cursor.is_none()); +} + +#[tokio::test] +async fn list_empty_store_returns_empty_first_page() { + let store = Store::new(MemoryBackend::new()); + let page = store.list(craft_cloud_store::BlobKind::Manifest, None, 3).await.unwrap(); + assert!(page.blobs.is_empty() && page.foreign.is_empty() && page.next_cursor.is_none()); +} + +#[tokio::test] +async fn list_rejects_invalid_cursor_and_zero_limit() { + let store = Store::new(MemoryBackend::new()); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, Some(ListCursor::from_token("bad")), 1).await, Err(StoreError::Backend(message)) if message == "invalid listing cursor") + ); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, None, 0).await, Err(StoreError::Backend(message)) if message == "listing limit must be greater than zero") + ); +} + +#[tokio::test] +async fn list_clamps_large_limit() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + for n in 0..=MAX_LIST_PAGE_SIZE { + backend.overwrite_raw(&format!("chunks/foreign-{n:04}"), Bytes::new()).unwrap(); + } + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, MAX_LIST_PAGE_SIZE + 40).await.unwrap(); + assert_eq!(page.foreign.len(), MAX_LIST_PAGE_SIZE); + assert!(page.next_cursor.is_some()); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_skips_temps_and_symlinks_outside_root() { + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("escaped"), b"outside").unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + std::fs::write(root.join("chunks").join(format!(".tmp-{}", uuid::Uuid::now_v7())), b"temp").unwrap(); + std::fs::write(root.join("chunks/foreign-object"), b"foreign").unwrap(); + symlink(outside.path(), root.join("chunks/link")).unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(page.blobs.is_empty()); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); +} + +#[tokio::test] +async fn disk_list_rejects_prefix_outside_root() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(dir.path().join("outside"), b"outside").unwrap(); + let backend = DiskBackend::new(&root).unwrap(); + assert!(matches!(backend.list_raw("../", None, 10).await, Err(StoreError::Backend(_)))); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_does_not_follow_symlink_prefix() { + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(outside.join("sub")).unwrap(); + std::fs::write(outside.join("sub/leak"), b"outside").unwrap(); + symlink(&outside, root.join("link")).unwrap(); + let backend = DiskBackend::new(&root).unwrap(); + assert!(backend.list_raw("link", None, 10).await.unwrap().is_empty()); + assert!(backend.list_raw("link/sub", None, 10).await.unwrap().is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_reports_non_utf8_names() { + use std::os::unix::ffi::OsStrExt; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + let name = std::ffi::OsStr::from_bytes(b"bad-\xff"); + std::fs::write(root.join("chunks").join(name), b"foreign").unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await, Err(StoreError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData && error.to_string() == "store path is not valid UTF-8") + ); +} + +#[tokio::test] +async fn disk_list_includes_foreign_and_skips_temporary_files() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + std::fs::write(root.join("chunks").join(format!(".tmp-{}", uuid::Uuid::now_v7())), b"temp").unwrap(); + std::fs::write(root.join("chunks/foreign-object"), b"foreign").unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(page.blobs.is_empty()); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); +} + #[tokio::test] async fn disk_puts_into_new_shard_directories_survive() { let dir = tempfile::tempdir().unwrap(); @@ -140,3 +297,142 @@ async fn disk_writes_leave_no_temporary_files() { let names: Vec = std::fs::read_dir(shard).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); assert_eq!(names, vec![key.path().rsplit('/').next().unwrap().to_owned()]); } + +/// Walks a whole listing with `limit` per page through the backend and returns every path in order. +async fn list_all_raw(backend: &impl Backend, prefix: &str, limit: usize) -> Vec { + let mut all: Vec = Vec::new(); + loop { + let page = backend.list_raw(prefix, all.last().map(String::as_str), limit).await.unwrap(); + if page.is_empty() { + return all; + } + assert!(page.len() <= limit); + all.extend(page); + } +} + +#[tokio::test] +async fn disk_list_page_does_not_walk_finished_directories_again() { + let dir = tempfile::tempdir().unwrap(); + let backend = DiskBackend::new(dir.path()).unwrap(); + let mut paths = Vec::new(); + for n in 0..60 { + let path = chunk_key(format!("walk cost {n}").as_bytes()).path(); + backend.put_raw(&path, Bytes::from_static(b"\0x")).await.unwrap(); + paths.push(path); + } + paths.sort(); + let shards: std::collections::BTreeSet<&str> = paths.iter().filter_map(|path| path.split('/').nth(1)).collect(); + assert!(shards.len() >= 40, "{} shard directories", shards.len()); + + // Page 1 ends in the middle of the store; page 2 resumes there. + let page_one = backend.list_raw("chunks", None, 30).await.unwrap(); + assert_eq!(page_one, paths[..30]); + let before = backend.directories_read(); + let page_two = backend.list_raw("chunks", page_one.last().map(String::as_str), 2).await.unwrap(); + let read = backend.directories_read() - before; + assert_eq!(page_two, paths[30..32]); + // The root, the directories on the way to the cursor and the ones holding the next two objects. The + // shard directories that sort before the cursor are not opened. + assert!(read <= 8, "page 2 opened {read} directories"); + + // Paging by small steps still yields everything exactly once, in order. + assert_eq!(list_all_raw(&backend, "chunks", 7).await, paths); +} + +#[tokio::test] +async fn disk_and_memory_list_in_the_same_byte_order() { + // `a.b` sorts before `a/x` (`.` is 0x2e, `/` is 0x2f) although the directory name `a` sorts before `a.b`. + let names = ["chunks/a.b", "chunks/a/x", "chunks/a/y/z", "chunks/a-b", "chunks/a0", "chunks/B", "chunks/é"]; + let dir = tempfile::tempdir().unwrap(); + let disk = DiskBackend::new(dir.path()).unwrap(); + let memory = MemoryBackend::new(); + for name in &names { + disk.put_raw(name, Bytes::from_static(b"\0")).await.unwrap(); + memory.put_raw(name, Bytes::from_static(b"\0")).await.unwrap(); + } + let mut expected: Vec = names.iter().map(|name| (*name).to_owned()).collect(); + expected.sort(); + for limit in [1, 2, 3, 100] { + assert_eq!(list_all_raw(&disk, "chunks", limit).await, expected, "disk, limit {limit}"); + assert_eq!(list_all_raw(&memory, "chunks", limit).await, expected, "memory, limit {limit}"); + } +} + +#[tokio::test] +async fn disk_list_skips_only_the_writers_temporary_names() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + let writer_name = format!(".tmp-{}", uuid::Uuid::now_v7()); + std::fs::write(root.join("chunks").join(&writer_name), b"in progress").unwrap(); + let foreign_names = [ + ".tmp-not-a-uuid", + ".tmp-", + ".tmp-0190a1b2-c3d4-7e5f-8a6b-7c8d9e0f1a2bx", + ".tmp-0190A1B2-C3D4-7E5F-8A6B-7C8D9E0F1A2B", + ".tmp-6ba7b810-9dad-11d1-80b4-00c04fd430c8", + ]; + for foreign in foreign_names { + std::fs::write(root.join("chunks").join(foreign), b"foreign").unwrap(); + } + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 20).await.unwrap(); + let mut expected: Vec = foreign_names.iter().map(|name| format!("chunks/{name}")).collect(); + expected.sort(); + assert_eq!(page.foreign, expected); + assert!(!page.foreign.iter().any(|path| path.ends_with(&writer_name))); +} + +#[tokio::test] +async fn memory_list_ignores_other_namespaces() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + let chunk = BlobKey::chunk(&ChunkId::from_bytes([0x22; 32])); + let manifest = BlobKey::manifest(&ManifestId::from_bytes([0x33; 32])); + for path in ["a/before", "chunks/zzz-foreign", "chunksx/sibling", "manifests/zzz-foreign", "zzz/after"] { + backend.overwrite_raw(path, Bytes::new()).unwrap(); + } + backend.overwrite_raw(&chunk.path(), Bytes::new()).unwrap(); + backend.overwrite_raw(&manifest.path(), Bytes::new()).unwrap(); + + let first = store.list(craft_cloud_store::BlobKind::Chunk, None, 1).await.unwrap(); + assert_eq!(first.blobs, vec![chunk]); + let second = store.list(craft_cloud_store::BlobKind::Chunk, first.next_cursor, 1).await.unwrap(); + assert_eq!(second.foreign, vec!["chunks/zzz-foreign"]); + assert!(second.next_cursor.is_none()); + let manifests = store.list(craft_cloud_store::BlobKind::Manifest, None, 10).await.unwrap(); + assert_eq!(manifests.blobs, vec![manifest]); + assert_eq!(manifests.foreign, vec!["manifests/zzz-foreign"]); + // A cursor from before the namespace starts at the namespace; one past it finds nothing. + assert_eq!(backend.list_raw("chunks", Some("a/before"), 10).await.unwrap().len(), 2); + assert!(backend.list_raw("chunks", Some("zzz/after"), 10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn list_cursor_is_checked_before_it_is_decoded() { + let invalid = |result: Result| matches!(result, Err(StoreError::Backend(message)) if message == "invalid listing cursor"); + let store = Store::new(MemoryBackend::new()); + let kind = craft_cloud_store::BlobKind::Chunk; + // Otherwise valid cursors (`chunks/` and then filler) that are far longer than any real object path. + let hex = |path: &str| path.bytes().map(|byte| format!("{byte:02x}")).collect::(); + let huge = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4 * 1024 * 1024)); + assert!(invalid(store.list(kind, Some(ListCursor::from_token(huge)), 1).await)); + let just_over = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4096 - "chunks/".len() + 1)); + assert!(invalid(store.list(kind, Some(ListCursor::from_token(just_over)), 1).await)); + let longest = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4096 - "chunks/".len())); + assert!(store.list(kind, Some(ListCursor::from_token(longest)), 1).await.is_ok()); + for garbage in + ["", ":", "1:c:", "1:c:zz", "1:c:6", "1:c:ff", "2:c:6368756e6b732f61", "1:x:6368756e6b732f61", "1:c:6d616e69666573742f61", "not a cursor"] + { + assert!(invalid(store.list(kind, Some(ListCursor::from_token(garbage)), 1).await), "{garbage:?}"); + } + // A real cursor of one kind is refused for the other kind. + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + backend.overwrite_raw("chunks/a", Bytes::new()).unwrap(); + backend.overwrite_raw("chunks/b", Bytes::new()).unwrap(); + let cursor = store.list(kind, None, 1).await.unwrap().next_cursor.unwrap(); + assert!(invalid(store.list(craft_cloud_store::BlobKind::Manifest, Some(cursor.clone()), 1).await)); + assert!(store.list(kind, Some(cursor), 1).await.is_ok()); +} diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index 9a78bb49..ab064918 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -44,6 +44,35 @@ impl Engine { self.state.base.remove(&id); } self.state.sent_deletes.clear(); + // The same for folders, whose deletes carry no version. A folder this machine asked to delete (the answer never + // recorded) that the server reports live is let go like a file whose delete went through (it was brought back + // after the delete, or changed by another machine first), and arrives as it is on the server, not deleted a + // second time. A folder the server no longer has went through, so it is let go too, once it is gone from here + // as well. This is settled deepest first, after the files above: a restored folder is only judged once the + // folders of this machine's own delete below it are settled, or a tombstoned subfolder still tracked in it + // would keep it and the next push would delete it again. A folder the feed did not mention was never touched: + // its delete is sent again. One that still has tracked nodes in it stays. + let live_dirs = std::mem::take(&mut self.state.live_dirs); + let mut asked: Vec<(usize, NodeId)> = + std::mem::take(&mut self.state.sent_dir_deletes).into_iter().map(|id| (self.state.base_path(id).map_or(0, |p| p.depth()), id)).collect(); + asked.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + for (_, id) in asked { + let is_dir = matches!(self.state.base.get(&id).map(|e| &e.kind), Some(BaseKind::Dir { .. })); + if !is_dir || self.state.base.values().any(|e| e.parent == Some(id)) { + continue; + } + let released = if live_dirs.contains(&id) { + true + } else if self.state.remote.contains_key(&id) { + false + } else { + // Deleted on the server; nothing of it is left here when its place is empty. + self.state.base_path(id).is_some_and(|path| matches!(self.stat(&path), Ok(None))) + }; + if released { + self.state.base.remove(&id); + } + } // Changed or new remote nodes, parents first. let mut work: Vec<(usize, NodeId)> = self.state.remote.keys().filter(|id| **id != root).filter_map(|id| Some((self.state.remote_path(*id)?.depth(), *id))).collect(); @@ -237,7 +266,7 @@ impl Engine { } /// Fails as a filesystem error on the node (it is skipped and waits) if taking `to` would act on a link. - fn refuse_link_toward(&self, to: &RelPath) -> Result<(), SyncError> { + pub(super) fn refuse_link_toward(&self, to: &RelPath) -> Result<(), SyncError> { match self.link_toward(to)? { Some(link) => Err(io_err(&link, std::io::Error::other(OTHER_REASON))), None => Ok(()), @@ -862,7 +891,7 @@ impl Engine { } /// Creates `dir` and any missing folders above it, moving aside anything in the way. - fn make_dirs(&mut self, dir: &RelPath, report: &mut Report) -> Result<(), SyncError> { + pub(super) fn make_dirs(&mut self, dir: &RelPath, report: &mut Report) -> Result<(), SyncError> { let mut current = RelPath::root(); for name in dir.names() { current = current.join(name); @@ -894,7 +923,7 @@ impl Engine { /// Makes sure nothing sits at `path`, not even a name that differs only by case: whatever is there is /// moved aside as a conflict copy. - fn ensure_free(&mut self, path: &RelPath, report: &mut Report) -> Result<(), SyncError> { + pub(super) fn ensure_free(&mut self, path: &RelPath, report: &mut Report) -> Result<(), SyncError> { // Checked before anything is made or moved aside. self.refuse_link_toward(path)?; if let Some(parent) = path.parent() { @@ -1050,6 +1079,10 @@ impl Engine { /// When the agent can't tell yet which save is newer, the file waits ([`LiveWinner::Wait`]); after /// `save_order_wait_ms` of that, `None` (a conflict copy). A winner is never guessed. fn live_winner(&mut self, id: NodeId, file: &FileState, at: &RelPath) -> Result)>, SyncError> { + // Where nothing can go up, "ours goes over theirs" is not on offer: both versions are kept as a conflict copy. + if self.read_only { + return Ok(None); + } let checked = self.stat(at)?; let hash = self.content_hash(at); // Our content is a Craft save whose record is on its way: never a conflict copy, it waits for the record. @@ -1096,7 +1129,7 @@ impl Engine { Some(LiveWinner::Wait) } - fn holds_base_content(&self, entry: &BaseEntry, at: &RelPath) -> bool { + pub(super) fn holds_base_content(&self, entry: &BaseEntry, at: &RelPath) -> bool { match &entry.kind { BaseKind::File { hash, .. } => self.content_hash(at) == Some(*hash), BaseKind::Dir { .. } => false, diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index 55f2335b..817efe6e 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -32,6 +32,8 @@ use crate::transport::{Transport, TransportError}; mod apply; mod push; +mod readonly; +pub use readonly::{READ_ONLY_EDITED, READ_ONLY_NEW}; /// Names that never sync (matched against the raw name, ignoring ASCII case): exact names, `prefix*` and `*suffix`. pub const DEFAULT_IGNORES: &[&str] = &[".craft-cloud", ".DS_Store", "Thumbs.db", "desktop.ini", "~$*", "*.tmp", "*.part", ".~lock.*"]; @@ -75,6 +77,9 @@ pub enum SyncError { /// Saves the engine's state during a pass (see [`Engine::on_checkpoint`]). pub type Checkpoint = Box io::Result<()> + Send + Sync>; +/// Asked before every commit (see [`Engine::on_guard`]). +pub type Guard = Box io::Result<()> + Send + Sync>; + fn io_err(path: &RelPath, source: io::Error) -> SyncError { SyncError::Io { path: path.to_string(), source } } @@ -204,6 +209,11 @@ pub struct Report { pub held_nodes: Vec, /// Operations the server refused this pass; they are retried next pass. pub refused: Vec, + /// Whether the pass ran in read-only mode (see [`Engine::set_read_only`]): nothing was sent to the server. + pub read_only: bool, + /// Files and folders deleted here that the server still has, put back this pass (read-only mode only: a delete + /// there can't go up, so the server's copy comes back, and the report says so). + pub restored: Vec, } impl Report { @@ -216,6 +226,7 @@ impl Report { && self.moved_remotely.is_empty() && self.created_dirs.is_empty() && self.conflict_copies.is_empty() + && self.restored.is_empty() && self.refused.is_empty()) } } @@ -350,6 +361,7 @@ pub struct Engine { /// (see [`Engine::twins_in`]). Behind a lock because the checks only read the engine. listings: Mutex>, checkpoint: Option, + guard: Option, /// The new folder IDs the last look at the start of a pass found but didn't take (see [`Engine::learn_dir_ids`]). declined_dir_ids: BTreeSet<(NodeId, u64)>, /// The account's storage on the server as the last refusal this pass gave it (`used`, `limit`): a later file @@ -366,6 +378,18 @@ pub struct Engine { /// Since when each such file has waited, across passes (a monotonic clock: on Linux and macOS it stops while the /// machine sleeps; on Windows it may not). unknown_since: HashMap, + /// Whether this space is read-only for the account (a viewer): the engine downloads and applies the server's changes + /// and never sends anything (see [`Engine::set_read_only`]). + read_only: bool, + /// The files only here that the last read-only pass listed (path and reason); a pass lists them again only when this + /// changes. Cleared when the mode ends. + local_only_listed: BTreeSet<(String, &'static str)>, + /// What read-only mode last found out about tracked files of the same size as the agreed content: whether they hold + /// something else, by the file's size, time and ID and the agreed content, so an unchanged file is read once. + read_only_verdicts: BTreeMap, + /// The files whose new content a commit of the running push took, so a push cut short by a refusal reports as uploaded + /// only those (content sent for the rest was not committed). + committed_files: BTreeSet, } /// What Craft apps saved from live documents, as the agent learnt it from them (`craft_cloud_proto::SaveRecord`). @@ -436,15 +460,43 @@ impl Engine { fs_changes: 0, listings: Mutex::new(BTreeMap::new()), checkpoint: None, + guard: None, declined_dir_ids: BTreeSet::new(), quota: None, saves: None, attached: BTreeSet::new(), deferred: BTreeSet::new(), unknown_since: HashMap::new(), + read_only: false, + local_only_listed: BTreeSet::new(), + read_only_verdicts: BTreeMap::new(), + committed_files: BTreeSet::new(), }) } + /// Puts the engine in or out of read-only mode, for a space where the account's role is `viewer` (known from + /// `GET /spaces`, or from a `403 read_only` the engine learns by itself). + /// + /// In read-only mode a pass applies the server's changes as usual and sends nothing: a local edit is kept and never + /// overwritten (when the server's version of the file changes too, the server's lands under the original name and the + /// edit stays as a conflict copy), a new local file stays local, and a file or folder deleted here is downloaded again + /// (reported in [`Report::restored`]). Everything kept here is listed in [`Report::skipped`] with the reason. While in + /// this mode each pass asks the server for the role (`GET /space`): once it is `editor` or better the mode ends, and + /// the next push uploads the local files as ordinary new files and edits. An engine not told anything learns the mode + /// from the first write the server refuses, then makes no further write. + pub fn set_read_only(&mut self, read_only: bool) { + self.read_only = read_only; + if !read_only { + self.local_only_listed.clear(); + self.read_only_verdicts.clear(); + } + } + + /// Whether the engine is in read-only mode ([`Engine::set_read_only`]). + pub fn is_read_only(&self) -> bool { + self.read_only + } + /// Where the engine learns what Craft apps saved from live documents: a version uploaded with content an app /// saved carries its record, and two versions of a live-linked file that both do never make a conflict copy /// (see [`SaveRecords`]). @@ -479,6 +531,20 @@ impl Engine { self.checkpoint = Some(Box::new(save)); } + /// Asks `check` before every commit of a push, whatever it holds: an error stops the pass ([`SyncError::Checkpoint`]) and the + /// batch is not sent. A caller whose folder can change under the pass (a link pointed elsewhere) uses it so that nothing + /// scanned in one folder is committed after the folder has changed. + pub fn on_guard(&mut self, check: impl FnMut() -> io::Result<()> + Send + Sync + 'static) { + self.guard = Some(Box::new(check)); + } + + fn guard(&mut self) -> Result<(), SyncError> { + match &mut self.guard { + Some(check) => check().map_err(SyncError::Checkpoint), + None => Ok(()), + } + } + fn checkpoint(&mut self) -> Result<(), SyncError> { match &mut self.checkpoint { Some(save) => save(&self.state).map_err(SyncError::Checkpoint), @@ -544,6 +610,7 @@ impl Engine { report.skipped.extend(self.fs.notices()); // Before anything can fail (an offline server, a download), so a pass that fails still leaves it behind. self.learn_dir_ids()?; + self.learn_role().await; let mode = match self.pull().await? { Mode::Normal if self.state.reconcile_pending => Mode::Reconcile, mode => mode, @@ -552,20 +619,49 @@ impl Engine { self.state.reconcile_pending = false; self.discard_stale_downloads(); self.checkpoint()?; - self.push(&mut report).await?; + // A first write the server refuses ends the push with nothing stored: the engine switches to read-only mode and + // the same pass goes on in it, so the local changes are reported as kept and nothing is retried. + self.committed_files.clear(); + match self.push(&mut report).await { + Err(SyncError::Transport(TransportError::ReadOnly)) => { + self.read_only = true; + // What the refused push did before the refusal stays in the report: batches committed before the role + // changed, and conflict copies already moved aside. Content sent for files no commit took is not + // reported as uploaded (it stays here, and is listed as kept). + let committed: BTreeSet = self.committed_files.iter().filter_map(|id| self.state.base_path(*id)).collect(); + report.uploaded.retain(|path| committed.contains(path)); + self.push(&mut report).await?; + } + other => other?, + } + report.read_only = self.read_only; self.approval = Approval::None; self.release.clear(); self.attach_late_saves().await; Ok(report) } + /// While in read-only mode, asks the server for the account's role: once it may write again (a viewer made an editor), + /// the mode ends and this pass pushes. An answer that doesn't come or doesn't say changes nothing: the pull that follows + /// meets the same trouble, and a missing role (a server from before team spaces) never refuses a write. + async fn learn_role(&mut self) { + if !self.read_only { + return; + } + if let Ok(space) = self.transport.space().await + && space.role.is_some_and(|role| role >= craft_cloud_proto::api::SpaceRole::Editor) + { + self.set_read_only(false); + } + } + /// Save records that came after their version went up (an app saved while the agent was away, and told it once /// back): each is added to its version on the server (appended; the same again changes nothing), so the version /// counts as a Craft save everywhere. Only a file whose server version is the one this machine agreed on, with the /// content the record names, and none yet. Sent once per version per run; an older server (`NotFound`) is let be, /// anything else tried again next pass. async fn attach_late_saves(&mut self) { - if self.saves.is_none() { + if self.saves.is_none() || self.read_only { return; } let mut late = Vec::new(); @@ -609,6 +705,11 @@ impl Engine { if node.deleted { self.state.remote.remove(&node.id); } else { + // A folder this machine asked to delete (the answer never recorded) that the server reports + // changed and live: noted here, judged in `apply_remote` once the files it held are settled. + if self.state.sent_dir_deletes.contains(&node.id) { + self.state.live_dirs.insert(node.id); + } self.state.remote.insert(node.id, RemoteNode::from_state(&node)); } } @@ -672,6 +773,10 @@ impl Engine { } } let (epoch, seq) = first.ok_or_else(|| SyncError::Corrupt("empty listing".into()))?; + // A folder this machine asked to delete (the answer never recorded) that the listing has live is never pushed as + // deleted from that stale request: a listing can't tell a delete that never arrived from one followed by a + // restore, so the folder is let go and arrives as the server has it (the conservative side of a doubt). + self.state.live_dirs.extend(self.state.sent_dir_deletes.iter().filter(|id| remote.contains_key(*id)).copied()); self.state.remote = remote; self.state.epoch = Some(epoch); self.state.seq = seq; diff --git a/crates/sync/src/engine/push.rs b/crates/sync/src/engine/push.rs index 470e36f2..af8607ef 100644 --- a/crates/sync/src/engine/push.rs +++ b/crates/sync/src/engine/push.rs @@ -37,6 +37,9 @@ fn too_big(size: u64, limit: u64) -> String { impl Engine { pub(super) async fn push(&mut self, report: &mut Report) -> Result<(), SyncError> { + if self.read_only { + return self.push_read_only(report).await; + } let root = self.state.root.ok_or_else(|| SyncError::Corrupt("no root".into()))?; let (local, mut unseen) = self.scan(report)?; // Inside a folder that couldn't be listed, or at or below a link, nothing is known, so nothing tracked there @@ -649,21 +652,36 @@ impl Engine { _ => None, }) .collect(); - if !deletes.is_empty() { + let dir_deletes: Vec = only_ops + .iter() + .filter_map(|op| match op { + Op::Delete { node, expected: None } => Some(*node), + _ => None, + }) + .collect(); + if !deletes.is_empty() || !dir_deletes.is_empty() { self.state.sent_deletes.extend(deletes.iter().copied()); + self.state.sent_dir_deletes.extend(dir_deletes.iter().copied()); self.checkpoint()?; } + self.guard()?; let result = self.transport.commit(&only_ops).await; // Applied or refused as a whole, the batch's deletes are no longer in doubt; any other answer leaves // them written down. - if matches!(result, Ok(_) | Err(TransportError::Rejected(_))) { + if matches!(result, Ok(_) | Err(TransportError::Rejected(_) | TransportError::ReadOnly)) { for (node, _) in &deletes { self.state.sent_deletes.remove(node); } + for node in &dir_deletes { + self.state.sent_dir_deletes.remove(node); + } } match result { Ok(_) => { for (_, effect) in batch.drain(..) { + if let Effect::File { id, .. } | Effect::Update { id, .. } = &effect { + self.committed_files.insert(*id); + } self.record(effect, report); } if queue.is_empty() diff --git a/crates/sync/src/engine/readonly.rs b/crates/sync/src/engine/readonly.rs new file mode 100644 index 00000000..3894a9da --- /dev/null +++ b/crates/sync/src/engine/readonly.rs @@ -0,0 +1,202 @@ +//! The push of a space that is read-only for the account (a viewer; see [`Engine::set_read_only`]). +//! +//! Nothing goes to the server. The folder is only ever changed to match the server, and only where that loses +//! nothing the user made: +//! +//! - A local edit stays where it is. When the server's version of that file changes too, the pull side already keeps +//! both (the server's version lands under the original name, the edit moves to a conflict copy). +//! - A new local file or folder stays local. +//! - A file or folder deleted here that the server still has is downloaded again. Deleting it on the server is not on +//! offer, and leaving the folder different from the space without saying so would hide that. The report names what +//! came back ([`Report::restored`]). +//! - A file moved or renamed here is the same case: the server's copy comes back at its old place, and the moved file +//! stays as a new local file. +//! +//! Everything that stays only here is listed in [`Report::skipped`] with the reason, on every pass. + +use std::collections::{BTreeMap, BTreeSet}; + +use craft_cloud_proto::{FileHash, NodeId}; + +use crate::fs::{FileMeta, Fs, FsKind}; +use crate::path::RelPath; +use crate::state::{BaseEntry, BaseKind}; +use crate::transport::Transport; + +use super::{Engine, Local, Report, SyncError, per_node}; + +/// Why a file that is only here (new, or a conflict copy) is not uploaded in read-only mode. +pub const READ_ONLY_NEW: &str = "not uploaded: this space is read-only for you; the file stays on this computer"; +/// Why an edited file is not uploaded in read-only mode. +pub const READ_ONLY_EDITED: &str = "edited here, not uploaded: this space is read-only for you; the file stays on this computer"; + +impl Engine { + /// The push of read-only mode: puts back what was deleted here and lists what stays local. Sends nothing. + /// + /// Known limit, kept on purpose: `finish_push` (which records the links the push saw and forgets where nodes were + /// before they moved) never runs in this mode, so right after the role is raised to editor the first push may hold + /// back a delete of a user's that it would have sent a pass later. A delete held is never a lost file; it is sent + /// by a following pass, once that push has finished. + pub(super) async fn push_read_only(&mut self, report: &mut Report) -> Result<(), SyncError> { + let before = report.restored.len(); + self.restore_deleted(report).await?; + if report.restored.len() != before { + self.checkpoint()?; + } + self.list_local_only(report) + } + + /// Downloads again every tracked node the server still has that is missing here, parents first. + async fn restore_deleted(&mut self, report: &mut Report) -> Result<(), SyncError> { + let root = self.state.root.ok_or_else(|| SyncError::Corrupt("no root".into()))?; + let mut nodes: Vec<(usize, NodeId)> = self + .state + .base + .iter() + .filter(|(id, _)| **id != root && self.state.remote.contains_key(*id)) + .filter_map(|(id, _)| Some((self.state.base_path(*id)?.depth(), *id))) + .collect(); + nodes.sort(); + // Nodes skipped this pass (a filesystem error); what is below one waits for it too. + let mut skipped: BTreeSet = BTreeSet::new(); + for (_, id) in nodes { + if self.state.base.get(&id).and_then(|e| e.parent).is_some_and(|p| skipped.contains(&p)) { + skipped.insert(id); + continue; + } + let result = self.restore_one(id, report).await; + // `Ok(true)`: something that is not the node sits where it was (a folder's place held by a file or a link): + // left alone, and what was below it waits. An error skips it the same way. + let held = match result { + Ok(held) => held, + other => per_node(other.map(drop), report)?, + }; + if held { + skipped.insert(id); + } + } + Ok(()) + } + + /// Puts back one node that is gone from here. Returns whether the node's own place is held by something else that + /// stays (the nodes below it are then left alone too). + async fn restore_one(&mut self, id: NodeId, report: &mut Report) -> Result { + let (Some(entry), Some(remote), Some(path)) = + (self.state.base.get(&id).cloned(), self.state.remote.get(&id).cloned(), self.state.base_path(id)) + else { + return Ok(false); + }; + // Anything there (the file, a link, another file) is left alone: only what is gone comes back. A folder's place + // held by anything but a folder holds back what was in it: a file or a link there is the user's, and the + // server's files go nowhere near it. + if let Some(here) = self.stat(&path)? { + return Ok(matches!(entry.kind, BaseKind::Dir { .. }) && !matches!(here, FsKind::Dir { .. })); + } + // `stat` of a path below a file or a link says "nothing there": looked at before anything is made or moved + // aside, or the file or link would be taken for something in the way. + if self.held_above(&path)? { + return Ok(true); + } + match (&entry.kind, &remote.file) { + (BaseKind::Dir { .. }, None) => { + self.refuse_link_toward(&path)?; + let made = report.created_dirs.len(); + self.make_dirs(&path, report)?; + // Reported once, as restored (not as a folder the pass made as well). + let mut kept = report.created_dirs.drain(made..).filter(|dir| *dir != path).collect(); + report.created_dirs.append(&mut kept); + report.restored.push(path); + } + (BaseKind::File { .. }, Some(file)) => { + self.ensure_free(&path, report)?; + // Saved here meanwhile: not taken this pass; the next one finds the file and keeps it. + if let Some((meta, hash)) = self.download(id, file, &path, None, report).await? { + let kind = self.base_file(file, meta, hash); + self.state.base.update(&id, |base| base.kind = kind); + report.restored.push(path); + } + } + // A node can't change kind; treat it as damaged state and leave it. + _ => {} + } + Ok(false) + } + + /// Whether a folder above `path` is not a folder here (a file, or a link or other entry): nothing below it is put + /// back, and nothing is moved aside to make room. + fn held_above(&self, path: &RelPath) -> Result { + let Some(parent) = path.parent() else { return Ok(false) }; + let mut current = RelPath::root(); + for name in parent.names() { + current = current.join(name); + if matches!(self.stat(¤t)?, Some(FsKind::File(_) | FsKind::Other)) { + return Ok(true); + } + } + Ok(false) + } + + /// Lists, in [`Report::skipped`], every file that exists only here: edited tracked files and untracked files + /// (new ones and conflict copies). A tracked file that was swapped for another tracked file counts as missing here + /// ([`Local::Missing`]), so it is put back and not listed (a listing gap only: both files stay). The list is given in full when the set of such files (or a reason) differs from the + /// one the last pass gave, and left out otherwise, so a quiet viewer's folder produces a quiet report. + fn list_local_only(&mut self, report: &mut Report) -> Result<(), SyncError> { + let (local, unseen) = self.scan(report)?; + let index = self.state.base_index(); + let mut found: BTreeSet<(String, &'static str)> = BTreeSet::new(); + let mut looked_at: BTreeMap = BTreeMap::new(); + for (path, kind) in &local { + let FsKind::File(meta) = kind else { continue }; + if unseen.unknown.iter().any(|u| path.starts_with(u)) { + continue; + } + let reason = match index.get(path).and_then(|id| self.state.base.get(id)).cloned() { + None => READ_ONLY_NEW, + Some(entry) => { + if !self.edited_here(&entry, path, *meta, &mut looked_at)? { + continue; + } + READ_ONLY_EDITED + } + }; + found.insert((path.to_string(), reason)); + } + self.read_only_verdicts = looked_at; + if found != self.local_only_listed { + report.skipped.extend(found.iter().map(|(path, reason)| (path.clone(), (*reason).to_owned()))); + self.local_only_listed = found; + } + Ok(()) + } + + /// Whether the tracked file at `path` holds something other than what both sides agreed on. A different size says so + /// without reading the file; the same size needs its content hashed, and the answer is kept for as long as the file + /// (its size, time and ID) and the agreed content stay the same, so a quiet viewer's pass does not read every edited + /// file again. Kept answers are carried in `looked_at`, which replaces the cache after the pass (files that went + /// are forgotten). + fn edited_here( + &self, + entry: &BaseEntry, + path: &RelPath, + meta: FileMeta, + looked_at: &mut BTreeMap, + ) -> Result { + if self.local_state(entry, path)? != Local::Modified { + return Ok(false); + } + let BaseKind::File { size, hash, .. } = &entry.kind else { return Ok(false) }; + if meta.size != *size { + return Ok(true); + } + if let Some((seen, base, edited)) = self.read_only_verdicts.get(path) + && *seen == meta + && base == hash + { + looked_at.insert(path.clone(), (meta, *hash, *edited)); + return Ok(*edited); + } + let edited = !self.holds_base_content(entry, path); + looked_at.insert(path.clone(), (meta, *hash, edited)); + Ok(edited) + } +} diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index c6c59451..75b7305f 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -13,8 +13,8 @@ pub mod state; pub mod transport; pub use engine::{ - Config, DEFAULT_IGNORES, Engine, NOT_NFC_REASON, OTHER_REASON, Report, SAVE_ORDER_WAITS, SAVE_RECORD_WAITS, STILL_CHANGING, SaveRecords, - SyncError, + Config, DEFAULT_IGNORES, Engine, NOT_NFC_REASON, OTHER_REASON, READ_ONLY_EDITED, READ_ONLY_NEW, Report, SAVE_ORDER_WAITS, SAVE_RECORD_WAITS, + STILL_CHANGING, SaveRecords, SyncError, }; pub use fs::{FileMeta, Fs, FsEntry, FsKind, LandError, MemFs, ReadOnly, is_read_only, read_only_error, rel}; pub use path::RelPath; diff --git a/crates/sync/src/state.rs b/crates/sync/src/state.rs index f88b8cab..323a4daf 100644 --- a/crates/sync/src/state.rs +++ b/crates/sync/src/state.rs @@ -36,6 +36,18 @@ pub struct State { /// the server's changes. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub sent_deletes: BTreeMap, + /// Folders this machine asked the server to delete in a commit whose answer it never recorded. A folder has no + /// version to tell a delete that went through and a later restore from one that never arrived, so the pull judges + /// it by the change feed: a folder the server reports changed and live (brought back from the trash after the + /// delete, or changed by another machine first) is let go, once the files it held are settled, and arrives as it is + /// on the server, instead of being deleted a second time; one the feed doesn't mention was never touched, and its delete is sent again. Cleared + /// once a pass has applied the server's changes. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub sent_dir_deletes: BTreeSet, + /// Of `sent_dir_deletes`, the folders the pull of this pass saw the server report changed and live. Only for the + /// rest of the pass: never saved. + #[serde(skip)] + pub live_dirs: BTreeSet, /// Server deletes held behind a link: tracked nodes missing here while a link or special file that appeared /// since the last finished push carries a name from the node's last local path, so it may be the node, moved with /// its folder (seed 112337's no-pass order; see the push). Each is held for as long as any of its links stands diff --git a/crates/sync/src/transport.rs b/crates/sync/src/transport.rs index f5d63147..9dd88b0a 100644 --- a/crates/sync/src/transport.rs +++ b/crates/sync/src/transport.rs @@ -19,6 +19,11 @@ pub enum TransportError { /// less than a chunk's worth is ([`Config::quota_floor`](crate::Config::quota_floor)). #[error("the account's storage on the server is full or nearly ({used} of {limit} bytes used); the admin can allow more")] QuotaExceeded { used: u64, limit: u64 }, + /// The server refused a write because the account's role in this space doesn't allow one (`403 read_only`: a + /// viewer, or a space that was archived). Nothing was stored. The engine switches the space to read-only mode + /// ([`Engine::set_read_only`](crate::Engine::set_read_only)) and keeps every local change. + #[error("this space is read-only for you: your role in it doesn't allow changes")] + ReadOnly, /// The change feed can't continue from our sequence number; list the whole tree. #[error("the change feed needs a full listing")] ResyncRequired, diff --git a/crates/sync/tests/common/mod.rs b/crates/sync/tests/common/mod.rs index 68877bbb..91b455a2 100644 --- a/crates/sync/tests/common/mod.rs +++ b/crates/sync/tests/common/mod.rs @@ -12,10 +12,10 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; use bytes::Bytes; -use craft_cloud_meta::{Meta, MetaError, Role, UserRecord}; -use craft_cloud_proto::api::{CommitResponse, SpaceResponse}; +use craft_cloud_meta::{Meta, MetaError, Role, SpaceActor, UserId, UserRecord, WriteGuard}; +use craft_cloud_proto::api::{CommitResponse, SpaceResponse, SpaceRole}; use craft_cloud_proto::{ - ChangeSet, ChunkId, Conflict, ConflictKind, ConflictStamp, DeviceId, FileHash, Manifest, ManifestId, Op, PageCursor, SpaceId, + ChangeSet, ChunkId, Conflict, ConflictKind, ConflictStamp, DeviceId, FileHash, Manifest, ManifestId, Op, PageCursor, SpaceId, SpaceName, }; use craft_cloud_store::memory::MemoryBackend; use craft_cloud_store::{BlobKey, Store}; @@ -41,6 +41,21 @@ impl Server { self.meta.create_user(name, "$argon2id$test", Role::Member, 0).await.unwrap() } + /// A team space called `name` with `owner` as its first owner. + pub async fn team_space(&self, name: &str, owner: &UserRecord) -> SpaceId { + self.meta.create_team_space(&SpaceActor::Cli, &SpaceName::new(name).unwrap(), owner.id, None, 0).await.unwrap() + } + + /// `user` is a member of `space` with `role` (added on the command line: accepted at once). + pub async fn add_member(&self, space: SpaceId, user: &UserRecord, role: SpaceRole) { + self.meta.invite_member(&SpaceActor::Cli, space, user.id, role, 0).await.unwrap(); + } + + /// Gives the member `user` of `space` another role. + pub async fn set_role(&self, space: SpaceId, user: &UserRecord, role: SpaceRole) { + self.meta.change_member_role(&SpaceActor::Cli, space, user.id, role, 0).await.unwrap(); + } + /// The whole-file hash of every version the server ever stored, for "nothing was lost" checks. /// /// These are the versions of the commits the transport saw succeed (recorded when `Meta::commit` returns @@ -80,6 +95,11 @@ pub struct Faults { pub every: AtomicU64, /// Make the next commit apply on the server but report failure, as if its reply was lost. pub lose_commit_reply: AtomicBool, + /// Fail the next commit before it reaches the server. + pub fail_next_commit: AtomicBool, + /// Make the next read of the change feed answer `ResyncRequired`, as if the server's history had been cut short: + /// the machine then lists the tree again. + pub resync_next_changes: AtomicBool, /// Make every commit come back refused, naming an operation the batch doesn't have. pub bad_conflict_index: AtomicBool, /// The machine died (see `MemFs::die_after`): every call fails. @@ -87,10 +107,16 @@ pub struct Faults { /// The machine dies right after the n-th commit from now applies on the server, before its answer arrives /// (0 = never): that commit's reply is lost, and every call after it fails. pub die_on_commit: AtomicU64, + /// The n-th commit from now on (1 = the next) and every one after it is refused with `ReadOnly`, as when the account's + /// role was lowered in between (0 = never). Nothing is stored by a refused commit. + pub refuse_commits_from: AtomicU64, /// How many chunks this device downloaded (counted, not a fault). pub chunk_downloads: AtomicU64, /// How many chunks this device uploaded (counted, not a fault). pub chunk_uploads: AtomicU64, + /// How many calls that write (missing-chunk query, chunk, manifest, commit, save record) the device made, however + /// they were answered (counted, not a fault): a viewer's engine makes none once it knows its role. + pub write_calls: AtomicU64, /// The next this many chunk uploads are answered as stored but never stored, as by a proxy that answered for /// the server and lost them. pub lose_chunks: AtomicU64, @@ -119,6 +145,8 @@ pub struct Local { device: DeviceId, username: String, page: u32, + /// The account when the space is a team space: the server's role checks run for it, as the routes do. + member: Option, pub faults: Arc, clock: Arc>, committed: Arc>>, @@ -130,12 +158,32 @@ fn meta_err(e: MetaError) -> TransportError { MetaError::MissingChunks(m) => TransportError::MissingChunks(m), MetaError::ResyncRequired => TransportError::ResyncRequired, MetaError::QuotaExceeded { used, limit } => TransportError::QuotaExceeded { used, limit }, + MetaError::ReadOnly => TransportError::ReadOnly, MetaError::NotFound(_) => TransportError::NotFound, other => TransportError::Protocol(other.to_string()), } } impl Local { + /// The metadata handle a write goes through: in a team space it checks the role again inside its transaction. + fn writer(&self) -> Meta { + match self.member { + Some(user) => self.meta.guarded(WriteGuard { user, min: SpaceRole::Editor }), + None => self.meta.clone(), + } + } + + /// The layer in front of a write route: counts the call and refuses an account whose role is too low. + async fn write_gate(&self) -> Result<(), TransportError> { + self.faults.write_calls.fetch_add(1, Ordering::Relaxed); + let Some(user) = self.member else { return Ok(()) }; + match self.meta.space_access(user, self.space).await.map_err(meta_err)? { + Some(role) if role >= SpaceRole::Editor => Ok(()), + Some(_) => Err(TransportError::ReadOnly), + None => Err(TransportError::NotFound), + } + } + fn now(&self) -> i64 { let mut c = self.clock.lock().unwrap(); *c += 1; @@ -148,7 +196,19 @@ impl Transport for Local { async fn space(&self) -> Result { self.faults.check()?; let info = self.meta.space_info(self.space).await.map_err(meta_err)?; - Ok(SpaceResponse { space: self.space, root: info.root, epoch: info.epoch, seq: info.seq, username: self.username.clone() }) + Ok(SpaceResponse { + space: self.space, + root: info.root, + epoch: info.epoch, + seq: info.seq, + username: self.username.clone(), + kind: None, + name: None, + role: match self.member { + Some(user) => self.meta.space_access(user, self.space).await.map_err(meta_err)?, + None => None, + }, + }) } async fn tree(&self, after: Option) -> Result { @@ -158,16 +218,21 @@ impl Transport for Local { async fn changes(&self, since: u64, after: Option) -> Result { self.faults.check()?; + if self.faults.resync_next_changes.swap(false, Ordering::Relaxed) { + return Err(TransportError::ResyncRequired); + } self.meta.changes(self.space, since, after, self.page).await.map_err(meta_err) } async fn missing_chunks(&self, ids: &[ChunkId]) -> Result, TransportError> { self.faults.check()?; + self.write_gate().await?; self.meta.missing_chunks(self.space, ids).await.map_err(meta_err) } async fn put_chunk(&self, id: &ChunkId, data: Bytes) -> Result<(), TransportError> { self.faults.check()?; + self.write_gate().await?; self.faults.chunk_uploads.fetch_add(1, Ordering::Relaxed); // One engine drives a transport at a time, so a load and a store are enough. if let Some(left) = self.faults.lose_chunks.load(Ordering::Relaxed).checked_sub(1) { @@ -175,7 +240,7 @@ impl Transport for Local { return Ok(()); } self.store.put(BlobKey::chunk(id), &data).await.map_err(|e| TransportError::Protocol(e.to_string()))?; - self.meta.add_chunk(self.space, id, data.len() as u64 + 1, self.now()).await.map_err(meta_err) + self.writer().add_chunk(self.space, id, data.len() as u64 + 1, self.now()).await.map_err(meta_err) } async fn get_chunk(&self, id: &ChunkId) -> Result { @@ -189,12 +254,13 @@ impl Transport for Local { async fn put_manifest(&self, id: &ManifestId, data: Bytes) -> Result<(), TransportError> { self.faults.check()?; + self.write_gate().await?; let manifest = Manifest::decode(&data).map_err(|e| TransportError::Protocol(e.to_string()))?; if manifest.id() != *id { return Err(TransportError::Protocol("manifest ID mismatch".into())); } self.store.put(BlobKey::manifest(id), &data).await.map_err(|e| TransportError::Protocol(e.to_string()))?; - self.meta.add_manifest(self.space, &manifest, data.len() as u64 + 1, self.now()).await.map_err(meta_err) + self.writer().add_manifest(self.space, &manifest, data.len() as u64 + 1, self.now()).await.map_err(meta_err) } async fn get_manifest(&self, id: &ManifestId) -> Result { @@ -207,10 +273,19 @@ impl Transport for Local { async fn commit(&self, ops: &[Op]) -> Result { self.faults.check()?; + self.write_gate().await?; + if self.faults.fail_next_commit.swap(false, Ordering::Relaxed) { + return Err(TransportError::Unavailable("injected before commit".into())); + } + match self.faults.refuse_commits_from.load(Ordering::Relaxed) { + 0 => {} + 1 => return Err(TransportError::ReadOnly), + n => self.faults.refuse_commits_from.store(n - 1, Ordering::Relaxed), + } if self.faults.bad_conflict_index.load(Ordering::Relaxed) { return Err(TransportError::Rejected(vec![Conflict { op: ops.len() + 5, kind: ConflictKind::NoSuchNode }])); } - let out = self.meta.commit(self.space, self.device, ops, self.now()).await.map_err(meta_err); + let out = self.writer().commit(self.space, self.device, ops, self.now()).await.map_err(meta_err); if let (Err(TransportError::Rejected(conflicts)), Ok(_)) = (&out, std::env::var("SIM_TRACE")) { eprintln!("TRACE {} refused {conflicts:?} of {ops:?}", self.name); } @@ -239,7 +314,8 @@ impl Transport for Local { save: &craft_cloud_proto::SaveRecord, ) -> Result<(), TransportError> { self.faults.check()?; - self.meta.attach_save(self.space, node, version, save).await.map(|_| ()).map_err(meta_err) + self.write_gate().await?; + self.writer().attach_save(self.space, node, version, save).await.map(|_| ()).map_err(meta_err) } } @@ -281,6 +357,7 @@ pub async fn machine_on( device, username: user.username.clone(), page, + member: None, faults: Arc::new(Faults::default()), clock: Arc::new(Mutex::new(1_760_000_000_000)), committed: server.committed.clone(), @@ -290,6 +367,18 @@ pub async fn machine_on( Engine::new(fs, transport, State::default(), config, || STAMP).unwrap() } +/// A machine of `user` in the team `space`, with an empty folder: its transport checks the account's role in `space` as +/// the server's routes do (a viewer's writes are refused with `ReadOnly`), and counts the write calls it makes. +pub async fn team_machine(server: &Server, user: &UserRecord, space: SpaceId, name: &str) -> Machine { + let mut m = machine(server, user, name).await; + let mut transport = m.transport().clone(); + transport.space = space; + transport.member = Some(user.id); + let config = Config::new(name, transport.device); + m = Engine::new(MemFs::new(), transport, State::default(), config, || STAMP).unwrap(); + m +} + /// The same machine after a crash and a restart: its folder and device as they are, a fresh engine that starts /// from `state` (what the agent saved last), with the settings `tweak` gives. pub fn restart(m: &Machine, state: State, tweak: impl FnOnce(&mut Config)) -> Machine { diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index a8570f7d..71d1ecf6 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3153,3 +3153,432 @@ async fn a_live_reset_doc_leaves_the_file_sync_alone() { assert!(!files(&a).iter().any(|(p, _)| p == "gone.txt"), "A's delete was undone"); assert_eq!(read(&b, "a.txt"), b"v2 from the laptop"); } + +// ---- restore from the trash, after a delete whose answer a machine never heard ---- + +/// The node of the file or folder `name` the server holds (deleted or not), in the space of `user`. +async fn node_named(server: &Server, user: &craft_cloud_meta::UserRecord, name: &str) -> craft_cloud_proto::NodeId { + let feed = server.meta.changes(user.space, 0, None, 5000).await.unwrap(); + let found: Vec<_> = feed.nodes.iter().filter(|n| n.name.as_ref().is_some_and(|n| n.as_str() == name)).collect(); + assert_eq!(found.len(), 1, "one node named {name}: {found:?}"); + found[0].id +} + +/// Brings `node` back from the trash, as the web or an admin tool would: not one of the syncing machines. +async fn restore_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + let token = *blake3::hash(craft_cloud_proto::NodeId::generate().as_uuid().as_bytes()).as_bytes(); + let device = server.meta.create_device(user.id, "web", &token, 0).await.unwrap(); + server.meta.restore_from_trash(user.space, device, node, 1_760_000_000_000).await.unwrap(); +} + +/// Takes `node` out of the trash (marks it for removal), as the web or an admin tool would. +async fn remove_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + server.meta.request_purge(user.space, node, 1_760_000_000_000).await.unwrap(); +} + +/// Brings the nodes named in `steps` back from the trash, in order; a name with a `-` in front is taken out of the trash +/// instead (and with it what is below it). Restoring a folder brings back all that is in the trash below it, so a test +/// that wants a folder alone removes the rest first. +async fn trash_steps(server: &Server, user: &craft_cloud_meta::UserRecord, steps: &[&str]) { + for step in steps { + match step.strip_prefix('-') { + Some(name) => remove_from_trash(server, user, node_named(server, user, name).await).await, + None => restore_from_trash(server, user, node_named(server, user, step).await).await, + } + } +} + +/// A machine deleted a file, the server applied it, the answer was lost; the file is brought back from the trash +/// before the machine syncs again. The machine's delete must not be sent again: the file comes back, on both machines. +#[tokio::test] +async fn a_file_restored_after_a_lost_delete_answer_comes_back_and_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "f.txt").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + assert_eq!(files(&b), files(&a)); + // And it stays: nothing deletes it on a later pass, on either machine. + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + let c = machine(&server, &ana, "studio").await; + let mut c = c; + c.sync_once().await.unwrap(); + assert_eq!(files(&c), files(&a)); +} + +/// The same, but the name was taken meanwhile by another file: the restored one comes back under a conflict name and +/// both files reach both machines. +#[tokio::test] +async fn a_file_restored_under_a_conflict_name_after_a_lost_delete_answer_keeps_both_files() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"the first"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + // The desktop sees the delete, and its user makes a new f.txt. + b.sync_once().await.unwrap(); + write(&mut b, "f.txt", b"a new file"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a, &mut b], 6).await; + let on_a = files(&a); + assert_eq!(on_a, files(&b)); + assert_eq!(on_a.len(), 2, "{on_a:?}"); + assert!(on_a.contains(&("f.txt".to_owned(), b"a new file".to_vec())), "{on_a:?}"); + let restored = on_a.iter().find(|(p, _)| p.contains("(conflict, web, ")).expect("the restored file under a conflict name"); + assert_eq!(restored.1, b"the first"); +} + +/// A folder that came back from the trash is merged into a live folder of the same name: the restored file lands in +/// it, and the machine whose delete answer was lost ends up with it, not deleted again. +#[tokio::test] +async fn a_file_restored_into_a_live_folder_after_a_lost_delete_answer_comes_back() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let file = node_named(&server, &ana, "f.txt").await; + // The desktop sees both deletes; its user makes a new folder d with another file. + b.sync_once().await.unwrap(); + b.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut b, "d/other.txt", b"other"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, file).await; + + settle(&mut [&mut a, &mut b], 6).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec()), ("d/other.txt".to_owned(), b"other".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); +} + +/// A folder deleted here, the answer lost, and brought back from the trash: it is not deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} + +/// A folder deleted here whose answer was lost and that nobody brought back is still deleted: the delete is not undone +/// by the doubt about it. +#[tokio::test] +async fn a_folder_deleted_with_a_lost_answer_and_not_restored_stays_deleted() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + settle(&mut [&mut a, &mut b], 6).await; + assert!(a.fs().dirs().is_empty(), "{:?}", a.fs().dirs()); + assert!(b.fs().dirs().is_empty(), "{:?}", b.fs().dirs()); +} + +/// A folder with a file in it, both deleted here, the answer lost, then brought back from the trash in different ways. The +/// folder this machine deleted is never deleted a second time, whichever of the two (or both) comes back. +async fn lost_delete_of_a_folder_with_a_file(restore: &[&str]) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, restore).await; + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +#[tokio::test] +async fn a_non_empty_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let (a, b, server, ana) = lost_delete_of_a_folder_with_a_file(&["-f.txt", "d"]).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has the folder"); +} + +/// A folder restored brings back the file that was deleted with it, which is not deleted a second time either. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_comes_back_with_its_file() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["d"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} + +#[tokio::test] +async fn only_the_file_restored_after_a_lost_delete_answer_brings_its_folder_back_too() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["f.txt"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} + +/// A tree of nested folders with a file at its bottom, all deleted here in one commit whose answer is lost, then +/// brought back from the trash in different ways (by node name). Whatever comes back, nothing is deleted a second time. +async fn lost_delete_of_a_nested_tree( + dirs: &[&str], + file: &str, + restore: &[&str], +) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + for dir in dirs { + a.fs_mut().mkdir(&path(dir)).unwrap(); + } + write(&mut a, file, b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path(file)).unwrap(); + for dir in dirs.iter().rev() { + a.fs_mut().remove_dir(&path(dir)).unwrap(); + } + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, restore).await; + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +/// Both machines and a fresh one have exactly the folders `dirs` and no file, and the server still has them. +async fn assert_only_dirs(m: (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord), dirs: &[&str]) { + let (a, b, server, ana) = m; + let expected: Vec<_> = dirs.iter().map(|d| path(d)).collect(); + assert_eq!(a.fs().dirs(), expected, "the laptop"); + assert_eq!(b.fs().dirs(), expected, "the desktop"); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), expected, "the server still has the folders"); +} + +#[tokio::test] +async fn a_nested_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-sub", "d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_nested_folder_and_its_subfolder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-f.txt", "d"]).await; + assert_only_dirs(m, &["d", "d/sub"]).await; +} + +#[tokio::test] +async fn only_the_deepest_file_restored_after_a_lost_delete_answer_brings_the_whole_chain_back() { + let (a, b, _server, _ana) = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["f.txt"]).await; + let expected = vec![("d/sub/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d"), path("d/sub")]); +} + +#[tokio::test] +async fn a_three_deep_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-a", "d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_three_deep_folder_and_its_middle_folder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-b", "d"]).await; + assert_only_dirs(m, &["d", "d/a"]).await; +} + +#[tokio::test] +async fn only_the_deepest_folder_restored_after_a_lost_delete_answer_brings_its_parents_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-f.txt", "b"]).await; + assert_only_dirs(m, &["d", "d/a", "d/a/b"]).await; +} + +/// The change feed is cut short after the lost delete answer and the restore: the machine lists the tree again, at the +/// same epoch. A restored empty folder is in that listing, and must not be deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_and_a_relist_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} + +/// A folder delete recorded before sending but rejected before reaching the server is treated conservatively after a relist. +#[tokio::test] +async fn an_unsent_folder_delete_and_relist_keeps_the_live_folder() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "keep.txt", b"keep"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + let before = server.meta.space_info(ana.space).await.unwrap().seq; + a.transport().faults.fail_next_commit.store(true, Ordering::Relaxed); + let error = a.sync_once().await.unwrap_err(); + assert!(error.to_string().contains("injected before commit"), "{error}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "the failed request never reached the server"); + + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + let report = a.sync_once().await.unwrap(); + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(files(&a), vec![("keep.txt".to_owned(), b"keep".to_vec())]); + assert_eq!(files(&b), files(&a)); + assert!(report.removed_remotely.is_empty(), "{report:?}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "no delete was sent again"); +} + +/// A same-epoch relist restores the live parent while keeping its deleted descendants deleted. +#[tokio::test] +async fn a_nested_relist_keeps_only_the_restored_parent() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + a.fs_mut().mkdir(&path("d/sub")).unwrap(); + write(&mut a, "d/sub/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/sub/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d/sub")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, &["-sub", "d"]).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty()); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has only the restored parent"); +} + +/// A tracked child keeps an in-doubt folder in the base when a relist sees it live. +#[tokio::test] +async fn a_tracked_child_keeps_an_in_doubt_folder_from_release() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"original"); + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "e/g.txt", b"other"); + settle(&mut [&mut a, &mut b], 5).await; + let folder = node_named(&server, &ana, "d").await; + let file = node_named(&server, &ana, "f.txt").await; + a.fs_mut().rename(&path("d/f.txt"), &path("e/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.fs_mut().rename(&path("e"), &path("d")).unwrap(); + // Restore a checkpoint with D's delete in doubt. D's tracked file F is still at d/f.txt, inside E's local folder. + let mut state = a.state().clone(); + state.sent_dir_deletes.insert(folder); + let mut a = common::restart(&a, state, |_| {}); + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + a.sync_once().await.unwrap(); + assert!(a.state().base.contains_key(&folder), "the folder is still tracked with its child"); + assert!(a.state().base.contains_key(&file), "the live file is still tracked"); + assert_eq!(files(&a), vec![("d/f.txt".to_owned(), b"original".to_vec()), ("d/g.txt".to_owned(), b"other".to_vec())]); + let listing = server.meta.tree(ana.space, None, 5000).await.unwrap(); + assert!(listing.nodes.iter().any(|node| node.id == folder && !node.deleted), "folder node was deleted"); + assert!(listing.nodes.iter().any(|node| node.id == file && !node.deleted), "tracked file node was deleted"); +} + +/// The local replacement becomes a conflict copy, and the restored file returns at its original path. +#[tokio::test] +async fn a_local_replacement_after_a_lost_delete_reply_keeps_its_content() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + write(&mut a, "f.txt", b"original"); + a.sync_once().await.unwrap(); + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + write(&mut a, "f.txt", b"local replacement"); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a], 6).await; + let on_a = files(&a); + assert!(on_a.contains(&("f.txt".to_owned(), b"original".to_vec())), "{on_a:?}"); + assert!(on_a.iter().any(|(p, bytes)| p.contains("(conflict, laptop, ") && bytes == b"local replacement"), "{on_a:?}"); + let mut b = machine(&server, &ana, "desktop").await; + b.sync_once().await.unwrap(); + assert_eq!(files(&b), files(&a)); +} diff --git a/crates/sync/tests/read_only.rs b/crates/sync/tests/read_only.rs new file mode 100644 index 00000000..a7cd7bd5 --- /dev/null +++ b/crates/sync/tests/read_only.rs @@ -0,0 +1,409 @@ +//! A space where the account is a viewer: the engine downloads and applies the server's changes and never sends a +//! write. A local edit is kept (and reported), never overwritten and never lost; a role change in either direction is +//! followed cleanly, and a viewer's folder never reaches the server. + +mod common; + +use std::sync::atomic::Ordering; + +use common::sim::Rng; +use common::{Machine, Server, path, settle, team_machine}; +use craft_cloud_proto::SpaceId; +use craft_cloud_proto::api::SpaceRole; +use craft_cloud_sync::{Fs, READ_ONLY_EDITED, READ_ONLY_NEW, Report}; + +fn write(m: &mut Machine, p: &str, data: &[u8]) { + m.fs_mut().write(&path(p), data, false, 0).unwrap(); +} + +fn read(m: &Machine, p: &str) -> Option> { + m.fs().read(&path(p)).ok() +} + +fn files(m: &Machine) -> Vec<(String, Vec)> { + m.fs().files().into_iter().map(|(p, d)| (p.to_string(), d)).collect() +} + +fn writes(m: &Machine) -> u64 { + m.transport().faults.write_calls.load(Ordering::Relaxed) +} + +fn kept(report: &Report, reason: &str) -> Vec { + report.skipped.iter().filter(|(_, why)| why == reason).map(|(p, _)| p.clone()).collect() +} + +/// An editor (`ana`, the owner) and a viewer (`ben`) of one team space, each with a machine. +struct Team { + server: Server, + space: SpaceId, + ben: craft_cloud_meta::UserRecord, + ana: Machine, + viewer: Machine, +} + +async fn team() -> Team { + let server = Server::new().await; + let (ana, ben) = (server.user("ana").await, server.user("ben").await); + let space = server.team_space("Studio", &ana).await; + server.add_member(space, &ben, SpaceRole::Viewer).await; + let ana_machine = team_machine(&server, &ana, space, "ana-pc").await; + let viewer = team_machine(&server, &ben, space, "ben-pc").await; + Team { server, space, ben, ana: ana_machine, viewer } +} + +/// Ana puts a document and a folder with a file into the space; the viewer downloads them. +async fn seeded() -> Team { + let mut t = team().await; + write(&mut t.ana, "doc.txt", b"base"); + t.ana.fs_mut().mkdir(&path("Shots")).unwrap(); + write(&mut t.ana, "Shots/a.psd", b"layers"); + t.ana.sync_once().await.unwrap(); + let report = t.viewer.sync_once().await.unwrap(); + assert_eq!(report.downloaded.len(), 2, "{report:?}"); + assert!(!report.read_only, "nothing was written yet, so the engine has not met the refusal"); + assert_eq!(files(&t.ana), files(&t.viewer)); + t +} + +#[tokio::test] +async fn a_viewers_edit_is_kept_and_never_uploaded() { + let mut t = seeded().await; + write(&mut t.viewer, "doc.txt", b"my edit"); + // The first push meets `read_only` once, learns it, and the same pass reports the edit as kept. + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.read_only && t.viewer.is_read_only(), "{report:?}"); + assert!(report.uploaded.is_empty() && report.removed_remotely.is_empty() && report.refused.is_empty(), "{report:?}"); + assert_eq!(kept(&report, READ_ONLY_EDITED), ["doc.txt"], "{report:?}"); + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"my edit", "the edit is where it was"); + // The server's copy is unchanged: the owner's machine sees nothing new. + let after = t.ana.sync_once().await.unwrap(); + assert!(!after.changed(), "{after:?}"); + assert_eq!(read(&t.ana, "doc.txt").unwrap(), b"base"); + + // No upload loop: more passes make no write call at all, and say nothing new about the same edit. + let calls = writes(&t.viewer); + for _ in 0..5 { + let again = t.viewer.sync_once().await.unwrap(); + assert!(again.read_only && !again.changed() && again.skipped.is_empty(), "{again:?}"); + } + assert_eq!(writes(&t.viewer), calls, "a viewer's engine sends nothing once it knows its role"); + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"my edit"); +} + +#[tokio::test] +async fn a_viewer_told_up_front_never_makes_a_write_call() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + write(&mut t.viewer, "doc.txt", b"my edit"); + write(&mut t.viewer, "new.txt", b"mine"); + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.read_only); + assert_eq!(writes(&t.viewer), 0); + assert_eq!(kept(&report, READ_ONLY_EDITED), ["doc.txt"]); + assert_eq!(kept(&report, READ_ONLY_NEW), ["new.txt"]); +} + +#[tokio::test] +async fn new_local_files_stay_local_and_are_listed_when_the_set_changes() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + write(&mut t.viewer, "new.txt", b"mine"); + t.viewer.fs_mut().mkdir(&path("Mine")).unwrap(); + write(&mut t.viewer, "Mine/b.txt", b"also mine"); + let report = t.viewer.sync_once().await.unwrap(); + let mut listed = kept(&report, READ_ONLY_NEW); + listed.sort(); + assert_eq!(listed, ["Mine/b.txt", "new.txt"]); + assert!(report.uploaded.is_empty() && report.created_dirs.is_empty()); + // The same set is not listed again; a changed set is listed in full. + assert!(t.viewer.sync_once().await.unwrap().skipped.is_empty()); + write(&mut t.viewer, "third.txt", b"3"); + let mut listed = kept(&t.viewer.sync_once().await.unwrap(), READ_ONLY_NEW); + listed.sort(); + assert_eq!(listed, ["Mine/b.txt", "new.txt", "third.txt"]); + // None of it is on the server. + let after = t.ana.sync_once().await.unwrap(); + assert!(!after.changed(), "{after:?}"); + assert_eq!(files(&t.ana).len(), 2); +} + +#[tokio::test] +async fn a_server_change_over_a_viewers_edit_keeps_both_and_uploads_nothing() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + write(&mut t.viewer, "doc.txt", b"viewer edit"); + t.viewer.sync_once().await.unwrap(); + write(&mut t.ana, "doc.txt", b"owner edit"); + t.ana.sync_once().await.unwrap(); + + let report = t.viewer.sync_once().await.unwrap(); + assert_eq!(report.conflict_copies.len(), 1, "{report:?}"); + // The server's version lands under the original name; the edit is the conflict copy, kept and marked local-only. + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"owner edit"); + let copy = report.conflict_copies[0].to_string(); + assert!(copy.contains("(conflict"), "{copy}"); + assert_eq!(read(&t.viewer, ©).unwrap(), b"viewer edit"); + assert_eq!(kept(&report, READ_ONLY_NEW), std::slice::from_ref(©)); + assert!(report.uploaded.is_empty()); + // It stays local: the owner never gets it, and the viewer stays quiet about it afterwards. + let after = t.ana.sync_once().await.unwrap(); + assert!(!after.changed(), "{after:?}"); + assert!(t.ana.fs().files().keys().all(|p| !p.to_string().contains("conflict"))); + assert!(t.viewer.sync_once().await.unwrap().skipped.is_empty()); + assert_eq!(read(&t.viewer, ©).unwrap(), b"viewer edit"); +} + +#[tokio::test] +async fn a_file_the_server_deleted_that_the_viewer_edited_is_kept() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + write(&mut t.viewer, "doc.txt", b"viewer edit"); + t.viewer.sync_once().await.unwrap(); + t.ana.fs_mut().remove_file(&path("doc.txt")).unwrap(); + t.ana.sync_once().await.unwrap(); + let report = t.viewer.sync_once().await.unwrap(); + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"viewer edit", "the edit is never deleted: {report:?}"); + assert!(report.removed_locally.is_empty() && report.uploaded.is_empty()); + assert_eq!(writes(&t.viewer), 0); +} + +#[tokio::test] +async fn a_local_delete_is_downloaded_again_with_a_notice() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + t.viewer.fs_mut().remove_file(&path("doc.txt")).unwrap(); + t.viewer.fs_mut().remove_file(&path("Shots/a.psd")).unwrap(); + t.viewer.fs_mut().remove_dir(&path("Shots")).unwrap(); + let report = t.viewer.sync_once().await.unwrap(); + let mut restored: Vec = report.restored.iter().map(ToString::to_string).collect(); + restored.sort(); + assert_eq!(restored, ["Shots", "Shots/a.psd", "doc.txt"], "{report:?}"); + assert!(report.removed_remotely.is_empty() && report.pending_deletes.is_empty()); + assert_eq!(files(&t.ana), files(&t.viewer)); + assert_eq!(t.viewer.fs().dirs(), vec![path("Shots")]); + assert_eq!(writes(&t.viewer), 0); + // The server never saw a delete. + let after = t.ana.sync_once().await.unwrap(); + assert!(!after.changed(), "{after:?}"); + assert!(!t.viewer.sync_once().await.unwrap().changed()); +} + +#[tokio::test] +async fn a_moved_file_comes_back_at_its_place_and_the_moved_one_stays_local() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + t.viewer.fs_mut().rename(&path("doc.txt"), &path("renamed.txt")).unwrap(); + let report = t.viewer.sync_once().await.unwrap(); + assert_eq!(report.restored, vec![path("doc.txt")], "{report:?}"); + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"base"); + assert_eq!(read(&t.viewer, "renamed.txt").unwrap(), b"base"); + assert_eq!(kept(&report, READ_ONLY_NEW), ["renamed.txt"]); + assert_eq!(writes(&t.viewer), 0); +} + +#[tokio::test] +async fn a_role_raised_to_editor_uploads_the_local_files() { + let mut t = seeded().await; + write(&mut t.viewer, "doc.txt", b"viewer edit"); + write(&mut t.viewer, "new.txt", b"mine"); + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.read_only); + assert_eq!(writes(&t.viewer), 1, "one refused call, then none"); + t.viewer.sync_once().await.unwrap(); + + t.server.set_role(t.space, &t.ben, SpaceRole::Editor).await; + let report = t.viewer.sync_once().await.unwrap(); + assert!(!report.read_only && !t.viewer.is_read_only(), "{report:?}"); + let mut uploaded: Vec = report.uploaded.iter().map(ToString::to_string).collect(); + uploaded.sort(); + assert_eq!(uploaded, ["doc.txt", "new.txt"], "the edit and the new file go up as ordinary changes: {report:?}"); + t.ana.sync_once().await.unwrap(); + assert_eq!(read(&t.ana, "doc.txt").unwrap(), b"viewer edit"); + assert_eq!(read(&t.ana, "new.txt").unwrap(), b"mine"); + settle(&mut [&mut t.ana, &mut t.viewer], 5).await; + assert_eq!(files(&t.ana), files(&t.viewer)); +} + +#[tokio::test] +async fn a_role_lowered_to_viewer_stops_uploads_cleanly_and_keeps_every_edit() { + let mut t = seeded().await; + t.server.set_role(t.space, &t.ben, SpaceRole::Editor).await; + write(&mut t.viewer, "doc.txt", b"edit while editor"); + let report = t.viewer.sync_once().await.unwrap(); + assert_eq!(report.uploaded, vec![path("doc.txt")]); + assert!(!report.read_only); + + // Lowered between two passes: the next write is refused, the engine switches over in the same pass and nothing is lost. + t.server.set_role(t.space, &t.ben, SpaceRole::Viewer).await; + write(&mut t.viewer, "doc.txt", b"edit after the change"); + write(&mut t.viewer, "new.txt", b"mine"); + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.read_only, "{report:?}"); + assert!(report.uploaded.is_empty() && report.refused.is_empty(), "{report:?}"); + assert_eq!(kept(&report, READ_ONLY_EDITED), ["doc.txt"]); + assert_eq!(kept(&report, READ_ONLY_NEW), ["new.txt"]); + assert_eq!(read(&t.viewer, "doc.txt").unwrap(), b"edit after the change"); + t.ana.sync_once().await.unwrap(); + assert_eq!(read(&t.ana, "doc.txt").unwrap(), b"edit while editor", "only what went up while the role allowed it"); + assert!(read(&t.ana, "new.txt").is_none()); + let calls = writes(&t.viewer); + t.viewer.sync_once().await.unwrap(); + t.viewer.sync_once().await.unwrap(); + assert_eq!(writes(&t.viewer), calls); + + // And back: the same files go up. + t.server.set_role(t.space, &t.ben, SpaceRole::Owner).await; + let report = t.viewer.sync_once().await.unwrap(); + assert!(!report.read_only); + t.ana.sync_once().await.unwrap(); + assert_eq!(read(&t.ana, "doc.txt").unwrap(), b"edit after the change"); + assert_eq!(read(&t.ana, "new.txt").unwrap(), b"mine"); +} + +/// Review 1 of P1-4: a file or a link where a tracked folder was is the user's. Nothing that was in the folder is put +/// back through it, and nothing is moved aside to make room. +#[tokio::test] +async fn a_file_or_a_link_where_a_folder_was_is_left_alone_with_everything_below() { + for link in [false, true] { + let mut t = seeded().await; + t.viewer.set_read_only(true); + t.viewer.fs_mut().remove_file(&path("Shots/a.psd")).unwrap(); + t.viewer.fs_mut().remove_dir(&path("Shots")).unwrap(); + if link { + t.viewer.fs_mut().make_link(&path("Shots")).unwrap(); + } else { + write(&mut t.viewer, "Shots", b"my file, where the folder was"); + } + for _ in 0..3 { + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.conflict_copies.is_empty(), "link {link}: nothing moved aside: {report:?}"); + assert!(report.restored.is_empty() && report.created_dirs.is_empty(), "link {link}: {report:?}"); + } + let at = t.viewer.fs().stat(&path("Shots")).unwrap().unwrap(); + if link { + assert!(matches!(at, craft_cloud_sync::FsKind::Other), "the link is still there: {at:?}"); + } else { + assert_eq!(read(&t.viewer, "Shots").unwrap(), b"my file, where the folder was"); + } + assert_eq!(files(&t.viewer).iter().filter(|(p, _)| p.starts_with("Shots/")).count(), 0, "nothing was put back below it"); + assert_eq!(writes(&t.viewer), 0); + assert_eq!(read(&t.ana, "Shots/a.psd").unwrap(), b"layers", "the server's copy is as it was"); + } +} + +/// Review 5: a folder put back is reported once, as restored. +#[tokio::test] +async fn a_restored_folder_is_counted_once() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + t.viewer.fs_mut().remove_file(&path("Shots/a.psd")).unwrap(); + t.viewer.fs_mut().remove_dir(&path("Shots")).unwrap(); + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.restored.contains(&path("Shots")), "{report:?}"); + assert!(report.created_dirs.is_empty(), "{report:?}"); +} + +/// Review 2: a quiet viewer's pass reads an edited file at most once while it stays as it is, and not at all when its +/// size differs from the agreed content. +#[tokio::test] +async fn edited_files_are_not_read_again_every_pass() { + let mut t = seeded().await; + t.viewer.set_read_only(true); + // A different size: found out without reading the file. + t.viewer.fs_mut().write(&path("doc.txt"), b"a much longer edit than the base", false, 1).unwrap(); + // The same size (6 bytes), saved later: read once, then remembered. + t.viewer.fs_mut().write(&path("Shots/a.psd"), b"LAYERS", false, 1).unwrap(); + let reads = |m: &craft_cloud_sync::MemFs| (m.reads_of(&path("doc.txt")), m.reads_of(&path("Shots/a.psd"))); + let first = t.viewer.sync_once().await.unwrap(); + assert_eq!(kept(&first, READ_ONLY_EDITED).len(), 2, "{first:?}"); + let after_first = reads(t.viewer.fs()); + for _ in 0..5 { + let again = t.viewer.sync_once().await.unwrap(); + assert!(again.skipped.is_empty(), "{again:?}"); + } + assert_eq!(reads(t.viewer.fs()), after_first, "no file is read again while it stays as it is"); + assert!(after_first.1 <= 2, "the same size is read once or twice, not every pass: {after_first:?}"); +} + +/// Review 3: a refusal that comes after earlier batches were committed keeps what they did in the report. +#[tokio::test] +async fn a_refusal_in_a_later_batch_keeps_the_earlier_batches_in_the_report() { + let mut t = team().await; + t.server.set_role(t.space, &t.ben, SpaceRole::Editor).await; + for i in 0..1_100 { + write(&mut t.viewer, &format!("f{i:04}.txt"), format!("file {i}").as_bytes()); + } + // The first batch (1000 operations) goes up; the account is a viewer by the time the second is sent. + t.viewer.transport().faults.refuse_commits_from.store(2, Ordering::Relaxed); + let report = t.viewer.sync_once().await.unwrap(); + assert!(report.read_only && t.viewer.is_read_only(), "{report:?}"); + assert_eq!(report.uploaded.len(), craft_cloud_proto::limits::MAX_BATCH_OPS, "the committed batch is reported: {report:?}"); + assert_eq!(kept(&report, READ_ONLY_NEW).len(), 100, "the rest stays here, listed: {report:?}"); + // What the server has is exactly what was reported. + let seen = t.ana.sync_once().await.unwrap(); + assert_eq!(seen.downloaded.len(), craft_cloud_proto::limits::MAX_BATCH_OPS, "{seen:?}"); +} + +/// Random edits, new files and deletes on a viewer's folder, and random edits on the owner's, over many seeds: the viewer +/// never makes a write call, nothing it wrote reaches the server, and nothing it wrote is lost from its disk. +#[tokio::test] +async fn simulated_viewers_never_send_a_write_and_never_lose_an_edit() { + let seeds: u64 = std::env::var("SIM_SEEDS").ok().and_then(|s| s.parse().ok()).unwrap_or(30); + for seed in 0..seeds { + let mut rng = Rng::new(seed); + let mut t = team().await; + t.viewer.set_read_only(true); + let names = ["a.txt", "b.txt", "x/c.txt", "x/y/d.txt"]; + // What the viewer wrote, by path, last. + let mut mine: std::collections::BTreeMap> = std::collections::BTreeMap::new(); + for step in 0..40 { + let name = names[rng.below(names.len())]; + let unique = format!("seed {seed} step {step} {}", rng.next()).into_bytes(); + let who = rng.below(4); + let parent = name.rsplit_once('/').map(|(dir, _)| dir.to_owned()); + let m = if who == 0 { &mut t.ana } else { &mut t.viewer }; + if let Some(dir) = &parent { + let mut at = String::new(); + for part in dir.split('/') { + at = if at.is_empty() { part.to_owned() } else { format!("{at}/{part}") }; + let _ = m.fs_mut().mkdir(&path(&at)); + } + } + match rng.below(5) { + 0 | 1 => { + write(m, name, &unique); + if who != 0 { + mine.insert(name.to_owned(), unique); + } + } + 2 => { + let _ = m.fs_mut().remove_file(&path(name)); + if who != 0 { + mine.remove(name); + } + } + 3 => { + t.ana.sync_once().await.unwrap(); + } + _ => { + t.viewer.sync_once().await.unwrap(); + } + } + } + for _ in 0..4 { + t.ana.sync_once().await.unwrap(); + t.viewer.sync_once().await.unwrap(); + } + assert_eq!(writes(&t.viewer), 0, "seed {seed}: the viewer made a write call"); + // Nothing the viewer wrote is on the server. + let on_server: Vec> = t.ana.fs().files().into_values().collect(); + for (name, data) in &mine { + let still_here = t.viewer.fs().files().values().any(|d| d == data); + let reached_server = on_server.iter().any(|d| d == data); + // Content the viewer wrote last at a path is on its disk (as the file, or as a conflict copy of it) unless the + // owner's later change replaced the file after the viewer's pass did not see it: then it is a conflict copy. + assert!(still_here, "seed {seed}: the viewer's content of {name} is gone"); + assert!(!reached_server, "seed {seed}: the viewer's content of {name} reached the server"); + } + } +} diff --git a/docs/agent.md b/docs/agent.md index 18e2a394..5ebe8b3c 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -25,6 +25,10 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ `run`'s output: `sync --approve-deletes=` approves exactly that set if it is the one waiting, without `status` first, and otherwise shows the waiting set, approves nothing and exits 2. A plain `sync` says the agent is already running. +- A team space's held deletions are approved with `sync --approve-deletes[=] --space ` (the same rules, + with the space's own list and `spaces//approval.json`; it needs the running agent), or by the local API's + `POST /v1/spaces/{id}/approve-deletes`. Each computer approves its own side: deletions made on one computer wait for + its approval, and again on the others when they arrive. - A delete held back because a link appeared that may be the file, moved with its folder ("held while … is a link", or "held while any of … is a link" when several links appeared with that name), comes with a set of its own. If you really deleted the file and none of the links listed is it, `sync --approve-deletes=` @@ -59,9 +63,25 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ and `--adopt` as for `login`) signs it in. An admin's invite (`user invite`, in [`server.md`](server.md)) works the same way; for a new account, `join` asks for its password twice on standard input. A code that doesn't work says so, without saying why: make a new one. +- An account with two-factor sign-in (turned on in the app's Security page; see "Two-factor sign-in" in + [`server.md`](server.md)) asks for a second step at `login` and at `join`: after the password, the command prints + `This account has two-factor sign-in. Type the code from your authenticator app (or a recovery code):` and reads one line + from standard input, so a script gives the password line and then the code line. The code is never an argument and + is never stored; a wrong one may be typed again twice (the server itself allows five wrong codes per sign-in). Six digits + are a code from the app; anything else is taken as a recovery code, which works once. Too many wrong codes lock the + account's second step for a while: the command says how long, and the lock does not sign anything out. For a `join` that + needs a code the invite isn't used up until the code is right. A running agent (`run`) does the same through its local + API; the Craft Cloud app asks for the code in its own window. +- Rolling this out: update the server first, then the agents and the app. An agent from before two-factor sign-in can't + complete a sign-in for an account that has it, and it reports that as a wrong password; update the agent, nothing is + broken. The app on a server that doesn't know two-factor sign-in says so on its Security page instead of failing. - Through the running agent: `pause` and `resume` syncing (a pause holds across restarts; recents keep working), `quit` the agent, and `devices` to list the account's machines (`devices remove ` takes one out; to take this machine out, use `logout`). +- The desktop reads storage through the authenticated local API, `GET /v1/usage`. It asks the server for every + request and returns `AccountUsage` unchanged. Signed out, it returns `409 not_signed_in`; when the server cannot be + reached, it returns `503 offline`. The desktop shows trash and old-version values as overlapping estimates and + never adds them to stored bytes. - Recent files, per app, shared by every machine of the account (through the running agent, as an app sees them): @@ -85,3 +105,160 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ The first machine to open a file makes it a live document from the file's content (UTF-8 text only); the others join it. Changes go to every machine that has it open as they are made, and edits made offline merge when the machine is back. `doc` never writes the file: what it shows is the live document. + +## The local API's admin calls + +For the Craft Cloud app's admin pages the running agent passes the server's admin API on (`/api/v1/admin/…`, see +[`server.md`](server.md), "Managing accounts over HTTP") under `/v1/admin/…`, with the signed-in device's token. They +have the same checks as every local call (the `Host`, no `Origin`, the token in `Authorization: Bearer`), bodies of at +most 8 KiB, and need the agent to be signed in (`not_signed_in` otherwise). + +| Local call | Server call | Body, answer | +| --- | --- | --- | +| `GET /v1/admin/server` | `GET /admin/server` | the server's counters and kinds; a member is refused `forbidden` | +| `GET /v1/admin/storage` | `GET /admin/storage` | bounded storage overview: totals, quota counts, top ten accounts, and manifest-index counts | +| `GET /v1/admin/users?cursor&limit&query` | `GET /admin/users` | a page of accounts by username; `next` is the cursor of the following page; `query` finds a username containing it | +| `POST /v1/admin/users` | `POST /admin/users` | `{username, role, password, totp_code?, recovery_code?}` → the account and its invite code | +| `PATCH /v1/admin/users/{id}` | `PATCH /admin/users/{id}` | `{role?, quota_bytes?, password, totp_code?, recovery_code?}` → the account (`null` removes the quota) | +| `POST /v1/admin/users/{id}/disable`, `…/enable` | the same | `{password}` → 204 | +| `DELETE /v1/admin/users/{id}` | the same | `{password}` → 202 | +| `POST /v1/admin/users/{id}/invite`, `…/password-reset` | the same | `{password}` (a reset code: also `totp_code?`, `recovery_code?`, `reset_two_factor?`) → `{code, join_url?, expires_ms}` | +| `DELETE /v1/admin/users/{id}/invite` | the same | `{password}` → 204 | +| `GET /v1/admin/users/{id}/devices` | the same | every device of the account | +| `GET /v1/admin/users/{id}/usage` | `GET /admin/users/{id}/usage` | the account's stored bytes, effective quota, and labeled trash/history estimates; unknown or removed account is `not_found` | +| `DELETE /v1/admin/users/{id}/devices/{device_id}` | the same | `{password}` → 204 | +| `POST /v1/admin/users/{id}/reset-two-factor` | the same | `{password, totp_code?, recovery_code?}` → 204: takes the account's two-factor sign-in away and signs its computers out | +| `GET /v1/account` | `GET /account` | the signed-in account: `role`, `two_factor_enabled` (how the app finds out whether to show its admin pages) | +| `GET /v1/account/activity?cursor&before_ms&limit` | the same | the account's own security events, newest first, any account (`user` and `action` are not read) | +| `GET /v1/admin/audit?cursor&before_ms&limit&user&action` | `GET /admin/audit` | the whole log, newest first; `user` an account ID, `action` an event name or a prefix like `auth.` of the closed list; a cursor is passed back as it came | +| `GET /v1/admin/audit/export?before_ms&user&action` | `GET /admin/audit/export` | the log as NDJSON, **streamed** through (chunked): event lines, then a last line `{"complete":true,"exported":n}` or `{"truncated":true,"exported":n}` | + +- **The server decides.** The agent checks no role and makes no decision of its own; it reads each body as the shared + type, writes it to the server again and answers what the server answered, with the server's HTTP status, error code and + message. A wrong password stays `403 wrong_password`, a member's call `403 forbidden`, a disabled admin's + `403 account_disabled`, the last enabled admin's demotion `409 last_admin`, too many tries `429 rate_limited` with the + server's `Retry-After` header. The one status the agent does not pass on is the server's `401`: on a signed-in route it + says this device was revoked, and the local API answers it as every other call does (`409 needs_you`); a local `401` always means a wrong token. +- **The confirmation is not kept.** Every change carries the acting admin's current password in its body, from the app + through the agent to the server. The agent holds it only for the length of the request, never writes or logs it, and + the `Debug` output of every body hides it. The codes the server makes (an invite, a password reset code) are shown by + the app once and kept by neither side. +- **A second factor** joins the confirmation when the admin's account has two-factor sign-in: `totp_code` (a code from the + authenticator app) or `recovery_code`, in the same body as the password, passed on as typed fields and never kept. The server + answers a missing or wrong one `403 two_factor_invalid` and too many `429 two_factor_locked` with the wait (the body's + `retry_after_secs` and the `Retry-After` header); neither is a revoked device, so the agent stays signed in. A password reset + code takes the whole `AdminPasswordReset` (the same fields and `reset_two_factor`, which makes redeeming the code turn the + account's two-factor sign-in off). +- **The audit export is never held whole.** The agent passes the server's stream through chunk by chunk; a client that + reads it (`craft-cloud-app`'s `admin_audit_export`) counts it as whole only with the server's `complete` line and the number + of events it says, and takes a stream that ends without it (a broken connection, a server that failed in the middle) for a + failed export. `truncated` is a good export of the newest events that is not the whole log. Filters and limits are checked + before anything is sent: an `action` outside the closed list is `400 invalid`, as is a `limit` of 0 or over 200 (100 for an + account's own activity). +## Local API: file history + +The local API exposes server history to desktop clients. Paths are relative to the synced folder and resolve through +the agent's saved sync state; unknown or unsynced paths return `404 not_found`, paths outside the root return +`400 invalid`, and folders return `400 invalid`. + +| Method and path | Request | Success | +|---|---|---| +| `GET /v1/files/history?path=&cursor=&limit=` | Query parameters; `cursor` and `limit` optional | `200` with `{versions, next}`; newest first | +| `POST /v1/files/restore` | `{"path":"Notes/todo.txt","version":"…","expected_version":"…"}` | `200` with the new version, its parent, sequence and epoch | + +Restore requires the file's current version to match `expected_version`; otherwise the API answers `409 conflict`. +The agent does not write restored bytes directly. Its normal change feed picks up the new server version and syncs it +to disk. + +## Local API: trash + +Trash items are addressed by their server node ID because deleted nodes are absent from the agent's sync tree. The +agent validates that each ID has the expected UUID form before forwarding it. List responses may include `path`, a +display hint formed from a currently synced parent folder and the item's name; it is `null` when that parent is not +live on this machine. A missing hint does not fail the request. + +| Method and path | Request | Success | +|---|---|---| +| `GET /v1/trash?cursor=&limit=` | Optional paging query parameters | `200` with `{items, next}`, newest deletion first | +| `POST /v1/trash/restore` | `{"id":""}` | `200` with restored node, parent, name, rename flag, version (for files), sequence and epoch | +| `POST /v1/trash/remove` | `{"id":""}` | `204` | + +A successful restore asks the agent for a normal sync pass; it never writes files directly. Removing an item only +marks it out of the trash. It cannot be restored from the trash afterward, but stored space is not freed yet. Server +HTTP status and error code are preserved; a server `401 unauthorized` becomes local `409 needs_you`. + +## Team spaces + +The running agent syncs the personal space as ever, and any team space the person turns on. Everything below is off for +a server that speaks HTTP protocol 1 (before team spaces): the agent asks that server about its version only, sends no +`craft-space` header, and lists no team space. + +- **Which spaces.** `GET /spaces` is asked at the start, then every two long-poll waits, and at once after an invitation is + accepted or declined here or a space answers `404`. A failed look changes nothing. +- **Folders.** A team space syncs to `Craft Cloud Shared//`, a folder beside the personal one (so synced folders never + nest). The name is the space's name with what a file name can't hold made `_`; a name that is taken, by another space or + by a folder that is already there, gets ` (2)`, ` (3)`. The folder is chosen when the space is first seen and kept when + the space is renamed on the server. Each folder has its own root marker (`.craft-cloud/root.json`). +- **State.** `account.json` gains `spaces: [{space, folder, sync, name, root}]` (left out while there are none, so a data + folder from before team spaces is not rewritten). Each synced team space keeps `spaces//state.json` in the + data folder; the personal space's files stay where they were. The state records the space and the canonical folder it + was made for, and is used for that pair only: for any other folder (the space moved, or turned off and on elsewhere, + and back) the space starts afresh as a first sync, merging the folder with the space and deleting nothing on either + side. The same goes when the server's stamp of the account's membership (`joined_ms` in `GET /spaces`, new each time + someone is removed and invited again) differs from the one the state was made under, which also covers an agent that was + not running while access was lost and regained; a server that gives no stamp is not compared. Turning a space off + forgets its state, so turning it on again, in the same folder or another, is a first sync too. Moving a space that + syncs, or turning it off, ends its session and waits for its running pass before anything else changes; the agent goes + on meanwhile (other spaces, the local API, stopping), and the call is answered when the session is closed (after 50 + seconds at most). A pass that was queued behind another pass when its space was closed (or the agent began to stop) is + not run. Another change of the same space meanwhile is refused with `409` ("this space is being changed, try again in a + moment"). Turning on a space that has no session or pass here does not wait for another space's pass; if the space's own + last pass has not ended after 50 seconds the answer is "this space is still finishing its last pass, try again in a + moment". If the agent stops while a space is being turned off, the space stays off. Signing out removes the team spaces' states too; their folders, markers and files stay. +- **A state is trusted only when nothing doubtful happened to it.** Each team space's setting in `account.json` has a + `generation` (left out while 0), and `state.json` records the generation it was made under; a state under another + generation is not used. The generation is raised and saved before a state is retired (turning the space off, losing + access) and before a new folder's marker is written, so a state that survives its removal, or a crash between the marker + and the removal, is never trusted again. If the state cannot be removed (something holds the file), turning the space off + fails with that error instead of reporting success. Before every pass the session also compares the folder's canonical path + and ID with those it opened on: a link or junction pointed at another folder meanwhile ends the session, drops the state and + starts the space afresh there as a first sync. It does the same just before the pass sends deletions, and a saved state also + records the folder's ID, so a folder restored from a backup at the same path is not the folder the state was made for. If a + raised generation cannot be saved, it is saved again at the next look at the list and that space is not started meanwhile (its state says why); other spaces go on. +- **A first sync merges, and says what that means.** Without a saved state, nothing is known to have been deleted, so + nothing is deleted on either side. A file that teammates deleted on the server meanwhile, and that is still in the + folder, is uploaded again. A file that exists on both sides with different content is not overwritten: the local one is + moved aside as a conflict copy and the server's version takes its path (the local file no longer sits at its path). A + file only here goes up, a file only on the server comes down, and equal files are adopted. +- **Off by default.** An accepted space does not sync until `POST /v1/spaces/{id}/sync` with `{"on": true}` (optionally + `folder`, an absolute path that lies outside every synced folder and the data folder, and `adopt` for a folder that already + has files: what differs from the space's becomes conflict copies, nothing is deleted). +- **One engine per space.** Each synced space has its own session (state, root, marker, guards), scheduler, folder watcher + and long-poll, over a transport bound to the space. Passes run one at a time across all spaces. A problem in one space + never pauses another or the personal space. `pause`, `resume` and `quit` apply to every space. +- **The echo.** Every answer for a team space must carry `craft-space` with that space (success, `403`, `404`, `409`, `410`). + A missing or different one stops that space with "the server is too old for team spaces": a server that ignored the header + would otherwise answer about the personal space. +- **Read-only.** A role of viewer, or an archived space, puts the space's engine in read-only mode (see the sync design, + "Read-only spaces"); `read_only` in the space's entry says so. The server lists an archived space with the member's + stored role (an editor stays `editor`) although their access is capped at viewer; read `archived`, not `role` alone, to + know whether the member may write. +- **Losing access.** A space the server no longer lists, or answers `404`, stops syncing and shows + `run_state: {"kind": "no_access"}` with "You no longer have access to . Your local copy is kept and no longer + syncs." Nothing in its folder is touched or deleted. Its saved sync state is dropped, so if the account is added again the + folder is merged with the space afresh (a first sync, above): no local file is deleted, a file deleted here meanwhile is + not deleted on the server, one deleted on the server meanwhile is not deleted here (and goes up again), and a file that + differs on both sides is kept twice (one as a conflict copy). A member of an archived space has not lost access: the space stays + listed (`archived: true`) and syncs read-only. + +| Local call | Answer | +| --- | --- | +| `GET /v1/spaces` | `{spaces: [{space, kind, name?, role, state, archived, folder?, sync, run_state?, last_sync_ms?, read_only}]}`: the personal space first, then the team spaces, invitations (`state: "invited"`, no folder) among them. `sync` is `on`, `off` or `unavailable` | +| `POST /v1/spaces/{id}/accept` | the server's answer: the space as it lists it | +| `POST /v1/spaces/{id}/decline` | 204 | +| `POST /v1/spaces/{id}/sync` | `{on, folder?, adopt?}` → the space as `GET /v1/spaces` lists it | +| `POST /v1/spaces/{id}/approve-deletes` | `{set?}` → `{approved, message}`: approves the deletions the mass-delete guard holds back in a synced team space (`spaces//approval.json`), as `craft-cloud-agent sync --approve-deletes[=] --space ` does. With a `set` only that set, if it is the one waiting; without, the set the previous call listed | +| `GET /v1/status` | gains `spaces: [{space, run_state, last_sync_ms?, read_only}]`, one per synced space or one that lost access; the top-level fields describe the personal space | + +Accept and decline are passed on to the server, which decides; its refusals keep their status and code. An agent from before +team spaces answers these calls with `not_found` ("no such call"), which `craft-cloud-app` reports as `agent_too_old`. diff --git a/docs/design/live.md b/docs/design/live.md index 75e9a4ee..3aa598df 100644 --- a/docs/design/live.md +++ b/docs/design/live.md @@ -24,6 +24,9 @@ per app connection for live sessions. `run` serves it while signed out too. Call one), `GET /v1/pairings/{id}` → `{state: waiting|used|expired, device_name?}`, `DELETE /v1/pairings/{id}` → 204; - `GET /v1/devices` → the account's devices (`current` marks this one); `DELETE /v1/devices/{id}` revokes another one and answers the list after (this one is refused with `cannot_remove_self`: that is `logout`); +- `GET`, `POST`, `PATCH` and `DELETE` `/v1/admin/…`, and `GET /v1/account` and `/v1/account/activity`: the server's admin API + and the account's own audit events passed on for the Craft Cloud app's admin pages, with the server's status and error codes (listed in [`../agent.md`](../agent.md), "The local API's admin + calls"); - `POST /v1/pause` / `POST /v1/resume` → status: while paused no sync pass starts (one under way finishes), and the long-poll, the folder watcher and the live connection hold; the local API and the apps' recents keep working (adds reach the server after resume). Kept in `state.json`, so a paused agent starts paused; `run_state` is diff --git a/docs/design/sync.md b/docs/design/sync.md index 52d60652..ce2cd861 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -578,6 +578,230 @@ changes locally, then pushes local changes. These rules came out of the simulati database (not a supported setup) would each count their own, so twice the sign-in attempts and twice the writes. - **Setup**: `craft-cloud-server init` writes the config, creates the data directory and the first admin. Solo use is that one admin account and nothing more. +- **Device sessions** (migration 0007): a device token has no expiry and its format is unchanged (43 characters, stored as a + BLAKE3 hash); what changed is what a device records and how an account ends them. A device records the `client_kind`, + `platform` and `app_version` it said at sign-in (display only; the agent says `agent`, its operating system and its version on every + sign-in and join), and, once revoked, when (`revoked_ms`) and why + (`revoke_reason`). The `revoked` flag is still the check: the three are set together in one transaction and only by + the first revoke, so a second revoke changes neither. A user keeps `password_changed_ms`, and `disabled_ms`, which the + admin API sets and the token check, sign-in, join and pairing read (see "Admin API and roles"). + - **Changing the password** (`POST /account/password`) checks the current password under one of the server-wide + password-check slots (all busy: `429` with `Retry-After`, and the try is given back), then in one transaction sets + the new hash and `password_changed_ms`, revokes every other device (`revoke_reason = password`) and deletes the + account's open invite and all its pairings, so no code made before the change works after it; if the password + changed meanwhile nothing changes (`409 conflict`), and neither does it if the calling device was signed out meanwhile (`401`). Afterwards each revoked device's live connections end at once. The same two + steps run for `user reset-password` (all devices, reason `reset`, unless `--keep-devices`) and `device revoke-all` + (reason `admin`) on the command line, which cannot reach the server's live connections: those end at the next ping + (within about 30 s) and a change-feed long-poll within 60 s. + - **Sign-in and join under a change**: a password sign-in creates its device only if the stored hash is still the one + it checked (the account is locked first), and answers as for a wrong password otherwise, so a device is never + added after a reset or change that signed the others out. Every transaction that revokes devices, changes a + password, redeems an invite or makes a pairing takes the account's row first, which keeps them from deadlocking. + - **Limits**: `password_changes` counts every try, a wrong password included, 5 per account per 15 minutes; the + password and device-name bodies are read up to 4 KiB (the sign-in bodies stay at 1 MiB). +- **Admin API and roles** (migration 0008): an account is an `admin` or a `member`, and every route under + `/api/v1/admin` is behind the `require_admin` layer, which runs after the device check and reads the role the + database gave for that very request (nothing is cached, so a demoted admin is out with the next request). A member + gets `403 forbidden`, a missing or revoked token `401`. The layer is only the first look: every change checks the + acting admin again inside its own transaction. + - **Confirmation.** A device token never expires, so a stolen admin token alone must not be enough to create an + admin, issue a code or sign others out. Every change (creating, changing, disabling, enabling, removing an account, + issuing or revoking an invite, issuing a password reset code, revoking another user's device) carries the acting + admin's current `password` in its JSON body, also the `DELETE` routes (at most 4 KiB). A missing or mistyped body + field is `400 invalid`; a wrong password is `403 wrong_password`, never 401, because the agent reads every 401 on a + signed-in route as a revoked device. Tries are counted per account in the password change's limiter (5 wrong + passwords in 15 minutes, then `429`; a right password is given back), and every check takes one of the + server-wide password-check slots. **An admin whose account has two-factor sign-in** adds `totp_code` (or a + `recovery_code`) to the same body: missing or wrong is `403 two_factor_invalid` (the field is a credential there, not + a form field; a code that is not six digits is `400 invalid`), too many wrong ones `429 two_factor_locked` with + `Retry-After`. The code is checked and used up by the confirmation, on the same counters, limiter and time-step rule + as sign-in's second step (below): **one 30-second step serves one change**, the next change takes the next code or + a recovery code. The password is told first: a wrong one is `403 wrong_password` whatever the code. Resetting another + **administrator's** password or two-factor sign-in takes the acting admin's own two-factor sign-in (`403 forbidden` + without it; the command line on the host is the way then). The change compares the admin's two-factor state with + the one the confirmation saw, under the account's lock, so a confirmation given before they turned it on or off is + `409 conflict`. + - **The acting admin in the transaction.** After the password check (a slow Argon2id run) the admin's role, device or + password can have changed. Every change locks, in this order, (1) the `last_admin` lock if it can take an + administrator away, (2) the user rows of the actor and the target in ascending ID, (3) the rows derived from them, + and then checks that the actor's device is not revoked and the account neither disabled nor marked for removal + (`401`), that the role is still `admin` (`403 forbidden`) and that the password hash is the one verified (`409 + conflict`). Sign-ins, joins and pairings take the user row first too, so none of these can deadlock. + - **The last administrator.** Demoting, disabling or removing an admin who is the last enabled one is `409 + last_admin`. A count cannot be locked with `FOR UPDATE`, and locking each target's row would not stop two admins + from each taking the other away, so every change that can reduce the admins first takes one transaction-wide lock + and counts after it (Postgres `pg_advisory_xact_lock` with a fixed key, SQLite the write transaction, which is + the same thing there). Nobody can disable or remove their own account (`400 invalid`) but may step down when + another admin is left. The command line does not enforce the rule or the confirmation: the operator has the files. + - **Disabled accounts.** Disabling signs every device out (`revoke_reason = disabled`), deletes the open invite or + reset code and the pairings, and sets `disabled_ms`; enabling clears it and brings back no device. A disabled + account's old tokens are `401 unauthorized`; sign-in and join answer `403 account_disabled`, but only after the + password or code was right (a wrong one is `401` or `404 invalid_code` as ever). Issuing is serialised with + disabling: a sign-in, a redeem, a pairing and a new device lock the user row first and look at `disabled_ms` and + `removed_ms` in their own transaction, so a password checked before the disable never turns into a device after + it, and enabling later cannot make such a device usable. **Removing** an account (`DELETE /admin/users/{id}`, `202`) + also sets `removed_ms`: it is disabled, left out of the lists and cannot be enabled, its username stays taken, + and nothing it stored is deleted (freeing it comes with garbage collection). + - **Codes.** `POST /admin/users` makes an account without a password (role `admin` or `member`) and a 7-day invite; + the person chooses the password when they redeem it. An invite for an existing account is only made for one + that still has no password: for an account with a password it would add a computer without asking for it, so + `409 conflict`, and a password reset code is the way in. A **password reset code** (`purpose = reset`, 16 + characters, one hour, one per account like an invite, so each replaces the other) is redeemed at `POST /join`: it + always asks for a new password (`password_required`, `weak_password`), sets it, signs **every** device of the + account out (`revoke_reason = reset`), deletes its pairings, and, as every redeem does, signs the redeeming computer + in. Codes are shown once in the answer (with the whole join link when `public_url` is set) and never logged. + - **Quota.** An admin may set `quota_bytes` for one account (from one chunk, 4 MiB, to `i64::MAX`; `null` removes + it). The limit of a space is its quota, else `stored_bytes_per_user`, else none, and it holds in the look before a + chunk or manifest is stored (`check_room`, also when there is no server-wide cap) and in the count that follows. + A quota under what the account stores deletes nothing and refuses new bytes with `507 quota_exceeded`. + - **Live connections.** The process that revokes a device ends its live connections at once; another server process on + the same Postgres database, and the command line, are noticed at the connection's next ping (about 30 s). + - Logs carry the kind of event and the target's ID at `info`, never a username, password, code or link. +- **Security audit log** (migration 0009): the events that matter for who may do what are rows of `audit_events`, + written by the change they record, in its transaction and before its commit, so a change whose event cannot be written + is not made (fail closed). The names are a closed list in the code: `auth.login`, `auth.login_failed`, + `auth.second_factor_failed`, `auth.recovery_code_used`, `auth.two_factor_locked`, `device.revoked`, + `device.revoked_others`, `account.password_changed`, `account.two_factor_enabled`, `account.two_factor_disabled`, + `account.recovery_codes_regenerated`, `admin.user_created`, `admin.user_disabled`, `admin.user_enabled`, + `admin.user_removed`, `admin.role_changed`, `admin.quota_changed`, `admin.invite_issued`, `admin.invite_revoked`, + `admin.password_reset_issued`, `admin.two_factor_reset`, `admin.device_revoked`, `join.invite_redeemed`, + `join.reset_redeemed`, `join.pairing_redeemed`, `cli.password_reset`, `cli.two_factor_reset`, `cli.device_revoked`, + `cli.devices_revoked_all`, `cli.user_removed`, and the team space events (`target_kind = space`, the space's ID as + `target_id`): `space.created`, `space.renamed`, `space.archived`, `space.unarchived`, `space.quota_changed`, + `space.member_invited`, `space.member_joined`, `space.member_declined`, `space.member_role_changed`, + `space.member_removed`, `space.member_left`, `cli.space_created`, `cli.space_member_changed` and `cli.space_archived` + (see the collaboration design). A name whose change does not exist yet (`user remove`, the renaming, + archiving and quota changes of a space) is listed so the vocabulary a client filters by does not change; its writer + arrives with that change. + - **Rows.** `id` (UUIDv7), `at_ms`, `actor_user` and `actor_device` (`NULL` for the command line, and for a sign-in that + failed: nobody proved who they were), `action`, `target_kind` (`user`, `device`, `space` or empty) and `target_id`, `outcome` + (`success`, or `failure` for a failed sign-in), `details`, and for aggregated rows `agg_key` and `agg_count` (the API + calls it `count`). There are no foreign keys: an event keeps the ID of an account that was removed, and the name is + gone because it was never stored. + - **What each operation writes.** A sign-in `auth.login` (actor: the account and its new device); a failed one + `auth.login_failed`. `DELETE /devices/{id}` `device.revoked`; `POST /devices/revoke-others` **one** + `device.revoked_others` with the count (none when no device was left); `POST /account/password` one + `account.password_changed` whose `revoked` counts the devices that went with it. Admin changes write their + `admin.*` event, a `PATCH` with a role and a quota one event for each field that changed, in one transaction; the + devices a disable or removal signs out are counted in its one event. Redeeming an invite, a reset code or a + pairing writes `join.invite_redeemed`, `join.reset_redeemed` (with `revoked`) or `join.pairing_redeemed`. The + command line writes `cli.password_reset`, `cli.device_revoked` and `cli.devices_revoked_all`, with no actor. **A + repeat or a no-op writes nothing**, since nothing changed: disabling a disabled account, signing out a device that is + out, giving a role or quota the account has, revoking an invite that is not there, and so a retried request after a + dropped connection leaves the event of its first run only. Files, uploads and live documents are not audited. + The admin and command line events about a device are about its **account** (`target_kind = user`) and carry the device + in `details`, so the account's own activity shows them. + - **`details` is typed.** Each event has its own few fields, each a closed list of words or a number: `client_kind` + (`agent`, `desktop`, `cli`, `web`) and `platform` (`windows`, `macos`, `linux`, `android`, `ios`, `other`) for a sign-in or + join, `reason` (as in the device list), `step` (`password`, `totp`, `recovery`, `challenge_expired`), `role_from` and + `role_to` (an account's role, or a role in a team space: `viewer`, `editor`, `owner`), `user` (the account a team space event is about, by ID), `quota_bytes` (a number, `null` when the quota was removed), `revoked` (a count), `device` (an ID) and `two_factor_removed` (`true`, only on `join.reset_redeemed`). A + device may call its platform any printable text of 32 bytes, an address or a file name included, so anything not in + the list is `other`; `web` means a browser session of the admin pages and cannot be named by a device. Nothing a client typed ever reaches a row. The text is at most 2,048 bytes: if it would be more, + whole fields go, the least important first, and `"truncated": true` is added; bytes are never cut off, which would + leave invalid JSON. **No address, file name, path, device name, name of an account that does not exist, password, + code, challenge, token, key or e-mail address is stored anywhere in a row.** + - **Two-factor sign-in** writes these, all in the transaction of the change unless said otherwise. A sign-in that made a + device writes `auth.login` whichever way it went (no second factor, a code, a recovery code); the challenge `POST + /auth/login` gives writes nothing, since nobody is signed in yet. Turning it on, new recovery codes and turning it off write + `account.two_factor_enabled`, `account.recovery_codes_regenerated` and `account.two_factor_disabled` (the account, from the + device that asked). A recovery code that is used up writes `auth.recovery_code_used` in the transaction that uses it up + (sign-in, join, turning it off, an admin's confirmation, which commits on its own before the change it confirms), by the + account and the device where there is one; never the code. An admin's reset writes `admin.two_factor_reset` (admin, target) + and the command line `cli.two_factor_reset` (no actor, with `revoked`); a reset code made with `reset_two_factor` that is + redeemed writes `join.reset_redeemed` with `two_factor_removed: true`, by the account that redeemed it: the code does not + say which admin made it, and none is made up. The wrong second step of a sign-in or a join is `auth.second_factor_failed` + (`step`: `totp` or `recovery`; `challenge_expired` for a challenge that is unknown, expired, used or out of tries, with no + account when it was not found), aggregated and written after the answer is decided, like `auth.login_failed`. **A wrong proof + on a signed-in route** (turning it off, new recovery codes, an admin's confirmation) **is not written one by one**; the lock + it can cause is: `auth.two_factor_locked`, once when a lock is set or lengthened and never for a refusal while it holds. That + event is written like a failed sign-in's, in a transaction of its own after the counters and the lock are committed, so a + fault in the log can neither undo the lock nor change the answer; it is logged without content and the event is lost. + - **Failed sign-ins** are one row per account, action, client kind and minute, `agg_key` = action + client kind + account + minute, written + after the answer is decided and in a transaction of its own: `INSERT … ON CONFLICT (agg_key) DO UPDATE SET agg_count = + audit_events.agg_count + 1` (the column is qualified, since a bare one is ambiguous against `excluded` on Postgres), + so many tries at one moment make one row that counts them all and keeps the identity, time and details of the first. + Tries with a name no account has share one row per client kind and minute with no account in it, so a guesser can neither put text in + the log nor make it grow without bound. A failure to write this row is logged (no content) and changes neither the + answer nor the limits on tries, which are not stored in the log at all. + - **Reading.** Pages are newest first by `(at_ms, id)` descending. `cursor` is that pair, opaque, and exclusive: the next + page starts strictly behind it, so events with one timestamp are neither repeated nor lost. The export reads 500 rows + per query through a channel of two pages, so a slow client slows the reading instead of filling memory; the end of the + log is fixed when it starts (`at_ms` up to then), at most 100,000 event rows are sent, and the reading stops when the + client goes away. The last line says how it ended; a database error in the middle breaks the connection (and is logged + without content) so that no client can take half an export for a whole one. + - **Append-only, in the code.** `Meta` has no function that changes or deletes a row of `audit_events` except + `audit_prune`, which deletes events older than the retention, and the counter of an aggregated row. A test reads the + sources of `craft-cloud-meta` and allows exactly those two statements. **This deviates from a design with + database triggers and separate roles**: it is a design choice so that the same code runs on both databases (SQLite has no database roles; triggers exist but are not used), and an operator + with access to the database can still change the log. Protection against that is outside this server. + - **Retention.** `[audit] retention_days` (1 to 3650, default 365, the section is optional). Old events are deleted by + `audit prune` (`--dry-run` only counts) and by a write that finds the last complete pruning an hour or more past (kept + in memory, set only when a pruning deleted everything that was due), after its commit and never inside the security + transaction, in batches of 1,000 rows each in a transaction of its own and at most 20 batches a time. A server + that nobody writes to keeps expired events until `audit prune` runs: schedule it (cron, the task scheduler). There + is no periodic task. + - **Personal data.** A row holds account and device IDs and times, which is personal data. `docs/server.md` says how long + it is kept and why; the server stores no address. + - Reading the log is not itself an event. +- **Two-factor sign-in** (migration 0010, after the audit log's 0009): an account may turn on a TOTP second factor (RFC 6238: HMAC-SHA-1, six + digits, 30-second steps, one step either way for a clock that is a little off; the codes authenticator apps make) with + ten recovery codes. With it on, a right password gives **no token**: `POST /auth/login` answers `401 + two_factor_required` with `two_factor: {challenge, expires_ms}`, and `POST /auth/login/second-factor` with the + challenge and a `totp_code` (or `recovery_code`) makes the device. + - **The secret** is 160 random bits, shown once (Base32 and an `otpauth://` URI) and stored only as an AES-256-GCM + ciphertext: a key-name byte, a 12-byte nonce and the ciphertext, with `craft-cloud:totp:v1:` and the account's ID as + associated data, so a ciphertext moved to another account does not open. The key is 32 random bytes in the file the + config names (`[two_factor] key_file`, relative to `config.toml`), which `init` makes and the **server never makes** + (a second server on the same Postgres database, or a restore without the key, would otherwise seal secrets + nobody can open); the AES key is BLAKE3 `derive_key("craft-cloud 2026 totp secret v1", key)` of it. The key's name is + the first byte of the BLAKE3 hash of the key file: one of 256 values, only there to tell the current key from the old one + (the server refuses to start when they are equal). It does not identify a key (two different keys can share a name), and + the GCM authentication tag refuses a wrong key whatever the name says. + **Fail closed**: when the database holds secrets, the server opens **every** one at start and refuses to run if the key + is missing or one does not open, naming the file. With no secrets and no key it runs, and set-up answers `503 + two_factor_unavailable`. **Rotation**: the new key in `key_file`, the old in `key_file_old`, every instance restarted with + both (a running server holds the keys it started with), then `two-factor rekey` rewrites the secrets under the new one + (one account at a time, each read fresh under the account's lock, a final scan, repeatable, refusing when a secret has a + key that is neither), then `key_file_old` goes, after `rekey` reported that nothing is left under another key. Without the key the secrets of a backup are unusable: keep it apart from backups. + - **Recovery codes** are ten of 16 characters in the join codes' alphabet (80 bits, shown in groups of four), stored as + the BLAKE3 hash of the normalised code, no pepper: 80 bits can't be searched even from a leaked database, and the codes + work whatever happens to the key. Each works once; new ones replace all the old. A recovery code stands in for a + code at sign-in, join, turning it off and the admin's confirmation. + - **Set-up** (`POST /account/two-factor/setup`, the password again) keeps a new secret for 10 minutes, bound to the + device that asked and apart from any active secret; another device gets `409 two_factor_pending`; five wrong codes use + it up. `confirm` with the first code turns it on, shows the recovery codes and, in the same transaction, signs every + other device out (`revoke_reason = two_factor`), raises `users.auth_gen` and deletes the account's invite, + pairings and challenges. Two confirmations at once turn it on once. Turning it off (`disable`: password and a code or + recovery code) does the same, and so do `user reset-2fa` and the admin's `reset-two-factor` (which sign **all** devices out). + - **The transaction contract.** Everything that issues a challenge or a device, or uses a proof up, locks the account's + row first (the order every other change of the account uses), checks again under the lock what the handler checked + before (the device still signed in, the password hash still the verified one, `auth_gen`, the account's **current** + two-factor state, the challenge), and only then acts. Sign-in's challenge is made in that transaction: a stale hash + is `401 unauthorized` like a wrong password, an account with it on gets a challenge and no device. The second + step finds the challenge by a plain read to learn the owner, locks the account, checks the challenge again, then uses up + the time step (or recovery code) **and** the challenge and makes the device in one transaction with one commit. + `join` does the same for an invite or reset code. A **time step is accepted once and only a later one after it** + (`last_accepted_step`, updated `WHERE last_accepted_step < step`, shared by sign-in and the admin's confirmations), so + two requests with one code can't both succeed. A challenge is 32 random bytes, kept as a BLAKE3 hash, good 5 minutes, + five tries, once; at most five are open per account (the oldest goes); the address it was made for is only an extra + binding (behind a proxy that hides addresses all clients share one), the secret token is what binds it. A password + change, a reset, turning it off or on and a disable delete the account's challenges. + - **Wrong proofs** are counted and committed in the transaction that refused them, under the account's lock (every check + runs before anything is consumed, so the invite stays, the challenge stays, no device is made): the next request sees the + attempts and the lock, so a challenge takes five wrong proofs and no more, and the tenth sets the lock. Ten wrong ones in a row lock the account's second factor for 15 + minutes; each wrong one after a lock ran out doubles it, up to 24 hours; a right one resets it; `user reset-2fa` lifts + it. While it lasts every path (second step, join, turning off, new recovery codes, the admin's confirmation) answers + `429 two_factor_locked` with `Retry-After`. It is never permanent, so nobody with the bare password can lock an + account out for good. Separate limiter `second_factors`: 10 wrong codes + per account and 30 per address in 5 minutes (a right one is given back), and they count against the sign-in limiter. + - **Join**: an invite or reset code of an account with it on needs `totp_code` or `recovery_code` in the same body; + without it the answer is `401 two_factor_required` (its `challenge` is empty: `/join` has no challenge row, the same + body is sent again with the code) and the code is **not** used up; a wrong factor is `401 two_factor_invalid`, is not + given back to the address's join tries and counts against the account. A reset code made with `reset_two_factor` + turns it off instead of asking. **A pairing needs no second factor** (only a signed-in device can make one, and turning + two-factor sign-in on signs the other devices out), so **any valid device token can add devices**. + - **Status codes**: a wrong code is `401` at sign-in and join (not signed in) and `403` on every signed-in route, where a + 401 would read as a revoked device. The 2FA bodies are at most 4 KiB and a malformed one is `400 invalid`. - **Join codes**: a machine can also get its device token with a code instead of a password. Codes are Crockford base32 (`0-9` and letters without I, L, O, U; on input case is ignored, O reads as 0, I and L as 1, dashes and spaces are dropped), shown in groups of four (`K7M4-Q9TX`), and stored only as their BLAKE3 hash (migration 0004: @@ -630,25 +854,100 @@ changes locally, then pushes local changes. These rules came out of the simulati admins who back up the bucket themselves (bucket versioning or replication); GC must stay paused until that bucket backup is done, or the admin keeps the bucket's own deleted-object retention longer than the GC grace. **Restore** sets a new epoch for every space. -- **Admin CLI**: `init`, `serve`, `user add|reset-password|invite [--revoke]`, `device list|revoke` and `reset-epoch` - exist; `user remove|quota`, `gc`, `verify` (re-hash every chunk), `backup ` and `restore ` come with M4. +- **Web admin and browser sessions** (migration 0011, off unless `[web_admin] enabled = true`): the admin API's operations + also have pages in the browser, under `/web`, **server-rendered HTML in Rust**: no script, no build step, no Node, one + stylesheet served by the server itself (`GET /web/style.css`). The pages are an adapter over the same `craft_cloud_meta` calls + and the same confirmation as the JSON routes, so the rules (last administrator, who may reset whom, quotas) have one home, and the + JSON routes stay as they were. This replaces "no web UI in v1" for **administrators only**: there is no page for a member and + none for files. + - **Setup is explicit.** The pages exist only with `[web_admin] enabled = true`, which the config accepts only with + `behind_proxy = true` and `public_url = "https://host[:port]"` without a path (a TLS proxy in front; `public_url` is the + origin the pages check forms against). Otherwise `/web` is `404` like any unknown route. `idle_minutes` (5 to 1440, default 480) + and `absolute_hours` (1 to 720, default 168) set the session lifetimes; unknown keys are refused. + - **Sessions** (`web_sessions`): the cookie `__Host-craft_cloud_session` holds 32 random bytes (base64url, 43 characters); the + table keeps only their BLAKE3 hash, the account, the account's `auth_gen` at sign-in, `created_ms`, `last_seen_ms`, + `absolute_expires_ms`, `revoked_ms` and `mfa_verified_ms` (when a second factor was given; `NULL` for an account without one). + The cookie is `Secure; HttpOnly; SameSite=Strict; Path=/`, with no `Domain` and no lifetime of its own (the server decides, so + a copied cookie can't outlive its session). A session is valid while it is not logged out, was made less than + `absolute_hours` ago, was used within `idle_minutes` (its last-seen time is moved at most once a minute, by the next + request), the account is neither disabled nor marked for removal, **and the account's `auth_gen` is still the one it + was made under**. Every change that raises `auth_gen` therefore ends the account's sessions with no code of its own: a password + change or reset (device, command line or redeemed code), a disable, an enable, a removal, turning two-factor sign-in on or off, + and a reset of it by an admin or the command line. The role is read from the database for every request, so a demoted admin + gets `403` at once. A change made on a session checks, in its transaction under the account's lock, that the session is still + logged in under the account's current `auth_gen` (the same re-check a device gets) **and still inside its idle and absolute lifetimes at the time + of the change** (the time is read after the change has taken its locks, so a wait for them does not extend a session), so a request that began before the session ended (a slow body, a wait for a lock or the password check) cannot + change anything after it. + - **Sign-in** (`POST /web/login`, then `POST /web/login/second-factor` for an account with two-factor sign-in): the JSON + sign-in's own steps in the same order (name and password shape first, a password-check slot, then the counts, a dummy + hash for a name nobody has), with limiters of their own (`AppState::web_logins`: 30 per client address and 10 per account + in five minutes), so an attack on the pages does not lock the API and the other way round. **Only administrators get a + session**, and every refusal of the password step (unknown name, wrong password, a member, a disabled or removed account) + is the same `401` page, so nothing says which accounts exist or what role they have. The second step uses the second factor's + own code (`totp_code` or `recovery_code`, a time step serving once, five wrong codes deleting the challenge, ten locking the + account for 15 minutes and doubling after that, with the per-account and per-address limits of the device sign-in): the + lock is the account's, so it holds for pages and devices alike. The challenge is a row of `login_challenges` of kind `web`; a + web challenge can never make a device and a device's can never make a session. The sign-in writes `auth.login` with no + device; a failed one the usual `auth.login_failed`. + - **One credential at a time.** The pages know the cookie only; the JSON admin routes know the bearer token only (no JSON + route takes the cookie). A request to either with an `Authorization` header **and** the session cookie is `400 invalid`, + so a client can't be unsure which credential acted. + - **CSRF and origin.** Every form carries `csrf`, a value derived from the session token under a key-derivation context of + its own (so the stored hash can't make it), compared in constant time; missing, wrong or another session's is `403`. Every + request that is not a `GET` or `HEAD`, the sign-in too, must carry an `Origin` equal to the origin of `public_url` or, + without one, a `Referer` on it; an `Origin` that differs (including `null`) is `403` whatever the `Referer`. No `GET` + changes anything (the audit export is a read). The sign-in form has no CSRF token, as no session exists to bind one to: + the origin check is what stops another site from signing a browser in. + - **Headers on every answer under `/web`**, errors and redirects included: `Content-Security-Policy: default-src 'self'; + script-src 'none'; style-src 'self'; img-src 'self'; connect-src 'none'; object-src 'none'; base-uri 'none'; form-action + 'self'; frame-ancestors 'none'`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: no-referrer`, `X-Frame-Options: DENY`, + `Cross-Origin-Opener-Policy` and `Cross-Origin-Resource-Policy: same-origin`, a restrictive `Permissions-Policy`, and + `Cache-Control: no-store` (the stylesheet `no-cache`). No page has a script, an inline style, an event handler or a file from + another host. `Strict-Transport-Security` is the proxy's to send. + - **Statuses** are one contract for the pages: `401` for a missing, expired or unknown session (a `GET` is sent to `/web/login` + with `303` instead; a form post gets the sign-in page), for a wrong username or password at sign-in, and for a sign-in step + (challenge) that is unknown, used up or expired; `403` for a member or demoted admin, a wrong second factor (the page of the + second step again, with the same challenge), a wrong confirmation password or code, a bad CSRF token or origin; `400` for a + malformed form, **including a body that is not a form (no `415`)**; `429` for the limits and the lock, **always with + `Retry-After`** (the end of the window that refused, or the lock's); `413` over 4 KiB; `409`/`404` as the store says. A sign-in + for an account that has no password yet (an open invite) spends the same password check as an unknown name and fails the + same way, in the pages and in `POST /auth/login`. `/web/` has the same headers and origin check as the other paths. Every change asks for the **confirmation in its form**: the admin's password, and `totp_code` or + `recovery_code` when their account has two-factor sign-in (the server asks whatever the page showed), checked by the + JSON routes' function with their counters, so a wrong password counts against the same five-in-fifteen-minutes window. + - **Pages**: overview (counters, kinds), users (search, 50 a page by cursor, create), one user (role and quota, disable, enable, + remove, invite, password reset code, end the open code, reset two-factor, devices with revoke), the audit log (filters + `user`, `action`, 50 a page by cursor, and `GET /web/audit/export`, the JSON export's NDJSON download). Invite and reset + codes are the answer to the `POST` that made them, shown once and kept nowhere; a reload asks the browser to post the form again. + Every value not written by the server's code is HTML-escaped on the way in, including a device's name and platform. + - **Cleanup without a periodic task.** A sign-in deletes up to 100 sessions that are over (logged out, past their lifetime, + idle); `craft-cloud-server web-sessions prune [--dry-run]` deletes all of them (run it from cron, as `audit prune`). Neither + touches a valid session. + - Nothing secret is logged (cookie, CSRF token, challenge, password, codes); events are logged at `info` with kind and target ID. +- **Admin CLI** (the way in without HTTP: it asks for no confirmation and does not enforce `last_admin`, since whoever runs it + has the files): `init` (which also makes the two-factor key file), `serve`, `user add|reset-password [--keep-devices]|invite [--revoke]|reset-2fa`, + `device list|revoke|revoke-all`, `web-sessions prune [--dry-run]`, `two-factor rekey` and `reset-epoch` exist; `user remove|quota`, `gc`, `verify` (re-hash every chunk), `backup ` and `restore ` come with M4. `user invite` prints the join string and link when `public_url` is in the config, else the code and how to finish it; it logs that an invite was made (the username at debug only, never the code). -- No web UI in v1. No telemetry, no update check. +- No web UI for files, and none for members. No telemetry, no update check. API (`/api/v1`, JSON except chunk and manifest bodies; every signed-in response carries the space's epoch in the `craft-epoch` header, and commits/tree/changes carry it in the body too). Errors are -`{error, message, conflicts?, missing?, quota?}` with stable codes (`conflict` 409, `missing_chunks` 409, +`{error, message, conflicts?, missing?, quota?, two_factor?}` with stable codes (`conflict` 409, `missing_chunks` 409, `resync_required` 410, `unauthorized` 401, `rate_limited` 429, `invalid` 400, `not_found` 404, `too_large` 413, `timeout` 408, `quota_exceeded` 507, -`username_taken` 409, `invalid_code` 404, `password_required` 400, `weak_password` 400, `internal` 500; the codes are constants in `craft-cloud-proto::api::codes`). Unknown routes and +`username_taken` 409, `invalid_code` 404, `password_required` 400, `weak_password` 400, `wrong_password` 403, `forbidden` 403 (an +admin route called by a member), `account_disabled` 403 (sign-in and join only), `last_admin` 409, `two_factor_required` 401 (with +`two_factor`), `two_factor_invalid` (401 at sign-in and join, 403 on signed-in routes), `two_factor_unavailable` 503, +`two_factor_pending` 409, `two_factor_locked` 429 (with `Retry-After`), `internal` 500; the codes are constants in `craft-cloud-proto::api::codes`). Unknown routes and methods answer `not_found` / `invalid` (405) the same way: | Call | Purpose | |---|---| | `GET /info` (also unversioned at `/api/info`) | protocol versions, chunker parameter sets, limits (no auth); a client probes `/api/info` first | | `GET /space` | the signed-in user's space: space ID, root node ID, epoch, current seq, username | -| `POST /auth/login` | username + password + device name → device ID + token (shown once); 10 tries per account and 30 per address per 5 minutes; a wrong password and an unknown user look the same | +| `POST /auth/login` | username + password + device name (+ optional `client_kind` `agent`\|`desktop`\|`cli`, `platform` and `app_version`, each at most 32 bytes, shown in the device list and never a basis for a permission; an unknown kind or a control character is `invalid`, checked before the try is counted) → device ID + token (shown once); 10 tries per account and 30 per address per 5 minutes; a wrong password and an unknown user look the same. For an account with two-factor sign-in a right password gives **no token**: `401 two_factor_required` with `two_factor: {challenge, expires_ms}` (the challenge is good 5 minutes, five tries, once) | +| `POST /auth/login/second-factor` | `{challenge, totp_code}` or `{challenge, recovery_code}` (no auth, at most 4 KiB) → device ID + token as a sign-in; a challenge that is unknown, expired, used or no longer fit is `401 unauthorized` (the same for each), a wrong code `401 two_factor_invalid`, a malformed one `400 invalid`, a lock `429 two_factor_locked` with `Retry-After`; `second_factors` limiter | +| `GET /account/usage` | `{stored_bytes, quota_bytes, trash_bytes, history_bytes, estimates: {index_complete, reason?}}`: stored bytes and labeled storage estimates for this device's account | | `GET /tree?after_seq=…&after_node=…&limit=…` | full listing for a first sync or a reconcile, in pages of up to 5,000 (cursor = changed seq + node ID) | | `GET /changes?since=N&after_seq=…&after_node=…&limit=…&wait=30` | changes after N, in pages, long-poll up to 60 s; `resync_required` if N is too old or ahead of the server | | `POST /chunks/missing` | which of these chunk IDs (up to 10,000) this space lacks | @@ -656,10 +955,150 @@ methods answer `not_found` / `invalid` (405) the same way: | `PUT /manifests/{id}` / `GET /manifests/{id}` | one manifest (binary); the server checks the ID, the limits and that every chunk is present | | `POST /commit` | batch of version/mkdir/move/delete operations, one transaction; `409 conflict` listing every conflicting operation (a version whose manifest isn't uploaded is a `missing_manifest` conflict) | | `GET /nodes/{id}/history` / `POST /nodes/{id}/restore` | list versions / make an old one current (M4) | +| `GET /account` | the signed-in account, flat: `{user_id, username, role, created_ms, password_changed_ms, two_factor_enabled}` (`password_changed_ms` is `null` until the password changes through the server) | +| `GET /account/two-factor` | `{enabled, enabled_ms, recovery_codes_left, pending}` | +| `POST /account/two-factor/setup` | `{password}` → `{secret, otpauth_uri, expires_ms}` (secret Base32 without padding, shown once; the set-up lasts 10 minutes, belongs to this device, replaces none that is on). `409 conflict` when it is on, `409 two_factor_pending` when another device has a set-up open, `503 two_factor_unavailable` without a key, `403 wrong_password`; the password's tries are the password change's (5 wrong per account per 15 minutes) and take a check slot | +| `POST /account/two-factor/confirm` | `{totp_code}` → `{recovery_codes: [10]}` shown once. Turns it on, signs every other device out, deletes the invite, pairings and challenges. At most 5 wrong codes per set-up; `403 two_factor_invalid`; `409 conflict` for no set-up, an expired one, or one another device started | +| `POST /account/two-factor/disable` | `{password, totp_code}` or `{password, recovery_code}` → 204. Deletes the secret, codes and challenges and signs every other device out; wrong code `403 two_factor_invalid`, `429 two_factor_locked` | +| `POST /account/two-factor/recovery-codes` | `{password, totp_code}` → `{recovery_codes: [10]}`; the old ones stop working | +| `POST /account/password` | `{current_password, new_password}` → 204. Signs every *other* device out and deletes the account's open invite and pairings, in one transaction; the device that asked stays signed in. A wrong current password is `403 wrong_password` (never 401: see below), a new one that breaks the rules or equals the current one `400 weak_password`; 5 tries per account per 15 minutes (`429 rate_limited`); body at most 4 KiB, passwords at most 1,024 bytes | +| `GET /devices` | your devices: `{id, name, created_ms, last_used_ms, revoked, current, client_kind, platform, app_version, revoked_ms, revoke_reason}`; the last five are new and a client may ignore them (`revoke_reason` is `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`) | +| `PATCH /devices/{id}` | `{name}` (at most 4 KiB, the device-name rule) → 200 with the device. Only your own devices that are not revoked, otherwise `404 not_found` | +| `POST /devices/revoke-others` | signs out every other device of the account (and deletes their pairings) → `{revoked: n}`, 0 when there is none | +| `DELETE /devices/{id}` | signs one of your devices out → 204; its time and reason (`user`) are kept, and a second call changes neither | +| `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | +| `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | +| `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash and whose folder is not in the trash itself (those are listed through the folder), newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | +| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders and, for a folder, all that is in the trash below it, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | +| `DELETE /trash/{id}` | takes a node out of the trash (204), and, for a folder, all that is in the trash below it: only marks them for removal, nothing is freed yet; up to 10,000 items below a folder at once, shallowest first; the rest is then listed and goes by a further call | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | -| `POST /join` | code + device name (+ password for an invite to a new account) → device ID + token, as a sign-in (no auth); `invalid_code`, `password_required`, `weak_password`, `rate_limited` | +| `POST /join` | code + device name (+ password for an invite to a new account **and always for a password reset code**, + the optional client fields of a sign-in) → device ID + token, as a sign-in (no auth); `invalid_code`, `password_required`, `weak_password`, `account_disabled`, `rate_limited`. A reset code signs every other device of the account out. Optional `totp_code` or `recovery_code`: an invite or reset code of an account with two-factor sign-in needs one (`401 two_factor_required` without, code not used up; `401 two_factor_invalid`, not given back to the address's tries), unless the reset code was made with `reset_two_factor`; a pairing code never does | +| `GET /admin/server` | (admin) `{version, users, devices_active, stored_bytes, live_bytes, storage_kind (disk\|s3\|memory), database_kind (sqlite\|postgres)}`: counters and kinds only, no path, address or setting | +| `GET /admin/storage` | (admin) bounded overview: account/storage totals, quota counts, top ten accounts, and manifest-index counts | +| `GET /admin/users/{id}/usage` | (admin) `{stored_bytes, quota_bytes, trash_bytes, history_bytes, estimates}` for one account; `quota_bytes` is effective quota; unknown or removed account is `not_found` | +| `GET /admin/users?cursor&limit&query` | (admin) `{users: [{id, username, role, created_ms, disabled_ms, two_factor_enabled, devices_active, stored_bytes, quota_bytes, invite_pending}], next?}` by username; `limit` 1 to 200 (default 50, otherwise `invalid`); `cursor` is opaque (the last username, base64url) and `next` is missing on the last page; `query` finds usernames containing it (case-insensitive, at most 64 bytes of printable characters, `%` and `_` are ordinary characters). Accounts marked for removal are left out | +| `POST /admin/users` | (admin, confirmation) `{username, role, password}` → `{user, invite: {code, join_url?, expires_ms}}`: an account with no password and a 7-day invite; `username_taken` 409 (also for an account marked for removal), `invalid` for a bad name or role | +| `PATCH /admin/users/{id}` | (admin, confirmation) `{role?, quota_bytes?, password}` → the user. Neither field is `invalid`; a missing `quota_bytes` leaves it, `null` removes it; `last_admin` 409 | +| `POST /admin/users/{id}/disable`, `/enable` | (admin, confirmation) → 204. Disable signs every device out and deletes the codes and pairings; not oneself (`invalid`), `last_admin`; enabling an account marked for removal is `conflict` | +| `DELETE /admin/users/{id}` | (admin, confirmation) → 202. Marks the account for removal as above; nothing is deleted | +| `POST /admin/users/{id}/invite`, `DELETE …/invite` | (admin, confirmation) → `{code, join_url?, expires_ms}` / 204: an invite only for an account that has no password (`conflict` otherwise); the delete ends the account's invite or reset code (`not_found` when there is none) | +| `POST /admin/users/{id}/password-reset` | (admin, confirmation) `{reset_two_factor?: bool}` → `{code, join_url?, expires_ms}`: a one-hour reset code, redeemed at `POST /join`; with `reset_two_factor` redeeming it also turns the account's two-factor sign-in off, else an account that has it needs its second factor to redeem. Another admin's needs two-factor sign-in on the acting admin's own (`forbidden`), one's own is `invalid` | +| `POST /admin/users/{id}/reset-two-factor` | (admin, confirmation) → 204: takes the account's two-factor sign-in away, deletes its challenges and signs **every** device out (`revoke_reason = two_factor`); not one's own (`invalid`), none to reset is `409 conflict`, another admin's needs the acting admin's own two-factor sign-in (`forbidden`) | +| `GET /admin/users/{id}/devices`, `DELETE …/devices/{device_id}` | (admin; the delete with confirmation) → `{devices: [device as in GET /devices]}`, revoked ones too / 204 (`revoke_reason = admin`); another user's device or an unknown user is `not_found` | +| `GET /account/activity?cursor&before_ms&limit` | the signed-in account's own security events, newest first: `{events: [event], next?}`; the events it acted in or was the target of, with the device shown only for its own acts, and the details only of events about itself (not of what a former administrator once did to others). `limit` 1 to 100 (default 50, otherwise `invalid`); `cursor` is opaque and exclusive and wins over `before_ms`, which only limits the first page; a cursor this server did not make is `invalid` | +| `GET /admin/audit?cursor&before_ms&limit&user&action` | (admin) the whole log in the same form (`limit` up to 200); `user` is an account ID (events it acted in or was the target of), `action` an event name or a group ending in a dot like `auth.`; anything else is `invalid` | +| `GET /admin/audit/export?user&action&before_ms` | (admin) the same filters (`before_ms` must not be negative) as NDJSON (`application/x-ndjson`, always offered as `craft-cloud-audit.ndjson`), streamed: one event per line, newest first, then one last line, `{"complete":true,"exported":n}`, or `{"truncated":true,"exported":n}` when there were more events than the export holds (100,000). A stream that ends without either line broke off and is not a whole export | + +**Account usage estimates.** `stored_bytes` keeps the existing unit: the stored size of the space's chunks and manifests, each once, including each format byte. `quota_bytes` is the effective quota: the per-account override when set, otherwise the configured `stored_bytes_per_user` cap, otherwise `null`. `trash_bytes` is the stored size of blobs referenced only by versions of deleted nodes still in the space; no live node version or live-document snapshot/import snapshot may reference them. A pending `purge_requested_ms` still counts: it hides the node from the trash list but frees no data yet. `history_bytes` is the stored size of blobs referenced only by non-current versions of live nodes; no current version, saved version (one that carries a save record), trashed node or live-document snapshot/import snapshot may reference them; a chunk counts only if every manifest naming it qualifies, so a chunk shared with a saved version stays out. A saved version of a deleted node still counts in `trash_bytes`, because purging the node removes it. Both include each shared blob once per space. Unrooted manifests are not counted. Both numbers are labeled, overlapping estimates; never add either to the other or to `stored_bytes`. If any manifest belonging to the space has an incomplete reference index, both estimates are `null` and `estimates` is `{index_complete: false, reason: "index_incomplete"}`; the server does not guess. + +**History and restore.** Versions are never removed today, so the history lists every version of a file, and `pruned` +is always `false`: the field is in the contract for the day versions can be thinned (a thinned version will stay +in the list with `pruned: true`, its metadata intact, and can't be restored). A restore is an ordinary `update` made +by the server: it checks, under the same lock as a commit, that the node is a file of the caller's space that is not +deleted and has no live document, that `version` is a version of this node, that its manifest is still in the space, +and that `expected_version` is still the file's current version; then it records a new version with a fresh ID whose +parent is the current version, with the manifest, modification time and executable bit of the old one, at the space's +next sequence number. The epoch does not change (only `reset-epoch` and backup restores do that), the change feed +reports the file like any other update, and a restore needs no quota because the manifest is already the space's. +The new version carries **no save record**: a save record names a state of a live document, and a restore +copies none and invents none. A restore is an ordinary new version, so agents take it in under the normal rules, +including the conflict rules for a machine with unsynced edits of its own. A file that has a live document is +refused, so file versions and live-document state stay separate. Unknown node, foreign node, a version of +another file and a missing manifest are all `not_found`; a deleted file, a folder and a file with a live document are +`invalid`; a stale `expected_version` is `conflict` (409, with the conflict list as for a commit). + +**Trash.** The trash is the set of tombstones with the time of the deletion. `nodes.deleted_ms` is set when a node goes +from live to deleted (not when a deleted node is deleted again) and cleared when it comes back, by an edit or by a restore +from the trash, so the next deletion starts its time afresh; `nodes.purge_requested_ms` is set by `DELETE /trash/{id}` +and cleared when the node comes back. Nodes deleted before the migration that added them are dated at the migration. A +deleted node without a time (damaged data) is dated at the next read of the trash, logged without names, and never +counts as expired. `DELETE /trash/{id}` frees nothing: versions, manifests, chunks and the tombstone stay, the change feed +and the sequence number do not change, and the node's history stays readable; the node only leaves the trash list and can +no longer be restored from it. (An edit that brings it back still wins, as for any deleted file.) Asking twice is fine. +A node is *in the trash* when it is deleted and not marked by `purge_requested_ms`. The list shows a node only if its +parent is not in the trash, so a deleted folder stands for everything below it. (A folder can only be deleted once it is +empty, so what is below it was deleted before it, in the same change or earlier; the rule does not depend on the times.) +A restore from the trash brings back the node asked for and the deleted folders above it, root-most first, keeping their +IDs; for a folder it also brings back every node below it that is in the trash, recursively, whenever it was deleted +(a file deleted on its own earlier inside the folder comes back with it); a node marked for removal, and what hangs +only from it, stays deleted. Each of them is revived like a single restore (new version for a file, `deleted_ms` and +the mark cleared, `changed_seq` set); a node below the folder keeps its name, except that of two deleted nodes of one +name in one folder the one deleted last keeps it and the other gets a conflict name. A removal of a folder marks +everything in the trash below it the same way. Both take at most 10,000 nodes below the folder per call, in the order +depth, then newest deletion first, then ID, so a node never comes back or is marked without its folder; there is no error +for more. What is beyond has its folder live (restore) or marked (removal), or hangs under a folder that does, so it is +listed on its own and a further call takes it, bounded the same way. A restore also makes the per-row work under the write +lock at most that many nodes. It creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: +where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted +folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back +as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is +taken too, and a restore with no free name is `conflict` and changes nothing. A restored *file* gets a **new version**: a copy of its last one (the same manifest, modification time and +executable bit, no save record), after it, in the same change. A machine that deleted the file, whose answer it never +heard (the server applied the delete), still names the old version in its delete; after a restore the server refuses +that delete as a version mismatch, and the engine, which already treats a node that came back at another version as one +whose delete went through, lets the node go and downloads the restored file as a new one. Without this the machine would +send the delete again and delete the restored file. A folder has no version, so a deleted folder that was brought back +looks the same as one that was never deleted; the engine therefore remembers the folder deletes it sent without hearing +the answer (`State::sent_dir_deletes`) and, when the next pull reports such a folder changed and live, lets it go instead +of deleting it again (a folder the feed does not mention was never touched, and its delete is sent again). A restore is +otherwise an ordinary change to live nodes, possibly with a new parent or name. Older agents lack the folder rule: with +one, a folder restored in the short window after a delete whose answer was lost can be deleted a second time (the folder +is empty and can be restored again), so update agents with or soon after the server. Restoring a *version* of a deleted file is still refused: bring the file back +from the trash first. + +**Reference index and `verify`.** The chunks a manifest names live in its object in the store. `manifest_chunks` +(manifest, ordinal, chunk) is a copy of that list in the database, and `manifest_index` records per manifest how many +chunks it names, how many edges are written (ordinals 0 up to that number, never a gap) and whether the list was +compared with the manifest (state 1) or is still being built (state 0). A manifest with no row is not indexed. The index +is derived data and is never the authority; it exists so the database can answer "which chunks does this manifest need" and +"which manifests use this chunk". It is written in the transaction that records a manifest (`add_manifest`), up to a budget +of 5,000 edges per transaction; a longer manifest (up to about a million chunks) is left being built, and the upload +handler finishes it in batches of at most that many edges before it answers, so no write lock is held for a long list. +`craft-cloud-server index build` indexes manifests recorded before the index existed (and any left unfinished), reading +each from the store and checking it against its ID first; it resumes where it stopped and may run beside the server. +A manifest is marked complete only after every edge was compared, ordinal by ordinal, with the manifest it was made from: +equal counts are not enough, and repeated chunks, wrong IDs, swapped order, missing and extra edges are all found. An +index that does not match its manifest while it is still being built is thrown away and written again; a complete index +that is wrong is only reported. Throwing the edges away is itself done in batches of the same size (a marker on the index row says it is under way, so an interrupted run resumes), never in one transaction; an edge at a negative ordinal is a mismatch like any other. Nothing in this slice frees anything: no deletion, no claims, no collection, no retention. + +`craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing: the database and the +storage directory must exist already (a wrong path is an error, not a new empty server with "no problems found"), and a +database whose schema is older than the program is refused, not upgraded (start the server once to upgrade it). A SQLite database is opened read-only (`Meta::open_existing`, `mode=ro`): the database file and the WAL keep their bytes, also when a crashed process left committed data in the WAL, which the command reads; it may leave an empty `-wal` and a `-shm` file (the rebuilt index of the WAL, not data). It lists +problems (which make it exit with an error): a manifest +object that is missing, damaged or not a manifest; a complete index that does not equal its manifest; edges whose manifest +has no index row; an index row without a recorded manifest; an edge or a manifest that names a chunk that is not recorded; +a chunk object that is missing or damaged; a size the database records that is not what the object has (the stored length of a chunk or manifest object, a chunk length the manifest names against the length recorded for that chunk, the file size recorded for a manifest); a version, snapshot or import snapshot that names a manifest that is not +recorded; blob memberships without a blob. It also gives notices (which do not): manifests whose index is not complete, +manifests no version or snapshot names, and chunks no complete manifest names. **The manifests no version or snapshot +names are not garbage.** A live document's binary assets (images and the like, uploaded through the agent's blob +interface as a manifest of their own) are named only by the document's state, which the server stores as opaque bytes, so +the database holds no reference to them. Until that gap is closed (the owner decides how assets are rooted), anything that +frees storage has to treat these manifests, and the chunks they name, as in use. + +**GC inventory and dry run.** `craft-cloud-server gc --dry-run [--json] --config ` lists each store namespace in +pages of at most 1,000 objects and checks each page against `blobs` in one batched query. It reports recorded objects, +orphans (store objects without a `blobs` row), and foreign keys (invalid names for that namespace). Listing errors name +the namespace and page; they are errors, never an empty inventory. Orphans are only candidates after a first-sight grace +period, which cannot be measured without recording state; a future collector would need two runs apart. Their size comes +from a metadata-only stored-size read because `Store::list` does not include sizes. + +Recorded chunks no complete-indexed manifest names are candidates only when all indexes are complete and database checks +report no problems. `Meta::index_audit` reads chunk rows, index states and roots under one read snapshot (Postgres uses +`REPEATABLE READ READ ONLY`); the store listing is a separate time-point diagnostic. These content-addressed objects +are immutable. A future collector must recheck roots before claiming because a reference can be added after this +snapshot. Summary equality across separate audits is not a concurrency guarantee: compensating writes can preserve +counts and examples. Incomplete indexes protect unknown dependencies. Existing reference roots are versions +(including saved versions and trash history), live snapshots and import snapshots, as queried by `Meta::index_audit` in +`crates/meta/src/index.rs`; this command adds no root kinds. Manifests with no version or snapshot root are never +candidates: they are `protected: may be a live document's asset`. Chunk roots that a complete index reports as used stay +protected. A dry run writes no rows, claims, inventory, generation, run record, store object or config. It prints +`nothing was changed`. On SQLite the command follows the rule of `verify`: it opens the existing database, may leave an +empty `-wal` and a `-shm` file (the WAL's rebuilt shared-memory index, not data), and changes neither the database file +nor the WAL's bytes, which its tests compare, with a crashed process's non-empty WAL too. On PostgreSQL the tests compare a +fingerprint of every table and the sequences. Both compare the whole store tree. Grace-period measurement, claims and +deletion belong to a later owner-approved slice. **Client contract** (what the sync agent relies on): @@ -672,7 +1111,13 @@ methods answer `not_found` / `invalid` (405) the same way: `version_exists` conflicts for exactly the retried operations mean the first attempt applied. - A long-poll refused because 16 are already waiting on the space answers `429 rate_limited` with `Retry-After`; wait that long before polling again. -- Unknown JSON fields are ignored in both directions, so later protocol versions can add fields. +- Unknown JSON fields are ignored in both directions, so later protocol versions can add fields. The device list's client + fields and revoke time and reason were added that way: a body without them reads as empty. +- **A signed-in route never answers 401 for anything but the token.** The agent reads every 401 on a signed-in route as + "this device was revoked" and signs out. A wrong password on a route that asks for one (`POST /account/password`) is + therefore `403 wrong_password`; 401 is for a missing, unknown or revoked token (a disabled account's tokens are revoked), and for + the sign-in and join routes. The admin routes follow the same rule: a wrong confirmation password is 403, a member is 403. + A wrong code or recovery code is `403 two_factor_invalid` on a signed-in route and `401` only at sign-in's second step and join. - After an admin restores a backup they run `craft-cloud-server reset-epoch --all`; agents then reconcile. ## Agent and launcher API @@ -691,6 +1136,16 @@ methods answer `not_found` / `invalid` (405) the same way: Calls: `status`, `login`, `logout`, `folders` (get/set synced or cloud-only), `history`, `restore`, `conflicts`, `pending-deletes` (approve or reject a mass-delete pause), `pause`, `resume`. + + Two-factor sign-in (see [`docs/agent.md`](../agent.md)): `login` and `join` answer 401 `two_factor_required` with the + server's challenge when the account has it; `POST /v1/login/second-factor {challenge, totp_code | recovery_code}` finishes + a login, and `join` is sent again with `totp_code` or `recovery_code` (its challenge is empty: the invite isn't consumed + by the first try). `GET /v1/account/two-factor`, `POST .../setup`, `.../confirm`, `.../disable`, `.../recovery-codes`, + `POST /v1/account/password` and `POST /v1/devices/revoke-others` pass through to the server for the app's Security + page. The local answers keep the server's code: `two_factor_invalid` (401 at sign-in, 403 when signed in), + `two_factor_locked` (429 with `Retry-After`), `wrong_password`, `two_factor_pending`, `two_factor_unavailable`; a gone + challenge is 409 `two_factor_expired`. Only a 401 on a signed-in call means this device was signed out; a 403 or 429 + never does. Codes, passwords and secrets are never stored and are redacted in every `Debug`. - **CLI**: `craft-cloud-agent login | status | pin | unpin | history | restore | approve-deletes | pause | resume`. Same operations as the local API, so it is usable and testable before any launcher work. @@ -1014,9 +1469,10 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** - **Names**: property tests that the validator rejects every unsafe form, and that case-fold + NFC uniqueness matches expectations for tricky cases (`ß`/`SS`, Turkish dotted I, combining accents). - **Server**: integration tests against an in-process server, **run once per backend combination** - (SQLite + disk always; Postgres and S3 in CI through service containers, locally when - `CRAFT_CLOUD_TEST_PG` / `CRAFT_CLOUD_TEST_S3_*` are set; CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS` so a - missing backend fails instead of skipping). The S3 test server is SeaweedFS (Apache-2.0), run as a + (SQLite + disk always; Postgres + disk locally when `CRAFT_CLOUD_TEST_PG` is set; Postgres + S3 + when `CRAFT_CLOUD_TEST_PG` / `CRAFT_CLOUD_TEST_S3_*` are set, including CI through service + containers; CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS` so a missing backend fails instead of skipping). + The S3 test server is SeaweedFS (Apache-2.0), run as a container, never linked. Cases: auth, quota, batch commit atomicity, `missing_chunks`, GC never removes a referenced or protected chunk while commits run, thinning, `backup` + `restore` gives the same state with a new epoch, `resync_required`. @@ -1061,7 +1517,7 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** | M3b2b1 | Agent on its own: seed 112337, `run` loop, watcher, long-poll, stability window, backoff, status file | DONE (2026-10-14) | | M3b2b2 | Simulated network (`turmoil`): partitions, delays, dropped connections | DONE (2026-10-15) | | M3c | Fixes from the full repo review | DONE (2026-10-15) | -| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | TODO | +| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | IN PROGRESS: history and restore of file versions, and the trash (list, restore, mark for removal), done (server API only); the rest TODO | | M5 | Selective sync, large-media polish (a few chunks in flight at once), ignore rules | TODO | | M6 | Packaging: binaries, Docker, systemd, built-in TLS, self-hosting guide | TODO | | M7 | Local control API + launcher integration PR | TODO | @@ -1158,7 +1614,7 @@ From the reviews of M3c, less what their fix rounds did: ## Out of scope for Phase 1 -Hosted service; collaboration and sharing; locking; web UI; mobile; virtual files and on-demand download; +Hosted service; collaboration and sharing; locking; a web UI for users' files (the administrators' pages, see "Web admin and browser sessions", are in); mobile; virtual files and on-demand download; end-to-end encryption (not built, but chunk IDs are opaque and stored chunks have a format byte so it can come later); per-chunk compression (same format byte); a packed small-chunk upload endpoint (one request per chunk until measurements say otherwise); compatibility with any other sync service. diff --git a/docs/server.md b/docs/server.md index bf2a6492..351fd9e5 100644 --- a/docs/server.md +++ b/docs/server.md @@ -25,13 +25,23 @@ For a non-loopback `--listen`, `init` also needs `--behind-proxy` and the proxy' For Postgres or S3, edit `[database]` and `[storage]` in `config.toml`. +`init` also makes `two-factor.key` next to `config.toml` and names it in the config (`[two_factor] key_file`). It is the key +for two-factor sign-in (below). **Back it up, apart from your database backups.** + To cap what one account may store, set `stored_bytes_per_user` (in bytes; there is no cap by default) in `config.toml`. It counts every chunk and manifest an account uploaded once, and nothing frees space until garbage collection arrives, including what a refused file stored before it was refused. A file that would go over it is skipped by the agent, which says so; the rest of the folder keeps syncing. Nearly full counts as full: with less than 4 MiB (one chunk) left, the agent pauses and says the account's storage is full. -A cap under 4 MiB is refused when the server starts, since not even one chunk would fit. Per-user quotas on -file sizes come later, with history. +A cap under 4 MiB is refused when the server starts, since not even one chunk would fit. An admin may override +the cap for an individual account as described below. + +Signed-in devices can read `GET /api/v1/account/usage` for the space's stored bytes, the effective quota, and labeled +estimates of bytes referenced only by trash or old versions. The estimates deduplicate shared blobs per space, exclude +live-document snapshots and unrooted app assets, and are never added to stored bytes. The history estimate treats a +version an app saved (one with a save record) as kept. The effective quota is the account override when set, otherwise +the configured `stored_bytes_per_user` cap; it is `null` when neither applies. An incomplete manifest index makes both +estimates `null`. Live documents have a cap of their own: `live_bytes_per_user` (in bytes; 1 GiB, `1_073_741_824`, by default) is the most one account's live documents (recent-file lists and documents of files) may hold together. It counts @@ -55,12 +65,352 @@ invite replaces the old one, and `user invite --revoke name` ends it. Every late that is signed in (`craft-cloud-agent add-computer`, see [`agent.md`](agent.md)). Codes that don't work are refused alike (wrong, expired or used), and an address that tries ten in a minute is held off for the rest of it. +## Devices, passwords and signing out + +An account's devices (computers and apps) each hold a token that does not expire. Accounts end them like this: + +- A person changes their own password in a Craft Cloud app that offers it (`POST /api/v1/account/password`). That signs + out every other device of the account, deletes its open invite and add-a-computer codes, and keeps the device + that asked. Five tries (wrong passwords count) are allowed per account every 15 minutes. +- `craft-cloud-server user --config cc-data/config.toml reset-password name` sets a new password (read from standard + input), **signs out every device of the account** and deletes its open invite and add-a-computer codes: a token that + was stolen does not survive the reset. It prints how many devices were signed out. Add `--keep-devices` to leave the + devices signed in (the old behaviour); the codes are deleted either way. +- `craft-cloud-server device --config cc-data/config.toml revoke-all name` signs out every device of an account + without touching the password. `device revoke name DEVICE_ID` signs out one; `device list name` shows them. + +The command line runs in its own process, so it cannot reach the server's open connections. A signed-out device's live +connection ends at its next ping (within about 30 seconds), and a change-feed long-poll within 60 seconds (when it +ends, it is answered `401 unauthorized`, never with changes); every other request is refused at once (`401 +unauthorized`). Devices signed out through the API end their live connections +immediately. + +## Two-factor sign-in + +An account can turn on a second factor in an app that offers it: a code from an authenticator app (TOTP: Aegis, 1Password, +Google Authenticator and the like), with ten recovery codes to keep. With it on, a right password does not sign anyone in: +the app also asks for the code, and so does joining a computer with an invite or an admin's password reset code. A +computer joined with an add-a-computer code (a pairing) asks for nothing, since only a computer that is already signed in +can make one: **any signed-in computer can add more**, and turning two-factor sign-in on signs every other one out. + +- **The key.** The authenticator's secret is stored only encrypted (AES-256-GCM, bound to its account), with the key in the + file `[two_factor] key_file` names (32 random bytes; a relative path is relative to `config.toml`). `init` makes it; **the + server never makes it** (a second server on the same database, or a restore without the key, would otherwise seal + secrets nobody can open). **Without this key the secrets in a database backup are unusable**: keep a copy apart from + the backups, and keep it out of everyone's reach. On Unix `init` makes it owner-only; + **on Windows the server sets no file permissions**, so put it somewhere only the account that runs the server can read. + The recovery codes do not depend on the key, so they still work if it is lost; the authenticator codes do not. +- **Failing closed.** When the database holds two-factor secrets, `serve` opens every one of them at start and refuses to + run if the key is missing, unreadable, or does not open all of them; the error names the key file the config expects. + A server without a key and without any secrets runs, and setting it up answers `503 two_factor_unavailable`. Several + servers on one Postgres database must all have the same key. +- **Rotating the key.** A running server keeps the keys it started with, so rotate in this order: (1) make a new key file, + put it in `key_file` and the old one in `key_file_old`; (2) **restart every server instance on the database** with that + config (until then an instance still holds only the old key and goes on sealing new secrets with it); (3) run + `craft-cloud-server two-factor rekey --config cc-data/config.toml` (it rewrites every secret, active and set-up, under the + new key, one account at a time, reads each account's secrets fresh under its lock so a set-up confirmed meanwhile is not + missed, scans again at the end, can be run again, prints only counts, and refuses without changing anything if a secret + was sealed with a key that is neither); (4) remove `key_file_old` and restart **only after `rekey` has reported that every + secret is under the current key** (an error that says secrets are still under another key means it is not done: restart the + instances you missed and run it again). The server runs in the middle of a rotation, with both keys. +- **A locked-out account.** A person who lost their authenticator and their recovery codes: `craft-cloud-server user + --config cc-data/config.toml reset-2fa name` deletes its two-factor secret, recovery codes and sign-in challenges, lifts + any lock, **signs every device out** (it prints how many) and deletes its open invite and add-a-computer codes. They sign + in with their password and can set it up again. Over HTTP an admin does the same with `POST /admin/users/{id}/reset-two-factor` + (below). Both need no key. +- **Wrong codes lock for a while.** Ten wrong codes or recovery codes in a row (at sign-in, joining, turning it off, + making new recovery codes, an admin's confirmation) lock the account's second factor for 15 minutes; each wrong one after + that doubles the time, up to 24 hours; a right one resets it. While it lasts everything answers `429 two_factor_locked` + with `Retry-After`. The lock is never for good, so someone who only has a password can not lock an account out forever. + Wrong codes also count against the sign-in limit (10 per account and 30 per address in 5 minutes, then `429`). +- **Recovery codes** are ten of 16 letters and digits (shown in groups of four), each good once. Only a hash is stored. +- **Older apps.** An agent from before this release shows every `401` at sign-in as "wrong password", so an account with + two-factor sign-in can't sign in with it (it never gets a token). Update the server first, then the Craft Cloud + desktop package with its agent, then stand-alone agents. +- **Logs** never carry a secret, code, recovery code, challenge, password or key. + +## Managing accounts over HTTP + +Accounts have a role, `admin` or `member`, and the routes under `/api/v1/admin` are for admins only (the first admin is +still made by `init`). They are an API for a tool or an admin app; the server's own admin pages are an option, see "The admin +pages in the browser" below. Every change asks +for **a fresh confirmation in its body**, the acting admin's own current password (and, for an admin who has two-factor +sign-in, a `totp_code` or `recovery_code` too), because a device's token never expires and must not be enough on its own: + +```sh +curl -s -X POST https://cloud.example.org/api/v1/admin/users/USER_ID/disable \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' -d '{"password":"your password"}' +``` + +- `GET /admin/server` shows counters (accounts, signed-in devices, stored bytes, live-document bytes) and the kinds of + database and storage, nothing else. `GET /admin/users` lists accounts 50 at a time (`limit` up to 200, `cursor` from + the last answer's `next`, `query` to find a username). +- `GET /admin/storage` returns bounded storage totals, effective quota totals and near/over-quota counts, the ten + largest accounts, and manifest-index counts. `GET /admin/users/{id}/usage` returns one account's stored bytes, + effective quota, and labeled trash/history estimates; unknown and removed accounts return `404 not_found`. +- `POST /admin/users` makes an account (role `admin` or `member`) with no password and gives an **invite**, shown once: + the person chooses their password when they join with it (a whole join link is included when `public_url` is set). + `POST /admin/users/{id}/invite` makes a new invite, but only for an account that still has no password; for one with a + password, make a **password reset code** (`POST …/password-reset`, good for an hour). Whoever redeems + it in Craft Cloud chooses a new password, **every device of the account is signed out, and the computer that redeems + it is signed in**. If the account has two-factor sign-in, redeeming the code needs its second factor too, unless the + admin asked for `"reset_two_factor": true`, which turns it off with the redeem. Codes are only shown in the answer, + never in the log. +- `PATCH /admin/users/{id}` changes the role and/or `quota_bytes`, a limit on what that account may store that replaces + `stored_bytes_per_user` for it (at least 4 MiB; `null` removes it; effective quota is the override, else configured + cap, else unlimited; it also holds when there is no server-wide cap; one + below what the account stores deletes nothing and refuses new bytes). +- `POST …/disable` signs every device of an account out and refuses its sign-in and joins (`account_disabled`); + `POST …/enable` allows them again but signs nothing back in. `DELETE /admin/users/{id}` marks an account for removal: + it is disabled for good and leaves the list, its name stays taken, and **nothing it stored is deleted yet** (that comes with + garbage collection). You cannot disable or remove yourself, and the last enabled admin cannot be demoted, disabled or + removed (`last_admin`). +- `GET /admin/users/{id}/devices` lists an account's devices and `DELETE …/devices/{device_id}` signs one out. +- `POST …/reset-two-factor` takes an account's two-factor sign-in away (it lost its authenticator and its recovery codes): + its secret, recovery codes and sign-in challenges go, and **every device is signed out**. Not your own account (turn + that off in your account), and nothing to reset is `409 conflict`. +- Resetting another **admin's** password or two-factor sign-in over HTTP needs two-factor sign-in on your own account + (`403 forbidden` without it); the command line on the host is the way if no admin has it. +- **A code serves one change.** The server accepts one 30-second step of an account's authenticator once, and only a later + step after it; sign-in and the admin's confirmations share that. So a second change in the same 30 seconds takes the + next code (wait for it) or an unused recovery code. + +A wrong confirmation password is `403 wrong_password` (never 401); five wrong ones in 15 minutes hold the account off for +the rest of the window. A wrong or missing code for an admin who has two-factor sign-in is `403 two_factor_invalid`. A signed-out device's live connection ends at once in the server process that did it; another +server process on the same database, or the command line, finds out at the connection's next ping (within about 30 seconds). +The command line stays as the way in without HTTP and enforces neither the confirmation nor `last_admin`. + +## Team spaces + +A **team space** is a versioned tree like an account's own space, shared between accounts of this server. Every account keeps +its personal space exactly as before; a server that never creates a team space behaves as it did. An administrator makes a +team space and names its first owner; owners then invite other accounts, who accept or decline. The roles are `viewer` +(reads, sees history and trash), `editor` (also writes) and `owner` (also manages the members, removes from the trash, renames +and archives). **An administrator has no access to a team space's files** unless they are a member; they can create, list, +archive and give a space an owner, set its limit, and add themselves as a member (which is audited and visible). + +A request names its space with the header `craft-space: ` (without it, the account's personal space, so older agents +and apps work unchanged); every answer carries the header back, so a client can tell an older server that ignored it. A space the +account has no access to is `404 not_found`, the same answer as for a space that does not exist; a member whose role does not +allow the call is `403 read_only`. The server reads the role from the database on every request. + +- `GET /api/v1/spaces` lists the account's spaces (the personal one first, then the team spaces by name, with invitations + as `state: invited`). `POST /spaces/{id}/accept` and `…/decline` answer an invitation. `GET /spaces/{id}/members` and + `…/usage` are for members. Owners invite (`POST …/members`, 30 a minute per account, `429 rate_limited` beyond), change a + role (`PATCH …/members/{user}`) and remove a member or withdraw an invitation (`DELETE …/members/{user}`). A member removes + their own row to leave. The last owner cannot be lowered, removed or leave (`409 last_owner`). An unknown, disabled or + removed account is one answer, `404 not_found`. +- `PATCH /spaces/{id}` renames (an owner) and changes the space's limit (an administrator: `quota_bytes`, `null` removes it). + `POST /spaces/{id}/archive` and `…/unarchive` are the owner's, with their password. An archived space is read-only for every + member and stays in their lists with `archived: true` (an invitation to it is not listed); nothing is deleted. +- An administrator's changes carry the same confirmation as the account routes above (password, and the second factor when on): + `POST /admin/spaces` (`name`, `owner`, `quota_bytes`), `GET /admin/spaces` (`limit`, `cursor`, `query`), + `POST /admin/spaces/{id}/archive` and `…/unarchive`, and `PUT /admin/spaces/{id}/owner` (makes an account an accepted + owner, for a space whose owners are gone). An administrator who is not an owner uses the same confirmation on the member routes. +- On the command line: `space create --owner `, `space list`, `space members `, + `space add --role ` (added and accepted at once), `space archive ` (a space is its ID or its + name), and `reset-epoch --space ` for one space. The command line enforces nothing of the above, as for accounts. +- Every change is in the audit log (`space.…`, `cli.space_…`), with IDs and role words only, never a space name. An account's own + activity (`GET /account/activity`) also shows the events that name it as the member concerned. +- Until live documents learn team spaces, a team space's live connection is refused (`404`); files in it sync as ordinary files. + +## The admin pages in the browser + +The server can serve admin pages: users, invites and reset codes, a user's devices, team spaces, the audit log and its download, in +plain server-rendered HTML (no script, no Node, nothing to build). They are **off by default** and only administrators can +sign in. They do the same things as the admin API with the same confirmation (your password again for every change, and a +code when your account has two-factor sign-in). +The Craft Cloud desktop app also has admin storage views: a server overview with quota and manifest-index summaries, +and per-account usage details. Both use the authenticated admin API; members do not see admin pages. + +The **Team spaces** page lists the team spaces (name, owners, members, use against the limit, active or archived) with a search +and paging like the accounts page, makes a space with its first owner, and has a page for each space: its members and roles (read +only), its limit, archive and unarchive, and naming an owner. Every change asks for the same confirmation as the others, acts as +the signed-in administrator in the audit log (no device), and wakes the space's long-polls like the admin API's routes. Members +of a space are managed by its owners in the Craft Cloud app, not here. + +```toml +public_url = "https://cloud.example.org" # the address of the pages, https, no path +behind_proxy = true +trusted_proxies = ["10.0.0.2"] + +[web_admin] +enabled = true +idle_minutes = 480 # a session ends when unused for this long (5 to 1440, default 8 hours) +absolute_hours = 168 # and at the latest this long after signing in (1 to 720, default 7 days) +``` + +Open `https://cloud.example.org/web` and sign in as an admin. The server refuses to start with `enabled = true` unless +`behind_proxy = true` and `public_url` is an `https://` address with no path: the pages' cookie is `Secure` (a browser only +sends it over HTTPS) and every form is checked against the origin of `public_url`. + +**Behind the proxy.** + +- The proxy terminates TLS and forwards `/web` and `/api` (and `/join`) to the server's `listen` address. It must pass the + `Origin` and `Cookie` headers through and must not rewrite the `Host` to something that changes the origin the browser + sees: browsers send `Origin: https://cloud.example.org`, and anything else is refused with `403`. Serving the pages under a + path prefix is not supported (`public_url` may not have a path). +- Put the proxy in `trusted_proxies` and let it **overwrite** `X-Forwarded-For` with the client's address; the sign-in limits + count the address that yields. A proxy that passes a client's own `X-Forwarded-For` along lets a visitor pick the address + the limits count. A misconfigured server shows itself at start (the config is refused) or as `403` on every form (the + proxy changed the origin). +- Send `Strict-Transport-Security` from the proxy; the server does not. Do not cache `/web` in the proxy: every answer says + `Cache-Control: no-store`. +- Bind `listen` to the proxy-facing address only. The cookie never leaves the browser over plain HTTP, but the plain HTTP + port must not be reachable by anyone but the proxy. + +**Sessions.** The session is a random value in the cookie `__Host-craft_cloud_session` (`Secure`, `HttpOnly`, +`SameSite=Strict`); the database keeps only its hash. A session ends when you sign out, after `idle_minutes` without use, +after `absolute_hours`, and **whenever the account changes**: a password change or reset, disabling, enabling or removing it, +and turning its two-factor sign-in on or off or resetting it. A demoted admin is out at the next request. Signing in again +makes a new session; there is no "remember me". Because the cookie is `SameSite=Strict`, opening the pages from a link on +another site shows the sign-in page once, and the next click on the pages themselves works. + +**Cleaning up.** Expired sessions are deleted a few at a time when someone signs in. On a server that is signed in to rarely, +or to empty the table: + +```sh +craft-cloud-server web-sessions --config cc-data/config.toml prune # deletes what is over, prints how many +craft-cloud-server web-sessions --config cc-data/config.toml prune --dry-run # only counts +``` + +Run it from cron or the task scheduler (daily is plenty), like `audit prune`. A valid session is never touched. There is no +background task. + +**What the log shows.** A sign-in to the pages is `auth.login` without a device; changes made there are the usual +`admin.*` events with the admin as the actor and no device. Failed sign-ins, second-factor failures and the lock are the +usual `auth.*` events. Nothing secret is logged: not the cookie, the page's forms, a code or a password. + +**Accessibility.** The pages are plain HTML: labelled fields, tables with captions and header cells, a skip link, visible +focus and no script, so they work with a keyboard and a screen reader and scale with the browser's zoom. They have been +checked by reading the markup and the tests, **not yet with a screen reader**: report what does not work. + +## The security audit log + +The server keeps a log of the events that matter for who may do what, in its database: sign-ins and failed sign-ins (a +row per account, client kind and minute, with a count), devices signed out, password changes, an administrator's changes to accounts, +roles, quotas, invites, reset codes and devices, codes redeemed on a new computer, and what the command line does to an +account (`user reset-password`, `user reset-2fa`, `device revoke`, `device revoke-all`). Two-factor sign-in writes its +changes, a recovery code's use, a lock and the wrong second steps of a sign-in or a join (never a code), but not each wrong code +on a signed-in route. Reading it: + +- Every account sees its own events, and what was done to it, with `GET /api/v1/account/activity`. +- Admins read all of it with `GET /api/v1/admin/audit` (filters `user`, an account ID, and `action`, an event name like + `auth.login` or a group like `auth.`; pages by `cursor`) and export it with `GET /api/v1/admin/audit/export`, newline + delimited JSON, newest first. The **last line** says whether the export is whole: `{"complete":true,"exported":n}`, or + `{"truncated":true,"exported":n}` when the log holds more than the 100,000 events an export carries (narrow it with + `user` and `action`). An export that ends without either line broke off: do not treat it as complete. Reading the log is + not logged. +- On the server's host: `craft-cloud-server audit --config cc-data/config.toml list [--user NAME] [--action auth.] [--limit N]` + prints the newest events as text (accounts by ID, not by name). + +**What is in it, and what is not.** A row has a time, the IDs of the account and device that acted and of what it acted +on, an event name, success or failure, and a few typed values (how many devices were signed out, the role before and after, +which kind of client and which platform out of a short list). Client kinds are `agent`, `desktop`, `cli` and `web`; `web` +means a browser session of the admin pages. It has **no IP addresses, no file names, no device names, no +names of accounts that do not exist, no passwords, codes, tokens or other secrets**; what a client types is never copied +into it. Because it holds account IDs and times it is personal data about the people who use the server: tell them +what you keep it for (finding out who changed what and who signed in or tried to), and for how long. An event keeps the ID +of an account that was removed; the name is not stored. + +**How long.** Events are kept 365 days. Set `retention_days` (1 to 3650) in an optional `[audit]` section of `config.toml`: + +```toml +[audit] +retention_days = 365 +``` + +`craft-cloud-server audit --config cc-data/config.toml prune` deletes the events older than that and prints how many +(`--dry-run` only counts). The server also deletes expired events when it writes a new one, at most once an hour and in small +batches that never hold up a sign-in; **a server nobody uses keeps its expired events** until `prune` runs, so run it from cron +or the task scheduler, for example daily. There is no background task. + +The log is append-only in the server's code, not enforced by the database (a design choice: the same code runs on SQLite and Postgres, and SQLite has no database roles; triggers exist but are not used). Someone +who can write to the database can change it; keep the database file and Postgres user private. + +## Upgrading + +This release adds migrations (`0007`, device sessions, `0008`, the admin API: reset codes, per-account quotas, the +removal mark, `0009`, the audit log, `0010`, two-factor sign-in: the secrets, recovery codes and sign-in challenges, +`0011`, the admin pages' browser sessions, `0012`, the history index, `0013`, the trash, `0014`, the manifest reference +index, and `0015`, team spaces: the kind, name and archive mark of a space, and its members; on SQLite it rebuilds the `spaces` table +in one transaction with a foreign-key check). The server applies them when it starts. A database it +has migrated cannot be opened by an older server: that one refuses to start ("the database was created by a newer Craft +Cloud"). Rolling back therefore means restoring a backup taken before the upgrade, so take one first. Upgrade the +server first: agents and apps keep working unchanged, **except that an account that turns on two-factor sign-in can't sign in +with an agent from before this release** (see "Two-factor sign-in"). Then update the desktop package with its matching agent, +then agents installed on their own. The `init` of a data directory made before this release did not make a key: add one +with `[two_factor] key_file` in `config.toml` (32 random bytes, owner-only) before anyone sets up two-factor sign-in. A config +that has the new `[audit]` or `[web_admin]` section is refused by an older server (it rejects unknown settings), so remove the section before +going back to one. To turn the admin pages on after the upgrade: take the backup, start the new server (it migrates), then add +`[web_admin]` as described above and restart behind the proxy. Several servers on one Postgres database must all run this +release before any of them is given `[web_admin]`, as for every migration. +## Version history + +The server keeps every version of every file; nothing is thinned or removed yet. A signed-in device can list a file's +versions (`GET /api/v1/nodes/{id}/history`) and make an old one current again (`POST /api/v1/nodes/{id}/restore`), +which adds a new version at the end of the file's history rather than rewinding it, so other computers pick it up +through the normal change feed. This needs no setting and no database migration, and an older server answers +`not_found` for both paths. The agent and the apps do not use them yet. + +## The trash + +A deleted file or folder stays in the trash, and in the server's records, until it is removed for good. A signed-in +device lists the trash (`GET /api/v1/trash`), brings a node back (`POST /api/v1/trash/{id}/restore`, which names it +differently if its name has been taken meanwhile) or takes it out of the trash (`DELETE /api/v1/trash/{id}`). Taking +it out only marks it: **nothing is deleted from the server yet**, no version or chunk is freed, and the node can't be +brought back from the trash afterwards. A folder in the trash is listed alone: everything that is in the trash below it +(deleted and not removed from the trash) comes back with it when you bring the folder back, and goes with it when you +take the folder out, **also a file that was deleted on its own, earlier, inside that folder**. Those items are listed +only through the folder. A folder brings back, or takes out, up to 10,000 items below it at once, the shallowest first; if +there are more, the rest then appears in the trash list on its own (its folder is live again or out of the trash), and +you bring it back or remove it by a further call. The only setting is the number of days a deleted node waits, shown to clients as +`purge_after_ms`; it removes nothing yet: + +```toml +[retention] +trash_days = 30 # 1 to 3650 +``` + +A file brought back gets a new version with its old content, so another computer's delete that was sent just before can't delete it again. Update the agents with or soon after the server: an older agent can delete a *folder* a second time if that folder was brought back right after a delete whose answer it never heard (the folder is empty and can be brought back again). + +The change adds two columns and an index to the metadata (migration 0013). An older server refuses the +upgraded database and rejects the `[retention]` section, so take a backup before upgrading and do not roll back. + +## Checking the storage + +`craft-cloud-server verify --config cc-data/config.toml` checks the database and the blob storage against each other +and prints what it finds; `--json` prints a machine-readable report and `--skip-chunk-reads` skips reading every chunk +object (which hashes it, and takes as long as reading all your data). It changes nothing and exits with an error if +it found a problem. It opens a SQLite database read-only: the database file and its WAL keep their bytes, also after a +crash that left committed data in the WAL, which it reads. Opening may leave an empty `-wal` and a `-shm` file behind +(SQLite's rebuilt index of the WAL, not data). The report also says how many manifests have a complete reference index, which is what a later +cleanup would need before it could free anything. After upgrading, run `craft-cloud-server index build --config +cc-data/config.toml` once: it indexes the manifests stored before the index existed, writing only that index, in small +steps, and it can be stopped and run again. Nothing in the server frees storage yet. + +The change adds two tables to the metadata (migration 0014). An older server refuses the upgraded database, +so take a backup before upgrading and do not roll back. + +`craft-cloud-server gc --dry-run --config cc-data/config.toml` inventories stored chunks and manifests, compares them with +the database and prints a preview; `--json` prints schema version 1. It changes no data, on SQLite or on PostgreSQL. +Like `verify`, it opens a SQLite database that exists already and creates nothing; opening a WAL-mode database may leave +an empty `-wal` file and a `-shm` file behind (SQLite's rebuilt index of the WAL, not data), and the database file and +the WAL's contents stay as they were, also after a crash that left data in the WAL. Store objects with no `blobs` row are orphan candidates, but need two runs apart to establish a future +first-sight grace period. Recorded chunks no complete indexed manifest names are candidates only when every manifest index +is complete and database checks pass. They come from one read-only database snapshot; the store listing is a separate, +time-point diagnostic. A future collector must recheck roots before claiming a chunk. Current versions, saved versions, +trash items, live snapshots and import snapshots protect their chunks. Unrooted manifests are protected because they may be live document assets. Without +`--dry-run`, the command refuses: deleting is not implemented yet. Listing errors fail the command; they do not mean the +store is empty. Nothing is deleted. + ## Restoring a backup -With the server stopped, restore the database and the chunk storage from the backup, then run +With the server stopped, restore the database and the chunk storage from the backup (and the two-factor key file, if +accounts use two-factor sign-in: the server will not start without it), then run `craft-cloud-server reset-epoch --config cc-data/config.toml --all`, and only then start `serve` again. Every account then starts a new epoch, so each synced machine reconciles with the restored state instead of trusting its -own change counter. `reset-epoch --config … name` resets one account. (A reset run while the server is up still +own change counter. `reset-epoch --config … name` resets one account, and `--space ` one team space. (A reset run while the server is up still works: open live connections end at their next ping, within 30 s, and connect again.) ## A full recents document @@ -71,3 +421,156 @@ every machine then sends up what the emptied log lacks, so nothing any machine h affected. A document whose own history is over 16 MiB stays full until compacting the log is supported. An account may have 64 live documents holding updates and 1,024 in all (one per app today); past that, new ones stay on the machines. Live documents can't be deleted yet. + +## Docker + +`packaging/docker/` holds a hardened container image of the server and a Docker Compose deployment: the server +behind Caddy, which terminates TLS. By default the database (SQLite) and the blobs live on a Docker volume; Postgres +and S3 are overlays. It needs Docker Compose 2.32 or newer on Linux and builds the image locally: there is no image +to pull and nothing is published to a registry. The first build wants about 4 GiB of memory for the Docker engine (it +uses two compiler jobs). + +### Quick start + +```sh +cd packaging/docker +./install.sh --domain cloud.example.org --admin you +``` + +`install.sh` checks Docker and the free disk space, writes `.env` (mode 0600, never committed) with random database +passwords, builds the image, starts the stack and creates the first admin. The admin's password is asked for without +echo, or read from a pipe, and reaches the server through standard input only. On a terminal it asks for the +settings you did not give; `./install.sh --help` lists the options (`--tls acme|internal|custom`, `--email`, +`--db sqlite|postgres`, `--http-port`, `--https-port`, `--no-admin`, `--no-start`). It never overwrites an existing +`.env`. Afterwards `docker compose ps` and `docker compose logs -f server` show the state, and the domain's +`/api/info` answers through Caddy. + +To set it up by hand, copy `.env.example` to `.env`, fill it in, and run `docker compose build` and +`docker compose up -d --wait`. + +### What runs + +| Service | Image | Notes | +|---|---|---| +| `server` | built from `packaging/docker/Dockerfile` | distroless `cc`, uid 65532, plain HTTP on port 8080 inside the stack | +| `caddy` | `caddy:2.11-alpine` | the only service with published ports (80 and 443); TLS and the redirect from HTTP | +| `secrets-init` | `caddy:2.11-alpine` | one-shot, no network: renders `config.toml` and writes the secret files | +| `db`, `db-init` | `postgres:17-alpine` | only with `compose.postgres.yaml` | + +Every base image is pinned by the digest of its multi-arch index, with the tag in a comment next to it. To update +one, change the tag and the digest together (`docker buildx imagetools inspect :` prints the index +digest). + +Every service has a read-only root filesystem, no capabilities (`cap_drop: [ALL]`), `no-new-privileges`, memory and +process limits, and a `noexec` tmpfs for `/tmp`. The exceptions are explained next to each service: Postgres and +`secrets-init` start as root and need a few capabilities to fix ownership, Caddy needs to bind ports 80 and 443, and +`secrets-init` (in the Postgres and S3 overlays) is not read-only because Compose writes its secret files into the +root filesystem. + +Three networks keep the parts apart: `edge` (published ports and outbound access, Caddy only), `frontend` (internal; +Caddy to the server) and `backend` (internal; the server to Postgres, Caddy is not attached). + +### Configuration and secrets + +The server reads only its TOML file. `config.toml.in` is the template and `render-config.sh` fills it. The +`secrets-init` service renders it into the `secrets` volume (mode 0400, owner 65532) and the server mounts it +read-only at `/etc/craft-cloud/config.toml`. The file holds the database URL (with the password, for Postgres) and, +for S3, the keys. The server's `Debug` output hides the URL's credentials, and `docker inspect` of the running +services shows no secret, since none is in an environment or a Compose file. The values are checked against short +lists of allowed characters before they are written, so a value cannot break the TOML or add a setting. + +To change a setting in `.env` (`CC_*`, see `.env.example`), edit the file, run `docker compose up -d --force-recreate secrets-init` +(it must be recreated to see the new values: Compose gives a container its secrets when it is created, and a plain +`up` would start the old container again and write the old values back), then `docker compose restart server`. + +The server believes `X-Forwarded-For` only from Caddy. `trusted_proxies` takes single addresses, no networks, so +Caddy has a fixed address (`CC_CADDY_IP`, `172.29.215.2` in the network `CC_FRONTEND_SUBNET`, `172.29.215.0/24`). If +these defaults clash with a network on your host, change both together, and `CC_FRONTEND_POOL` (where the other +containers get addresses from) with them. + +### TLS modes + +Set `CC_TLS_MODE` in `.env`: + +- `acme` (default): a public certificate from Let's Encrypt. The domain must resolve to the host, and the ports must + be reachable from the internet (with other ports, set `CC_HTTP_PORT`, `CC_HTTPS_PORT` and a matching + `CC_PUBLIC_URL`). +- `internal`: a certificate from Caddy's own CA, for tests. Clients do not trust it. There is no HSTS in this mode. +- `custom`: your own certificate. Put `fullchain.pem` and `private.key` in `packaging/docker/tls/` (or `CC_TLS_DIR`), + readable by root only, and restart Caddy to load a renewed one. + +Caddy sets a request header timeout of 10 seconds, as the checklist above asks. It writes no access log, because URLs hold invite +codes (`/join/`); its error log leaves out the request's URI and headers for the same reason. + +### Postgres + +```sh +./install.sh --domain cloud.example.org --db postgres +``` + +or add `COMPOSE_FILE=compose.yaml:compose.postgres.yaml` to `.env`. This adds a Postgres container and a one-shot +`db-init` that creates the role `craft_cloud`. The server migrates its database when it opens it, so that role owns +the database `craft_cloud` and its schema and may change them. That is less separation than an owner role plus a +runtime role. The role is not a superuser and has no other privileges; `PUBLIC` may not connect to the database or +create in its schema, and the role cannot connect to `postgres` or `template1`. The superuser password exists only +for the Postgres container and `db-init`. Blobs stay on the data volume. + +To rotate the database password, change `CC_SECRET_DB_PASSWORD` in `.env` (characters from `A-Za-z0-9._~-`, at least +32), then run `docker compose up -d --force-recreate secrets-init`, +`docker compose up -d --force-recreate db-init` and `docker compose restart server`. The superuser password cannot be changed that way: Postgres applies it only when +the data directory is created. Change it with `ALTER ROLE postgres` first and then in `.env`. + +The connection between the server and Postgres is not encrypted (the server's `sqlx` has no TLS feature). It never +leaves the internal `backend` network. An external Postgres over TLS is not covered. + +### S3 + +`compose.s3.yaml` replaces the disk storage with an S3 bucket: fill in the `CC_S3_*` and `CC_SECRET_S3_*` lines of +`.env.example` in `.env` and add `compose.s3.yaml` to `COMPOSE_FILE`. The keys end up in `config.toml` like the +database password, not in the server's environment. The server joins the `edge` network to reach the endpoint. The +image holds the CA certificates the server needs for HTTPS endpoints. The overlay has been tried with a local S3 mock +over plain HTTP (`CC_S3_ALLOW_HTTP=true`), not with a real S3 service or an HTTPS endpoint. + +### The first admin and more accounts + +`install.sh` creates the first admin. By hand, with the password on standard input: + +```sh +printf '%s\n' 'a password' | docker compose run --rm --no-deps -T server user --config /etc/craft-cloud/config.toml add --admin you +``` + +More accounts: the same line without `--admin`, or `docker compose run --rm --no-deps server user --config +/etc/craft-cloud/config.toml invite name`, which prints the whole join string because `public_url` is set. The +other commands of this guide (`device`, `reset-epoch`, `live`) run the same way. + +### Data, upgrading and backups + +The data is in the volume `data` (`/var/lib/craft-cloud` in the container): `meta.db` (with SQLite) and `blobs/`. +The image creates the directory for uid 65532 with mode 0700, so a new named volume gets that owner. A bind mount +instead of the volume must be prepared first: `chown 65532:65532 DIR && chmod 0700 DIR`. + +To upgrade, update the source, then run `docker compose build server` and `docker compose up -d --wait`. The server +migrates the database when it opens it and refuses a database newer than itself, so back up before an upgrade and do +not go back to an older image without the backup. + +There is no backup automation yet. Stop the server (`docker compose stop server`), copy the volume (SQLite: +`meta.db`, `meta.db-wal`, `meta.db-shm` and `blobs/` together; Postgres: `pg_dump` from the `db` container plus +`blobs/`), and start it again. Restoring is described above, including `reset-epoch`. + +`docker compose down` keeps the volumes. `docker compose down -v` deletes all data. + +### Troubleshooting + +- `secrets-init` fails with a message such as `PUBLIC_URL has a value that is empty or has characters that are not + allowed`: a setting in `.env` is wrong. The message names the setting and never shows the value. +- The server exits at once with `unable to open database file`: the data directory is not owned by 65532 (see bind + mounts above). +- `server` is not healthy: `docker compose logs server`. The check is `craft-cloud-server healthcheck`, a + `GET /api/info` request to `127.0.0.1:8080` inside the container. +- All devices share one sign-in budget: `trusted_proxies` does not contain Caddy's address. `CC_CADDY_IP` must lie in + `CC_FRONTEND_SUBNET`; run `docker compose up -d --force-recreate secrets-init` and `docker compose up -d` after changing either. + +### Not covered + +Postgres over TLS, TLS in the server itself (Caddy terminates it), Kubernetes or Helm files, a registry image and +release automation, and backup automation (copy the data as above until the server can create backups itself). diff --git a/packaging/docker/.env.example b/packaging/docker/.env.example new file mode 100644 index 00000000..119b3e63 --- /dev/null +++ b/packaging/docker/.env.example @@ -0,0 +1,44 @@ +# Settings of the Docker deployment. install.sh writes a real `.env` from this list (mode 0600, never committed); +# copy this file to `.env` and fill it in by hand if you do not use install.sh. Nothing in this file is a secret. + +# --- required --------------------------------------------------------------- +# The domain computers reach the server at, and the matching public URL (add :port if HTTPS is not on 443). +CC_DOMAIN=cloud.example.org +CC_PUBLIC_URL=https://cloud.example.org + +# --- TLS --------------------------------------------------------------------- +# acme (Let's Encrypt), internal (Caddy's own CA, for tests) or custom (files in CC_TLS_DIR). +CC_TLS_MODE=acme +CC_ACME_EMAIL= +CC_TLS_DIR=./tls +CC_HTTP_PORT=80 +CC_HTTPS_PORT=443 + +# --- image ------------------------------------------------------------------- +CC_IMAGE=craft-cloud:local +CC_VERSION=0.0.0 +CC_REVISION=unknown + +# --- networks ------------------------------------------------------------------ +# Caddy's fixed address on the internal frontend network. The server believes X-Forwarded-For from this address only +# (trusted_proxies takes single addresses, no networks), so it must lie inside CC_FRONTEND_SUBNET. +CC_FRONTEND_SUBNET=172.29.215.0/24 +CC_CADDY_IP=172.29.215.2 +# Where the other containers get their addresses from: inside CC_FRONTEND_SUBNET, and not containing CC_CADDY_IP. +CC_FRONTEND_POOL=172.29.215.128/25 +CC_EDGE_SUBNET=172.29.216.0/24 + +# --- secrets (the Postgres profile; install.sh fills them with random values) ----------------------------------- +# Only characters A-Z a-z 0-9 . _ ~ - and at least 32 of them: the database password becomes part of a database URL. +CC_SECRET_DB_SUPERUSER_PASSWORD= +CC_SECRET_DB_PASSWORD= + +# --- S3 storage (compose.s3.yaml) ---------------------------------------------------------------------------- +CC_S3_BUCKET= +CC_S3_PREFIX= +CC_S3_REGION=us-east-1 +# Leave empty for Amazon S3; set it for MinIO and other S3-compatible services. +CC_S3_ENDPOINT= +CC_S3_ALLOW_HTTP=false +CC_SECRET_S3_ACCESS_KEY_ID= +CC_SECRET_S3_SECRET_ACCESS_KEY= diff --git a/packaging/docker/.gitattributes b/packaging/docker/.gitattributes new file mode 100644 index 00000000..0c6475c3 --- /dev/null +++ b/packaging/docker/.gitattributes @@ -0,0 +1,2 @@ +# Scripts and configs are mounted into Linux containers and must keep LF line endings. +* text eol=lf diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 00000000..b6a3e0b7 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,63 @@ +# Container image of craft-cloud-server: a Debian trixie build stage and a distroless runtime that holds the binary +# and nothing else (no shell, no package manager). It runs as uid 65532. +# +# The build context is the repository root, because Cargo needs the manifests of all workspace members: +# docker build -f packaging/docker/Dockerfile -t craft-cloud:local . +# docker build -f packaging/docker/Dockerfile --build-arg CRAFT_CLOUD_VERSION=0.1.0 \ +# --build-arg CRAFT_CLOUD_REVISION="$(git rev-parse HEAD)" -t craft-cloud:0.1.0 . +# +# Every base image is pinned by the digest of its multi-arch index; the tag in the comment is for humans. Update +# the tag and the digest together (see "Docker" in docs/server.md). + +# rust:1.95-trixie (egui 0.36 needs Rust 1.95; glibc 2.41 like the runtime image below) +ARG RUST_IMAGE=rust:1.95-trixie@sha256:f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3 +# debian:trixie-slim (only used to create the data directory with its owner and mode) +ARG DEBIAN_IMAGE=debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4 +# gcr.io/distroless/cc-debian13:nonroot (glibc, libgcc and CA certificates; no shell, no package manager; uid 65532) +ARG RUNTIME_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:08476c244888bfb425bae1ae4dd7c17f093e972aab8426883d97f3c6f280f7c1 + +# ---- build --------------------------------------------------------------------- +FROM ${RUST_IMAGE} AS build +# Docker builds often run in a small VM: keep the compiler's parallelism and memory use low. +ENV CARGO_BUILD_JOBS=2 \ + CARGO_INCREMENTAL=0 \ + CARGO_PROFILE_RELEASE_STRIP=symbols +WORKDIR /src +COPY Cargo.toml Cargo.lock ./ +# Cargo resolves every workspace member, including the apps and xtask, even when only the server package is built. +# Nothing of them but the server enters the image. +COPY .cargo .cargo +COPY crates crates +COPY apps apps +COPY xtask xtask +RUN cargo build --release --locked -p craft-cloud-server \ + && install -D target/release/craft-cloud-server /out/craft-cloud-server + +# ---- files that need a shell to create --------------------------------------------- +FROM ${DEBIAN_IMAGE} AS files +# The data directory belongs to the runtime user from the start, so a fresh named volume inherits the owner and the +# mode 0700. +RUN install -d -m 0700 -o 65532 -g 65532 /rootfs/var/lib/craft-cloud + +# ---- runtime ----------------------------------------------------------------------- +FROM ${RUNTIME_IMAGE} +ARG CRAFT_CLOUD_VERSION=0.0.0 +ARG CRAFT_CLOUD_REVISION=unknown +LABEL org.opencontainers.image.title="craft-cloud-server" \ + org.opencontainers.image.description="Self-hosted sync and storage server for the Craft App Suite" \ + org.opencontainers.image.source="https://github.com/itrycodinganditsbad/craft-cloud" \ + org.opencontainers.image.licenses="MIT OR Apache-2.0" \ + org.opencontainers.image.version="${CRAFT_CLOUD_VERSION}" \ + org.opencontainers.image.revision="${CRAFT_CLOUD_REVISION}" +COPY --from=files --chown=65532:65532 /rootfs/var/lib/craft-cloud /var/lib/craft-cloud +COPY --from=build /out/craft-cloud-server /usr/local/bin/craft-cloud-server +USER 65532:65532 +EXPOSE 8080 +# The SQLite database and the blob storage (when it is on disk) live here: the only writable place besides /tmp. +VOLUME ["/var/lib/craft-cloud"] +# The image has no shell and no curl: the binary asks its own listener (GET /api/info on 127.0.0.1:8080). +HEALTHCHECK --interval=15s --timeout=5s --start-period=30s --retries=3 \ + CMD ["/usr/local/bin/craft-cloud-server", "healthcheck"] +ENTRYPOINT ["/usr/local/bin/craft-cloud-server"] +# The config file is mounted by Compose; see packaging/docker/compose.yaml. +CMD ["serve", "--config", "/etc/craft-cloud/config.toml"] diff --git a/packaging/docker/caddy/Caddyfile b/packaging/docker/caddy/Caddyfile new file mode 100644 index 00000000..3d522b18 --- /dev/null +++ b/packaging/docker/caddy/Caddyfile @@ -0,0 +1,72 @@ +# Reverse proxy of the Docker deployment: TLS in front of craft-cloud-server, which speaks plain HTTP only. +# +# CC_DOMAIN, CC_PUBLIC_URL, CC_TLS_MODE and CC_ACME_EMAIL come from the Compose environment: +# acme a public certificate from Let's Encrypt (the domain must resolve here, ports 80 and 443 be reachable) +# internal a certificate from Caddy's own CA, for local tests (browsers warn) +# custom your own certificate: fullchain.pem and private.key in the directory CC_TLS_DIR + +{ + # Not exposed: the admin API listens on loopback inside this container only. + admin localhost:2019 + # Caddy cannot install its internal CA into a read-only container and would log an error about it. + skip_install_trust + email "{$CC_ACME_EMAIL}" + + # Caddy logs a failed request (an upstream error, for example) with its full URI, which would hold an invite code + # (/join/) and any query. The log keeps neither the URI nor the headers (Authorization, Cookie). + log { + output stderr + format filter { + wrap console + fields { + request>uri delete + request>headers delete + } + } + } + + servers { + timeouts { + # The server cuts off slow request bodies itself, but asks the proxy to bound the request headers. + read_header 10s + } + } +} + +# Plain HTTP: redirect to the configured public URL, which keeps a port other than 443. An explicit site for the HTTP +# address replaces Caddy's automatic redirect, which would drop the port. ACME HTTP challenges are answered before this. +http://{$CC_DOMAIN} { + redir {$CC_PUBLIC_URL}{uri} 308 +} + +{$CC_DOMAIN} { + import tls-{$CC_TLS_MODE}.caddy + + # Only fills in the generic headers where the server sent none (the leading `?`). HSTS comes from the tls file of + # a real certificate; a test certificate must not pin the browser to HTTPS. + header { + ?X-Content-Type-Options "nosniff" + -Server + } + + # No `request_body { max_size ... }` on purpose: uploads are streamed and the server limits each route itself. + reverse_proxy server:8080 { + # Pass response bytes on at once: downloads stream and nothing is held back. + flush_interval -1 + + # Send requests only while the server answers its info endpoint. + health_uri /api/info + health_status 200 + health_interval 10s + health_timeout 3s + + transport http { + dial_timeout 5s + # The time until the server starts to answer; there is no limit on how long a body takes to transfer. + response_header_timeout 60s + } + } + + # No access log on purpose: a log of full URLs would keep invite codes (/join/). The error log above leaves + # out the URI and the headers. If you add an access log, filter them out the same way. +} diff --git a/packaging/docker/caddy/tls-acme.caddy b/packaging/docker/caddy/tls-acme.caddy new file mode 100644 index 00000000..676383aa --- /dev/null +++ b/packaging/docker/caddy/tls-acme.caddy @@ -0,0 +1,3 @@ +# A certificate from Caddy's default public issuers (Let's Encrypt, ZeroSSL). Nothing to configure. +# HSTS only with a real certificate, so not in the internal mode. +header Strict-Transport-Security "max-age=31536000" diff --git a/packaging/docker/caddy/tls-custom.caddy b/packaging/docker/caddy/tls-custom.caddy new file mode 100644 index 00000000..e26ac61d --- /dev/null +++ b/packaging/docker/caddy/tls-custom.caddy @@ -0,0 +1,4 @@ +# Your own certificate: the directory CC_TLS_DIR holds fullchain.pem and private.key and is mounted read-only at +# /certs. Keep the key readable by root only. Caddy reads the files at start; restart it to load a renewed one. +tls /certs/fullchain.pem /certs/private.key +header Strict-Transport-Security "max-age=31536000" diff --git a/packaging/docker/caddy/tls-internal.caddy b/packaging/docker/caddy/tls-internal.caddy new file mode 100644 index 00000000..06e1f32c --- /dev/null +++ b/packaging/docker/caddy/tls-internal.caddy @@ -0,0 +1,2 @@ +# A certificate from Caddy's internal CA. For tests only: browsers and agents do not trust it. No HSTS. +tls internal diff --git a/packaging/docker/compose.postgres.yaml b/packaging/docker/compose.postgres.yaml new file mode 100644 index 00000000..d21108de --- /dev/null +++ b/packaging/docker/compose.postgres.yaml @@ -0,0 +1,99 @@ +# Overlay for Postgres instead of SQLite: +# docker compose -f compose.yaml -f compose.postgres.yaml up -d --wait +# Adds the `db` container (on the internal `backend` network, which Caddy is not attached to) and `db-init`, which +# creates the server's database role. The data volume still holds the blobs. +# +# The connection is not encrypted: the server's sqlx has no TLS feature. That is acceptable on an internal network of +# one host only. An external Postgres over TLS is not covered. + +x-postgres-image: &postgres-image postgres:17-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b + +services: + # Compose puts a file secret into the container's root filesystem, so this one service cannot be read-only here. + secrets-init: + read_only: false + environment: + DB_KIND: postgres + secrets: + - db_superuser_password + - db_password + + db: + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + image: *postgres-image + environment: + POSTGRES_DB: craft_cloud + POSTGRES_USER: postgres + POSTGRES_PASSWORD_FILE: /run/secrets/db_superuser_password + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums" + volumes: + - pgdata:/var/lib/postgresql/data + # Read by the image's entrypoint while it still runs as root. + - type: volume + source: secrets + target: /run/secrets + read_only: true + volume: {subpath: db} + depends_on: + secrets-init: {condition: service_completed_successfully} + tmpfs: + - /tmp + - /run/postgresql + # The entrypoint starts as root to fix the ownership of the data directory and then drops to the postgres user; it + # needs exactly these capabilities. + cap_add: [CHOWN, FOWNER, DAC_OVERRIDE, SETUID, SETGID] + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "postgres", "-d", "craft_cloud"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + networks: [backend] + mem_limit: 1g + pids_limit: 200 + stop_grace_period: 60s + + # Creates the database role of the server, or refreshes its password. Runs on every `up`; safe to run again. + db-init: + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + image: *postgres-image + # The postgres user of the alpine image. + user: "70:70" + entrypoint: ["/bin/sh", "/db-init.sh"] + environment: + PGHOST: db + PGDATABASE: craft_cloud + volumes: + - ./db-init.sh:/db-init.sh:ro + - type: volume + source: secrets + target: /run/secrets + read_only: true + volume: {subpath: db-init} + depends_on: + db: {condition: service_healthy} + # For the private password file db-init.sh writes. + tmpfs: + - /tmp:size=1m,mode=1777,noexec,nosuid,nodev + healthcheck: {disable: true} + networks: [backend] + mem_limit: 128m + pids_limit: 50 + + server: + depends_on: + db-init: {condition: service_completed_successfully} + networks: [backend] + +volumes: + pgdata: {} + +secrets: + db_superuser_password: {environment: CC_SECRET_DB_SUPERUSER_PASSWORD} + db_password: {environment: CC_SECRET_DB_PASSWORD} diff --git a/packaging/docker/compose.s3.yaml b/packaging/docker/compose.s3.yaml new file mode 100644 index 00000000..dc3561b4 --- /dev/null +++ b/packaging/docker/compose.s3.yaml @@ -0,0 +1,29 @@ +# Overlay for S3-compatible storage instead of the local disk (combine it with compose.postgres.yaml if you like): +# docker compose -f compose.yaml -f compose.s3.yaml up -d --wait +# Set CC_S3_* and CC_SECRET_S3_* in .env. The keys go into config.toml (mode 0400, readable by the server only) like +# the database password; they are not in the server's environment. +# +# The server needs a route to the S3 endpoint, so it also joins the `edge` network, the only one with outbound access. +# That network publishes no port of the server. + +services: + # Compose puts a file secret into the container's root filesystem, so this one service cannot be read-only here. + secrets-init: + read_only: false + environment: + STORAGE_KIND: s3 + S3_BUCKET: ${CC_S3_BUCKET:?set CC_S3_BUCKET in .env} + S3_PREFIX: ${CC_S3_PREFIX:-} + S3_REGION: ${CC_S3_REGION:-us-east-1} + S3_ENDPOINT: ${CC_S3_ENDPOINT:-} + S3_ALLOW_HTTP: ${CC_S3_ALLOW_HTTP:-false} + secrets: + - s3_access_key_id + - s3_secret_access_key + + server: + networks: [edge] + +secrets: + s3_access_key_id: {environment: CC_SECRET_S3_ACCESS_KEY_ID} + s3_secret_access_key: {environment: CC_SECRET_S3_SECRET_ACCESS_KEY} diff --git a/packaging/docker/compose.yaml b/packaging/docker/compose.yaml new file mode 100644 index 00000000..9ffa4982 --- /dev/null +++ b/packaging/docker/compose.yaml @@ -0,0 +1,149 @@ +# Docker Compose deployment of Craft Cloud: the server (SQLite and local disk) behind Caddy, which terminates TLS. +# Postgres and S3 are overlays: -f compose.yaml -f compose.postgres.yaml [-f compose.s3.yaml]. See "Docker" in +# docs/server.md, and run ./install.sh for a first setup. +# +# ./install.sh # writes .env, builds the image, starts the stack +# docker compose up -d --wait # after that +# +# Secrets never appear in this file or in the environment of a service that runs. Compose reads them from the +# gitignored .env and gives them to the one-shot `secrets-init` service only, which writes them as files (mode 0400, +# owned by the reading user) into the `secrets` volume. Every other service mounts just its own subdirectory +# read-only. +# +# Volume subpaths (used below) need Docker Compose 2.32 or newer. + +name: craft-cloud + +x-hardening: &hardening + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + +services: + # Renders config.toml and distributes the secrets (see secrets-init.sh). It has no network at all, runs as root for + # about a second and needs only the capabilities to change the owner and mode of the files it writes. + secrets-init: + <<: *hardening + # Any image with busybox will do; the one of Caddy is needed anyway. + image: caddy:2.11-alpine@sha256:e70b164da81ef698b3ad19af799ea6086ce6d9e48ed4079eb9fe26f6fbf14ee5 + user: "0:0" + entrypoint: ["/bin/sh", "/secrets-init.sh"] + network_mode: none + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER] + # Not secrets: what render-config.sh puts into config.toml. + environment: + PUBLIC_URL: ${CC_PUBLIC_URL:?set CC_PUBLIC_URL in .env, or run ./install.sh} + TRUSTED_PROXY: ${CC_CADDY_IP:-172.29.215.2} + DB_KIND: sqlite + STORAGE_KIND: disk + volumes: + - ./secrets-init.sh:/secrets-init.sh:ro + - ./render-config.sh:/render-config.sh:ro + - ./config.toml.in:/config.toml.in:ro + - secrets:/out + healthcheck: {disable: true} + mem_limit: 64m + pids_limit: 50 + + server: + <<: *hardening + restart: unless-stopped + image: ${CC_IMAGE:-craft-cloud:local} + build: + context: ../.. + dockerfile: packaging/docker/Dockerfile + args: + CRAFT_CLOUD_VERSION: ${CC_VERSION:-0.0.0} + CRAFT_CLOUD_REVISION: ${CC_REVISION:-unknown} + user: "65532:65532" + command: ["serve", "--config", "/etc/craft-cloud/config.toml"] + volumes: + # The SQLite database and the blobs. A new named volume inherits the owner (65532) and mode (0700) of the image's + # directory. A bind mount must be given both first: chown 65532:65532 DIR && chmod 0700 DIR. + - data:/var/lib/craft-cloud + - type: volume + source: secrets + target: /etc/craft-cloud + read_only: true + volume: {subpath: server} + tmpfs: + - /tmp:size=16m,mode=1777,noexec,nosuid,nodev + depends_on: + secrets-init: {condition: service_completed_successfully} + # The image has no shell and no curl: the binary asks its own listener. + healthcheck: + test: ["CMD", "/usr/local/bin/craft-cloud-server", "healthcheck", "--addr", "127.0.0.1:8080"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 30s + networks: [frontend] + mem_limit: 1g + pids_limit: 512 + stop_grace_period: 30s + + caddy: + <<: *hardening + restart: unless-stopped + image: caddy:2.11-alpine@sha256:e70b164da81ef698b3ad19af799ea6086ce6d9e48ed4079eb9fe26f6fbf14ee5 + ports: + - "${CC_HTTP_PORT:-80}:80" + - "${CC_HTTPS_PORT:-443}:443" + - "${CC_HTTPS_PORT:-443}:443/udp" + environment: + CC_DOMAIN: ${CC_DOMAIN:?set CC_DOMAIN in .env, or run ./install.sh} + CC_TLS_MODE: ${CC_TLS_MODE:-acme} + CC_PUBLIC_URL: ${CC_PUBLIC_URL:?set CC_PUBLIC_URL in .env, or run ./install.sh} + CC_ACME_EMAIL: ${CC_ACME_EMAIL:-} + volumes: + - ./caddy:/etc/caddy:ro + - caddy_data:/data + - caddy_config:/config + # Only read in the `custom` TLS mode. + - ${CC_TLS_DIR:-./tls}:/certs:ro + tmpfs: + - /tmp + # Runs as root inside the container (the official image does), with no capability except binding ports 80 and 443. + cap_add: [NET_BIND_SERVICE] + depends_on: + server: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "http://127.0.0.1:2019/config/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + edge: {} + # The fixed address the server lists in trusted_proxies. + frontend: + ipv4_address: ${CC_CADDY_IP:-172.29.215.2} + mem_limit: 256m + pids_limit: 200 + +networks: + # Published ports and outbound access (ACME); only Caddy is attached. + edge: + ipam: + config: + - subnet: ${CC_EDGE_SUBNET:-172.29.216.0/24} + # Caddy to the server. `internal`: no route to the outside world. The other containers get their addresses from the + # upper half, so none of them can take Caddy's fixed address (CC_CADDY_IP) first. + frontend: + internal: true + ipam: + config: + - subnet: ${CC_FRONTEND_SUBNET:-172.29.215.0/24} + ip_range: ${CC_FRONTEND_POOL:-172.29.215.128/25} + # Server and database in the Postgres profile. `internal`, and Caddy is not attached. + backend: + internal: true + +volumes: + # The SQLite database and, with disk storage, the blobs: the data to back up. + data: {} + # config.toml and the secret files written by `secrets-init`, one directory per consuming service. + secrets: {} + caddy_data: {} + caddy_config: {} diff --git a/packaging/docker/config.toml.in b/packaging/docker/config.toml.in new file mode 100644 index 00000000..d5b684ea --- /dev/null +++ b/packaging/docker/config.toml.in @@ -0,0 +1,33 @@ +# Template of the server's config.toml in the Docker deployment. render-config.sh fills the placeholders (a name +# between two at signs) and keeps the lines whose marker prefix is active: the database kind, the storage kind, and +# for S3 the optional endpoint and plain-HTTP lines. Every other marker line is dropped. The values are checked +# before they are written. +# The rendered file is written by the secrets-init service into the secrets volume (mode 0400, owner 65532); it can +# hold the database password and S3 keys, so nobody but the server can read it. + +# The server speaks plain HTTP inside the stack. Caddy terminates TLS and has the fixed address in trusted_proxies, +# whose X-Forwarded-For the server believes (it takes single addresses, no networks). +listen = "0.0.0.0:8080" +behind_proxy = true +trusted_proxies = ["@TRUSTED_PROXY@"] +public_url = "@PUBLIC_URL@" + +# Optional limits, in bytes. See docs/server.md. +# stored_bytes_per_user = 2_000_000_000_000 +# live_bytes_per_user = 1_073_741_824 + +[database] +#sqlite#url = "sqlite:/var/lib/craft-cloud/meta.db" +#postgres#url = "postgres://craft_cloud:@DB_PASSWORD@@db:5432/craft_cloud" + +[storage] +#disk#kind = "disk" +#disk#path = "/var/lib/craft-cloud/blobs" +#s3#kind = "s3" +#s3#bucket = "@S3_BUCKET@" +#s3#prefix = "@S3_PREFIX@" +#s3#region = "@S3_REGION@" +#s3#access_key_id = "@S3_ACCESS_KEY_ID@" +#s3#secret_access_key = "@S3_SECRET_ACCESS_KEY@" +#s3endpoint#endpoint = "@S3_ENDPOINT@" +#s3http#allow_http = true diff --git a/packaging/docker/db-init.sh b/packaging/docker/db-init.sh new file mode 100755 index 00000000..49de4d13 --- /dev/null +++ b/packaging/docker/db-init.sh @@ -0,0 +1,52 @@ +#!/bin/sh +case $- in *x*) set +x ;; esac +# One-shot role setup for the Postgres profile (service `db-init`). It runs as the postgres user of the Postgres +# image, reads the passwords from secret files and never prints them. Safe to run again: every run applies the +# password and the role's attributes, which is also how the password is rotated. +# +# The server migrates its database when it opens it, so its role owns the database and the schema `public` and may +# change them. That is less separation than an owner role plus a runtime role would give. The role is not a +# superuser and has no other privileges, and nobody else may connect to its database or to the other databases. +set -eu +umask 077 + +# The superuser password goes into a private password file (mode 0600, on the tmpfs at /tmp, removed on exit), so it +# is neither in a process's environment nor on a command line. The password only holds the characters +# A-Za-z0-9._~-, which need no escaping in this file format. +passfile=$(mktemp /tmp/pgpass.XXXXXX) +trap 'rm -f "$passfile"' EXIT INT TERM +printf '*:*:*:postgres:%s\n' "$(cat /run/secrets/db_superuser_password)" >"$passfile" +PGUSER=postgres +PGPASSFILE=$passfile +export PGUSER PGPASSFILE + +# POSTGRES_PASSWORD_FILE only applies when the data directory is first created, so a changed superuser password in +# .env cannot log in here. +if ! psql --no-psqlrc --quiet --command 'SELECT 1' >/dev/null 2>&1; then + cat >&2 <<'MSG' +db-init: cannot log in as the Postgres superuser "postgres" (or the database is unreachable). +If CC_SECRET_DB_SUPERUSER_PASSWORD in .env was changed after the first start, the database still has the old +password: the image applies it only when the data directory is created. Put the old value back in .env. +MSG + exit 1 +fi + +# psql reads the password itself (backticks), so it is not on a command line. +psql --no-psqlrc --set ON_ERROR_STOP=1 --quiet <<'SQL' +\set db_password `cat /run/secrets/db_password` + +SELECT 'CREATE ROLE craft_cloud LOGIN' + WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'craft_cloud')\gexec + +ALTER ROLE craft_cloud LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'db_password'; + +ALTER DATABASE craft_cloud OWNER TO craft_cloud; +ALTER SCHEMA public OWNER TO craft_cloud; +REVOKE ALL ON DATABASE craft_cloud FROM PUBLIC; +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +-- The role must not be able to connect to the maintenance databases either. +REVOKE CONNECT ON DATABASE postgres FROM PUBLIC; +REVOKE CONNECT ON DATABASE template1 FROM PUBLIC; +SQL + +echo "db-init: the database role craft_cloud is ready" diff --git a/packaging/docker/install.sh b/packaging/docker/install.sh new file mode 100755 index 00000000..98ad74e2 --- /dev/null +++ b/packaging/docker/install.sh @@ -0,0 +1,231 @@ +#!/bin/sh +# Sets up the Docker deployment of Craft Cloud in this directory: checks the requirements, writes `.env` (mode 0600) +# with random database passwords, builds the image, starts the stack and, if asked, creates the first admin. +# +# ./install.sh [--domain cloud.example.org] [--tls acme|internal|custom] [--email you@example.org] +# [--db sqlite|postgres] [--http-port 80] [--https-port 443] [--admin name | --no-admin] [--no-start] +# +# Missing values are asked for when a terminal is attached; without one, --domain is required. The admin's password +# is read from standard input without echo (or from a pipe: printf '%s\n' "$PASSWORD" | ./install.sh ... --admin you) +# and goes to the server over a pipe: never into an argument, the environment or a log. +# +# It never overwrites an existing `.env`. It runs from a clone of the repository or an unpacked release archive; it +# downloads nothing itself (Docker pulls the pinned base images). +set -eu +umask 077 + +cd "$(dirname "$0")" + +die() { + echo "install.sh: $*" >&2 + exit 1 +} + +domain= +tls_mode= +acme_email= +db_kind= +http_port= +https_port= +admin= +no_admin=0 +no_start=0 +interactive=0 +[ -t 0 ] && interactive=1 + +while [ $# -gt 0 ]; do + case $1 in + --domain | --tls | --email | --db | --http-port | --https-port | --admin) + [ $# -ge 2 ] || die "$1 needs a value" + case $1 in + --domain) domain=$2 ;; + --tls) tls_mode=$2 ;; + --email) acme_email=$2 ;; + --db) db_kind=$2 ;; + --http-port) http_port=$2 ;; + --https-port) https_port=$2 ;; + --admin) admin=$2 ;; + esac + shift 2 + ;; + --no-admin) + no_admin=1 + shift + ;; + --no-start) + no_start=1 + shift + ;; + -h | --help) + sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) die "unknown option $1 (see --help)" ;; + esac +done + +# --- requirements --------------------------------------------------------------------------------------------------- +command -v docker >/dev/null 2>&1 || die "docker is not installed" +compose_version=$(docker compose version --short 2>/dev/null) || die "docker compose (version 2) is not available" +# Volume subpaths, which the secret distribution relies on, need Compose 2.32. +major=${compose_version%%.*} +rest=${compose_version#*.} +minor=${rest%%.*} +case "$major$minor" in + *[!0-9]*) die "can't read the Docker Compose version '$compose_version'" ;; +esac +if [ "$major" -lt 2 ] || { [ "$major" -eq 2 ] && [ "$minor" -lt 32 ]; }; then + die "Docker Compose 2.32 or newer is needed (found $compose_version)" +fi +[ ! -e .env ] || die ".env exists already; edit it, or move it away to set up again" + +free_kib=$(df -Pk . | awk 'NR == 2 { print $4 }') +case $free_kib in + "" | *[!0-9]*) ;; + *) [ "$free_kib" -ge 4194304 ] || die "less than 4 GiB of free disk space here; the image build needs room" ;; +esac + +# --- settings ------------------------------------------------------------------------------------------------------- +ask() { + # ask + if [ "$interactive" -eq 1 ]; then + printf '%s [%s]: ' "$2" "$3" + IFS= read -r answer || answer= + eval "$1=\${answer:-\$3}" + else + eval "$1=\$3" + fi +} + +if [ -z "$domain" ]; then + [ "$interactive" -eq 1 ] || die "--domain is required without a terminal" + printf 'Domain the server is reached at (for example cloud.example.org): ' + IFS= read -r domain || domain= +fi +[ -n "$tls_mode" ] || ask tls_mode "TLS (acme, internal, custom)" acme +[ -n "$db_kind" ] || ask db_kind "Database (sqlite, postgres)" sqlite +[ -n "$http_port" ] || ask http_port "HTTP port" 80 +[ -n "$https_port" ] || ask https_port "HTTPS port" 443 +if [ "$tls_mode" = acme ] && [ -z "$acme_email" ] && [ "$interactive" -eq 1 ]; then + printf 'Email for Let'\''s Encrypt notices (optional): ' + IFS= read -r acme_email || acme_email= +fi + +printf '%s' "$domain" | grep -Eq '^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$' || die "the domain has characters that are not allowed" +case $tls_mode in acme | internal | custom) ;; *) die "--tls must be acme, internal or custom" ;; esac +case $db_kind in sqlite | postgres) ;; *) die "--db must be sqlite or postgres" ;; esac +for port in "$http_port" "$https_port"; do + case $port in "" | *[!0-9]* | 0*) die "ports are numbers from 1 to 65535" ;; esac + [ "$port" -le 65535 ] || die "ports are numbers from 1 to 65535" +done +if [ -n "$acme_email" ]; then + printf '%s' "$acme_email" | grep -Eq '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$' || die "the email address has characters that are not allowed" +fi +if [ -n "$admin" ]; then + printf '%s' "$admin" | grep -Eq '^[A-Za-z0-9._-]{1,64}$' || die "usernames are 1-64 letters, digits, '.', '_' or '-'" +fi + +if command -v ss >/dev/null 2>&1; then + for port in "$http_port" "$https_port"; do + if ss -ltn 2>/dev/null | awk '{ print $4 }' | grep -Eq "[:.]$port\$"; then + die "port $port is already in use; pick another with --http-port / --https-port" + fi + done +fi + +public_url="https://$domain" +[ "$https_port" = 443 ] || public_url="$public_url:$https_port" + +version=$(sh ./workspace-version.sh) +revision=$(git rev-parse HEAD 2>/dev/null || true) +[ -n "$revision" ] || revision=unknown + +# --- .env --------------------------------------------------------------------------------------------------------- +# A random password of 40 URL-safe characters (the database password becomes part of a database URL). +random_password() { + value=$(LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null | head -c 40 || true) + [ "${#value}" -eq 40 ] || die "can't read random bytes from /dev/urandom" + printf '%s' "$value" +} + +compose_file=compose.yaml +[ "$db_kind" != postgres ] || compose_file=compose.yaml:compose.postgres.yaml + +{ + echo "# Written by install.sh. Holds the database passwords: keep it private (mode 0600) and out of version control." + echo "COMPOSE_PATH_SEPARATOR=:" + echo "COMPOSE_FILE=$compose_file" + echo "CC_DOMAIN=$domain" + echo "CC_PUBLIC_URL=$public_url" + echo "CC_TLS_MODE=$tls_mode" + echo "CC_ACME_EMAIL=$acme_email" + echo "CC_HTTP_PORT=$http_port" + echo "CC_HTTPS_PORT=$https_port" + echo "CC_IMAGE=craft-cloud:$version" + echo "CC_VERSION=$version" + echo "CC_REVISION=$revision" + if [ "$db_kind" = postgres ]; then + echo "CC_SECRET_DB_SUPERUSER_PASSWORD=$(random_password)" + echo "CC_SECRET_DB_PASSWORD=$(random_password)" + fi +} >.env.new +chmod 600 .env.new +mv .env.new .env +echo "Wrote .env (mode 0600)." + +if [ "$tls_mode" = custom ]; then + mkdir -p tls + chmod 700 tls + echo "Put fullchain.pem and private.key in $(pwd)/tls before the stack starts, then run: docker compose up -d --wait" + no_start=1 +fi + +[ "$no_start" -eq 0 ] || { + echo "Not started. Next: docker compose build && docker compose up -d --wait" + exit 0 +} + +# --- build and start ---------------------------------------------------------------------------------------------- +echo "Building the image craft-cloud:$version (this takes a while the first time)..." +docker compose build server +echo "Starting the stack..." +docker compose up -d --wait + +# --- first admin ---------------------------------------------------------------------------------------------------- +if [ "$no_admin" -eq 0 ] && [ -z "$admin" ] && [ "$interactive" -eq 1 ]; then + printf 'Name of the first admin (empty to skip): ' + IFS= read -r admin || admin= + printf '%s' "$admin" | grep -Eq '^([A-Za-z0-9._-]{1,64})?$' || die "usernames are 1-64 letters, digits, '.', '_' or '-'" +fi + +if [ "$no_admin" -eq 0 ] && [ -n "$admin" ]; then + password= + if [ "$interactive" -eq 1 ]; then + trap 'stty echo 2>/dev/null || true' EXIT INT TERM + stty -echo + printf 'Password for %s: ' "$admin" + IFS= read -r password || password= + printf '\nRepeat the password: ' + IFS= read -r again || again= + stty echo + printf '\n' + [ "$password" = "$again" ] || die "the passwords differ; create the admin later (see docs/server.md)" + unset again + else + IFS= read -r password || password= + fi + [ -n "$password" ] || die "no password given; create the admin later (see docs/server.md)" + # The password travels through a pipe into the container's standard input. A refusal (a password the server's rules + # do not accept) is shown by the server without the password. + if printf '%s\n' "$password" | docker compose run --rm --no-deps -T server user --config /etc/craft-cloud/config.toml add --admin "$admin"; then + unset password + else + unset password + die "the admin was not created; see the message above, then run it by hand (see docs/server.md)" + fi +fi + +echo +echo "Craft Cloud runs at $public_url" +[ "$tls_mode" != internal ] || echo "TLS mode 'internal': the certificate is from Caddy's own CA; clients must be told to trust it (curl -k for a test)." +echo "Next: docker compose ps, and docker compose logs -f server" diff --git a/packaging/docker/render-config.sh b/packaging/docker/render-config.sh new file mode 100755 index 00000000..3231df7c --- /dev/null +++ b/packaging/docker/render-config.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# Renders the server's config.toml from config.toml.in and writes it to standard output. +# +# render-config.sh