diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..72b5ef0a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# The build context of packaging/docker/Dockerfile is the repository root, because Cargo needs the manifests of +# every workspace member. Nothing here may reach the image: no history, no local data, no secrets. +.git +.github +target +**/target +/dist +/temp +/data +docs +packaging +**/*.md +.env +.env.* +**/.env +**/.env.* +*.sqlite +*.sqlite-wal +*.sqlite-shm +*~ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc9c0ee2..be5e6063 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,3 +127,26 @@ jobs: - name: Test storage, metadata and the server on every backend run: cargo test -p craft-cloud-store -p craft-cloud-meta -p craft-cloud-server --locked + + # The other jobs build with the newest stable Rust, so code or a dependency that needs a newer compiler than the + # `rust-version` in Cargo.toml would go unnoticed. Keep the toolchain here equal to that `rust-version`. + msrv: + name: minimum Rust version + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - name: Install Linux system dependencies (egui/winit) + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev + + - uses: dtolnay/rust-toolchain@1.95 + + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Check + run: cargo check --workspace --all-targets --locked diff --git a/.gitignore b/.gitignore index 6ea3d416..714e91fe 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ target/ *.sqlite-shm # a snapshot test's own render when it doesn't match *.new.png +# the Docker deployment's settings and secrets (written by packaging/docker/install.sh), and a custom TLS certificate +/packaging/docker/.env +/packaging/docker/tls/ diff --git a/AGENTS.md b/AGENTS.md index a0d3a8e6..b4e67997 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ The design lives in `docs/design/sync.md`; the live protocols (the server's live ```text crates/ - api craft-cloud-api what apps need of the protocol, with no postcard: IDs, names, limits, the agent's local API (`local.rs`, `/v1/blobs` too), error codes and the account's devices (`api.rs`) + api craft-cloud-api what apps need of the protocol, with no postcard: IDs, names, limits, the agent's local API (`local.rs`, `/v1/blobs` too), error codes, the account's devices and the admin API's bodies (`api.rs`) proto craft-cloud-proto wire types: hashes, manifests, commit ops, API bodies, join codes and join strings (`join.rs`), the live protocol (`live.rs`); re-exports `craft-cloud-api` under the same paths chunker craft-cloud-chunker FastCDC `fastcdc-v1` + BLAKE3, frozen by golden vectors store craft-cloud-store blob store: disk, S3, memory; checks every blob in and out @@ -34,14 +34,14 @@ crates/ sync craft-cloud-sync the sync engine over `Fs`/`Transport` traits; `MemFs`; scenario tests and the simulation kit craft-cloud-kit the app kit (`docs/kit.md`): an app's model live through the agent; the `LiveDocument` trait and `Session` (`session.rs`: open, the `local_change` hook, `poll`, undo through Loro's undo manager, reconnect and the 64 MiB queue, presence, saves), the change writer that cuts commits under 512 KiB (`writer.rs`), the mappings (`tree.rs`, `text.rs`, `table.rs`, `timeline.rs`, `opaque.rs`, `list.rs`), values, chunk references and blob stores (`value.rs`) egui craft-cloud-egui Craft Cloud's piece of an app's window: the account line, the shared recents, a live document's badge, and `CloudWatch` that keeps them current - app craft-cloud-app what apps link: a blocking client of the agent's local API (status, sign-in, joining, computers, pause, autostart, recents, live documents at the level of bytes: `live.rs`) + app craft-cloud-app what apps link: a blocking client of the agent's local API (status, sign-in, joining, computers, pause, autostart, recents, live documents at the level of bytes: `live.rs`, the admin calls: `admin.rs`) apps/ - craft-cloud-server HTTP API (`/api/v1`), the live service (`/api/v1/live`, `live.rs`: logs, documents by import, snapshots and compaction, presence), join codes (`join.rs`: pairings, join, the `/join/{code}` page), auth, admin CLI (`init`, `serve`, `user` with `invite`, `device`, `reset-epoch`, `live`) - craft-cloud-agent the sync agent: real disk `Fs`, HTTP `Transport` over a `Connection`, saved account and state, root marker, CLI (`login`, `join`, `run`, `sync`, `status`, `logout`, `recents`, `doc` (`doc.rs`), `add-computer`, `devices`, `pause`, `resume`, `quit`, `autostart`), the `run` loop, the local API (`local_api.rs`), the live hub (`live/`: a document's Loro copy, recents document, replica (recents and documents of files), store (with the documents' journals), link, hub, the local API's docs socket), start at login (`autostart/`: Windows, macOS, Linux), the background build (`src/bin/bg.rs`, logging to `runlog.rs`) + craft-cloud-server HTTP API (`/api/v1`), the live service (`/api/v1/live`, `live.rs`: logs, documents by import, snapshots and compaction, presence), join codes (`join.rs`: pairings, join, the `/join/{code}` page), auth, admin pages (`web/`: server-rendered HTML with browser sessions, off unless `[web_admin]` is on), admin CLI (`init`, `serve`, `user` with `invite`, `device`, `reset-epoch`, `live`, `web-sessions`) + craft-cloud-agent the sync agent: real disk `Fs`, HTTP `Transport` over a `Connection`, saved account and state, root marker, CLI (`login`, `join`, `run`, `sync`, `status`, `logout`, `recents`, `doc` (`doc.rs`), `add-computer`, `devices`, `pause`, `resume`, `quit`, `autostart`), the `run` loop, local API (`local_api.rs`, including server admin calls in `admin.rs`), live hub (`live/`: a document's Loro copy, recents document, replica (recents and documents of files), store (with the documents' journals), link, hub, the local API's docs socket), start at login (`autostart/`: Windows, macOS, Linux), background build (`src/bin/bg.rs`, logging to `runlog.rs`) tests/network/ the simulation over a simulated network (turmoil): `main.rs` histories and scenarios, `harness.rs` server and agent hosts, `live.rs` live documents over WebSockets on it tests/recents_sim.rs the recents convergence simulation: replicas against the server's live sessions tests/documents_sim.rs the documents convergence simulation (shares `tests/docs_common/` with `documents.rs` and `live_speed.rs`) - craft-cloud-desktop the Craft Cloud app (`craft-cloud`, eframe/egui): the pure model (`model.rs`), the agent behind a trait and calls on threads (`agent.rs`), starting the agent (`start_agent.rs`), one app per user (`single_instance.rs`), the window (`ui/`, `app.rs`), the tray (`tray/`: tray-icon in the window's process on Windows and macOS; on Linux a StatusNotifierItem over zbus in a process of its own, `craft-cloud --tray`, `tray/process.rs`, since Wayland can't hide a window), the app's own start at login (`autostart.rs`), its icons (`assets/`, drawn by `cargo xtask icons`) + craft-cloud-desktop the Craft Cloud app (`craft-cloud`, eframe/egui): the pure model (`model.rs`, the admin pages' part of it in `admin.rs`), the agent behind a trait and calls on threads (`agent.rs`), starting the agent (`start_agent.rs`), one app per user (`single_instance.rs`), the window (`ui/`, `app.rs`), the tray (`tray/`: tray-icon in the window's process on Windows and macOS; on Linux a StatusNotifierItem over zbus in a process of its own, `craft-cloud --tray`, `tray/process.rs`, since Wayland can't hide a window), the app's own start at login (`autostart.rs`), its icons (`assets/`, drawn by `cargo xtask icons`) tests/snapshots/ every screen as kittest draws it (`UPDATE_SNAPSHOTS=1` redraws them) craft-cloud-setup the Windows setup (`craft-cloud-setup.exe`): install, upgrade, move from a hand-made install, `--uninstall`; the exes built in by `cargo xtask package` craft-cloud-kit-demo the app kit's demo (never packaged or installed): a story and a layer tree, live through the kit; its model and adapter (`model.rs`), its window (`ui.rs`); tests drive two windows on one fake hub, and time the kit on a big document diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index e0b2760c..a0c69d3c 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -14,3 +14,4 @@ next to them. | `apps/craft-cloud-desktop/assets/tray-*.png` | Craft Cloud's tray icon in each state (synced, syncing, offline, needs attention, paused): a cloud with a badge | Craft Cloud contributors | Drawn by `cargo xtask icons` (`xtask/src/icons.rs`) for this repository | MIT OR Apache-2.0 | | `apps/craft-cloud-desktop/tests/autostart/*` | Expected start-at-login entries of the app per OS (golden files) | Craft Cloud contributors | Written for this repository | MIT OR Apache-2.0 | | `apps/craft-cloud-desktop/tests/snapshots/*.png` | The Craft Cloud app's screens as its snapshot tests expect them (the text in egui's default fonts, see `epaint_default_fonts`) | Craft Cloud contributors | Rendered by the tests (`UPDATE_SNAPSHOTS=1`) | MIT OR Apache-2.0 | +| `apps/craft-cloud-server/src/web/style.css` | The admin pages' stylesheet: system fonts, no images, no external files | Craft Cloud contributors | Written for this repository | MIT OR Apache-2.0 | diff --git a/Cargo.lock b/Cargo.lock index 7b4cbe7c..54b8e93c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,6 +121,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -910,6 +945,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.7" @@ -1028,6 +1073,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1260,6 +1311,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio", + "tracing", "uuid", ] @@ -1283,6 +1335,7 @@ dependencies = [ name = "craft-cloud-server" version = "0.1.0" dependencies = [ + "aes-gcm", "argon2", "axum", "base64 0.23.1", @@ -1291,13 +1344,19 @@ dependencies = [ "clap", "craft-cloud-meta", "craft-cloud-proto", + "craft-cloud-server", "craft-cloud-store", + "fs4", "futures-util", "getrandom 0.4.3", + "hmac", + "percent-encoding", "reqwest", "serde", "serde_json", + "sha1 0.11.0", "sqlx", + "subtle", "tempfile", "thiserror 2.0.21", "tokio", @@ -1306,6 +1365,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zeroize", ] [[package]] @@ -1326,6 +1386,7 @@ dependencies = [ "blake3", "bytes", "craft-cloud-proto", + "futures-util", "object_store", "tempfile", "thiserror 2.0.21", @@ -1429,6 +1490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1441,6 +1503,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.3" @@ -1562,6 +1633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -2087,6 +2159,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e72ed92b67c146290f88e9c89d60ca163ea417a446f61ffd7b72df3e7f1dfd5" +dependencies = [ + "rustix 1.1.5", + "windows-sys 0.61.2", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2286,6 +2368,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gl_generator" version = "0.14.0" @@ -2894,6 +2986,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -4107,6 +4208,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -4395,6 +4502,18 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -6230,6 +6349,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 3c65eb18..05ce1349 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = ["crates/*", "apps/*", "xtask"] version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" -rust-version = "1.94" +rust-version = "1.95" repository = "https://github.com/itrycodinganditsbad/craft-cloud" [workspace.lints.rust] @@ -28,6 +28,13 @@ craft-cloud-agent = { path = "apps/craft-cloud-agent" } craft-cloud-app = { path = "crates/app" } craft-cloud-kit = { path = "crates/kit" } craft-cloud-egui = { path = "crates/egui" } +# Two-factor sign-in: AES-256-GCM for the stored TOTP secrets (RustCrypto, pure Rust), and HMAC-SHA-1 for the codes +# themselves (authenticator apps speak SHA-1). `subtle` compares codes in constant time, `zeroize` wipes secrets. +aes-gcm = "0.10.3" +hmac = "0.13" +sha1 = "0.11" +subtle = "2.6" +zeroize = "1.9" argon2 = "0.6" async-trait = "0.1.89" # `ws`: the server's live service and the agent's local API are WebSockets (tokio-tungstenite, MIT). @@ -47,6 +54,10 @@ reqwest = { version = "0.13", default-features = false, features = ["json", "rus file-id = "0.2.3" notify = { version = "8.2", default-features = false, features = ["macos_fsevent"] } jiff = { version = "0.2.38", default-features = false, features = ["std", "tz-system", "tzdb-zoneinfo", "tzdb-bundle-platform"] } +# Total and free bytes of a filesystem (the admin's storage overview); rustix on Unix, windows-sys on Windows. +fs4 = { version = "1.1", default-features = false, features = ["sync"] } +# Decoding a SQLite URL's path as sqlx does (the admin's storage overview finds the database file with it). +percent-encoding = "2.3" rustix = { version = "1.1.5", default-features = false, features = ["std", "fs"] } toml = "1.1" tracing = "0.1.44" diff --git a/README.md b/README.md index 48d61966..3f0ecc60 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ that is signed in, and enter its code on the new one. ## Documentation - [Installing the Craft Cloud app](docs/install.md) on Windows, macOS and Linux -- [Running the server](docs/server.md): reverse proxy, Postgres and S3, accounts, invites, backups +- [Running the server](docs/server.md): reverse proxy, Postgres and S3, accounts, invites, backups, Docker - [The agent on the command line](docs/agent.md): `login`, `run`, `sync`, `join`, start at login - [Development](docs/development.md): how it fits together, building, testing, the sync engine's rules - [The app kit](docs/kit.md): how a Craft app makes its documents live diff --git a/apps/craft-cloud-agent/src/account.rs b/apps/craft-cloud-agent/src/account.rs index 4a04ac33..023fa3e0 100644 --- a/apps/craft-cloud-agent/src/account.rs +++ b/apps/craft-cloud-agent/src/account.rs @@ -326,7 +326,13 @@ pub enum LoopState { /// The last pass failed for a reason that may pass by itself; the next try is at `next_try_ms`. Retrying { next_try_ms: i64, message: String }, /// Stopped until the user does something; looked at again by itself at `next_check_ms`, if set. - Paused { reason: String, next_check_ms: Option }, + Paused { + reason: String, + next_check_ms: Option, + /// The cause as the local API's `reason_code` says it, where there is one (`server_full`, `quota_exceeded`). + #[serde(default, skip_serializing_if = "Option::is_none")] + reason_code: Option, + }, /// Stopping after the running pass. Stopping, /// The loop ended. diff --git a/apps/craft-cloud-agent/src/admin.rs b/apps/craft-cloud-agent/src/admin.rs new file mode 100644 index 00000000..f18ae7ae --- /dev/null +++ b/apps/craft-cloud-agent/src/admin.rs @@ -0,0 +1,379 @@ +//! The local API's admin calls (`/v1/admin/…`): the server's admin API (`/api/v1/admin/…`, `craft-cloud-server`'s +//! `admin.rs`) for the Craft Cloud app's admin pages, passed on with the signed-in device's token. +//! +//! - **Nothing is decided here.** The agent checks no role and keeps no confirmation: the acting admin's password +//! travels in the body of each change, from the app through this agent to the server, and is dropped with the +//! request. Bodies are read as the shared types (so the `Debug` output of every one hides the password), at most +//! [`MAX_ADMIN_JSON_BYTES`] long, and written to the server again; a field this agent doesn't know is dropped. +//! Nothing of a body is logged. +//! - **The server's answer is the answer**: the same body and, for a refusal, the same HTTP status, error code and +//! message, and `Retry-After` for a `429`. A `403` (`wrong_password`, `forbidden`, `account_disabled`) stays a `403`, +//! a `409` (`last_admin`) a `409`. The one exception is the server's `401`: for a signed-in route it means this +//! device was revoked, and is told as every other call tells it (`409 needs_you`), not as `401`, which on the local +//! API means a wrong token. +//! - The same `Host`, `Origin` and token checks as every local call (`local_api::guard`) run first. + +use axum::body::Body; +use axum::extract::rejection::{JsonRejection, PathRejection, QueryRejection}; +use axum::extract::{DefaultBodyLimit, Path, Query, State}; +use axum::http::StatusCode; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{ + AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditPage, codes as api_codes, valid_audit_action, +}; +use craft_cloud_proto::local::{MAX_ADMIN_JSON_BYTES, paths}; +use reqwest::Method; +use serde::Deserialize; +use uuid::Uuid; + +use crate::cli::AgentError; +use crate::http::{CallError, MAX_DEVICES}; +use crate::local_api::{Api, LocalError, path_id, server_api}; + +/// Most accounts one page of the user list may hold (the server's own limit; a longer page is refused). +const MAX_PAGE_USERS: usize = 200; + +impl From for LocalError { + /// A refusal keeps the server's status, code and message and its `Retry-After` (never a 401: that is a revoked device, + /// told as every signed-in call tells it); anything else is what the sync paths make of it. + fn from(e: CallError) -> Self { + AgentError::from(e).into() + } +} + +type Answer = Result; + +/// The admin calls, to be merged into the local API's router (behind its `guard`). +pub(crate) fn routes() -> Router { + let small = || DefaultBodyLimit::max(MAX_ADMIN_JSON_BYTES); + let user = format!("{}/{{id}}", paths::ADMIN_USERS); + Router::new() + .route(paths::ACCOUNT, get(account)) + .route(paths::ACCOUNT_ACTIVITY, get(activity)) + .route(paths::ADMIN_AUDIT, get(audit)) + .route(paths::ADMIN_AUDIT_EXPORT, get(audit_export)) + .route(paths::ADMIN_SERVER, get(server)) + .route(paths::ADMIN_STORAGE, get(storage)) + .route(paths::ADMIN_USERS, get(list_users).post(create_user).layer(small())) + .route(&user, axum::routing::patch(update_user).delete(remove_user).layer(small())) + .route(&format!("{user}/usage"), get(user_usage)) + .route(&format!("{user}/disable"), post(disable_user).layer(small())) + .route(&format!("{user}/enable"), post(enable_user).layer(small())) + .route(&format!("{user}/invite"), post(issue_invite).delete(revoke_invite).layer(small())) + .route(&format!("{user}/password-reset"), post(issue_reset).layer(small())) + .route(&format!("{user}/reset-two-factor"), post(reset_two_factor).layer(small())) + .route(&format!("{user}/devices"), get(list_devices)) + .route(&format!("{user}/devices/{{device_id}}"), axum::routing::delete(revoke_device).layer(small())) +} + +async fn server(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/server", &[]).await?)) +} + +async fn storage(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/storage", &[]).await?)) +} + +async fn user_usage(State(state): State, id: Result, PathRejection>) -> Answer> { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get(&format!("admin/users/{id}/usage"), &[]).await?)) +} + +#[derive(Debug, Deserialize)] +struct ListQuery { + cursor: Option, + limit: Option, + query: Option, +} + +async fn list_users(State(state): State, q: Result, QueryRejection>) -> Answer> { + let Query(q) = q.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let (_, transport) = server_api(&state)?; + let limit = q.limit.map(|l| l.to_string()); + let mut pairs: Vec<(&str, &str)> = Vec::new(); + if let Some(cursor) = q.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(("cursor", cursor)); + } + if let Some(limit) = limit.as_deref() { + pairs.push(("limit", limit)); + } + if let Some(query) = q.query.as_deref().filter(|s| !s.is_empty()) { + pairs.push(("query", query)); + } + let list: AdminUserList = transport.admin_get("admin/users", &pairs).await?; + if list.users.len() > MAX_PAGE_USERS { + return Err(AgentError::from(craft_cloud_sync::TransportError::Protocol(format!( + "the server listed {} accounts on a page, more than the {MAX_PAGE_USERS} this agent reads", + list.users.len() + ))) + .into()); + } + Ok(Json(list)) +} + +async fn create_user(State(state): State, body: Result, JsonRejection>) -> Answer> { + let Json(body) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, "admin/users", &body).await?)) +} + +async fn update_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let id = path_id(id)?; + let Json(body) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::PATCH, &format!("admin/users/{id}"), &body).await?)) +} + +/// A change with only the confirmation as its body, answered without one: the server's own status (`204`, or `202`). +async fn confirmed(state: &Api, method: Method, path: String, body: Result, JsonRejection>) -> Answer { + let Json(body) = body?; + let (_, transport) = server_api(state)?; + let status = transport.admin_empty(method, &path, &body).await?; + Ok(StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::NO_CONTENT)) +} + +async fn disable_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/disable", path_id(id)?), body).await +} + +async fn enable_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/enable", path_id(id)?), body).await +} + +async fn remove_user( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::DELETE, format!("admin/users/{}", path_id(id)?), body).await +} + +async fn revoke_invite( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::DELETE, format!("admin/users/{}/invite", path_id(id)?), body).await +} + +async fn revoke_device( + State(state): State, + ids: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + let (user, device) = path_id(ids)?; + confirmed(&state, Method::DELETE, format!("admin/users/{user}/devices/{device}"), body).await +} + +/// A new invite: shown to the admin once, by the app; the agent keeps nothing of it and says nothing of it in its log. +async fn issue_invite( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let Json(body) = body?; + let path = format!("admin/users/{}/invite", path_id(id)?); + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, &path, &body).await?)) +} + +/// A password reset code, with the confirmation and whether redeeming it also turns the account's two-factor sign-in off +/// ([`AdminPasswordReset::reset_two_factor`]): shown once, kept and logged by nobody. +async fn issue_reset( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer> { + let Json(body) = body?; + let path = format!("admin/users/{}/password-reset", path_id(id)?); + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_json(Method::POST, &path, &body).await?)) +} + +async fn reset_two_factor( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Answer { + confirmed(&state, Method::POST, format!("admin/users/{}/reset-two-factor", path_id(id)?), body).await +} + +async fn list_devices(State(state): State, id: Result, PathRejection>) -> Answer> { + let id = path_id(id)?; + let (_, transport) = server_api(&state)?; + let list: AdminDevices = transport.admin_get(&format!("admin/users/{id}/devices"), &[]).await?; + if list.devices.len() > MAX_DEVICES { + return Err(AgentError::from(craft_cloud_sync::TransportError::Protocol(format!( + "the server listed {} devices, more than the {MAX_DEVICES} this agent reads", + list.devices.len() + ))) + .into()); + } + Ok(Json(list)) +} + +// ---- the account and the audit log ---- + +/// The most events a page of the account's own activity holds, and of the admin's audit list (the server's limits). +const ACCOUNT_MAX_PAGE: u32 = 100; +const ADMIN_MAX_PAGE: u32 = 200; +/// The longest cursor passed on, in bytes (the server's are a few dozen). +const MAX_CURSOR_BYTES: usize = 256; + +/// `GET /v1/account`: the signed-in account as the server says it (its role, and whether it has two-factor sign-in). +async fn account(State(state): State) -> Answer> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.account_info().await?)) +} + +/// The query of the audit routes. Unknown parameters are ignored. +#[derive(Debug, Deserialize)] +struct AuditParams { + cursor: Option, + before_ms: Option, + limit: Option, + user: Option, + action: Option, +} + +fn invalid(message: &str) -> LocalError { + LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, message) +} + +/// The query as pairs for the server, after checking what is checked here (the server checks it all again): a cursor is passed +/// on as it came, but bounded and made of the characters of the server's own; `user` is an account ID, `action` one of the +/// closed list or a prefix of it, `before_ms` a time, `limit` 1 to `max_limit`. `admin`: whether `user` and `action` are read +/// (an account's own activity has neither). +fn audit_pairs(params: &AuditParams, max_limit: u32, admin: bool, list: bool) -> Result, LocalError> { + let mut pairs = Vec::new(); + if let Some(cursor) = params.cursor.as_deref().filter(|c| !c.is_empty()) { + if !list || cursor.len() > MAX_CURSOR_BYTES || !cursor.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) { + return Err(invalid("the cursor is not one this server gave")); + } + pairs.push(("cursor", cursor.to_owned())); + } + if let Some(before) = params.before_ms { + if before < 0 { + return Err(invalid("before_ms is a time in milliseconds")); + } + pairs.push(("before_ms", before.to_string())); + } + if let Some(limit) = params.limit { + if !list || !(1..=max_limit).contains(&limit) { + return Err(invalid(&format!("limit is 1 to {max_limit}"))); + } + pairs.push(("limit", limit.to_string())); + } + if admin { + if let Some(user) = params.user.as_deref().filter(|u| !u.is_empty()) { + let id = Uuid::parse_str(user).map_err(|_| invalid("user is an account ID"))?; + pairs.push(("user", id.to_string())); + } + if let Some(action) = params.action.as_deref().filter(|a| !a.is_empty()) { + if !valid_audit_action(action) { + return Err(invalid("action is an event name like auth.login, or a prefix like auth.")); + } + pairs.push(("action", action.to_owned())); + } + } + Ok(pairs) +} + +fn as_refs<'a>(pairs: &'a [(&'static str, String)]) -> Vec<(&'static str, &'a str)> { + pairs.iter().map(|(k, v)| (*k, v.as_str())).collect() +} + +/// `GET /v1/account/activity?cursor&before_ms&limit`: the account's own security events (any account; `user` and `action` are +/// not read). +async fn activity(State(state): State, params: Result, QueryRejection>) -> Answer> { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ACCOUNT_MAX_PAGE, false, true)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("account/activity", &as_refs(&pairs)).await?)) +} + +/// `GET /v1/admin/audit?cursor&before_ms&limit&user&action`: the whole log, filtered; the server refuses a member. +async fn audit(State(state): State, params: Result, QueryRejection>) -> Answer> { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ADMIN_MAX_PAGE, true, true)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.admin_get("admin/audit", &as_refs(&pairs)).await?)) +} + +/// `GET /v1/admin/audit/export?before_ms&user&action`: the server's NDJSON export passed through as it comes, chunk by chunk: +/// the agent never holds more than the chunk in hand, and a connection that breaks in the middle ends this one without its +/// last chunk, so the client sees a broken stream. The last line (`complete` or `truncated`) is the server's. +async fn audit_export(State(state): State, params: Result, QueryRejection>) -> Answer { + let Query(params) = params.map_err(|e| invalid(&e.body_text()))?; + let pairs = audit_pairs(¶ms, ADMIN_MAX_PAGE, true, false)?; + let (_, transport) = server_api(&state)?; + let answer = transport.admin_export_stream(&as_refs(&pairs)).await?; + let stream = futures_util::stream::unfold(Some(answer.body), |body| async move { + let mut body = body?; + match body.next().await { + Ok(Some(bytes)) => Some((Ok::<_, std::io::Error>(bytes), Some(body))), + Ok(None) => None, + // The error ends the stream with it: the connection breaks, and nothing is made up for the missing end. + Err(e) => Some((Err(std::io::Error::other(e.to_string())), None)), + } + }); + let mut response = Response::new(Body::from_stream(stream)); + let headers = response.headers_mut(); + headers.insert(axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static("application/x-ndjson")); + headers.insert(axum::http::header::CACHE_CONTROL, axum::http::HeaderValue::from_static("no-store")); + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::http::Refusal; + use axum::response::IntoResponse; + use craft_cloud_proto::local::codes; + use craft_cloud_sync::TransportError; + + fn refused(status: u16, code: &str, retry: Option) -> LocalError { + LocalError::from(CallError::Refused(Refusal { status, error: code.into(), message: "m".into(), two_factor: None, retry_after_secs: retry })) + } + + #[test] + fn a_refusal_keeps_its_status_code_and_retry_after() { + let wrong = refused(403, "wrong_password", None); + assert_eq!((wrong.status, wrong.code.as_str()), (StatusCode::FORBIDDEN, "wrong_password")); + let invalid = refused(403, "two_factor_invalid", None); + assert_eq!((invalid.status, invalid.code.as_str()), (StatusCode::FORBIDDEN, "two_factor_invalid")); + let last = refused(409, "last_admin", None); + assert_eq!((last.status, last.code.as_str()), (StatusCode::CONFLICT, "last_admin")); + let locked = refused(429, "two_factor_locked", Some(30)).into_response(); + assert_eq!(locked.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(locked.headers().get(axum::http::header::RETRY_AFTER).and_then(|v| v.to_str().ok()), Some("30")); + } + + /// The server's `401` (this device was revoked) is never passed on as a `401`: on the local API that says the + /// token is wrong. + #[test] + fn a_revoked_device_is_not_a_wrong_token() { + let failure = LocalError::from(CallError::Transport(TransportError::Unauthorized)); + assert_ne!(failure.status, StatusCode::UNAUTHORIZED); + assert_eq!(failure.code, codes::NEEDS_YOU); + } +} diff --git a/apps/craft-cloud-agent/src/cli.rs b/apps/craft-cloud-agent/src/cli.rs index 194d4713..b6ac4301 100644 --- a/apps/craft-cloud-agent/src/cli.rs +++ b/apps/craft-cloud-agent/src/cli.rs @@ -33,11 +33,11 @@ use jiff::tz::TimeZone; use crate::account::{ACCOUNT_SCHEMA, Account, AccountError, DataDir, LastPass, LoopState, Outcome, PassSummary, STATE_SCHEMA, SavedState}; use crate::disk::{AGENT_DIR, DiskFs, create_private_dir}; -use crate::http::{self, DeviceToken, HttpTransport, JoinError, ReachError, ServerUrl}; +use crate::http::{self, CallError, DeviceToken, Factor, HttpTransport, JoinError, LoginStep, ReachError, Refusal, ServerUrl}; use crate::lock::{self, DATA_LOCK, FOLDER_LOCK, Lock, LockError}; use crate::root::{self, RootError, RootMarker}; use crate::settle::Stability; -use craft_cloud_proto::api::LoginResponse; +use craft_cloud_proto::api::{LoginResponse, TwoFactorChallenge}; use craft_cloud_proto::join::JoinString; #[derive(Debug, Parser)] @@ -248,7 +248,7 @@ impl AgentError { | AgentError::Lock(LockError::Busy(_) | LockError::NotAFile(_)) | AgentError::Revoked | AgentError::OtherAccountState(_) - | AgentError::Server(TransportError::Protocol(_) | TransportError::QuotaExceeded { .. }) + | AgentError::Server(TransportError::Protocol(_) | TransportError::QuotaExceeded { .. } | TransportError::ServerFull) | AgentError::Sync(SyncError::Corrupt(_)) | AgentError::Account(AccountError::Newer { .. } | AccountError::Damaged { .. }) | AgentError::NeedsYou(_) => Exit::NeedsYou, @@ -336,6 +336,24 @@ pub enum AgentError { .0.display() )] FolderHasFiles(PathBuf), + /// `login`: the account has two-factor sign-in. The challenge goes back with a code from its authenticator app (or a + /// recovery code) to finish the sign-in. For `join` the challenge is empty: the same join goes again with the code. + #[error("this account has two-factor sign-in: a code from its authenticator app (or a recovery code) is needed")] + TwoFactorRequired(TwoFactorChallenge), + /// The code or recovery code was wrong (the server's message). Never a revoked device. + #[error("{0}")] + TwoFactorInvalid(String), + /// Too many wrong codes: the second factor is locked for a while (`retry_after_secs`, the server's `Retry-After`). + #[error("{message}")] + TwoFactorLocked { message: String, retry_after_secs: Option }, + /// The second step of a sign-in found its challenge gone (it works five minutes and takes five tries, and a password + /// change ends it): the sign-in starts again. + #[error("this sign-in step has run out (it works for five minutes and takes five tries): sign in again")] + ChallengeGone, + /// The server refused a call of the account (a wrong password, a conflict, a lock, a limit): its status, code and + /// message as it said them, and its `Retry-After`. Never a revoked device (that is a 401, [`AgentError::Revoked`]). + #[error("{message}")] + Refused { status: u16, code: String, message: String, retry_after_secs: Option }, /// Something found as `run` started that only a person can put right (the message says what); the agent waits, /// serving the local API, until it is. #[error("{0}")] @@ -352,6 +370,61 @@ impl From for AgentError { } } +impl From for AgentError { + /// For a signed-in call of the account: the server's refusal with its status, code and message; anything else as sync's + /// transport errors are (a 401 is a revoked device, offline is offline). + fn from(e: CallError) -> Self { + match e { + CallError::Refused(Refusal { status, error, message, retry_after_secs, .. }) => { + AgentError::Refused { status, code: error, message, retry_after_secs } + } + CallError::Transport(t) => t.into(), + } + } +} + +/// What a refusal said, as the transport error the sign-in used to see for it (a refusal with a code of its own is handled +/// before this), so that messages and exits of the first step don't change. +fn legacy(refusal: &Refusal) -> TransportError { + if refusal.status == 429 { + TransportError::Unavailable(format!("the server is busy: {}", refusal.message)) + } else { + TransportError::Protocol(format!("the server or a proxy refused the request (HTTP {}): {}", refusal.status, refusal.message)) + } +} + +/// A wait in words, for a lock: "about 14 minutes". +fn wait_in_words(secs: u64) -> String { + match secs { + 0..=90 => format!("{secs} seconds"), + 91..=5_400 => format!("about {} minutes", secs.div_ceil(60)), + _ => format!("about {} hours", secs.div_ceil(3_600)), + } +} + +/// The errors of a sign-in's second step and of a join with a code, from what the server said. +fn factor_error(e: CallError) -> AgentError { + match e { + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_INVALID => AgentError::TwoFactorInvalid(r.message), + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_LOCKED => locked(r.message, r.retry_after_secs), + CallError::Refused(r) if r.error == craft_cloud_proto::api::codes::TWO_FACTOR_REQUIRED => { + AgentError::TwoFactorRequired(r.two_factor.unwrap_or(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })) + } + // The challenge doesn't work any more: not a revoked device (nothing is signed in yet). + CallError::Transport(TransportError::Unauthorized) => AgentError::ChallengeGone, + CallError::Refused(r) => legacy(&r).into(), + CallError::Transport(t) => t.into(), + } +} + +fn locked(message: String, retry_after_secs: Option) -> AgentError { + let message = match retry_after_secs { + Some(secs) => format!("{message} (try again in {})", wait_in_words(secs)), + None => message, + }; + AgentError::TwoFactorLocked { message, retry_after_secs } +} + impl From for AgentError { fn from(e: SyncError) -> Self { match e { @@ -384,15 +457,40 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re return Err(already_signed_in(&account)); } let password = read_password(input)?; - match login(&data, &args, &password, out).await { + let mut through_agent = false; + let first = match login(&data, &args, &password, out).await { // `run` holds the data folder: it signs in, through its local API. - Err(AgentError::Lock(LockError::Busy(path))) => login_through_agent(&data, &args, password, out).await.map_err(|e| match e { - AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), - other => other, - })?, - other => { - other?; + Err(AgentError::Lock(LockError::Busy(path))) => { + through_agent = true; + login_through_agent(&data, &args, password, out).await.map_err(|e| match e { + AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), + other => other, + }) + } + other => other.map(|_| ()), + }; + match first { + // An account with two-factor sign-in: the code comes from standard input, like the password (never from the + // command line), and a wrong one may be typed again twice. + Err(AgentError::TwoFactorRequired(challenge)) => { + let mut tries = 0; + loop { + let factor = read_factor(input, out)?; + let done = if through_agent { + second_step_through_agent(&data, &args, &challenge, &factor, out).await + } else { + login_second_factor(&data, &args, &challenge, &factor, out).await.map(|_| ()) + }; + match done { + Err(AgentError::TwoFactorInvalid(message)) if tries < MAX_CODE_TRIES - 1 => { + tries += 1; + say!(out, "{message}")?; + } + other => break other?, + } + } } + other => other?, } Ok(Exit::Done) } @@ -440,11 +538,13 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re return Err(already_signed_in(&account)); } let mut password: Option = None; + let mut factor: Option = None; + let mut wrong_codes = 0; loop { - let joined = match join(&data, &args, password.as_deref(), out).await { + let joined = match join_with(&data, &args, password.as_deref(), factor.as_ref(), out).await { // `run` holds the data folder: it joins, through its local API. Err(AgentError::Lock(LockError::Busy(path))) => { - join_through_agent(&data, &args, password.clone(), out).await.map_err(|e| match e { + join_through_agent(&data, &args, password.clone(), factor.clone(), out).await.map_err(|e| match e { AgentError::Local(craft_cloud_app::Error::NotRunning(_)) => AgentError::Lock(LockError::Busy(path)), other => other, }) @@ -453,6 +553,13 @@ pub async fn run(cli: Cli, input: &mut impl BufRead, out: &mut impl Write) -> Re }; match joined { Err(AgentError::PasswordRequired) if password.is_none() => password = Some(new_password(input, out)?), + // The code was not used up: the same join goes again with the second factor. + Err(AgentError::TwoFactorRequired(_)) if factor.is_none() => factor = Some(read_factor(input, out)?), + Err(AgentError::TwoFactorInvalid(message)) if wrong_codes < MAX_CODE_TRIES - 1 => { + wrong_codes += 1; + say!(out, "{message}")?; + factor = Some(read_factor(input, out)?); + } other => break other?, } } @@ -572,8 +679,29 @@ fn new_password(input: &mut impl BufRead, out: &mut impl Write) -> Result Result { + say!(out, "This account has two-factor sign-in. Type the code from your authenticator app (or a recovery code):")?; + out.flush().map_err(io_at(Path::new("standard output")))?; + let mut line = String::new(); + input.take(200).read_line(&mut line).map_err(|e| AgentError::Usage(format!("can't read the code from standard input: {e}")))?; + if line.trim().is_empty() { + return Err(AgentError::Usage("no code on standard input".into())); + } + Ok(Factor::typed(&line)) +} + /// `join` while `run` runs: the running agent joins. -async fn join_through_agent(data: &DataDir, args: &JoinArgs, password: Option, out: &mut impl Write) -> Result<(), AgentError> { +async fn join_through_agent( + data: &DataDir, + args: &JoinArgs, + password: Option, + factor: Option, + out: &mut impl Write, +) -> Result<(), AgentError> { let folder = args.folder.as_ref().map(|f| std::path::absolute(f).map_err(io_at(f))).transpose()?; let request = craft_cloud_app::JoinRequest { join: args.join.clone(), @@ -581,6 +709,8 @@ async fn join_through_agent(data: &DataDir, args: &JoinArgs, password: Option(f: impl FnOnce() -> Result AgentError { use craft_cloud_proto::local::codes; + // A refusal that carries the challenge or the wait of a lock. + if let AgentError::Local(inner @ craft_cloud_app::Error::RefusedWith { .. }) = &e { + let message = inner.to_string(); + return match inner.code() { + Some(codes::TWO_FACTOR_REQUIRED) => { + AgentError::TwoFactorRequired(inner.challenge().cloned().unwrap_or(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })) + } + Some(codes::TWO_FACTOR_LOCKED) => locked(message, inner.retry_after_secs()), + _ => AgentError::Usage(message), + }; + } match e { - AgentError::Local(craft_cloud_app::Error::Refused { code, message }) => match code.as_str() { + AgentError::Local(craft_cloud_app::Error::Refused { code, message, .. }) => match code.as_str() { + codes::TWO_FACTOR_INVALID => AgentError::TwoFactorInvalid(message), + codes::TWO_FACTOR_EXPIRED => AgentError::ChallengeGone, + codes::TWO_FACTOR_REQUIRED => AgentError::TwoFactorRequired(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 }), codes::WRONG_PASSWORD => AgentError::WrongPassword, codes::NOT_SIGNED_IN => AgentError::NotSignedIn, codes::OFFLINE => AgentError::Offline(message), @@ -749,6 +893,36 @@ async fn login_through_agent(data: &DataDir, args: &LoginArgs, password: String, ) } +/// The second step of `login` while `run` runs: the running agent finishes the sign-in. +async fn second_step_through_agent( + data: &DataDir, + args: &LoginArgs, + challenge: &TwoFactorChallenge, + factor: &Factor, + out: &mut impl Write, +) -> Result<(), AgentError> { + let folder = args.folder.as_ref().map(|f| std::path::absolute(f).map_err(io_at(f))).transpose()?; + let request = craft_cloud_app::SecondFactorLoginRequest { + server: args.url.clone(), + user: args.user.clone(), + folder: folder.map(|f| f.display().to_string()), + device_name: args.device_name.clone(), + adopt: args.adopt, + challenge: challenge.challenge.clone(), + totp_code: factor.totp_code.clone(), + recovery_code: factor.recovery_code.clone(), + }; + let data_path = data.path().to_owned(); + let status = blocking(move || craft_cloud_app::Client::connect_in(&data_path)?.login_second_factor(&request)).await.map_err(local_error)?; + say!( + out, + "signed in as {} on {} (through the running agent, which syncs {} now)", + status.account.unwrap_or_default(), + status.server.unwrap_or_default(), + status.folder.unwrap_or_default() + ) +} + /// `recents [add ] [--watch]`: a client of the running agent like an app. async fn recents(data: &DataDir, args: RecentsArgs, out: &mut impl Write) -> Result { let data_path = data.path().to_owned(); @@ -1095,20 +1269,65 @@ pub(crate) async fn login_locked(data: &DataDir, args: &LoginArgs, password: &st let server = ServerUrl::parse(&args.url).map_err(|e| AgentError::Usage(e.to_string()))?; let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; ask_server(&server, |_, message| AgentError::Offline(message)).await?; - let signed_in = match http::login(&server, &args.user, password, &prepared.device_name).await { - Err(TransportError::Unauthorized) => return Err(AgentError::WrongPassword), - other => other?, + let signed_in = match http::login_step(&server, &args.user, password, &prepared.device_name).await { + Ok(LoginStep::Device(signed_in)) => signed_in, + // An account with two-factor sign-in: no device yet. The caller asks for the code and finishes with + // [`login_second_factor_locked`], which needs nothing of this call but its arguments. + Ok(LoginStep::SecondFactor(challenge)) => return Err(AgentError::TwoFactorRequired(challenge)), + Err(CallError::Transport(TransportError::Unauthorized)) => return Err(AgentError::WrongPassword), + Err(CallError::Refused(refusal)) => return Err(legacy(&refusal).into()), + Err(CallError::Transport(e)) => return Err(e.into()), }; set_up(data, &server, signed_in, prepared).await } +/// The second step of a sign-in: the folder's checks again (the agent keeps nothing between the steps), then the challenge and +/// the second factor make the device, and the folder is set up as after [`login`]. Prints nothing. +pub(crate) async fn login_second_factor_locked(data: &DataDir, args: &LoginArgs, challenge: &str, factor: &Factor) -> Result { + if let Some(account) = data.load_account()? { + return Err(already_signed_in(&account)); + } + let server = ServerUrl::parse(&args.url).map_err(|e| AgentError::Usage(e.to_string()))?; + let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; + ask_server(&server, |_, message| AgentError::Offline(message)).await?; + let signed_in = http::second_factor(&server, challenge, factor).await.map_err(factor_error)?; + set_up(data, &server, signed_in, prepared).await +} + +/// [`login_second_factor_locked`] for the command line: holds the data folder and says it signed in. +pub async fn login_second_factor( + data: &DataDir, + args: &LoginArgs, + challenge: &TwoFactorChallenge, + factor: &Factor, + out: &mut impl Write, +) -> Result { + let _data_lock = hold_data(data)?; + let account = login_second_factor_locked(data, args, &challenge.challenge, factor).await?; + say!(out, "signed in as {} on {} (device \"{}\")", account.username, account.server, account.device_name)?; + say!(out, "syncing {}; run `craft-cloud-agent sync` to sync it", account.folder.display())?; + Ok(account) +} + /// Joins with a join string (an invite or an add-a-computer code), creates a device and sets up the folder, as /// [`login`] does. Everything that can be checked here is checked before the code is spent (the folder, the device /// name, what the server speaks); a failure after it (the folder is set up for another space) removes the new device /// again, and says the code is used up. `password`: the new account's, for an invite that needs one. pub async fn join(data: &DataDir, args: &JoinArgs, password: Option<&str>, out: &mut impl Write) -> Result { + join_with(data, args, password, None, out).await +} + +/// [`join`] with the second factor of an account that has two-factor sign-in (without one, such an account answers +/// [`AgentError::TwoFactorRequired`] and the code is not used up). +pub async fn join_with( + data: &DataDir, + args: &JoinArgs, + password: Option<&str>, + factor: Option<&Factor>, + out: &mut impl Write, +) -> Result { let _data_lock = hold_data(data)?; - let account = join_locked(data, args, password).await?; + let account = join_locked(data, args, password, factor).await?; say!(out, "joined as {} on {} (device \"{}\")", account.username, account.server, account.device_name)?; say!(out, "syncing {}; run `craft-cloud-agent run` to keep it in sync", account.folder.display())?; Ok(account) @@ -1116,7 +1335,7 @@ pub async fn join(data: &DataDir, args: &JoinArgs, password: Option<&str>, out: /// [`join`] for a caller that holds the data folder's lock already: the `run` loop, joining for the local API. /// Prints nothing. -pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Option<&str>) -> Result { +pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Option<&str>, factor: Option<&Factor>) -> Result { if let Some(account) = data.load_account()? { return Err(already_signed_in(&account)); } @@ -1125,8 +1344,13 @@ pub(crate) async fn join_locked(data: &DataDir, args: &JoinArgs, password: Optio let prepared = prepare(data, args.folder.as_deref(), args.device_name.as_deref(), args.adopt)?; let unreachable = |server: &ServerUrl, message| AgentError::Unreachable { server: server.to_string(), message }; ask_server(&server, unreachable).await?; - let signed_in = match http::join(&server, join.code.as_str(), &prepared.device_name, password).await { + let signed_in = match http::join_with_factor(&server, join.code.as_str(), &prepared.device_name, password, factor).await { Ok(signed_in) => signed_in, + Err(JoinError::TwoFactorRequired) => { + return Err(AgentError::TwoFactorRequired(TwoFactorChallenge { challenge: String::new(), expires_ms: 0 })); + } + Err(JoinError::TwoFactorInvalid(message)) => return Err(AgentError::TwoFactorInvalid(message)), + Err(JoinError::TwoFactorLocked { message, retry_after_secs }) => return Err(locked(message, retry_after_secs)), Err(JoinError::InvalidCode) => return Err(AgentError::InvalidCode), Err(JoinError::PasswordRequired) => return Err(AgentError::PasswordRequired), Err(JoinError::WeakPassword(message)) => return Err(AgentError::WeakPassword(message)), @@ -1708,8 +1932,8 @@ fn describe_loop(state: &LoopState) -> String { LoopState::Idle => "idle, waiting for changes".into(), LoopState::Syncing => "syncing".into(), LoopState::Retrying { next_try_ms, message } => format!("offline, retrying at {}: {message}", show_time(*next_try_ms)), - LoopState::Paused { reason, next_check_ms: Some(at) } => format!("paused: {reason} (looked at again at {})", show_time(*at)), - LoopState::Paused { reason, next_check_ms: None } => { + LoopState::Paused { reason, next_check_ms: Some(at), .. } => format!("paused: {reason} (looked at again at {})", show_time(*at)), + LoopState::Paused { reason, next_check_ms: None, .. } => { format!("paused: {reason} (stop it with Ctrl-C, put this right, then start `craft-cloud-agent run` again)") } LoopState::Stopping => "stopping after the running pass".into(), @@ -1920,6 +2144,8 @@ mod tests { (SyncError::Transport(TransportError::Protocol("a refusal naming no operation".into())), Exit::NeedsYou), // Only with no room left at all: a file that doesn't fit while others still may is skipped instead. (SyncError::Transport(TransportError::QuotaExceeded { used: 10, limit: 10 }), Exit::NeedsYou), + // The server itself refuses uploads: nothing fits, whatever the file. + (SyncError::Transport(TransportError::ServerFull), Exit::NeedsYou), (SyncError::Io { path: "an interrupted download".into(), source: std::io::Error::other("x") }, Exit::Failed), (SyncError::Corrupt("chunk x of a.txt is damaged".into()), Exit::NeedsYou), (SyncError::Checkpoint(std::io::Error::other("disk full")), Exit::Failed), @@ -1935,7 +2161,8 @@ mod tests { | TransportError::Unauthorized | TransportError::Unavailable(_) | TransportError::Protocol(_) - | TransportError::QuotaExceeded { .. }, + | TransportError::QuotaExceeded { .. } + | TransportError::ServerFull, ) | SyncError::Io { .. } | SyncError::Corrupt(_) diff --git a/apps/craft-cloud-agent/src/daemon.rs b/apps/craft-cloud-agent/src/daemon.rs index 0841eea7..7916610c 100644 --- a/apps/craft-cloud-agent/src/daemon.rs +++ b/apps/craft-cloud-agent/src/daemon.rs @@ -132,7 +132,7 @@ fn pass_end(result: &Result, recheck_in: Option PassEnd::Done { recheck_in, needs_approval: summary.needs_approval(), held: summary.held_set().is_some() }, - Err(e) if e.exit() == Exit::NeedsYou => PassEnd::NeedsYou { reason: e.to_string(), recheck: recheck_for(e) }, + Err(e) if e.exit() == Exit::NeedsYou => PassEnd::NeedsYou { reason: e.to_string(), reason_code: reason_code_for(e), recheck: recheck_for(e) }, Err(e) => PassEnd::Failed(e.to_string()), } } @@ -140,17 +140,30 @@ fn pass_end(result: &Result, recheck_in: Option Recheck { match e { AgentError::Root(_) | AgentError::DataInFolder { .. } | AgentError::Lock(_) => Recheck::Soon, - AgentError::Server(craft_cloud_sync::TransportError::Protocol(_) | craft_cloud_sync::TransportError::QuotaExceeded { .. }) => Recheck::Slowly, + AgentError::Server( + craft_cloud_sync::TransportError::Protocol(_) + | craft_cloud_sync::TransportError::QuotaExceeded { .. } + | craft_cloud_sync::TransportError::ServerFull, + ) => Recheck::Slowly, _ => Recheck::Never, } } +/// The machine-readable cause of a pause, for the local API's `reason_code`: only the causes an app treats on its own. +fn reason_code_for(e: &AgentError) -> Option { + match e { + AgentError::Server(craft_cloud_sync::TransportError::ServerFull) => Some(craft_cloud_proto::api::codes::SERVER_FULL.to_owned()), + AgentError::Server(craft_cloud_sync::TransportError::QuotaExceeded { .. }) => Some(craft_cloud_proto::api::codes::QUOTA_EXCEEDED.to_owned()), + _ => None, + } +} + /// The pass thread: opens the folder when a pass is asked for (again after a stop that needs the user, so a folder /// that came back is opened afresh), runs the pass on its own current-thread runtime and reports it. Ends when the /// loop drops its end of `requests`, releasing the folder's lock. @@ -409,7 +422,9 @@ fn to_wall(at: Ms, mono_now: Ms, wall_now: i64) -> i64 { fn without_times(status: &LoopStatus) -> LoopStatus { let state = match &status.state { LoopState::Retrying { message, .. } => LoopState::Retrying { next_try_ms: 0, message: message.clone() }, - LoopState::Paused { reason, next_check_ms } => LoopState::Paused { reason: reason.clone(), next_check_ms: next_check_ms.map(|_| 0) }, + LoopState::Paused { reason, next_check_ms, reason_code } => { + LoopState::Paused { reason: reason.clone(), next_check_ms: next_check_ms.map(|_| 0), reason_code: reason_code.clone() } + } other => other.clone(), }; LoopStatus { heartbeat_ms: 0, state, ..status.clone() } @@ -421,7 +436,7 @@ fn run_state(state: &LoopState) -> RunState { LoopState::Idle => RunState::Idle, LoopState::Syncing => RunState::Syncing, LoopState::Retrying { message, .. } => RunState::Retrying { message: message.clone() }, - LoopState::Paused { reason, .. } => RunState::Paused { reason: reason.clone() }, + LoopState::Paused { reason, reason_code, .. } => RunState::Paused { reason: reason.clone(), reason_code: reason_code.clone() }, LoopState::Stopping | LoopState::Stopped => RunState::Stopping, LoopState::SignedOut => RunState::SignedOut, LoopState::PausedByUser => RunState::PausedByUser, @@ -565,7 +580,9 @@ impl Status { Phase::Idle => LoopState::Idle, Phase::Syncing => LoopState::Syncing, Phase::Retrying { next_try, message } => LoopState::Retrying { next_try_ms: self.wall(next_try), message }, - Phase::Paused { reason, next_check } => LoopState::Paused { reason, next_check_ms: next_check.map(|at| self.wall(at)) }, + Phase::Paused { reason, reason_code, next_check } => { + LoopState::Paused { reason, reason_code, next_check_ms: next_check.map(|at| self.wall(at)) } + } Phase::Stopping => LoopState::Stopping, } } @@ -672,7 +689,7 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou status.write_now(match data.load_account() { Ok(Some(_)) => LoopState::Idle, Ok(None) => LoopState::SignedOut, - Err(e) => LoopState::Paused { reason: e.to_string(), next_check_ms: None }, + Err(e) => LoopState::Paused { reason: e.to_string(), next_check_ms: None, reason_code: None }, }); let api = match LocalApi::start(data, control.clone()).await { Ok(api) => api, @@ -994,7 +1011,7 @@ pub async fn run_loop(data: &DataDir, options: RunOptions, mut stop: mpsc::Unbou None => stop_open = false, }, command = commands.recv(), if commands_open => match command { - Some(Command::Login(_, reply) | Command::Join(_, reply)) => { + Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::LoginSecondFactor(_, reply)) => { let _ = reply.send(Err(crate::cli::already_signed_in(&account))); } // Signed out on the server first (an unreachable server changes nothing, and syncing goes on); then @@ -1164,7 +1181,7 @@ async fn wait_needing_you( let _ = out.flush(); let mut commands_open = true; loop { - status.write(LoopState::Paused { reason: reason.to_owned(), next_check_ms: None }, false, false); + status.write(LoopState::Paused { reason: reason.to_owned(), next_check_ms: None, reason_code: None }, false, false); let wake = Duration::from_millis(status.next_beat.saturating_sub(status.now())); tokio::select! { command = commands.recv(), if commands_open => match command { @@ -1178,7 +1195,7 @@ async fn wait_needing_you( let _ = reply.send(Err(e)); } }, - Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::Pause(reply) | Command::Resume(reply)) => { + Some(Command::Login(_, reply) | Command::Join(_, reply) | Command::LoginSecondFactor(_, reply) | Command::Pause(reply) | Command::Resume(reply)) => { let _ = reply.send(Err(AgentError::NeedsYou(reason.to_owned()))); } Some(Command::Quit(reply)) => { @@ -1227,6 +1244,20 @@ async fn wait_for_sign_in( return SignedOut::SignedIn; } } + Some(Command::LoginSecondFactor(request, reply)) => { + let args = LoginArgs { + url: request.server.clone(), + user: request.user.clone(), + folder: request.folder.as_ref().map(std::path::PathBuf::from), + device_name: request.device_name.clone(), + adopt: request.adopt, + }; + let factor = crate::http::Factor { totp_code: request.totp_code.clone(), recovery_code: request.recovery_code.clone() }; + let result = crate::cli::login_second_factor_locked(data, &args, &request.challenge, &factor).await; + if signed_in(result, "signed in", control, reply, out) { + return SignedOut::SignedIn; + } + } Some(Command::Join(request, reply)) => { let args = JoinArgs { join: request.join.clone(), @@ -1236,7 +1267,9 @@ async fn wait_for_sign_in( // The caller asks for it with `POST /v1/autostart` once joined. autostart: false, }; - let result = crate::cli::join_locked(data, &args, request.password.as_deref()).await; + let factor = (request.totp_code.is_some() || request.recovery_code.is_some()) + .then(|| crate::http::Factor { totp_code: request.totp_code.clone(), recovery_code: request.recovery_code.clone() }); + let result = crate::cli::join_locked(data, &args, request.password.as_deref(), factor.as_ref()).await; if signed_in(result, "joined", control, reply, out) { return SignedOut::SignedIn; } @@ -1358,7 +1391,18 @@ mod tests { assert!(matches!(pass_end(&Err(protocol), None), PassEnd::NeedsYou { recheck: Recheck::Slowly, .. })); // A full account: the admin may allow more at any time. let full = AgentError::Server(TransportError::QuotaExceeded { used: 10, limit: 10 }); - assert!(matches!(pass_end(&Err(full), None), PassEnd::NeedsYou { recheck: Recheck::Slowly, .. })); + assert!( + matches!(pass_end(&Err(full), None), PassEnd::NeedsYou { recheck: Recheck::Slowly, reason_code: Some(ref code), .. } if code == "quota_exceeded") + ); + // A full server: the admin may free space at any time, and the pause says it is the server, not the account. + let server_full = AgentError::Server(TransportError::ServerFull); + match pass_end(&Err(server_full), None) { + PassEnd::NeedsYou { reason, reason_code, recheck: Recheck::Slowly } => { + assert!(reason.contains("The server is full") && !reason.contains("account"), "{reason}"); + assert_eq!(reason_code.as_deref(), Some("server_full")); + } + other => panic!("{other:?}"), + } assert!(matches!(pass_end(&Err(AgentError::Revoked), None), PassEnd::NeedsYou { recheck: Recheck::Never, .. })); } @@ -1455,12 +1499,12 @@ mod tests { let next_check_ms = Some(at + jitter[(i + 1) % jitter.len()]); if i == 100 { // A pause in between is a real change: written at once, then left alone the same way. - status.write(LoopState::Paused { reason: "gone".into(), next_check_ms }, false, false); + status.write(LoopState::Paused { reason: "gone".into(), next_check_ms, reason_code: None }, false, false); assert!(file.exists(), "a new state is written at once"); std::fs::remove_file(&file).unwrap(); for _ in 0..50 { let next_check_ms = Some(at + jitter[i % jitter.len()]); - status.write(LoopState::Paused { reason: "gone".into(), next_check_ms }, false, false); + status.write(LoopState::Paused { reason: "gone".into(), next_check_ms, reason_code: None }, false, false); } assert!(!file.exists(), "an unchanged pause isn't written again before the heartbeat"); status.write(LoopState::Retrying { next_try_ms, message: "offline".into() }, false, false); diff --git a/apps/craft-cloud-agent/src/http.rs b/apps/craft-cloud-agent/src/http.rs index 16efff9f..21e27b45 100644 --- a/apps/craft-cloud-agent/src/http.rs +++ b/apps/craft-cloud-agent/src/http.rs @@ -35,8 +35,10 @@ use std::time::Duration; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use craft_cloud_proto::api::{ - CommitRequest, CommitResponse, DeviceInfo, ErrorBody, InfoResponse, JoinRequest, LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, - MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, PairingCreated, PairingStatus, SpaceResponse, codes, + AccountInfo, ChangePasswordRequest, CommitRequest, CommitResponse, DeviceInfo, ErrorBody, HistoryResponse, InfoResponse, JoinRequest, + LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, PairingCreated, PairingStatus, + RecoveryCodes, RenameDeviceRequest, RestoreRequest, RestoreResponse, RevokedOthers, SecondFactorRequest, SpaceResponse, TwoFactorChallenge, + TwoFactorConfirmRequest, TwoFactorProofRequest, TwoFactorSetupRequest, TwoFactorSetupResponse, TwoFactorStatus, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::local::MAX_MACHINE_BYTES; @@ -59,6 +61,8 @@ pub const MAX_DEVICES: usize = 1000; const MAX_CODE_BYTES: usize = 64; /// Longest server message shown to the user, in characters. const MAX_MESSAGE_CHARS: usize = 300; +/// The whole of an audit export, from connecting to the last byte (it is read as it comes, with the usual idle limit). +const EXPORT_TIMEOUT: Duration = Duration::from_secs(60 * 60); const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// No data at all for this long fails the request. const READ_TIMEOUT: Duration = Duration::from_secs(60); @@ -441,6 +445,8 @@ fn error_for(status: StatusCode, body: Option) -> TransportError { (Some(codes::QUOTA_EXCEEDED), Some(ErrorBody { quota: Some(quota), .. })) => { return TransportError::QuotaExceeded { used: quota.used, limit: quota.limit }; } + // By the code alone: the numbers in the body are only for the admin. + (Some(codes::SERVER_FULL), _) => return TransportError::ServerFull, _ => {} } match status { @@ -492,6 +498,25 @@ pub fn check_compatible(info: &InfoResponse) -> Result<(), TransportError> { Ok(()) } +/// The most bytes the server takes of a device's `platform` or `app_version`. +const MAX_DEVICE_DETAIL_BYTES: usize = 32; + +/// What this agent says about itself when it signs in or joins: `agent`, the operating system and the crate's +/// version, for the account's device list (display only). The server takes at most 32 bytes without control +/// characters per field, so a longer value is cut at a character boundary rather than refused. +fn device_details() -> (Option, Option, Option) { + fn limited(text: &str) -> String { + text.chars() + .filter(|c| !c.is_control()) + .scan(0usize, |len, c| { + *len += c.len_utf8(); + (*len <= MAX_DEVICE_DETAIL_BYTES).then_some(c) + }) + .collect() + } + (Some("agent".to_owned()), Some(limited(std::env::consts::OS)), Some(limited(env!("CARGO_PKG_VERSION")))) +} + /// Signs in and creates a device. `Unauthorized` means a wrong username or password. pub async fn login(server: &ServerUrl, username: &str, password: &str, device_name: &str) -> Result { login_over(&Reqwest::new(server)?, server, username, password, device_name).await @@ -505,11 +530,178 @@ pub async fn login_over( password: &str, device_name: &str, ) -> Result { - let body = LoginRequest { username: username.to_owned(), password: password.to_owned(), device_name: device_name.to_owned() }; + let (client_kind, platform, app_version) = device_details(); + let body = LoginRequest { + username: username.to_owned(), + password: password.to_owned(), + device_name: device_name.to_owned(), + client_kind, + platform, + app_version, + }; let req = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body)?; json(connection.send(req).await?, MAX_SMALL_JSON_BYTES).await } +/// A server's refusal of a call that the agent keeps whole (its status, code and message, the challenge of +/// `two_factor_required`, the `Retry-After` of a lock), unlike [`TransportError`], which sync relies on and which has no +/// room for them. Only the calls that need it return this ([`CallError`]); the sync paths, [`error_for`] and +/// `TransportError` are as they were. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Refusal { + pub status: u16, + pub error: String, + pub message: String, + pub two_factor: Option, + pub retry_after_secs: Option, +} + +/// Why a call that returns a [`Refusal`] failed: the server refused it (a wrong password or code, a conflict, a lock: never a +/// reason to think this device was revoked), or it failed as any call does (offline, a 401 on a signed-in call, a 404). +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum CallError { + #[error("{}", .0.message)] + Refused(Refusal), + #[error(transparent)] + Transport(TransportError), +} + +impl From for CallError { + fn from(e: TransportError) -> Self { + CallError::Transport(e) + } +} + +/// The server's `Retry-After` in seconds, cut to a sane size. +fn retry_after_of(headers: &HeaderMap) -> Option { + headers.get(RETRY_AFTER).and_then(|v| v.to_str().ok()).and_then(|s| s.trim().parse::().ok()).map(|secs| secs.min(24 * 60 * 60)) +} + +/// A failed answer as a [`CallError`]: a 400, 403, 409, 422 or 429 with an error body, a 401 that is `two_factor_required` or +/// `two_factor_invalid` (the not-signed-in routes' answers; on a signed-in route a 401 is only ever a revoked device) and a +/// 503 `two_factor_unavailable` are a [`Refusal`]; everything else is what [`error_for`] makes of it. +async fn failure(resp: Answer) -> CallError { + let status = resp.status; + let retry = retry_after_of(&resp.headers); + let body = body_capped(resp, MAX_SMALL_JSON_BYTES).await.ok().and_then(|b| serde_json::from_slice::(&b).ok()); + if let Some(b) = &body { + let refused = match status.as_u16() { + 400 | 403 | 409 | 422 | 429 => true, + 401 => matches!(b.error.as_str(), codes::TWO_FACTOR_REQUIRED | codes::TWO_FACTOR_INVALID), + 503 => b.error == codes::TWO_FACTOR_UNAVAILABLE, + _ => false, + }; + if refused { + return CallError::Refused(Refusal { + status: status.as_u16(), + error: b.error.clone(), + message: clean(&b.message), + two_factor: b.two_factor.clone(), + retry_after_secs: retry, + }); + } + } + CallError::Transport(error_for(status, body)) +} + +/// The second factor a person gave: a code from the authenticator app, or a recovery code. Its `Debug` output hides both. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Factor { + pub totp_code: Option, + pub recovery_code: Option, +} + +impl Factor { + /// Reads what a person typed: six digits (spaces allowed) are a code, anything else a recovery code. + pub fn typed(text: &str) -> Self { + let trimmed = text.trim(); + let digits: String = trimmed.chars().filter(|c| !c.is_whitespace()).collect(); + if digits.len() == 6 && digits.bytes().all(|b| b.is_ascii_digit()) { + Self { totp_code: Some(digits), recovery_code: None } + } else { + Self { totp_code: None, recovery_code: Some(trimmed.to_owned()) } + } + } +} + +impl std::fmt::Debug for Factor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Factor") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// The first step of a sign-in: a device, or the challenge of an account that has two-factor sign-in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LoginStep { + Device(LoginResponse), + SecondFactor(TwoFactorChallenge), +} + +/// Signs in as [`login`] does, but tells the account's second factor from a wrong password: `Unauthorized` (a wrong username +/// or password) is [`CallError::Transport`], a challenge is [`LoginStep::SecondFactor`]. +pub async fn login_step(server: &ServerUrl, username: &str, password: &str, device_name: &str) -> Result { + login_step_over(&Reqwest::new(server)?, server, username, password, device_name).await +} + +/// [`login_step`] over another connection (a simulated network, in tests). +pub async fn login_step_over( + connection: &C, + server: &ServerUrl, + username: &str, + password: &str, + device_name: &str, +) -> Result { + let (client_kind, platform, app_version) = device_details(); + let body = LoginRequest { + username: username.to_owned(), + password: password.to_owned(), + device_name: device_name.to_owned(), + client_kind, + platform, + app_version, + }; + let req = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body)?; + let resp = connection.send(req).await?; + if resp.status.is_success() { + return Ok(LoginStep::Device(json(resp, MAX_SMALL_JSON_BYTES).await?)); + } + match failure(resp).await { + CallError::Refused(Refusal { error, two_factor, .. }) if error == codes::TWO_FACTOR_REQUIRED => match two_factor { + Some(challenge) => Ok(LoginStep::SecondFactor(challenge)), + None => Err(CallError::Transport(TransportError::Protocol("the server asked for a second factor without a challenge".into()))), + }, + other => Err(other), + } +} + +/// The second step of a sign-in: the `challenge` and the second factor make the device. A challenge that doesn't work any +/// more (unknown, expired, used, from before a password change) is `Transport(Unauthorized)` here, which is no revoked device +/// (the caller says the sign-in has to start again); a wrong code is `Refused` with `two_factor_invalid`, a lock with +/// `two_factor_locked` and its wait. +pub async fn second_factor(server: &ServerUrl, challenge: &str, factor: &Factor) -> Result { + second_factor_over(&Reqwest::new(server)?, server, challenge, factor).await +} + +/// [`second_factor`] over another connection (a simulated network, in tests). +pub async fn second_factor_over( + connection: &C, + server: &ServerUrl, + challenge: &str, + factor: &Factor, +) -> Result { + let body = + SecondFactorRequest { challenge: challenge.to_owned(), totp_code: factor.totp_code.clone(), recovery_code: factor.recovery_code.clone() }; + let req = request(Method::POST, server.endpoint("auth/login/second-factor"), None, 0).json(&body)?; + let resp = connection.send(req).await?; + if resp.status.is_success() { + return Ok(json(resp, MAX_SMALL_JSON_BYTES).await?); + } + Err(failure(resp).await) +} + /// Why a join ([`join`]) was refused. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum JoinError { @@ -525,13 +717,34 @@ pub enum JoinError { /// Too many failed joins from this address; the server's message. #[error("{0}")] RateLimited(String), + /// The account has two-factor sign-in: join again with a code from its authenticator app or a recovery code. The code + /// was not used up. + #[error("the account has two-factor sign-in: a code from its authenticator app is needed")] + TwoFactorRequired, + /// The code or recovery code was wrong (the join code still works); the server's message. + #[error("{0}")] + TwoFactorInvalid(String), + /// Too many wrong codes: the second factor is locked for `retry_after_secs`. + #[error("{message}")] + TwoFactorLocked { message: String, retry_after_secs: Option }, #[error(transparent)] Transport(TransportError), } /// Redeems a join code (normalised) for a new device named `device_name`; `password` for an invite to a new account. pub async fn join(server: &ServerUrl, code: &str, device_name: &str, password: Option<&str>) -> Result { - join_over(&Reqwest::new(server).map_err(JoinError::Transport)?, server, code, device_name, password).await + join_with_factor(server, code, device_name, password, None).await +} + +/// [`join`] with the second factor of an account that has two-factor sign-in. +pub async fn join_with_factor( + server: &ServerUrl, + code: &str, + device_name: &str, + password: Option<&str>, + factor: Option<&Factor>, +) -> Result { + join_over_with(&Reqwest::new(server).map_err(JoinError::Transport)?, server, code, device_name, password, factor).await } /// [`join`] over another connection (a simulated network, in tests). @@ -542,15 +755,43 @@ pub async fn join_over( device_name: &str, password: Option<&str>, ) -> Result { - let body = JoinRequest { code: code.to_owned(), device_name: device_name.to_owned(), password: password.map(str::to_owned) }; + join_over_with(connection, server, code, device_name, password, None).await +} + +/// [`join_over`] with a second factor. +pub async fn join_over_with( + connection: &C, + server: &ServerUrl, + code: &str, + device_name: &str, + password: Option<&str>, + factor: Option<&Factor>, +) -> Result { + let (client_kind, platform, app_version) = device_details(); + let body = JoinRequest { + code: code.to_owned(), + device_name: device_name.to_owned(), + password: password.map(str::to_owned), + totp_code: factor.and_then(|f| f.totp_code.clone()), + recovery_code: factor.and_then(|f| f.recovery_code.clone()), + client_kind, + platform, + app_version, + }; let req = request(Method::POST, server.endpoint("join"), None, 0).json(&body).map_err(JoinError::Transport)?; let resp = connection.send(req).await.map_err(JoinError::Transport)?; if resp.status.is_success() { return json(resp, MAX_SMALL_JSON_BYTES).await.map_err(JoinError::Transport); } let status = resp.status; + let retry = retry_after_of(&resp.headers); let body = body_capped(resp, MAX_SMALL_JSON_BYTES).await.ok().and_then(|b| serde_json::from_slice::(&b).ok()); Err(match body.as_ref().map(|b| b.error.as_str()) { + Some(codes::TWO_FACTOR_REQUIRED) => JoinError::TwoFactorRequired, + Some(codes::TWO_FACTOR_INVALID) => JoinError::TwoFactorInvalid(body.as_ref().map(|b| clean(&b.message)).unwrap_or_default()), + Some(codes::TWO_FACTOR_LOCKED) => { + JoinError::TwoFactorLocked { message: body.as_ref().map(|b| clean(&b.message)).unwrap_or_default(), retry_after_secs: retry } + } Some(codes::INVALID_CODE) => JoinError::InvalidCode, Some(codes::PASSWORD_REQUIRED) => JoinError::PasswordRequired, Some(codes::WEAK_PASSWORD) => JoinError::WeakPassword(body.as_ref().map(|b| clean(&b.message)).unwrap_or_default()), @@ -582,6 +823,28 @@ pub struct HttpTransport { auth: HeaderValue, } +/// A history or restore refusal, retaining the server's HTTP status and stable error code. +#[derive(Debug)] +pub enum HistoryCallError { + Transport(TransportError), + Refused { status: StatusCode, body: Box }, +} + +async fn history_answer(answer: Answer) -> Result { + if !answer.status.is_success() { + let status = answer.status; + let body = body_capped(answer, MAX_SMALL_JSON_BYTES) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_else(|| ErrorBody { error: codes::INTERNAL.into(), message: status.to_string(), ..ErrorBody::default() }); + return Err(HistoryCallError::Refused { status, body: Box::new(body) }); + } + let bytes = body_capped(answer, MAX_SMALL_JSON_BYTES).await.map_err(HistoryCallError::Transport)?; + serde_json::from_slice(&bytes) + .map_err(|e| HistoryCallError::Transport(TransportError::Protocol(format!("the server's answer doesn't parse: {e}")))) +} + impl HttpTransport { pub fn new(server: ServerUrl, token: &DeviceToken) -> Result { Self::over(Reqwest::new(&server)?, server, token) @@ -598,6 +861,73 @@ impl HttpTransport { &self.server } + /// The server's newest-first history for one file node. + pub async fn file_history(&self, node: NodeId, cursor: Option<&str>, limit: Option) -> Result { + let mut url = self.server.endpoint(&format!("nodes/{node}/history")); + { + let mut query = url.query_pairs_mut(); + if let Some(cursor) = cursor { + query.append_pair("cursor", cursor); + } + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + } + let req = request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Makes a prior version the file's content as a new version. + pub async fn restore_file(&self, node: NodeId, version: VersionId, expected_version: VersionId) -> Result { + let req = self + .request(Method::POST, &format!("nodes/{node}/restore"), MAX_SMALL_JSON_BYTES) + .json(&RestoreRequest { version, expected_version }) + .map_err(HistoryCallError::Transport)?; + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// The server's newest-first trash page. + pub async fn trash(&self, cursor: Option<&str>, limit: Option) -> Result { + let mut url = self.server.endpoint("trash"); + { + let mut query = url.query_pairs_mut(); + if let Some(cursor) = cursor { + query.append_pair("cursor", cursor); + } + if let Some(limit) = limit { + query.append_pair("limit", &limit.to_string()); + } + } + let req = request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Restores a deleted node from the server's trash. + pub async fn restore_from_trash(&self, node: NodeId) -> Result { + let req = self.request(Method::POST, &format!("trash/{node}/restore"), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + history_answer(answer).await + } + + /// Marks a deleted node out of the server's trash. This does not free stored data yet. + pub async fn remove_from_trash(&self, node: NodeId) -> Result<(), HistoryCallError> { + let req = self.request(Method::DELETE, &format!("trash/{node}"), MAX_SMALL_JSON_BYTES); + let answer = self.send(req).await.map_err(HistoryCallError::Transport)?; + if answer.status.is_success() { + return Ok(()); + } + let status = answer.status; + let body = body_capped(answer, MAX_SMALL_JSON_BYTES) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_else(|| ErrorBody { error: codes::INTERNAL.into(), message: status.to_string(), ..ErrorBody::default() }); + Err(HistoryCallError::Refused { status, body: Box::new(body) }) + } + fn request(&self, method: Method, path: &str, max_body: usize) -> Request { request(method, self.server.endpoint(path), Some(&self.auth), max_body) } @@ -660,6 +990,67 @@ impl HttpTransport { } } + /// A signed-in call whose refusals the agent keeps whole. + async fn account_call(&self, req: Request) -> Result { + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(json(resp, MAX_SMALL_JSON_BYTES).await?); + } + Err(failure(resp).await) + } + + /// [`HttpTransport::account_call`] for an answer without a body. + async fn account_unit(&self, req: Request) -> Result<(), CallError> { + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(()); + } + Err(failure(resp).await) + } + + /// Whether the account has two-factor sign-in on (`GET /account/two-factor`). + pub async fn two_factor_status(&self) -> Result { + self.account_call(self.request(Method::GET, "account/two-factor", 0)).await + } + + /// Starts a set-up with the account's password: the new secret, shown once. + pub async fn two_factor_setup(&self, password: &str) -> Result { + let req = self.request(Method::POST, "account/two-factor/setup", 0).json(&TwoFactorSetupRequest { password: password.to_owned() })?; + self.account_call(req).await + } + + /// Turns it on with the first code: the recovery codes, shown once. + pub async fn two_factor_confirm(&self, totp_code: &str) -> Result { + let req = self.request(Method::POST, "account/two-factor/confirm", 0).json(&TwoFactorConfirmRequest { totp_code: totp_code.to_owned() })?; + self.account_call(req).await + } + + /// Turns it off: the password and a code or recovery code. + pub async fn two_factor_disable(&self, body: &TwoFactorProofRequest) -> Result<(), CallError> { + self.account_unit(self.request(Method::POST, "account/two-factor/disable", 0).json(body)?).await + } + + /// New recovery codes: the password and a code from the authenticator app. + pub async fn two_factor_recovery_codes(&self, body: &TwoFactorProofRequest) -> Result { + self.account_call(self.request(Method::POST, "account/two-factor/recovery-codes", 0).json(body)?).await + } + + /// Changes the account's password. + pub async fn change_password(&self, body: &ChangePasswordRequest) -> Result<(), CallError> { + self.account_unit(self.request(Method::POST, "account/password", 0).json(body)?).await + } + + /// Signs every other device of the account out. + pub async fn revoke_other_devices(&self) -> Result { + self.account_call(self.request(Method::POST, "devices/revoke-others", 0)).await + } + + /// Renames one of the account's devices. + pub async fn rename_device(&self, device: DeviceId, name: &str) -> Result { + let req = self.request(Method::PATCH, &format!("devices/{device}"), 0).json(&RenameDeviceRequest { name: name.to_owned() })?; + self.account_call(req).await + } + /// Revokes a device of this account (this one, to sign out). pub async fn revoke(&self, device: DeviceId) -> Result<(), TransportError> { self.no_content(self.request(Method::DELETE, &format!("devices/{device}"), 0)).await @@ -674,6 +1065,11 @@ impl HttpTransport { Ok(list) } + /// The account's stored bytes and labeled estimates, fetched without caching. + pub async fn usage(&self) -> Result { + json(self.send(self.request(Method::GET, "account/usage", MAX_SMALL_JSON_BYTES)).await?, MAX_SMALL_JSON_BYTES).await + } + /// A new add-a-computer code from this device (replacing its open one). pub async fn create_pairing(&self) -> Result { let created: PairingCreated = json(self.send(self.request(Method::POST, "pairings", 0)).await?, MAX_SMALL_JSON_BYTES).await?; @@ -696,6 +1092,51 @@ impl HttpTransport { pub async fn delete_pairing(&self, id: Uuid) -> Result<(), TransportError> { self.no_content(self.request(Method::DELETE, &format!("pairings/{id}"), 0)).await } + + /// The signed-in account (`GET /account`): its role and whether it has two-factor sign-in. + pub async fn account_info(&self) -> Result { + self.account_call(self.request(Method::GET, "account", MAX_SMALL_JSON_BYTES)).await + } + + /// The audit export (`GET /admin/audit/export?`) as the server sends it: the good answer, whose body is read as it + /// comes ([`AnswerBody`]), never held whole; a refusal is kept whole. The whole-request time is long (a big export over a + /// slow link), the idle time is the usual minute. + pub async fn admin_export_stream(&self, query: &[(&str, &str)]) -> Result { + let mut url = self.server.endpoint("admin/audit/export"); + if !query.is_empty() { + url.query_pairs_mut().extend_pairs(query); + } + let mut req = request(Method::GET, url, Some(&self.auth), 0); + req.timeout = EXPORT_TIMEOUT; + let resp = self.send(req).await?; + if resp.status.is_success() { + return Ok(resp); + } + Err(failure(resp).await) + } + + /// `GET /api/v1/?` of the admin API (or the account's activity), read as `T`; a refusal is kept whole. + pub async fn admin_get(&self, path: &str, query: &[(&str, &str)]) -> Result { + let mut url = self.server.endpoint(path); + if !query.is_empty() { + url.query_pairs_mut().extend_pairs(query); + } + self.account_call(request(Method::GET, url, Some(&self.auth), MAX_SMALL_JSON_BYTES)).await + } + + /// `method /api/v1/` of the admin API with `body` (the confirmation among it), answered with a body read as `T`. + pub async fn admin_json(&self, method: Method, path: &str, body: &B) -> Result { + self.account_call(self.request(method, path, MAX_SMALL_JSON_BYTES).json(body)?).await + } + + /// [`HttpTransport::admin_json`] for a call answered without a body (`204`, or `202`): the status it succeeded with. + pub async fn admin_empty(&self, method: Method, path: &str, body: &B) -> Result { + let resp = self.send(self.request(method, path, MAX_SMALL_JSON_BYTES).json(body)?).await?; + if resp.status.is_success() { + return Ok(resp.status); + } + Err(failure(resp).await) + } } #[async_trait] @@ -763,6 +1204,26 @@ mod tests { use super::*; use craft_cloud_proto::{Conflict, ConflictKind}; + #[test] + fn a_second_factor_is_read_as_typed_and_hidden_in_debug() { + assert_eq!(Factor::typed(" 123 456 "), Factor { totp_code: Some("123456".into()), recovery_code: None }); + assert_eq!(Factor::typed("abcd-efgh-jkmn-pqrs"), Factor { totp_code: None, recovery_code: Some("abcd-efgh-jkmn-pqrs".into()) }); + assert_eq!(Factor::typed("12345").recovery_code.as_deref(), Some("12345"), "five digits are not a code"); + let shown = format!("{:?}", Factor::typed("123456")); + assert!(!shown.contains("123456"), "{shown}"); + let shown = format!( + "{:?}", + Refusal { + status: 401, + error: "x".into(), + message: "m".into(), + two_factor: Some(TwoFactorChallenge { challenge: "chal-secret".into(), expires_ms: 1 }), + retry_after_secs: None + } + ); + assert!(!shown.contains("chal-secret"), "{shown}"); + } + #[test] fn https_is_required_except_on_this_machine() { assert!(ServerUrl::parse("https://cloud.example.com").is_ok()); @@ -811,7 +1272,12 @@ mod tests { #[test] fn requests_keep_bodies_and_tokens_out_of_debug_output() { let server = ServerUrl::parse("http://127.0.0.1:1").unwrap(); - let body = LoginRequest { username: "ana".into(), password: "correct horse battery".into(), device_name: "laptop".into() }; + let body = LoginRequest { + username: "ana".into(), + password: "correct horse battery".into(), + device_name: "laptop".into(), + ..LoginRequest::default() + }; let login = request(Method::POST, server.endpoint("auth/login"), None, 0).json(&body).unwrap(); let shown = format!("{login:?}"); assert!(!shown.contains("correct horse") && !shown.contains("ana"), "{shown}"); @@ -823,6 +1289,22 @@ mod tests { assert!(!shown.contains("secret-token") && shown.contains("authorization"), "{shown}"); } + /// An admin change carries the admin's password in its body, to the server only: no `Debug` output of the request + /// or of the bodies shows it. + #[test] + fn admin_confirmations_stay_out_of_debug_output() { + use craft_cloud_proto::api::{AdminConfirm, AdminCreateUser, AdminUpdateUser}; + let server = ServerUrl::parse("http://127.0.0.1:1").unwrap(); + let confirm = AdminConfirm { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), ..AdminConfirm::default() }; + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + let update = AdminUpdateUser { password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + let sent = request(Method::POST, server.endpoint("admin/users/x/disable"), None, 0).json(&confirm).unwrap(); + assert!(sent.body.as_deref().is_some_and(|b| String::from_utf8_lossy(b).contains("admin-secret-pw")), "the body carries it"); + let shown = format!("{sent:?} {confirm:?} {create:?} {update:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + } + fn body(code: &str) -> ErrorBody { ErrorBody { error: code.into(), message: "m".into(), ..ErrorBody::default() } } @@ -839,6 +1321,13 @@ mod tests { assert_eq!(error_for(StatusCode::INSUFFICIENT_STORAGE, Some(full)), TransportError::QuotaExceeded { used: 9, limit: 10 }); // Without the numbers it is only a server error: tried again later. assert!(matches!(error_for(StatusCode::INSUFFICIENT_STORAGE, Some(body(codes::QUOTA_EXCEEDED))), TransportError::Unavailable(_))); + // The server itself is full: its own error, with or without the numbers. + assert_eq!(error_for(StatusCode::INSUFFICIENT_STORAGE, Some(body(codes::SERVER_FULL))), TransportError::ServerFull); + let room = craft_cloud_proto::api::ServerRoom { free_bytes: Some(1), total_bytes: Some(2), capacity_bytes: None }; + assert_eq!( + error_for(StatusCode::INSUFFICIENT_STORAGE, Some(ErrorBody { server: Some(room), ..body(codes::SERVER_FULL) })), + TransportError::ServerFull + ); assert_eq!(error_for(StatusCode::UNAUTHORIZED, Some(body(codes::UNAUTHORIZED))), TransportError::Unauthorized); assert_eq!(error_for(StatusCode::NOT_FOUND, None), TransportError::NotFound); assert!(matches!(error_for(StatusCode::TOO_MANY_REQUESTS, Some(body(codes::RATE_LIMITED))), TransportError::Unavailable(_))); diff --git a/apps/craft-cloud-agent/src/lib.rs b/apps/craft-cloud-agent/src/lib.rs index ace6aa3e..8742bfb4 100644 --- a/apps/craft-cloud-agent/src/lib.rs +++ b/apps/craft-cloud-agent/src/lib.rs @@ -11,6 +11,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod account; +pub mod admin; pub mod autostart; pub mod cli; pub mod daemon; diff --git a/apps/craft-cloud-agent/src/live/copy.rs b/apps/craft-cloud-agent/src/live/copy.rs index cd8cc722..205558f8 100644 --- a/apps/craft-cloud-agent/src/live/copy.rs +++ b/apps/craft-cloud-agent/src/live/copy.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use craft_cloud_proto::live::MAX_UPDATE_BYTES; -use loro::{ExportMode, IdSpan, LoroDoc, VersionVector}; +use loro::{ExportMode, IdSpan, LoroDoc}; /// Why an update or a stored document was refused. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -271,17 +271,39 @@ impl LiveCopy { } /// A snapshot of the document as it was at `version` (its history up to there, and the state then; nothing - /// later). `Invalid` if this copy doesn't hold every operation `version` names. + /// later). `Invalid` if this copy doesn't hold every operation `version` names, if those operations need others + /// past `version`, or if the document is shallow (Loro can't take a historical snapshot of one). + /// + /// Built by replaying the history up to `version` into a fresh document and taking its snapshot: Loro's own + /// `SnapshotAt` checks the document out at `version`, which on a long text history can need many GiB. pub fn snapshot_at(&self, version: &Version) -> Result, UpdateError> { - self.guard(|doc| { + let updates = self.guard(|doc| { let have = doc.oplog_vv(); if version.iter().any(|(peer, end)| *end > 0 && have.get(peer).is_none_or(|h| h < end)) { return Err(UpdateError::Invalid("the copy doesn't hold that version")); } - let vv: VersionVector = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| (*peer, *end)).collect(); - let frontiers = doc.vv_to_frontiers(&vv); - doc.export(ExportMode::SnapshotAt { version: std::borrow::Cow::Owned(frontiers) }).map_err(|e| UpdateError::Malformed(e.to_string())) - }) + if doc.is_shallow() { + return Err(UpdateError::Invalid("a shallow document has no historical snapshot")); + } + let spans: Vec<_> = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| IdSpan::new(*peer, 0, *end)).collect(); + doc.export(ExportMode::updates_in_range(spans)).map_err(|e| UpdateError::Malformed(e.to_string())) + })?; + let copy = Self::new(self.peer()); + let result = (|| { + copy.import_any_size(&updates)?; + let want: Version = version.iter().filter(|(_, end)| **end > 0).map(|(peer, end)| (*peer, *end)).collect(); + if copy.version()? != want { + return Err(UpdateError::Invalid("the version is not causally complete")); + } + copy.snapshot() + })(); + if copy.is_poisoned() { + // Dropping a poisoned Loro document panics again; this one is only a scratch copy, so it is let go of. + // A panic in it says nothing about this document, so it isn't reported as one. + std::mem::forget(copy); + return Err(UpdateError::Malformed("the historical snapshot couldn't be built".into())); + } + result } /// The changes (Loro commits: peer, first and end counter) of this document that `has` doesn't wholly hold, in @@ -645,4 +667,46 @@ mod tests { assert!(log.at(3).is_some()); assert_eq!(version_of(&snapshot).unwrap().get(&1), Some(&8)); } + + #[test] + fn historical_snapshot_preserves_large_text_and_accepts_later_updates() { + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + let copy = LiveCopy::new(9); + let text = "a".repeat(200_000); + let first = text_update(&source, 0, &text); + copy.import(&first).unwrap(); + let version = copy.version().unwrap(); + let later = text_update(&source, text.len(), "later"); + copy.import(&later).unwrap(); + let snapshot = copy.snapshot_at(&version).unwrap(); + let fresh = LiveCopy::load(&snapshot, 5).unwrap(); + assert_eq!(fresh.doc.get_text("text").to_string(), text); + fresh.import(&later).unwrap(); + assert_eq!(fresh.doc.get_text("text").to_string(), format!("{text}later")); + } + + /// A version whose operations depend on others past it (peer 2 typed after more of peer 1 than the version + /// holds) has no snapshot of its own: refused, never silently widened. + #[test] + fn a_historical_snapshot_of_a_version_that_needs_later_operations_is_refused() { + let (x, y) = (LoroDoc::new(), LoroDoc::new()); + x.set_peer_id(1).unwrap(); + y.set_peer_id(2).unwrap(); + let first = text_update(&x, 0, "ab"); + let second = text_update(&x, 2, "cd"); + y.import(&first).unwrap(); + y.import(&second).unwrap(); + let from_y = text_update(&y, 4, "e"); + let copy = LiveCopy::new(9); + for update in [&first, &second, &from_y] { + copy.import(update).unwrap(); + } + let partial: Version = [(1, 2), (2, 1)].into_iter().collect(); + assert!(matches!(copy.snapshot_at(&partial), Err(UpdateError::Invalid(_)))); + assert!(!copy.is_poisoned()); + let whole = copy.version().unwrap(); + let snapshot = copy.snapshot_at(&whole).unwrap(); + assert_eq!(LiveCopy::load(&snapshot, 5).unwrap().doc.get_text("text").to_string(), "abcde"); + } } diff --git a/apps/craft-cloud-agent/src/local_api.rs b/apps/craft-cloud-agent/src/local_api.rs index 3e2f8bb5..5a3c4bc6 100644 --- a/apps/craft-cloud-agent/src/local_api.rs +++ b/apps/craft-cloud-agent/src/local_api.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; use axum::extract::rejection::{JsonRejection, PathRejection}; -use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::extract::{DefaultBodyLimit, Path, Query, Request, State}; use axum::http::StatusCode; use axum::http::header::{AUTHORIZATION, HOST, ORIGIN}; use axum::middleware::{self, Next}; @@ -25,11 +25,15 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; use craft_cloud_proto::DeviceId; -use craft_cloud_proto::api::{DeviceInfo, ErrorBody, PairingStatus, codes as api_codes}; +use craft_cloud_proto::api::{ + AccountUsage, DeviceInfo, HistoryResponse, PairingStatus, RecoveryCodes, RestoreResponse, RevokedOthers, TwoFactorChallenge, + TwoFactorSetupResponse, TwoFactorStatus, codes as api_codes, +}; use craft_cloud_proto::join::{Code, JoinString, MAX_JOIN_BYTES, PAIRING_CHARS}; use craft_cloud_proto::local::{ - AutostartRequest, JoinRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_SCHEMA, LocalApiFile, LoginRequest, LogoutRequest, MAX_LOCAL_JSON_BYTES, - PairingInfo, RunState, Status, codes, paths, + AutostartRequest, ChangePasswordRequest, JoinRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_SCHEMA, LocalApiFile, LoginRequest, LogoutRequest, + MAX_LOCAL_JSON_BYTES, PairingInfo, PasswordRequest, RenameDeviceRequest, RunState, SecondFactorLoginRequest, Status, TotpRequest, + TwoFactorProofRequest, codes, paths, }; use tokio::sync::{mpsc, oneshot}; @@ -46,6 +50,8 @@ use crate::live::app_blobs::{self, BlobError}; pub enum Command { Login(LoginRequest, oneshot::Sender>), Join(JoinRequest, oneshot::Sender>), + /// The second step of a sign-in that answered `two_factor_required`. + LoginSecondFactor(SecondFactorLoginRequest, oneshot::Sender>), Logout(LogoutRequest, oneshot::Sender>), /// Hold syncing and the live connection until [`Command::Resume`], also after a restart. Pause(oneshot::Sender>), @@ -191,23 +197,45 @@ impl Control { } } -/// An error answer: the HTTP status and the [`ErrorBody`]. +/// An error answer: the HTTP status, the typed local API code and message, and optional details such as a two-factor +/// challenge, a lock wait, or a trash-restore name conflict. #[derive(Debug)] pub struct LocalError { pub status: StatusCode, - pub code: &'static str, + pub code: String, pub message: String, + pub two_factor: Option, + pub retry_after_secs: Option, + pub name_taken: Option, } impl LocalError { - pub fn new(status: StatusCode, code: &'static str, message: impl Into) -> Self { - Self { status, code, message: message.into() } + pub fn new(status: StatusCode, code: impl Into, message: impl Into) -> Self { + Self { status, code: code.into(), message: message.into(), two_factor: None, retry_after_secs: None, name_taken: None } + } + + fn with_code(status: StatusCode, code: String, message: impl Into) -> Self { + Self { status, code, message: message.into(), two_factor: None, retry_after_secs: None, name_taken: None } } } impl IntoResponse for LocalError { fn into_response(self) -> Response { - (self.status, Json(ErrorBody { error: self.code.to_owned(), message: self.message, ..ErrorBody::default() })).into_response() + let extras = craft_cloud_proto::local::ErrorExtras { two_factor: self.two_factor, retry_after_secs: self.retry_after_secs }; + let mut body = serde_json::json!({ "error": self.code, "message": self.message }); + if let (Some(object), Ok(serde_json::Value::Object(more))) = (body.as_object_mut(), serde_json::to_value(&extras)) { + object.extend(more); + } + if let Some(name_taken) = self.name_taken + && let Some(object) = body.as_object_mut() + { + object.insert("name_taken".to_owned(), serde_json::Value::Bool(name_taken)); + } + let mut response = (self.status, Json(body)).into_response(); + if let Some(secs) = self.retry_after_secs { + response.headers_mut().insert(axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from(secs)); + } + response } } @@ -217,7 +245,33 @@ impl From for LocalError { // A setup that failed after the code was spent answers as what failed, with the whole message. if let AgentError::JoinSetup(inner) = e { let inner = LocalError::from(*inner); - return Self::new(inner.status, inner.code, message); + return Self::with_code(inner.status, inner.code, message); + } + // The second factor: a challenge, a wrong code, a lock and a server's refusal keep what they carry. None of them is a + // revoked device (that is `Revoked`, from a 401 on a signed-in call). + match &e { + AgentError::TwoFactorRequired(challenge) => { + let mut error = Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, message); + // `join` has no challenge to hand on: the same join goes again with the code. + error.two_factor = Some(challenge.clone()); + return error; + } + AgentError::TwoFactorInvalid(_) => return Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_INVALID, message), + AgentError::ChallengeGone => return Self::new(StatusCode::CONFLICT, codes::TWO_FACTOR_EXPIRED, message), + AgentError::TwoFactorLocked { retry_after_secs, .. } => { + let mut error = Self::new(StatusCode::TOO_MANY_REQUESTS, codes::TWO_FACTOR_LOCKED, message); + error.retry_after_secs = *retry_after_secs; + return error; + } + AgentError::Refused { status, code, retry_after_secs, .. } => { + // The server's own status and code (never a 401: that is `Revoked`). + let status = + StatusCode::from_u16(*status).ok().filter(|s| (s.is_client_error() || s.is_server_error()) && *s != StatusCode::UNAUTHORIZED); + let mut error = Self::new(status.unwrap_or(StatusCode::BAD_REQUEST), code.clone(), message); + error.retry_after_secs = *retry_after_secs; + return error; + } + _ => {} } let (status, code) = match &e { AgentError::WrongPassword => (StatusCode::UNAUTHORIZED, codes::WRONG_PASSWORD), @@ -337,24 +391,39 @@ impl Drop for LocalApi { } } -type Api = Arc; +pub(crate) type Api = Arc; fn router(state: Api) -> Router { Router::new() .route(paths::STATUS, get(status)) .route(paths::LOGIN, post(login)) + .route(paths::LOGIN_SECOND_FACTOR, post(login_second_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route(paths::LOGOUT, post(logout)) + .route(paths::TWO_FACTOR, get(two_factor_status)) + .route(paths::TWO_FACTOR_SETUP, post(two_factor_setup).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_CONFIRM, post(two_factor_confirm).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_DISABLE, post(two_factor_disable).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::TWO_FACTOR_RECOVERY_CODES, post(two_factor_recovery_codes).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::ACCOUNT_PASSWORD, post(change_password).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route(paths::DEVICES_REVOKE_OTHERS, post(revoke_other_devices)) .route(paths::RECENTS, get(crate::live::recents_socket)) .route(paths::DOCS, get(crate::live::docs::docs_socket)) .route(paths::JOIN, post(join)) .route(paths::PAIRINGS, post(create_pairing)) .route(&format!("{}/{{id}}", paths::PAIRINGS), get(pairing_status).delete(delete_pairing)) .route(paths::DEVICES, get(devices)) - .route(&format!("{}/{{id}}", paths::DEVICES), delete(remove_device)) + .route(paths::USAGE, get(usage)) + .route(&format!("{}/{{id}}", paths::DEVICES), delete(remove_device).patch(rename_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route(paths::AUTOSTART, post(set_autostart)) .route(paths::PAUSE, post(pause)) .route(paths::RESUME, post(resume)) .route(paths::QUIT, post(quit)) + .merge(crate::admin::routes()) + .route(paths::FILE_HISTORY, get(file_history)) + .route(paths::FILE_RESTORE, post(file_restore)) + .route(paths::TRASH, get(trash)) + .route(paths::TRASH_RESTORE, post(trash_restore)) + .route(paths::TRASH_REMOVE, post(trash_remove)) .route(paths::BLOBS, axum::routing::put(put_blob).layer(DefaultBodyLimit::max(MAX_BLOB_BYTES))) .route(&format!("{}/{{hash}}", paths::BLOBS), get(get_blob)) .fallback(|| async { LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such call") }) @@ -363,6 +432,139 @@ fn router(state: Api) -> Router { .with_state(state) } +#[derive(serde::Deserialize)] +struct HistoryQuery { + path: String, + cursor: Option, + limit: Option, +} + +fn file_node(state: &ApiState, path: &str) -> Result { + if path.is_empty() || path.starts_with('/') || path.contains('\\') || path.contains(':') || path.as_bytes().contains(&0) { + return Err(LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "path must be relative to the sync root")); + } + let names = path + .split('/') + .map(craft_cloud_proto::Name::new) + .collect::, _>>() + .map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "path contains an invalid name"))?; + let saved = + DataDir::new(&state.data).load_state().map_err(|e| LocalError::new(StatusCode::INTERNAL_SERVER_ERROR, api_codes::INTERNAL, e.to_string()))?; + let rel = craft_cloud_sync::RelPath::from_names(names); + let node = + saved.state.base_at(&rel).ok_or_else(|| LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no synced file at that path"))?; + if !matches!(saved.state.base.get(&node).map(|entry| &entry.kind), Some(craft_cloud_sync::BaseKind::File { .. })) { + return Err(LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "history is only available for files")); + } + Ok(node) +} + +fn history_error(error: crate::http::HistoryCallError) -> LocalError { + match error { + crate::http::HistoryCallError::Refused { status: StatusCode::UNAUTHORIZED, body } => { + LocalError::new(StatusCode::CONFLICT, codes::NEEDS_YOU, body.message) + } + crate::http::HistoryCallError::Refused { status, body } => { + let name_taken = body + .conflicts + .as_ref() + .map(|conflicts| conflicts.iter().any(|conflict| matches!(conflict.kind, craft_cloud_proto::ConflictKind::NameTaken { .. }))); + LocalError { status, code: body.error, message: body.message, name_taken, two_factor: None, retry_after_secs: None } + } + crate::http::HistoryCallError::Transport(craft_cloud_sync::TransportError::NotFound) => { + LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such file or version") + } + crate::http::HistoryCallError::Transport(craft_cloud_sync::TransportError::Unauthorized) => { + LocalError::new(StatusCode::CONFLICT, codes::NEEDS_YOU, "sign in again to continue") + } + crate::http::HistoryCallError::Transport(error) => AgentError::from(error).into(), + } +} + +async fn file_history( + State(state): State, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result, LocalError> { + let Query(query) = query.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let node = file_node(&state, &query.path)?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.file_history(node, query.cursor.as_deref(), query.limit).await.map_err(history_error)?)) +} + +async fn file_restore( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let node = file_node(&state, &request.path)?; + let (_, transport) = server_api(&state)?; + let answer = transport.restore_file(node, request.version, request.expected_version).await.map_err(history_error)?; + state.control.request_pass(); + Ok(Json(answer)) +} + +#[derive(serde::Deserialize)] +struct TrashQuery { + cursor: Option, + limit: Option, +} + +fn trash_path_hint(saved: Option<&crate::account::SavedState>, parent: craft_cloud_proto::NodeId, name: &str) -> Option { + let saved = saved?; + let parent_path = saved.state.base_path(parent)?; + if parent_path.is_root() { Some(name.to_owned()) } else { Some(format!("{parent_path}/{name}")) } +} + +async fn trash( + State(state): State, + query: Result, axum::extract::rejection::QueryRejection>, +) -> Result, LocalError> { + let Query(query) = query.map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text()))?; + let (_, transport) = server_api(&state)?; + let response = transport.trash(query.cursor.as_deref(), query.limit).await.map_err(history_error)?; + let saved = DataDir::new(&state.data).load_state().ok(); + Ok(Json(craft_cloud_proto::local::TrashResponse { + items: response + .items + .into_iter() + .map(|item| craft_cloud_proto::local::TrashItem { + path: trash_path_hint(saved.as_ref(), item.parent, &item.name), + id: item.id, + parent: item.parent, + name: item.name, + is_dir: item.is_dir, + size: item.size, + deleted_ms: item.deleted_ms, + purge_after_ms: item.purge_after_ms, + }) + .collect(), + next: response.next, + })) +} + +async fn trash_restore( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let id = request.id.parse().map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "id must be a node ID"))?; + let (_, transport) = server_api(&state)?; + let answer = transport.restore_from_trash(id).await.map_err(history_error)?; + state.control.request_pass(); + Ok(Json(answer)) +} + +async fn trash_remove( + State(state): State, + body: Result, JsonRejection>, +) -> Result { + let Json(request) = body?; + let id = request.id.parse().map_err(|_| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, "id must be a node ID"))?; + let (_, transport) = server_api(&state)?; + transport.remove_from_trash(id).await.map_err(history_error)?; + Ok(StatusCode::NO_CONTENT) +} + /// Equal strings, compared in time that doesn't depend on where they differ. fn same_secret(a: &str, b: &str) -> bool { a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 @@ -408,6 +610,101 @@ async fn login(State(state): State, body: Result, JsonRe ask(&state, |reply| Command::Login(request, reply)).await } +/// Most bytes of the small bodies (a password, a code, a name): far less than a join string's megabyte. +const TINY_JSON_BYTES: usize = 4 * 1024; + +async fn login_second_factor( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + ask(&state, |reply| Command::LoginSecondFactor(request, reply)).await +} + +/// A call of the signed-in account to the server, its refusals kept whole: never mistaken for a revoked device. +fn account_call_error(e: crate::http::CallError) -> LocalError { + AgentError::from(e).into() +} + +async fn two_factor_status(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_status().await.map_err(account_call_error)?)) +} + +async fn two_factor_setup( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_setup(&request.password).await.map_err(account_call_error)?)) +} + +async fn two_factor_confirm(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + Ok(Json(transport.two_factor_confirm(&request.totp_code).await.map_err(account_call_error)?)) +} + +async fn two_factor_disable(State(state): State, body: Result, JsonRejection>) -> Result { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + transport + .two_factor_disable(&craft_cloud_proto::api::TwoFactorProofRequest { + password: request.password, + totp_code: request.totp_code, + recovery_code: request.recovery_code, + }) + .await + .map_err(account_call_error)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn two_factor_recovery_codes( + State(state): State, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let sent = craft_cloud_proto::api::TwoFactorProofRequest { + password: request.password, + totp_code: request.totp_code, + recovery_code: request.recovery_code, + }; + Ok(Json(transport.two_factor_recovery_codes(&sent).await.map_err(account_call_error)?)) +} + +async fn change_password(State(state): State, body: Result, JsonRejection>) -> Result { + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let sent = craft_cloud_proto::api::ChangePasswordRequest { current_password: request.current_password, new_password: request.new_password }; + transport.change_password(&sent).await.map_err(account_call_error)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn revoke_other_devices(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.revoke_other_devices().await.map_err(account_call_error)?)) +} + +async fn rename_device( + State(state): State, + id: Result, PathRejection>, + body: Result, JsonRejection>, +) -> Result, LocalError> { + let id = path_id(id)?; + let Json(request) = body?; + let (_, transport) = server_api(&state)?; + let renamed = transport.rename_device(id, &request.name).await.map_err(|e| match e { + // A device that isn't there (or isn't ours, or is signed out): the server's `not_found`. + crate::http::CallError::Transport(craft_cloud_sync::TransportError::NotFound) => { + LocalError::new(StatusCode::NOT_FOUND, api_codes::NOT_FOUND, "no such computer in this cloud") + } + other => account_call_error(other), + })?; + Ok(Json(renamed)) +} + async fn logout(State(state): State, body: Result, JsonRejection>) -> Result, LocalError> { let Json(request) = body?; ask(&state, |reply| Command::Logout(request, reply)).await @@ -475,14 +772,14 @@ async fn join(State(state): State, body: Result, JsonReje } /// The signed-in account's server API, for the calls the local API makes itself (they touch nothing here). -fn server_api(state: &ApiState) -> Result<(Account, HttpTransport), LocalError> { +pub(crate) fn server_api(state: &ApiState) -> Result<(Account, HttpTransport), LocalError> { let account = state.control.account().ok_or(AgentError::NotSignedIn)?; let transport = HttpTransport::new(account.server.clone(), &account.token).map_err(AgentError::from)?; Ok((account, transport)) } /// A path's ID, or `400 invalid`. -fn path_id(id: Result, PathRejection>) -> Result { +pub(crate) fn path_id(id: Result, PathRejection>) -> Result { id.map(|Path(id)| id).map_err(|e| LocalError::new(StatusCode::BAD_REQUEST, api_codes::INVALID, e.body_text())) } @@ -526,6 +823,12 @@ async fn devices(State(state): State) -> Result>, Loca Ok(Json(transport.devices().await.map_err(AgentError::from)?)) } +/// `GET /v1/usage`: asks the signed-in account's server each time; nothing is cached. +async fn usage(State(state): State) -> Result, LocalError> { + let (_, transport) = server_api(&state)?; + Ok(Json(transport.usage().await.map_err(AgentError::from)?)) +} + /// Removes (revokes) another computer of the account; answered with the list as it is then. This computer can't be /// removed this way: that is signing out. async fn remove_device(State(state): State, id: Result, PathRejection>) -> Result>, LocalError> { diff --git a/apps/craft-cloud-agent/src/schedule.rs b/apps/craft-cloud-agent/src/schedule.rs index e8a4e13b..a407817d 100644 --- a/apps/craft-cloud-agent/src/schedule.rs +++ b/apps/craft-cloud-agent/src/schedule.rs @@ -101,7 +101,7 @@ pub enum PassEnd { /// It failed for a reason that may pass by itself (exit code 1). Failed(String), /// It stopped for something that needs the user (exit code 3). - NeedsYou { reason: String, recheck: Recheck }, + NeedsYou { reason: String, reason_code: Option, recheck: Recheck }, } /// What the loop is doing, for the status file. @@ -114,7 +114,7 @@ pub enum Phase { /// The last pass failed; the next try is at `next_try`. Retrying { next_try: Ms, message: String }, /// Stopped until the user does something; `next_check` when it looks again by itself. - Paused { reason: String, next_check: Option }, + Paused { reason: String, reason_code: Option, next_check: Option }, /// Stopping: no new pass starts. Stopping, } @@ -149,7 +149,7 @@ pub struct Scheduler { running: bool, failures: u32, retry: Option<(Ms, String)>, - paused: Option<(String, Recheck, Option)>, + paused: Option<(String, Option, Recheck, Option)>, waiting_approval: bool, /// Deletes are held behind a link (as of the last pass that finished). holding: bool, @@ -280,7 +280,7 @@ impl Scheduler { self.failures = self.failures.saturating_add(1); self.retry = Some((now.saturating_add(delay), message)); } - PassEnd::NeedsYou { reason, recheck } => { + PassEnd::NeedsYou { reason, reason_code, recheck } => { self.retry = None; self.failures = 0; let next_check = match recheck { @@ -288,7 +288,7 @@ impl Scheduler { Recheck::Slowly => Some(now.saturating_add(ms(self.timing.recheck_slow))), Recheck::Never => None, }; - self.paused = Some((reason, recheck, next_check)); + self.paused = Some((reason, reason_code, recheck, next_check)); } } } @@ -313,7 +313,7 @@ impl Scheduler { /// When the next pass is due, if one is. fn pass_due(&self) -> Option { - if let Some((_, _, next_check)) = &self.paused { + if let Some((_, _, _, next_check)) = &self.paused { return *next_check; } if let Some((at, _)) = &self.retry { @@ -366,8 +366,8 @@ impl Scheduler { if self.running { return Phase::Syncing; } - if let Some((reason, _, next_check)) = &self.paused { - return Phase::Paused { reason: reason.clone(), next_check: *next_check }; + if let Some((reason, reason_code, _, next_check)) = &self.paused { + return Phase::Paused { reason: reason.clone(), reason_code: reason_code.clone(), next_check: *next_check }; } if let Some((next_try, message)) = &self.retry { return Phase::Retrying { next_try: *next_try, message: message.clone() }; @@ -538,18 +538,18 @@ mod tests { #[test] fn pauses_say_why_and_only_some_recheck_by_themselves() { let mut s = scheduler(false); - run(&mut s, 0, PassEnd::NeedsYou { reason: "the folder is missing".into(), recheck: Recheck::Soon }); - assert_eq!(s.phase(), Phase::Paused { reason: "the folder is missing".into(), next_check: Some(60_000) }); + run(&mut s, 0, PassEnd::NeedsYou { reason: "the folder is missing".into(), reason_code: None, recheck: Recheck::Soon }); + assert_eq!(s.phase(), Phase::Paused { reason: "the folder is missing".into(), reason_code: None, next_check: Some(60_000) }); s.local_change(10); s.urgent_change(); assert_eq!(s.next(20), Next::Wait(Some(60_000)), "nothing but the recheck runs a pass while paused"); run(&mut s, 60_000, done()); assert_eq!(s.phase(), Phase::Idle, "back once the folder is"); - run(&mut s, 660_000, PassEnd::NeedsYou { reason: "protocol".into(), recheck: Recheck::Slowly }); + run(&mut s, 660_000, PassEnd::NeedsYou { reason: "protocol".into(), reason_code: None, recheck: Recheck::Slowly }); assert_eq!(s.next(660_001), Next::Wait(Some(1_260_000))); - run(&mut s, 1_260_000, PassEnd::NeedsYou { reason: "revoked".into(), recheck: Recheck::Never }); + run(&mut s, 1_260_000, PassEnd::NeedsYou { reason: "revoked".into(), reason_code: None, recheck: Recheck::Never }); assert_eq!(s.next(1_260_001), Next::Wait(None)); assert_eq!(s.next(99_999_999), Next::Wait(None), "never by itself"); s.stop(); diff --git a/apps/craft-cloud-agent/tests/end_to_end.rs b/apps/craft-cloud-agent/tests/end_to_end.rs index 92d2fd97..4920700a 100644 --- a/apps/craft-cloud-agent/tests/end_to_end.rs +++ b/apps/craft-cloud-agent/tests/end_to_end.rs @@ -49,9 +49,24 @@ impl Backend for CountingBackend { self.inner.get_raw(path).await } + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + if path.starts_with("chunks/") { + self.chunk_reads.fetch_add(1, Ordering::Relaxed); + } + self.inner.get_raw_limited(path, max_bytes).await + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + self.inner.size_raw(path).await + } + async fn delete_raw(&self, path: &str) -> Result<(), StoreError> { self.inner.delete_raw(path).await } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + self.inner.list_raw(prefix, after, limit).await + } } async fn server() -> Server { @@ -163,6 +178,43 @@ async fn a_first_sync_goes_both_ways() { assert_eq!(a.files(), b.files()); } +/// Deleting a folder with files in it and restoring it from the trash: the folder and everything that was in the trash +/// below it, also a file deleted on its own before, come back to the disk of both computers by their normal pass. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_restored_from_the_trash_comes_back_with_its_files() { + let server = server().await; + let a = Agent::signed_in(&server, "desktop").await; + a.write("dir1/child.txt", "deleted with the folder"); + a.write("dir1/early.txt", "deleted first"); + a.write("dir1/sub/deep.txt", "deep"); + a.write("keep.txt", "stays"); + assert_eq!(a.sync().await, Exit::Done); + let b = Agent::signed_in(&server, "laptop").await; + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(b.files().len(), 4); + + std::fs::remove_file(a.path("dir1/early.txt")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + std::fs::remove_dir_all(a.path("dir1")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(a.files(), ["keep.txt"]); + assert_eq!(b.files(), ["keep.txt"]); + + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let listed = server.meta.trash(ana.space, None, 200, 0).await.unwrap().entries; + assert_eq!(listed.iter().map(|e| e.name.as_str()).collect::>(), ["dir1"], "the folder is listed alone"); + let device = server.meta.list_devices(ana.id).await.unwrap().into_iter().find(|d| d.name == "desktop").unwrap().id; + server.meta.restore_from_trash(ana.space, device, listed[0].id, 0).await.unwrap(); + + for agent in [&a, &b] { + assert_eq!(agent.sync().await, Exit::Done); + assert_eq!(agent.files(), ["dir1/child.txt", "dir1/early.txt", "dir1/sub/deep.txt", "keep.txt"]); + assert_eq!(agent.read("dir1/sub/deep.txt").as_deref(), Some("deep")); + assert_eq!(agent.read("dir1/early.txt").as_deref(), Some("deleted first")); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn big_files_travel_in_chunks_and_keep_their_exec_bit() { let server = server().await; @@ -761,6 +813,39 @@ async fn a_failed_pass_keeps_the_deletes_the_user_was_shown() { assert_eq!((exit, summary.removed_remotely), (Exit::Done, 12), "{summary:?}"); } +/// The agent tells the server what it is when it signs in and when it joins, so the account's device list shows it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn login_and_join_send_the_agents_device_details() { + use craft_cloud_agent::cli::JoinArgs; + use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; + let server = server().await; + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + async fn details(server: &Server, user: craft_cloud_meta::UserId, name: &str) -> (String, String, String) { + let devices = server.meta.list_devices(user).await.unwrap(); + let device = devices.into_iter().find(|d| d.name == name).unwrap_or_else(|| panic!("no device {name}")); + (device.client.kind, device.client.platform, device.client.app_version) + } + let expected = ("agent".to_owned(), std::env::consts::OS.to_owned(), env!("CARGO_PKG_VERSION").to_owned()); + + let a = Agent::signed_in(&server, "desk").await; + assert_eq!(details(&server, ana.id, "desk").await, expected, "after login"); + drop(a); + + let invite = Code::generate(INVITE_CHARS).unwrap(); + let now = i64::try_from(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()).unwrap(); + server.meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + let b = Agent::new(); + let args = JoinArgs { + join: JoinString::new(server.url.clone(), invite).link(), + folder: Some(b.folder.clone()), + device_name: Some("laptop".into()), + adopt: false, + autostart: false, + }; + cli::join(&b.data, &args, None, &mut Vec::new()).await.unwrap(); + assert_eq!(details(&server, ana.id, "laptop").await, expected, "after join"); +} + #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn login_makes_the_agent_folder_owner_only() { diff --git a/apps/craft-cloud-agent/tests/local_api.rs b/apps/craft-cloud-agent/tests/local_api.rs index 2839d706..539f2867 100644 --- a/apps/craft-cloud-agent/tests/local_api.rs +++ b/apps/craft-cloud-agent/tests/local_api.rs @@ -10,7 +10,7 @@ use craft_cloud_agent::cli::{self, AgentError, Exit}; use craft_cloud_agent::daemon::{RunOptions, run_loop}; use craft_cloud_agent::schedule::Timing; use craft_cloud_meta::{Meta, Role}; -use craft_cloud_proto::api::{DeviceInfo, ErrorBody, PairingState, PairingStatus}; +use craft_cloud_proto::api::{DeviceInfo, ErrorBody, HistoryResponse, PairingState, PairingStatus}; use craft_cloud_proto::local::{JoinRequest, LOCAL_API_FILE, LocalApiFile, LoginRequest, PairingInfo, RunState, Status, codes}; use craft_cloud_server::AppState; use craft_cloud_server::auth::hash_password; @@ -24,6 +24,7 @@ const PATIENCE: Duration = Duration::from_secs(30); struct Server { url: String, meta: Meta, + task: tokio::task::JoinHandle>, _dirs: (tempfile::TempDir, tempfile::TempDir), } @@ -34,8 +35,8 @@ async fn server() -> Server { let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]).unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); - Server { url, meta, _dirs: (db, blobs) } + let task = tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, task, _dirs: (db, blobs) } } fn quick() -> RunOptions { @@ -172,8 +173,42 @@ const ROUTES: &[(&str, &str)] = &[ ("GET", "/v1/pairings/0192f0c4-0000-7000-8000-000000000000"), ("DELETE", "/v1/pairings/0192f0c4-0000-7000-8000-000000000000"), ("GET", "/v1/devices"), + ("GET", "/v1/usage"), ("DELETE", "/v1/devices/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/files/history?path=Notes%2Ftodo.txt"), + ("POST", "/v1/files/restore"), + ("GET", "/v1/trash?limit=1"), + ("POST", "/v1/trash/restore"), + ("POST", "/v1/trash/remove"), ("POST", "/v1/autostart"), + ("POST", "/v1/login/second-factor"), + ("GET", "/v1/account/two-factor"), + ("POST", "/v1/account/two-factor/setup"), + ("POST", "/v1/account/two-factor/confirm"), + ("POST", "/v1/account/two-factor/disable"), + ("POST", "/v1/account/two-factor/recovery-codes"), + ("POST", "/v1/account/password"), + ("POST", "/v1/devices/revoke-others"), + ("PATCH", "/v1/devices/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/admin/server"), + ("GET", "/v1/admin/storage"), + ("GET", "/v1/admin/users"), + ("POST", "/v1/admin/users"), + ("PATCH", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000"), + ("GET", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/usage"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/disable"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/enable"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/invite"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/invite"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/password-reset"), + ("POST", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/reset-two-factor"), + ("GET", "/v1/account"), + ("GET", "/v1/account/activity"), + ("GET", "/v1/admin/audit"), + ("GET", "/v1/admin/audit/export"), + ("GET", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/devices"), + ("DELETE", "/v1/admin/users/0192f0c4-0000-7000-8000-000000000000/devices/0192f0c4-0000-7000-8000-000000000001"), ]; /// A user session without systemd: Linux's entry is the XDG autostart file. @@ -522,6 +557,53 @@ async fn signed_in(server: &Server, data: &DataDir, folder: &Path) -> (Running, (running, api) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn usage_reads_through_to_the_server_without_caching() { + let server = server().await; + let data = DataDir::new(tempfile::tempdir().unwrap().keep()); + let folder = tempfile::tempdir().unwrap(); + let (_running, api) = signed_in(&server, &data, folder.path()).await; + let account = data.load_account().unwrap().unwrap(); + let transport = craft_cloud_agent::http::HttpTransport::new(account.server.clone(), &account.token).unwrap(); + std::fs::write(folder.path().join("usage-first.bin"), b"first payload").unwrap(); + reaches_the_server(&server, "usage-first.bin").await; + let server_first = transport.usage().await.unwrap(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 200, "{body}"); + let local_first: craft_cloud_proto::api::AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!(local_first, server_first, "the local API passes nonzero numbers through unchanged"); + assert!(local_first.stored_bytes > 0); + + std::fs::write(folder.path().join("usage-second.bin"), b"a different payload for fresh usage").unwrap(); + reaches_the_server(&server, "usage-second.bin").await; + let server_second = transport.usage().await.unwrap(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 200, "{body}"); + let local_second: craft_cloud_proto::api::AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!(local_second, server_second, "a second request sees server changes; no cache"); + assert!(local_second.stored_bytes > local_first.stored_bytes); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn usage_refuses_signed_out_and_reports_server_unreachable() { + let server = server().await; + let data = DataDir::new(tempfile::tempdir().unwrap().keep()); + let running = Running::start(&data); + let api = api_file(data.path()).await; + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 409, "{body}"); + assert_eq!(error_code(&body), codes::NOT_SIGNED_IN); + let _ = running.stop().await; + + let folder = tempfile::tempdir().unwrap(); + let (running, api) = signed_in(&server, &data, folder.path()).await; + server.task.abort(); + let (code, body) = call(&api, "GET", "/v1/usage", ""); + assert_eq!(code, 503, "{body}"); + assert_eq!(error_code(&body), codes::OFFLINE); + drop(running); +} + async fn reaches_the_server(server: &Server, name: &str) { let deadline = Instant::now() + PATIENCE; while !server_has(server, name).await { @@ -530,6 +612,190 @@ async fn reaches_the_server(server: &Server, name: &str) { } } +async fn history_page(api: &LocalApiFile, path: &str, cursor: Option<&str>, limit: u32) -> HistoryResponse { + let mut query = format!("path={path}&limit={limit}"); + if let Some(cursor) = cursor { + query.push_str("&cursor="); + query.push_str(cursor); + } + let (code, body) = call(api, "GET", &format!("/v1/files/history?{query}"), ""); + assert_eq!(code, 200, "{body}"); + serde_json::from_str(&body).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_file_history_pages_restores_and_syncs_to_disk() { + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + let subdir = folder.join("Notes"); + std::fs::create_dir(&subdir).unwrap(); + let file = subdir.join("story.txt"); + + let mut previous = None; + for content in [b"first version".as_slice(), b"second version".as_slice(), b"third version".as_slice()] { + std::fs::write(&file, content).unwrap(); + let deadline = Instant::now() + PATIENCE; + loop { + let (code, body) = call(&api, "GET", "/v1/files/history?path=Notes%2Fstory.txt&limit=1", ""); + if code == 200 { + let page: HistoryResponse = serde_json::from_str(&body).unwrap(); + if page.versions.first().is_some_and(|version| Some(version.id) != previous) { + previous = page.versions.first().map(|version| version.id); + break; + } + } + assert!(Instant::now() < deadline, "version reaches server history"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + let newest = history_page(&api, "Notes%2Fstory.txt", None, 1).await; + let cursor = newest.next.clone().expect("more than one history page"); + let older = history_page(&api, "Notes%2Fstory.txt", Some(&cursor), 1).await; + let current = newest.versions.first().expect("current version").id; + let restore = older.versions.first().expect("older version").id; + let body = serde_json::json!({ "path": "Notes/story.txt", "version": restore, "expected_version": current }).to_string(); + let (code, answer) = call(&api, "POST", "/v1/files/restore", &body); + assert_eq!(code, 200, "{answer}"); + let restored: craft_cloud_proto::api::RestoreResponse = serde_json::from_str(&answer).unwrap(); + assert_eq!(restored.parent_version, current); + + let stale = serde_json::json!({ "path": "Notes/story.txt", "version": current, "expected_version": current }).to_string(); + let (code, answer) = call(&api, "POST", "/v1/files/restore", &stale); + assert_eq!((code, error_code(&answer).as_str()), (409, "conflict"), "{answer}"); + + let deadline = Instant::now() + PATIENCE; + loop { + if std::fs::read(&file).is_ok_and(|bytes| bytes.as_slice() == b"second version") { + break; + } + assert!(Instant::now() < deadline, "the change feed syncs restored bytes to disk"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let (exit, output) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{output}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_file_history_rejects_unknown_outside_and_directory_paths() { + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + std::fs::create_dir(folder.join("Folder")).unwrap(); + let deadline = Instant::now() + PATIENCE; + loop { + let (code, body) = call(&api, "GET", "/v1/files/history?path=Folder", ""); + if code == 400 { + assert_eq!(error_code(&body), "invalid"); + break; + } + assert!(Instant::now() < deadline, "directory enters sync state"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let (code, body) = call(&api, "GET", "/v1/files/history?path=missing.txt", ""); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = call(&api, "GET", "/v1/files/history?path=..%2Foutside.txt", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let _ = running.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn local_trash_pages_restores_renames_and_removes_nodes() { + use craft_cloud_proto::local::TrashResponse; + let server = server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in(&server, &data, &folder).await; + + std::fs::write(folder.join("first.txt"), b"first bytes").unwrap(); + std::fs::write(folder.join("second.txt"), b"second bytes").unwrap(); + reaches_the_server(&server, "first.txt").await; + reaches_the_server(&server, "second.txt").await; + std::fs::remove_file(folder.join("first.txt")).unwrap(); + std::fs::remove_file(folder.join("second.txt")).unwrap(); + let deadline = Instant::now() + PATIENCE; + let page = loop { + let (code, body) = call(&api, "GET", "/v1/trash?limit=1", ""); + if code == 200 { + let page: TrashResponse = serde_json::from_str(&body).unwrap(); + if page.items.len() == 1 && page.next.is_some() { + break page; + } + } + assert!(Instant::now() < deadline, "both deleted files enter the trash"); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + let first = page.items.first().unwrap(); + assert_eq!(first.path.as_deref(), Some(first.name.as_str()), "live root parent yields a path hint"); + let cursor = page.next.as_deref().unwrap(); + let (code, body) = call(&api, "GET", &format!("/v1/trash?limit=1&cursor={cursor}"), ""); + assert_eq!(code, 200, "{body}"); + let second_page: TrashResponse = serde_json::from_str(&body).unwrap(); + let second = second_page.items.first().unwrap(); + assert_ne!(first.id, second.id); + + let restore_body = serde_json::json!({ "id": first.id }).to_string(); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &restore_body); + assert_eq!(code, 200, "{body}"); + let restored: craft_cloud_proto::api::TrashRestoreResponse = serde_json::from_str(&body).unwrap(); + assert!(!restored.renamed); + let restored_path = folder.join(&restored.name); + let deadline = Instant::now() + PATIENCE; + loop { + if std::fs::read(&restored_path) + .is_ok_and(|bytes| bytes.as_slice() == if first.name == "first.txt" { &b"first bytes"[..] } else { &b"second bytes"[..] }) + { + break; + } + assert!(Instant::now() < deadline, "the agent syncs restored bytes to disk"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // Re-delete the restored ID, create a live node with its old name, then verify conflict rename on restore. + std::fs::remove_file(&restored_path).unwrap(); + let deadline = Instant::now() + PATIENCE; + let mut trashed_again = false; + while !trashed_again { + let (code, body) = call(&api, "GET", "/v1/trash?limit=50", ""); + if code == 200 { + let page: TrashResponse = serde_json::from_str(&body).unwrap(); + trashed_again = page.items.iter().any(|item| item.id == restored.node); + } + assert!(Instant::now() < deadline, "restored node is deleted again"); + if !trashed_again { + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + std::fs::write(&restored_path, b"replacement bytes").unwrap(); + reaches_the_server(&server, &restored.name).await; + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": restored.node }).to_string()); + assert_eq!(code, 200, "{body}"); + let renamed: craft_cloud_proto::api::TrashRestoreResponse = serde_json::from_str(&body).unwrap(); + assert!(renamed.renamed, "{renamed:?}"); + assert_ne!(renamed.name, restored.name); + + let (code, body) = call(&api, "POST", "/v1/trash/remove", &serde_json::json!({ "id": second.id }).to_string()); + assert_eq!(code, 204, "{body}"); + let (code, body) = call(&api, "GET", "/v1/trash?limit=50", ""); + assert_eq!(code, 200, "{body}"); + let remaining: TrashResponse = serde_json::from_str(&body).unwrap(); + assert!(remaining.items.iter().all(|item| item.id != second.id)); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": second.id }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": "invalid" }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let unknown = craft_cloud_proto::NodeId::generate(); + let (code, body) = call(&api, "POST", "/v1/trash/restore", &serde_json::json!({ "id": unknown }).to_string()); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let _ = running.stop().await; +} + /// Paused, nothing syncs and the status says so (not as a pause that needs someone); it stays paused after `run` /// starts again, until it is resumed. Quit then ends `run` cleanly, and its `local-api.json` with it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -614,7 +880,7 @@ async fn a_problem_found_at_start_keeps_the_agent_up_and_says_so() { let api = api_file(data.path()).await; let now = status(&api); assert!(!now.signed_in, "{now:?}"); - let RunState::Paused { reason } = &now.run_state else { panic!("{now:?}") }; + let RunState::Paused { reason, .. } = &now.run_state else { panic!("{now:?}") }; assert!(reason.contains("account.json"), "{reason}"); // Nothing else can be done until it is put right. let (code, body) = call(&api, "POST", "/v1/join", r#"{"join":"c.example/K7M4-Q9TX"}"#); @@ -694,6 +960,8 @@ impl Machine { device_name: Some("lptp1".into()), adopt, password: password.map(str::to_owned), + totp_code: None, + recovery_code: None, }) .unwrap() } @@ -775,7 +1043,7 @@ async fn a_second_computer_joins_with_a_code_from_the_first() { std::fs::write(b.folder.join("from-b.txt"), "made on B").unwrap(); eventually( "B says it was removed", - || matches!(status(&b.api).run_state, RunState::Paused { ref reason } if reason.contains("no longer accepts")), + || matches!(status(&b.api).run_state, RunState::Paused { ref reason, .. } if reason.contains("no longer accepts")), ) .await; assert!(status(&b.api).removed, "told apart from other stops, for the app"); @@ -851,3 +1119,453 @@ async fn an_invite_to_a_new_account_asks_for_a_password_through_the_agent() { assert_eq!(joined.account.as_deref(), Some("cleo")); b.stop().await; } + +// ---- the admin calls (`/v1/admin/…`) ---- + +use craft_cloud_proto::api::{ + AccountUsage, AdminCode, AdminCreatedUser, AdminDevices, AdminServerInfo, AdminStorageOverview, AdminUserInfo, AdminUserList, +}; + +/// The server with `root`, an administrator, beside `ana`, a member. +async fn admin_server() -> Server { + let server = server().await; + server.meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + server +} + +/// Signs `run` in as `user` through its local API and waits until it is idle. +async fn signed_in_as(server: &Server, user: &str, data: &DataDir, folder: &Path) -> (Running, LocalApiFile) { + let running = Running::start(data); + let api = api_file(data.path()).await; + let body = serde_json::to_string(&LoginRequest { + server: server.url.clone(), + user: user.into(), + password: PASSWORD.into(), + folder: Some(folder.display().to_string()), + device_name: Some(format!("{user}-pc")), + adopt: false, + }) + .unwrap(); + let (code, answer) = call(&api, "POST", "/v1/login", &body); + assert_eq!(code, 200, "{answer}"); + eventually("signed in and idle", || status(&api).run_state == RunState::Idle).await; + (running, api) +} + +/// A call with the confirmation `password` and the rest of the body in `fields`. +fn confirmed(api: &LocalApiFile, method: &str, path: &str, password: &str, fields: serde_json::Value) -> (u16, String) { + let mut body = fields; + body["password"] = password.into(); + call(api, method, path, &body.to_string()) +} + +/// Whether an admin disabled the account. +async fn is_disabled(server: &Server, name: &str) -> bool { + let user = server.meta.user_by_name(name).await.unwrap().unwrap(); + server.meta.admin_user(user.id, 0).await.unwrap().unwrap().disabled_ms.is_some() +} + +fn files_under(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + files_under(&path, out); + } else { + out.push(path); + } + } +} + +/// Every route passes the server's answer on with its status and code: reads, the changes of an account, the codes, +/// the devices; the confirmation is the admin's password in the body; a missing or wrong one is the server's `400 +/// invalid` and `403 wrong_password` (never 401), and too many wrong ones its `429`. Nothing of the password or of a +/// code is kept by the agent or printed by it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_calls_reach_the_servers_admin_api_and_pass_its_answers_through() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let folder = dir.path().join("Craft Cloud"); + let (running, api) = signed_in_as(&server, "root", &data, &folder).await; + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let ana_path = format!("/v1/admin/users/{}", ana.id.0); + + // Reads. + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!(code, 200, "{body}"); + let info: AdminServerInfo = serde_json::from_str(&body).unwrap(); + assert_eq!((info.users, info.devices_active, info.storage_kind.as_str()), (2, 1, "disk")); + let (code, body) = call(&api, "GET", "/v1/admin/storage", ""); + assert_eq!(code, 200, "{body}"); + let overview: AdminStorageOverview = serde_json::from_str(&body).unwrap(); + assert_eq!(overview.accounts, 2); + assert_eq!(overview.stored_bytes, 0); + // The room the server reports travels through the agent as well. + assert_eq!(overview.store_kind.as_deref(), Some("disk")); + let (code, body) = call(&api, "GET", &format!("{ana_path}/usage"), ""); + assert_eq!(code, 200, "{body}"); + let usage: AccountUsage = serde_json::from_str(&body).unwrap(); + assert_eq!((usage.stored_bytes, usage.quota_bytes, usage.trash_bytes, usage.history_bytes), (0, None, Some(0), Some(0))); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=1", ""); + assert_eq!(code, 200, "{body}"); + let page: AdminUserList = serde_json::from_str(&body).unwrap(); + assert_eq!(page.users.iter().map(|u| u.username.as_str()).collect::>(), ["ana"]); + let next = page.next.expect("a second page"); + let (code, body) = call(&api, "GET", &format!("/v1/admin/users?limit=1&cursor={next}"), ""); + assert_eq!(code, 200, "{body}"); + let second: AdminUserList = serde_json::from_str(&body).unwrap(); + assert_eq!((second.users.first().map(|u| u.username.as_str()), second.next), (Some("root"), None)); + let (code, body) = call(&api, "GET", "/v1/admin/users?query=AN", ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().users.len(), 1); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=0", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "the server's own refusal: {body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=x", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = call(&api, "GET", &format!("{ana_path}/devices"), ""); + assert_eq!(code, 200, "{body}"); + assert!(serde_json::from_str::(&body).unwrap().devices.is_empty()); + + // A missing confirmation is the server's `invalid`; a wrong one its `wrong_password`, a 403: never a 401. + let (code, body) = call(&api, "POST", &format!("{ana_path}/disable"), "{}"); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), "not the password", serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (403, "wrong_password"), "{body}"); + let (code, body) = call(&api, "POST", &format!("{ana_path}/disable"), "not json"); + assert_eq!(code, 400, "{body}"); + assert!(!is_disabled(&server, "ana").await, "nothing changed without a right confirmation"); + assert!(status(&api).signed_in && !status(&api).removed, "a 403 doesn't sign the agent out"); + + // The changes: with the confirmation, each is the server's answer. + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!(code, 200, "{body}"); + let created: AdminCreatedUser = serde_json::from_str(&body).unwrap(); + assert_eq!((created.user.username.as_str(), created.user.role.as_str(), created.user.invite_pending), ("bo", "member", true)); + let invite_code = created.invite.code.clone(); + assert!(!invite_code.is_empty()); + let bo_path = format!("/v1/admin/users/{}", created.user.id); + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, "username_taken"), "{body}"); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "quota_bytes": 50_000_000 })); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().quota_bytes, Some(50_000_000)); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "quota_bytes": null, "role": "admin" })); + assert_eq!(code, 200, "{body}"); + let changed: AdminUserInfo = serde_json::from_str(&body).unwrap(); + assert_eq!((changed.quota_bytes, changed.role.as_str()), (None, "admin")); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + let (code, body) = confirmed(&api, "PATCH", &ana_path, PASSWORD, serde_json::json!({ "role": "member" })); + assert_eq!(code, 200, "{body}"); + // The last administrator can't be demoted: the server's 409. + let root = server.meta.user_by_name("root").await.unwrap().unwrap(); + let (code, body) = confirmed(&api, "PATCH", &format!("/v1/admin/users/{}", root.id.0), PASSWORD, serde_json::json!({ "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, "last_admin"), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + assert!(is_disabled(&server, "ana").await); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/enable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/password-reset"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 200, "{body}"); + let reset: AdminCode = serde_json::from_str(&body).unwrap(); + assert!(!reset.code.is_empty()); + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (409, "conflict"), "ana has a password: the server says so: {body}"); + let (code, body) = confirmed(&api, "POST", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 200, "{body}"); + let second_invite: AdminCode = serde_json::from_str(&body).unwrap(); + let (code, body) = confirmed(&api, "DELETE", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (204, ""), "{body}"); + let (code, body) = confirmed(&api, "DELETE", &format!("{bo_path}/invite"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "nothing left to revoke: {body}"); + let unknown_device = craft_cloud_proto::DeviceId::generate(); + let (code, body) = confirmed(&api, "DELETE", &format!("{ana_path}/devices/{unknown_device}"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (404, "not_found"), "{body}"); + let (code, body) = confirmed(&api, "DELETE", &bo_path, PASSWORD, serde_json::json!({})); + assert_eq!((code, body.as_str()), (202, ""), "the server's own status: {body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users", ""); + assert_eq!(code, 200, "{body}"); + assert!(serde_json::from_str::(&body).unwrap().users.iter().all(|u| u.username != "bo"), "removed accounts are out of the list"); + let (code, body) = call(&api, "GET", "/v1/admin/users/not-an-id/devices", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + + // Wrong passwords are counted by the server: it says when it has had enough, and that stays a 429. + let mut limited = None; + for _ in 0..8 { + let (code, body) = confirmed(&api, "POST", &format!("{ana_path}/disable"), "still wrong", serde_json::json!({})); + if code == 429 { + limited = Some(error_code(&body)); + break; + } + assert_eq!((code, error_code(&body).as_str()), (403, "wrong_password"), "{body}"); + } + assert_eq!(limited.as_deref(), Some("rate_limited")); + assert!(status(&api).signed_in && !status(&api).removed); + + let (exit, out) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + // Nothing of the confirmation or the codes was kept or printed (the files are read once the agent has stopped: it + // holds the data folder's lock while it runs). + let mut written = Vec::new(); + files_under(data.path(), &mut written); + for file in written.iter().filter(|f| f.file_name().is_some_and(|n| n != LOCAL_API_FILE)) { + let text = String::from_utf8_lossy(&std::fs::read(file).unwrap()).into_owned(); + for secret in [PASSWORD, "still wrong", invite_code.as_str(), reset.code.as_str(), second_invite.code.as_str()] { + assert!(!text.contains(secret), "{} holds a secret", file.display()); + } + } + for secret in [PASSWORD, "still wrong", invite_code.as_str(), reset.code.as_str(), second_invite.code.as_str()] { + assert!(!out.contains(secret), "the agent printed a secret: {out}"); + } +} + +/// A member's calls are refused by the server as `403 forbidden`, and that is what the app is told; the agent is +/// not signed out by it (a 401 would mean "this device was revoked"). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_member_is_told_forbidden_and_stays_signed_in() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (running, api) = signed_in_as(&server, "ana", &data, &dir.path().join("Craft Cloud")).await; + let some_id = "0192f0c4-0000-7000-8000-000000000000"; + for (method, path) in + [("GET", "/v1/admin/server".to_owned()), ("GET", "/v1/admin/users".to_owned()), ("GET", format!("/v1/admin/users/{some_id}/devices"))] + { + let (code, body) = call(&api, method, &path, ""); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{method} {path}: {body}"); + } + for (method, path) in [ + ("POST", "/v1/admin/users".to_owned()), + ("PATCH", format!("/v1/admin/users/{some_id}")), + ("POST", format!("/v1/admin/users/{some_id}/disable")), + ("DELETE", format!("/v1/admin/users/{some_id}")), + ] { + let fields = serde_json::json!({ "username": "x", "role": "admin", "quota_bytes": 5_000_000 }); + let (code, body) = confirmed(&api, method, &path, PASSWORD, fields); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{method} {path}: {body}"); + } + let now = status(&api); + assert!(now.signed_in && !now.removed, "{now:?}"); + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + assert_eq!(ana.role, Role::Member); + let (exit, out) = running.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); +} + +/// The server's `401` for a revoked admin device is not a local `401` (which says the token is wrong): it is told as +/// every signed-in call tells it, as something that needs the person. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_revoked_admin_device_is_told_as_needs_you() { + let server = admin_server().await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (running, api) = signed_in_as(&server, "root", &data, &dir.path().join("Craft Cloud")).await; + let root = server.meta.user_by_name("root").await.unwrap().unwrap(); + for device in server.meta.list_devices(root.id).await.unwrap() { + server.meta.revoke_device(root.id, device.id).await.unwrap(); + } + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NEEDS_YOU), "{body}"); + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "x", "role": "member" })); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NEEDS_YOU), "{body}"); + running.stop().await.0.unwrap(); +} + +/// Signed out there is no server to ask; the bodies are small; the checks on every call are the other routes'. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn admin_calls_need_a_sign_in_and_small_bodies() { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let running = Running::start(&data); + let api = api_file(data.path()).await; + let id = "0192f0c4-0000-7000-8000-000000000000"; + let (code, body) = call(&api, "GET", "/v1/admin/server", ""); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NOT_SIGNED_IN), "{body}"); + let (code, body) = confirmed(&api, "POST", &format!("/v1/admin/users/{id}/disable"), PASSWORD, serde_json::json!({})); + assert_eq!((code, error_code(&body).as_str()), (409, codes::NOT_SIGNED_IN), "{body}"); + let (code, body) = call(&api, "GET", "/v1/admin/users?limit=9999999999", ""); + assert_eq!(code, 400, "a limit that is no number: {body}"); + let long = serde_json::json!({ "username": "a".repeat(9 * 1024), "role": "member", "password": PASSWORD }).to_string(); + let (code, body) = call(&api, "POST", "/v1/admin/users", &long); + assert_eq!((code, error_code(&body).as_str()), (413, "too_large"), "{body}"); + let (code, _) = call(&api, "POST", "/v1/admin/users", "{\"username\": 5}"); + assert_eq!(code, 422, "the shape of the body is checked here too"); + running.stop().await.0.unwrap(); +} + +// ---- the account, its activity and the audit log ---- + +use craft_cloud_proto::api::{AccountInfo, AuditEvent, AuditPage}; +use craft_cloud_server::audit::AuditExportLimits; + +/// [`admin_server`] with the export cut as `limits` say. +async fn admin_server_limited(limits: AuditExportLimits) -> Server { + let (db, blobs) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]).unwrap().with_audit_export(limits); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, task, _dirs: (db, blobs) } +} + +fn page_of(body: &str) -> AuditPage { + serde_json::from_str(body).unwrap_or_else(|e| panic!("not an audit page ({e}): {body}")) +} + +/// The account, its activity, the audit list and its export pass through the agent with the server's statuses and codes: a +/// member's `403 forbidden` is never a signed-out agent; filters are checked before they are sent; the export is the server's +/// stream, whole, and `truncated` when the server cuts it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_account_its_activity_and_the_audit_log_pass_through() { + let server = admin_server_limited(AuditExportLimits::default()).await; + let dir = tempfile::tempdir().unwrap(); + let root_data = DataDir::new(dir.path().join("root")); + let ana_data = DataDir::new(dir.path().join("ana")); + let (root_run, root) = signed_in_as(&server, "root", &root_data, &dir.path().join("Root Cloud")).await; + let (ana_run, ana) = signed_in_as(&server, "ana", &ana_data, &dir.path().join("Ana Cloud")).await; + let ana_user = server.meta.user_by_name("ana").await.unwrap().unwrap(); + + // Some more events: the admin creates an account, disables it and enables it again (not ana: that would sign her out). + let (code, body) = confirmed(&root, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": "bo", "role": "member" })); + assert_eq!(code, 200, "{body}"); + let bo = serde_json::from_str::(&body).unwrap().user.id; + for what in ["disable", "enable"] { + let (code, body) = confirmed(&root, "POST", &format!("/v1/admin/users/{bo}/{what}"), PASSWORD, serde_json::json!({})); + assert_eq!(code, 204, "{body}"); + } + + // The signed-in account, as the server says it. + let (code, body) = call(&root, "GET", "/v1/account", ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(serde_json::from_str::(&body).unwrap().role, "admin"); + let (code, body) = call(&ana, "GET", "/v1/account", ""); + assert_eq!((code, serde_json::from_str::(&body).unwrap().role.as_str()), (200, "member")); + + // The admin's list: pages with a cursor given back as it came, filters, the server's own refusals. + let (code, body) = call(&root, "GET", "/v1/admin/audit?limit=2", ""); + assert_eq!(code, 200, "{body}"); + let first = page_of(&body); + assert_eq!(first.events.len(), 2); + assert!(first.events[0].at_ms >= first.events[1].at_ms, "newest first"); + let next = first.next.clone().expect("more events than a page"); + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?limit=2&cursor={next}"), ""); + assert_eq!(code, 200, "{body}"); + let second = page_of(&body); + assert!(second.events.iter().all(|e| first.events.iter().all(|f| f.id != e.id)), "no event twice"); + let (code, body) = call(&root, "GET", "/v1/admin/audit?action=admin.", ""); + assert_eq!(code, 200, "{body}"); + let admin_events = page_of(&body).events; + assert!(admin_events.len() >= 3 && admin_events.iter().all(|e| e.action.starts_with("admin.")), "{admin_events:?}"); + let (code, body) = call(&root, "GET", "/v1/admin/audit?action=auth.login", ""); + let logins = page_of(&body).events; + assert!(code == 200 && logins.len() == 2 && logins.iter().all(|e| e.action == "auth.login"), "{body}"); + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?user={}", ana_user.id.0), ""); + assert_eq!(code, 200, "{body}"); + let about_ana = page_of(&body).events; + assert!(!about_ana.is_empty() && about_ana.iter().all(|e| e.actor_user == Some(ana_user.id.0) || e.target_id == Some(ana_user.id.0))); + let (code, body) = call(&root, "GET", "/v1/admin/audit?before_ms=1", ""); + assert!(code == 200 && page_of(&body).events.is_empty(), "{body}"); + // What is wrong is said before anything is sent, or by the server (a cursor it didn't make). + for query in [ + "action=nope.", + "action=auth.log", + "action=x%27%20OR%201=1", + "user=not-an-id", + "limit=0", + "limit=201", + "limit=x", + "before_ms=-1", + "cursor=a%20b", + "cursor=AAAA", + ] { + let (code, body) = call(&root, "GET", &format!("/v1/admin/audit?{query}"), ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{query}: {body}"); + } + + // A member's own activity: the events about her, whatever the admin's filters say; a member can't read the log. + let (code, body) = call(&ana, "GET", "/v1/account/activity", ""); + assert_eq!(code, 200, "{body}"); + let own = page_of(&body).events; + assert!(!own.is_empty(), "{own:?}"); + assert!(own.iter().all(|e| e.actor_user == Some(ana_user.id.0) || e.target_id == Some(ana_user.id.0)), "{own:?}"); + let (code, body) = + call(&ana, "GET", &format!("/v1/account/activity?user={}&action=admin.", server.meta.user_by_name("root").await.unwrap().unwrap().id.0), ""); + assert_eq!(code, 200, "{body}"); + assert_eq!(page_of(&body).events.len(), own.len(), "an account's activity has no admin filters"); + let (code, body) = call(&ana, "GET", "/v1/account/activity?limit=101", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + for path in ["/v1/admin/audit", "/v1/admin/audit/export"] { + let (code, body) = call(&ana, "GET", path, ""); + assert_eq!((code, error_code(&body).as_str()), (403, "forbidden"), "{path}: {body}"); + } + let now = status(&ana); + assert!(now.signed_in && !now.removed, "a 403 doesn't sign the agent out: {now:?}"); + + // The export: the server's lines, then its last line; whole through the typed client, which writes as it reads. + let client = craft_cloud_app::Client::connect_in(root_data.path()).unwrap(); + let all = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + let outcome = client.admin_audit_export(&craft_cloud_app::AuditQuery::default(), &mut out); + (outcome, out) + }) + .await + .unwrap(); + let (outcome, out) = all; + let outcome = outcome.unwrap(); + assert!(!outcome.truncated); + let lines: Vec = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len() as u64, outcome.events); + let (code, body) = call(&root, "GET", "/v1/admin/audit?limit=200", ""); + assert_eq!(code, 200, "{body}"); + assert!(page_of(&body).events.iter().all(|e| lines.iter().any(|l| l.id == e.id)), "the export holds every event of the list"); + // A filtered export; a refusal before the stream; a filter that is wrong. + let client = craft_cloud_app::Client::connect_in(root_data.path()).unwrap(); + let filtered = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + let query = craft_cloud_app::AuditQuery { action: Some("auth.".into()), ..Default::default() }; + let outcome = client.admin_audit_export(&query, &mut out).unwrap(); + (outcome, out) + }) + .await + .unwrap(); + assert_eq!(filtered.0.events, 2, "the two sign-ins"); + let ana_client = craft_cloud_app::Client::connect_in(ana_data.path()).unwrap(); + let refused = tokio::task::spawn_blocking(move || ana_client.admin_audit_export(&Default::default(), &mut Vec::new())).await.unwrap(); + assert_eq!(refused.unwrap_err().code(), Some("forbidden")); + let (code, body) = call(&root, "GET", "/v1/admin/audit/export?action=nope.", ""); + assert_eq!((code, error_code(&body).as_str()), (400, "invalid"), "{body}"); + + // Nothing of any of it was printed. + let (exit, out) = root_run.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + assert!(!out.contains(PASSWORD), "{out}"); + ana_run.stop().await.0.unwrap(); +} + +/// A server that holds more events than an export may: the last line is `truncated`, and the client says so (not whole). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_truncated_export_is_said_to_be() { + let server = admin_server_limited(AuditExportLimits { page: 1, channel: 1, max_rows: 2 }).await; + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("root")); + let (running, api) = signed_in_as(&server, "root", &data, &dir.path().join("Root Cloud")).await; + for name in ["bo", "cy", "di"] { + let (code, body) = confirmed(&api, "POST", "/v1/admin/users", PASSWORD, serde_json::json!({ "username": name, "role": "member" })); + assert_eq!(code, 200, "{body}"); + } + let client = craft_cloud_app::Client::connect_in(data.path()).unwrap(); + let (outcome, out) = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + (client.admin_audit_export(&Default::default(), &mut out), out) + }) + .await + .unwrap(); + let outcome = outcome.unwrap(); + assert_eq!((outcome.events, outcome.truncated), (2, true)); + assert_eq!(String::from_utf8(out).unwrap().lines().count(), 2, "the newest two, and no last line in what is written"); + running.stop().await.0.unwrap(); +} diff --git a/apps/craft-cloud-agent/tests/two_factor.rs b/apps/craft-cloud-agent/tests/two_factor.rs new file mode 100644 index 00000000..8cda152b --- /dev/null +++ b/apps/craft-cloud-agent/tests/two_factor.rs @@ -0,0 +1,795 @@ +//! Two-factor sign-in through the agent: the second step of a sign-in and the code of a join, the account's two-factor calls +//! passed on to the server, and the command line's prompts, against a real server (with a key for two-factor secrets) on a +//! loopback port and the server's clock held still, so that codes don't change under a test. +//! +//! What they pin down: a challenge, its expiry and a lock's `Retry-After` reach the app typed; a wrong code, a wrong +//! password or a lock never sign the agent out (a 403 or 429 on the account's calls, a 401 at the second step), while a real +//! 401 still does; nothing secret is kept on disk or printed. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::{Duration, Instant}; + +use clap::Parser; +use craft_cloud_agent::account::DataDir; +use craft_cloud_agent::cli::{self, AgentError, Exit}; +use craft_cloud_agent::daemon::{RunOptions, run_loop}; +use craft_cloud_agent::schedule::Timing; +use craft_cloud_app::{Client, Error, JoinRequest, LoginRequest, SecondFactorLoginRequest, TwoFactorProofRequest}; +use craft_cloud_meta::{Meta, Role}; +use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS}; +use craft_cloud_proto::local::{LOCAL_API_FILE, LocalApiFile}; +use craft_cloud_server::AppState; +use craft_cloud_server::auth::hash_password; +use craft_cloud_server::secretbox::{SecretBox, TwoFactorKey}; +use craft_cloud_server::totp; +use craft_cloud_store::Store; +use craft_cloud_store::disk::DiskBackend; +use tokio::sync::mpsc; + +const PASSWORD: &str = "correct horse battery"; +const PATIENCE: Duration = Duration::from_secs(30); + +struct Server { + url: String, + meta: Meta, + clock: Arc, + _dirs: (tempfile::TempDir, tempfile::TempDir), +} + +impl Server { + async fn start() -> Self { + let (db, blobs) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + // The clock stands still at the start of a TOTP step, until a test moves it. + let clock = Arc::new(AtomicI64::new(totp::step_at(craft_cloud_server::state::now_ms()) * 30_000 + 1_000)); + let ticking = clock.clone(); + let state = AppState::new(meta.clone(), Store::new(DiskBackend::new(blobs.path()).unwrap()), vec![]) + .unwrap() + .with_two_factor(Some(SecretBox::new(&TwoFactorKey::from_bytes([9; 32]), None).unwrap())) + .with_clock(Arc::new(move || ticking.load(Ordering::SeqCst))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending())); + Server { url, meta, clock, _dirs: (db, blobs) } + } + + fn now(&self) -> i64 { + self.clock.load(Ordering::SeqCst) + } + + fn advance(&self, ms: i64) { + self.clock.fetch_add(ms, Ordering::SeqCst); + } +} + +fn quick() -> RunOptions { + RunOptions { + timing: Timing { + debounce: Duration::from_millis(100), + max_delay: Duration::from_secs(1), + stability: Duration::from_millis(200), + periodic: Duration::from_secs(600), + unwatched_periodic: Duration::from_millis(500), + backoff_first: Duration::from_millis(200), + backoff_max: Duration::from_secs(1), + recheck: Duration::from_millis(300), + recheck_slow: Duration::from_secs(5), + approval_check: Duration::from_millis(100), + held_check: Duration::from_millis(300), + }, + heartbeat: Duration::from_millis(200), + poll_wait: Duration::from_secs(1), + poll_retry_first: Duration::from_millis(100), + poll_retry_max: Duration::from_secs(1), + poll_min_spacing: Duration::from_millis(100), + poll_busy: Duration::from_millis(200), + fail_watch: false, + record_delay: Duration::ZERO, + session_grace: Duration::from_secs(10), + announce_delay: Duration::ZERO, + autostart: craft_cloud_agent::autostart::Target::Off, + } +} + +/// A running agent (`run` on a thread of its own), signed out, and a client of its local API. +struct Machine { + data: DataDir, + folder: PathBuf, + stop: mpsc::UnboundedSender<()>, + thread: Option, String)>>, + _dir: tempfile::TempDir, +} + +impl Machine { + async fn start() -> Self { + let dir = tempfile::tempdir().unwrap(); + let data = DataDir::new(dir.path().join("data")); + let (stop, stops) = mpsc::unbounded_channel(); + let loop_data = data.clone(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let mut out = Vec::new(); + let exit = runtime.block_on(run_loop(&loop_data, quick(), stops, &mut out)); + (exit, String::from_utf8_lossy(&out).into_owned()) + }); + let file = data.path().join(LOCAL_API_FILE); + let deadline = Instant::now() + PATIENCE; + while !file.exists() { + assert!(Instant::now() < deadline, "the local API never came up"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + Self { folder: dir.path().join("Craft Cloud"), data, stop, thread: Some(thread), _dir: dir } + } + + fn client(&self) -> Client { + Client::connect_in(self.data.path()).unwrap() + } + + fn api(&self) -> LocalApiFile { + serde_json::from_slice(&std::fs::read(self.data.path().join(LOCAL_API_FILE)).unwrap()).unwrap() + } + + fn login_request(&self, server: &Server) -> LoginRequest { + LoginRequest { + server: server.url.clone(), + user: "ana".into(), + password: PASSWORD.into(), + folder: Some(self.folder.display().to_string()), + device_name: Some("desktop".into()), + adopt: true, + } + } + + fn second_step(&self, server: &Server, challenge: &str, totp_code: Option<&str>, recovery_code: Option<&str>) -> SecondFactorLoginRequest { + SecondFactorLoginRequest { + server: server.url.clone(), + user: "ana".into(), + folder: Some(self.folder.display().to_string()), + device_name: Some("desktop".into()), + adopt: true, + challenge: challenge.into(), + totp_code: totp_code.map(str::to_owned), + recovery_code: recovery_code.map(str::to_owned), + } + } + + async fn stop(mut self) -> (Result, String) { + let _ = self.stop.send(()); + let thread = self.thread.take().unwrap(); + tokio::task::spawn_blocking(move || thread.join().unwrap()).await.unwrap() + } +} + +impl Drop for Machine { + fn drop(&mut self) { + if self.thread.is_some() { + let _ = self.stop.send(()); + let _ = self.stop.send(()); + } + } +} + +/// A blocking call of the app client, off the runtime. +async fn off(f: impl FnOnce() -> Result + Send + 'static) -> Result { + tokio::task::spawn_blocking(f).await.unwrap() +} + +fn code_for(secret: &[u8], at_ms: i64) -> String { + totp::code_for(secret, totp::step_at(at_ms)).unwrap() +} + +fn wrong_code(code: &str) -> String { + format!("{:06}", (code.parse::().unwrap() + 1) % 1_000_000) +} + +/// Signs the machine in and turns two-factor sign-in on through the agent. The secret and the recovery codes come back. +async fn protect(server: &Server, machine: &Machine) -> (Vec, Vec) { + let client = machine.client(); + let request = machine.login_request(server); + let c = client.clone(); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + let c = client.clone(); + let setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let code = code_for(&secret, server.now()); + let c = client.clone(); + let codes = off(move || c.two_factor_confirm(&code)).await.unwrap(); + assert_eq!(codes.recovery_codes.len(), 10); + (secret, codes.recovery_codes) +} + +async fn sign_out(machine: &Machine) { + let c = machine.client(); + assert!(!off(move || c.logout(false)).await.unwrap().signed_in); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_second_step_of_a_sign_in_goes_through_the_agent_typed() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, recovery) = protect(&server, &machine).await; + sign_out(&machine).await; + let client = machine.client(); + + // The first step: a challenge, typed, with its expiry; the agent is still signed out and keeps nothing. + let request = machine.login_request(&server); + let c = client.clone(); + let refused = off(move || c.login(&request)).await.unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required")); + let challenge = refused.challenge().cloned().expect("a challenge"); + assert_eq!(challenge.challenge.len(), 43); + assert_eq!(challenge.expires_ms, server.now() + 5 * 60_000); + assert!(!client.status().unwrap().signed_in); + // A wrong password is still a wrong password, with nothing about two-factor sign-in. + let wrong = LoginRequest { password: "wrong password".into(), ..machine.login_request(&server) }; + let c = client.clone(); + let refused = off(move || c.login(&wrong)).await.unwrap_err(); + assert!(matches!(&refused, Error::Refused { code, .. } if code == "wrong_password"), "{refused:?}"); + + // A wrong code is `two_factor_invalid` and signs nothing in or out; a challenge takes five. + let right = code_for(&secret, server.now() + 30_000); + server.advance(30_000); + let step = machine.second_step(&server, &challenge.challenge, Some(&wrong_code(&right)), None); + let c = client.clone(); + let invalid = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + assert!(!client.status().unwrap().signed_in); + // The right code finishes it, and the agent syncs as after a password sign-in. + let step = machine.second_step(&server, &challenge.challenge, Some(&right), None); + let c = client.clone(); + let status = off(move || c.login_second_factor(&step)).await.unwrap(); + assert!(status.signed_in && status.account.as_deref() == Some("ana"), "{status:?}"); + // The challenge is used up: the same step again is `two_factor_expired`, never a signed-out agent. + let step = machine.second_step(&server, &challenge.challenge, Some(&right), None); + let c = client.clone(); + let again = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(client.status().unwrap().signed_in); + assert!(matches!(&again, Error::Refused { .. } | Error::RefusedWith { .. }), "{again:?}"); + + // A recovery code, once. + sign_out(&machine).await; + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + let step = machine.second_step(&server, &challenge.challenge, None, Some(&recovery[0].to_lowercase().replace('-', " "))); + let c = client.clone(); + assert!(off(move || c.login_second_factor(&step)).await.unwrap().signed_in); + sign_out(&machine).await; + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + let step = machine.second_step(&server, &challenge.challenge, None, Some(&recovery[0])); + let c = client.clone(); + let used = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&used, Error::Refused { code, .. } if code == "two_factor_invalid"), "{used:?}"); + + // A challenge that ran out is `two_factor_expired` (a 401 here would read as a revoked device). + server.advance(5 * 60_000 + 1); + let step = machine.second_step(&server, &challenge.challenge, Some(&code_for(&secret, server.now())), None); + let c = client.clone(); + let expired = off(move || c.login_second_factor(&step)).await.unwrap_err(); + assert!(matches!(&expired, Error::Refused { code, .. } if code == "two_factor_expired"), "{expired:?}"); + let (exit, out) = machine.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_accounts_calls_are_passed_on_and_a_refusal_never_signs_the_agent_out() { + let server = Server::start().await; + let machine = Machine::start().await; + let client = machine.client(); + let request = machine.login_request(&server); + let c = client.clone(); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + + let c = client.clone(); + let off_state = off(move || c.two_factor_status()).await.unwrap(); + assert_eq!((off_state.enabled, off_state.recovery_codes_left, off_state.pending), (false, 0, false)); + // A wrong password is 403 `wrong_password`, and the agent stays signed in, not even "removed". + let c = client.clone(); + let wrong = off(move || c.two_factor_setup("wrong password!")).await.unwrap_err(); + assert!(matches!(&wrong, Error::Refused { code, .. } if code == "wrong_password"), "{wrong:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + let c = client.clone(); + let setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + assert_eq!(setup.secret.len(), 32); + assert!(setup.otpauth_uri.starts_with("otpauth://totp/Craft%20Cloud:ana?secret="), "{}", setup.otpauth_uri); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let c = client.clone(); + assert!(off(move || c.two_factor_status()).await.unwrap().pending); + // A wrong first code: 403 `two_factor_invalid`; the agent is not signed out. + let right = code_for(&secret, server.now()); + let bad = wrong_code(&right); + let c = client.clone(); + let invalid = off(move || c.two_factor_confirm(&bad)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + assert!(client.status().unwrap().signed_in); + let c = client.clone(); + let codes = off(move || c.two_factor_confirm(&right)).await.unwrap(); + assert_eq!(codes.recovery_codes.len(), 10); + let c = client.clone(); + let on = off(move || c.two_factor_status()).await.unwrap(); + assert_eq!((on.enabled, on.recovery_codes_left), (true, 10)); + // Already on: a conflict, said as one. + let c = client.clone(); + let conflict = off(move || c.two_factor_setup(PASSWORD)).await.unwrap_err(); + assert!(matches!(&conflict, Error::Refused { code, .. } if code == "conflict"), "{conflict:?}"); + + // New recovery codes with a fresh step; the old ones die. + server.advance(30_000); + let fresh = code_for(&secret, server.now()); + let c = client.clone(); + let renewed = off(move || c.two_factor_recovery_codes(PASSWORD, &fresh)).await.unwrap(); + assert!(renewed.recovery_codes.iter().all(|code| !codes.recovery_codes.contains(code))); + + // Password change and the other computers: passed on too. + let c = client.clone(); + let weak = off(move || c.change_password(PASSWORD, "short")).await.unwrap_err(); + assert!(matches!(&weak, Error::Refused { code, .. } if code == "weak_password"), "{weak:?}"); + let c = client.clone(); + assert_eq!(off(move || c.revoke_other_devices()).await.unwrap().revoked, 0); + let c = client.clone(); + let devices = off(move || c.devices()).await.unwrap(); + let this = devices.iter().find(|d| d.current).unwrap().id; + let c = client.clone(); + let renamed = off(move || c.rename_device(this, "my desktop")).await.unwrap(); + assert_eq!(renamed.name, "my desktop"); + let c = client.clone(); + let missing = off(move || c.rename_device(craft_cloud_app::DeviceId::generate(), "x")).await.unwrap_err(); + assert!(matches!(&missing, Error::Refused { code, .. } if code == "not_found"), "{missing:?}"); + + // Ten wrong codes lock it: 429 `two_factor_locked`, typed with its wait, and still no sign-out. + for _ in 0..10 { + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(bad_code(&secret, &server)), recovery_code: None }; + let c = client.clone(); + let invalid = off(move || c.two_factor_disable(&proof)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + } + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(code_for(&secret, server.now())), recovery_code: None }; + let c = client.clone(); + let locked = off(move || c.two_factor_disable(&proof)).await.unwrap_err(); + assert_eq!((locked.code(), locked.retry_after_secs()), (Some("two_factor_locked"), Some(900)), "{locked:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + + // Turning it off with a recovery code works once the lock is over. + server.advance(900_001); + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: None, recovery_code: Some(renewed.recovery_codes[0].clone()) }; + let c = client.clone(); + off(move || c.two_factor_disable(&proof)).await.unwrap(); + let c = client.clone(); + assert!(!off(move || c.two_factor_status()).await.unwrap().enabled); + + // A real 401 (the device was signed out on the server) is still what it was: the agent can't go on. + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + server.meta.revoke_all_devices(ana.id, craft_cloud_meta::RevokeReason::Admin, server.now()).await.unwrap(); + let c = client.clone(); + let revoked = off(move || c.two_factor_status()).await.unwrap_err(); + assert!(matches!(&revoked, Error::Refused { code, .. } if code == "needs_you"), "{revoked:?}"); + let _ = machine.stop().await; +} + +fn bad_code(secret: &[u8], server: &Server) -> String { + wrong_code(&code_for(secret, server.now())) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn joining_an_account_with_two_factor_takes_the_code_in_a_second_call() { + let server = Server::start().await; + let first = Machine::start().await; + let (secret, recovery) = protect(&server, &first).await; + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + + let second = Machine::start().await; + let client = second.client(); + let join = JoinRequest { + join: format!("{}/{}", server.url, invite.display()), + folder: Some(second.folder.display().to_string()), + device_name: Some("laptop".into()), + adopt: true, + password: None, + totp_code: None, + recovery_code: None, + }; + // No second factor: `two_factor_required` (no challenge to give: the same join goes again), and the invite is not used up. + let c = client.clone(); + let j = join.clone(); + let refused = off(move || c.join(&j)).await.unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required"), "{refused:?}"); + assert!(!client.status().unwrap().signed_in); + // A wrong code is `two_factor_invalid`; the invite still works. + server.advance(30_000); + let right = code_for(&secret, server.now()); + let c = client.clone(); + let j = JoinRequest { totp_code: Some(wrong_code(&right)), ..join.clone() }; + let invalid = off(move || c.join(&j)).await.unwrap_err(); + assert!(matches!(&invalid, Error::Refused { code, .. } if code == "two_factor_invalid"), "{invalid:?}"); + let c = client.clone(); + let j = JoinRequest { totp_code: Some(right), ..join.clone() }; + assert!(off(move || c.join(&j)).await.unwrap().signed_in); + // A recovery code does it too, on a new invite. + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + let third = Machine::start().await; + let c = third.client(); + let j = JoinRequest { + join: format!("{}/{}", server.url, invite.display()), + folder: Some(third.folder.display().to_string()), + recovery_code: Some(recovery[2].clone()), + ..join + }; + assert!(off(move || c.join(&j)).await.unwrap().signed_in); + let _ = (first.stop().await, second.stop().await, third.stop().await); +} + +/// The new calls take small bodies only, and what is secret is kept nowhere: not in `account.json`, not in what `run` prints. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nothing_secret_is_stored_or_printed_and_bodies_are_small() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, recovery) = protect(&server, &machine).await; + sign_out(&machine).await; + let client = machine.client(); + let request = machine.login_request(&server); + let c = client.clone(); + let challenge = off(move || c.login(&request)).await.unwrap_err().challenge().cloned().unwrap(); + server.advance(30_000); + let used = code_for(&secret, server.now()); + let step = machine.second_step(&server, &challenge.challenge, Some(&used), None); + let c = client.clone(); + off(move || c.login_second_factor(&step)).await.unwrap(); + + // Big bodies are refused before they are read. + let api = machine.api(); + let big = format!(r#"{{"password":"{}"}}"#, "x".repeat(6000)); + for path in ["/v1/account/two-factor/setup", "/v1/login/second-factor", "/v1/account/password"] { + let (status, _) = raw( + api.port, + &format!("POST {path} HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nAuthorization: Bearer {}\r\nContent-Type: application/json", api.port, api.token), + &big, + ); + assert_eq!(status, 413, "{path}"); + } + let account_json = std::fs::read_to_string(machine.data.path().join("account.json")).unwrap(); + let mut every_file = String::new(); + collect(machine.data.path(), &mut every_file); + let (exit, out) = machine.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + for secret_text in std::iter::once(used).chain(std::iter::once(challenge.challenge)).chain(recovery).chain(std::iter::once(PASSWORD.to_owned())) { + assert!(!account_json.contains(&secret_text), "{secret_text} in account.json"); + assert!(!every_file.contains(&secret_text), "{secret_text} in the data folder"); + assert!(!out.contains(&secret_text), "{secret_text} in what run printed"); + } +} + +fn collect(dir: &Path, into: &mut String) { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + collect(&path, into); + } else if let Ok(text) = std::fs::read_to_string(&path) { + // The local API's own token is this folder's business, not a second factor. + if path.file_name().is_some_and(|n| n != LOCAL_API_FILE) { + into.push_str(&text); + } + } + } +} + +/// One raw HTTP/1.1 request, so a test controls every header. +fn raw(port: u16, head: &str, body: &str) -> (u16, String) { + use std::io::{Read, Write}; + let mut stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream.set_read_timeout(Some(PATIENCE)).unwrap(); + let request = format!("{head}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + stream.write_all(request.as_bytes()).unwrap(); + let mut answer = String::new(); + let _ = stream.read_to_string(&mut answer); + let status = answer.split(' ').nth(1).and_then(|s| s.parse().ok()).unwrap_or(0); + (status, answer.split_once("\r\n\r\n").map(|(_, b)| b.to_owned()).unwrap_or_default()) +} + +// ---- the command line ---- + +fn cli_login(url: &str, data: &Path, folder: &Path) -> cli::Cli { + cli::Cli::try_parse_from([ + "craft-cloud-agent", + "--data-dir", + data.to_str().unwrap(), + "login", + url, + "--user", + "ana", + "--folder", + folder.to_str().unwrap(), + "--adopt", + ]) + .unwrap() +} + +/// A signed-in, protected account from the command line's point of view: the secret and a recovery code. Another agent's data +/// folder made it, through the server's own routes. +async fn protected_account(server: &Server) -> (Vec, Vec) { + let machine = Machine::start().await; + let made = protect(server, &machine).await; + let _ = machine.stop().await; + made +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_command_line_asks_for_the_code_on_standard_input() { + let server = Server::start().await; + let (secret, recovery) = protected_account(&server).await; + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + + // A wrong code, then the right one: two lines after the password, and the code is never an argument. + server.advance(30_000); + let right = code_for(&secret, server.now()); + let mut out = Vec::new(); + let stdin = format!("{PASSWORD}\n{}\n{right}\n", wrong_code(&right)); + let exit = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("two-factor sign-in") && printed.contains("signed in as ana"), "{printed}"); + assert!(!printed.contains(&right) && !printed.contains(PASSWORD), "nothing secret is echoed: {printed}"); + + // Three wrong codes: it gives up with the server's message, and nothing is signed in. + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let wrong = wrong_code(&right); + let stdin = format!("{PASSWORD}\n{wrong}\n{wrong}\n{wrong}\n"); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + assert!(matches!(err, AgentError::TwoFactorInvalid(_)), "{err:?}"); + assert!(DataDir::new(data.clone()).load_account().unwrap().is_none()); + + // A recovery code instead of a code. + let stdin = format!("{PASSWORD}\n{}\n", recovery[0].to_lowercase()); + let mut out = Vec::new(); + let exit = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + assert_eq!(exit, Exit::Done, "{}", String::from_utf8_lossy(&out)); + + // Nothing on standard input where the code should be: a plain usage error, no panic. + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let stdin = format!("{PASSWORD}\n"); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + assert!(matches!(err, AgentError::Usage(ref m) if m.contains("no code")), "{err:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_lock_says_how_long_to_wait() { + let server = Server::start().await; + let machine = Machine::start().await; + let (secret, _) = protect(&server, &machine).await; + let client = machine.client(); + for _ in 0..10 { + let proof = TwoFactorProofRequest { password: PASSWORD.into(), totp_code: Some(bad_code(&secret, &server)), recovery_code: None }; + let c = client.clone(); + let _ = off(move || c.two_factor_disable(&proof)).await; + } + let _ = machine.stop().await; + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let stdin = format!("{PASSWORD}\n{}\n", code_for(&secret, server.now() + 30_000)); + let err = cli::run(cli_login(&server.url, &data, &folder), &mut stdin.as_bytes(), &mut Vec::new()).await.unwrap_err(); + match err { + AgentError::TwoFactorLocked { retry_after_secs, message } => { + assert_eq!(retry_after_secs, Some(900)); + assert!(message.contains("15 minutes"), "{message}"); + } + other => panic!("{other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_command_line_joins_with_a_code_and_through_a_running_agent() { + let server = Server::start().await; + let (secret, _) = protected_account(&server).await; + let invite = Code::generate(INVITE_CHARS).unwrap(); + server.meta.create_invite("ana", &invite.hash(), server.now(), INVITE_TTL_MS).await.unwrap(); + server.advance(30_000); + let right = code_for(&secret, server.now()); + let dir = tempfile::tempdir().unwrap(); + let (data, folder) = (dir.path().join("data"), dir.path().join("Craft Cloud")); + let join = cli::Cli::try_parse_from([ + "craft-cloud-agent", + "--data-dir", + data.to_str().unwrap(), + "join", + &format!("{}/{}", server.url, invite.display()), + "--folder", + folder.to_str().unwrap(), + "--adopt", + ]) + .unwrap(); + let mut out = Vec::new(); + let stdin = format!("{right}\n"); + let exit = cli::run(join, &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("two-factor sign-in") && printed.contains("joined as ana"), "{printed}"); + + // `login` while `run` holds the data folder goes through the running agent, second step and all. + let machine = Machine::start().await; + server.advance(30_000); + let right = code_for(&secret, server.now()); + let data = machine.data.path().to_path_buf(); + let stdin = format!("{PASSWORD}\n{right}\n"); + let mut out = Vec::new(); + let exit = cli::run(cli_login(&server.url, &data, &machine.folder), &mut stdin.as_bytes(), &mut out).await.unwrap(); + let printed = String::from_utf8(out).unwrap(); + assert_eq!(exit, Exit::Done, "{printed}"); + assert!(printed.contains("through the running agent"), "{printed}"); + assert!(machine.client().status().unwrap().signed_in); + let _ = machine.stop().await; +} + +// ---- the admin pages' confirmation with a second factor ---- + +use craft_cloud_app::{AdminConfirm, AdminPasswordReset, UserQuery}; + +impl Server { + /// The same server with `root`, an administrator with the same password. + async fn start_with_admin() -> Self { + let server = Self::start().await; + server.meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + server + } +} + +impl Machine { + fn login_as(&self, server: &Server, user: &str) -> LoginRequest { + LoginRequest { user: user.into(), device_name: Some(format!("{user}-pc")), ..self.login_request(server) } + } +} + +/// A confirmation with the password and, if given, a code (six digits) or recovery code. +fn confirm_with(code: Option<&str>) -> AdminConfirm { + let mut confirm = AdminConfirm { password: Some(PASSWORD.into()), ..AdminConfirm::default() }; + if let Some(code) = code { + if code.len() == 6 && code.bytes().all(|b| b.is_ascii_digit()) { + confirm.totp_code = Some(code.into()); + } else { + confirm.recovery_code = Some(code.into()); + } + } + confirm +} + +/// An admin whose account has two-factor sign-in confirms each change with a code too: the agent passes the code, and the +/// server's `two_factor_invalid` (403) and `two_factor_locked` (429, with the wait) come back typed, never as a signed-out +/// agent. A reset code with `reset_two_factor` and the reset of an account's two-factor sign-in carry their fields. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_admin_with_two_factor_confirms_changes_with_a_code_through_the_agent() { + let server = Server::start_with_admin().await; + let admin = Machine::start().await; + let client = admin.client(); + let c = client.clone(); + let request = admin.login_as(&server, "root"); + assert!(off(move || c.login(&request)).await.unwrap().signed_in); + + // The account says it is an admin, without two-factor sign-in yet. + let c = client.clone(); + let me = off(move || c.account()).await.unwrap(); + assert_eq!((me.username.as_str(), me.role.as_str(), me.two_factor_enabled), ("root", "admin", false)); + // Without it, a password is enough. + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let ana_id = ana.id.0; + let c = client.clone(); + off(move || c.admin_disable_user(ana_id, &confirm_with(None))).await.unwrap(); + let c = client.clone(); + off(move || c.admin_enable_user(ana_id, &confirm_with(None))).await.unwrap(); + + // Two-factor sign-in on for the admin. + let setup = { + let c = client.clone(); + off(move || c.two_factor_setup(PASSWORD)).await.unwrap() + }; + let secret = totp::base32_decode(&setup.secret).unwrap(); + let first = code_for(&secret, server.now()); + let c = client.clone(); + let recovery = off(move || c.two_factor_confirm(&first)).await.unwrap().recovery_codes; + let c = client.clone(); + assert!(off(move || c.account()).await.unwrap().two_factor_enabled); + + // A password alone, or with a wrong code: 403 `two_factor_invalid`; the agent is as it was. + let c = client.clone(); + let missing = off(move || c.admin_disable_user(ana_id, &confirm_with(None))).await.unwrap_err(); + assert_eq!(missing.code(), Some("two_factor_invalid"), "{missing:?}"); + server.advance(30_000); + let right = code_for(&secret, server.now()); + let wrong = wrong_code(&right); + let c = client.clone(); + let bad = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&wrong)))).await.unwrap_err(); + assert_eq!(bad.code(), Some("two_factor_invalid"), "{bad:?}"); + let status = client.status().unwrap(); + assert!(status.signed_in && !status.removed, "{status:?}"); + assert!(!server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_some(), "nothing changed"); + // The right code: done. + let c = client.clone(); + let fresh = right.clone(); + off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&fresh)))).await.unwrap(); + assert!(server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_some()); + // A recovery code, in the other field, once. + let c = client.clone(); + let code = recovery[0].clone(); + off(move || c.admin_enable_user(ana_id, &confirm_with(Some(&code)))).await.unwrap(); + assert!(server.meta.admin_user(ana.id, 0).await.unwrap().unwrap().disabled_ms.is_none()); + let c = client.clone(); + let code = recovery[0].clone(); + let used = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&code)))).await.unwrap_err(); + assert_eq!(used.code(), Some("two_factor_invalid"), "a recovery code works once: {used:?}"); + + // A reset code that also turns two-factor off when it is used: the fields travel. + server.advance(30_000); + let reset = AdminPasswordReset { + password: Some(PASSWORD.into()), + totp_code: Some(code_for(&secret, server.now())), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + let c = client.clone(); + let issued = off(move || c.admin_issue_reset(ana_id, &reset)).await.unwrap(); + assert!(!issued.code.is_empty()); + + // The list shows who has two-factor sign-in; resetting it for an account that has it. + let ana_machine = Machine::start().await; + let ana_client = ana_machine.client(); + let c = ana_client.clone(); + let ana_request = ana_machine.login_as(&server, "ana"); + // (Ana's password was not changed: the reset code is only issued.) + assert!(off(move || c.login(&ana_request)).await.unwrap().signed_in); + let c = ana_client.clone(); + let ana_setup = off(move || c.two_factor_setup(PASSWORD)).await.unwrap(); + let ana_secret = totp::base32_decode(&ana_setup.secret).unwrap(); + let ana_first = code_for(&ana_secret, server.now()); + let c = ana_client.clone(); + off(move || c.two_factor_confirm(&ana_first)).await.unwrap(); + let c = client.clone(); + let listed = off(move || c.admin_users(&UserQuery { query: Some("ana".into()), ..UserQuery::default() })).await.unwrap(); + assert!(listed.users.iter().any(|u| u.username == "ana" && u.two_factor_enabled), "{listed:?}"); + server.advance(30_000); + let c = client.clone(); + let fresh = code_for(&secret, server.now()); + off(move || c.admin_reset_two_factor(ana_id, &confirm_with(Some(&fresh)))).await.unwrap(); + let c = ana_client.clone(); + let off_state = off(move || c.two_factor_status()).await; + // Ana's device was signed out with it: her agent is told so, as a revoked device. + assert!(off_state.is_err() || !off_state.unwrap().enabled); + let c = client.clone(); + let listed = off(move || c.admin_users(&UserQuery { query: Some("ana".into()), ..UserQuery::default() })).await.unwrap(); + assert!(listed.users.iter().any(|u| u.username == "ana" && !u.two_factor_enabled), "{listed:?}"); + + // Too many wrong codes stop the confirmation: the server's per-account limit says `429 rate_limited` first (wrong codes + // count like wrong passwords), and its lock `429 two_factor_locked` with the wait comes after more; either is kept as the + // server said it (the code, the status, the wait if there is one), and the agent stays signed in. + let mut stopped = None; + for _ in 0..12 { + let c = client.clone(); + let wrong = wrong_code(&code_for(&secret, server.now())); + let refused = off(move || c.admin_disable_user(ana_id, &confirm_with(Some(&wrong)))).await.unwrap_err(); + if matches!(refused.code(), Some("rate_limited" | "two_factor_locked")) { + stopped = Some(refused); + break; + } + assert_eq!(refused.code(), Some("two_factor_invalid"), "{refused:?}"); + } + let stopped = stopped.expect("too many wrong codes are stopped"); + if stopped.code() == Some("two_factor_locked") { + assert!(stopped.retry_after_secs().is_some_and(|secs| secs > 0), "{stopped:?}"); + } + assert!(client.status().unwrap().signed_in); + + let (exit, out) = admin.stop().await; + assert_eq!(exit.unwrap(), Exit::Done, "{out}"); + for secret in [PASSWORD, issued.code.as_str()] { + assert!(!out.contains(secret), "the agent printed a secret: {out}"); + } + let _ = ana_machine.stop().await; +} diff --git a/apps/craft-cloud-desktop/src/admin.rs b/apps/craft-cloud-desktop/src/admin.rs new file mode 100644 index 00000000..3eaf17b4 --- /dev/null +++ b/apps/craft-cloud-desktop/src/admin.rs @@ -0,0 +1,1700 @@ +//! The admin pages' state, as a pure function of what happened, like [`crate::model`] (which hosts it: the pages are +//! [`Screen::Admin`], and [`Event::Admin`] and the admin [`Effect`]s go through the same [`Model::update`]). +//! +//! - **Who sees them**: the pages are offered only when the server answers the admin counters +//! ([`Effect::AdminServer`], asked at every sign-in; [`Access::Admin`]); a `forbidden` makes it [`Access::Member`]. +//! That is a convenience, not a check: the server decides every call again, and a refusal is shown as the server's. +//! - **Pages**: the users (a paged list with search), one user (role and quota, disable, enable, remove, invite and +//! reset code, the user's devices), and a new account (name and role; its quota is set afterwards, on the user's +//! page, as a change of its own). +//! - **Every change asks for the admin's password afresh** in a confirmation ([`Dialog`], holding a [`Confirmation`]): +//! its text goes into the call as the body's `password` and is cleared from the model at once. Whatever else the +//! server comes to want of a confirmation (a second factor's code) is another field of [`Confirmation`] and of +//! [`Confirmation::take`]: the dialog, the events and the calls stay as they are. +//! - **A code is shown once** ([`ShownCode`]): an invite or a reset code is kept in the model only until the admin +//! dismisses it (or leaves the pages), never written anywhere, and left out of `Debug` output. + +use craft_cloud_app::{ + AccountInfo, AccountUsage, AdminCode, AdminCreatedUser, AdminServerInfo, AdminStorageOverview, AdminUserInfo, AdminUserList, DeviceId, + DeviceInfo, Error, UserQuery, Uuid, +}; +use craft_cloud_proto::api::{AdminConfirm, codes}; + +use crate::model::{Effect, Event, Model, Overview, Screen}; +use crate::security::{digits, explain, is_code}; + +/// A number no earlier call of the admin pages had, for matching an answer to its call: the numbers never restart, so an +/// answer that comes late (after the pages were left and opened again, or the account signed out and in) matches +/// nothing and is ignored. +pub(crate) fn fresh() -> u64 { + static LAST: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + LAST.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) +} + +/// Accounts on one page of the list. +pub const PAGE_SIZE: u32 = 20; +/// The most a quota may be, in GiB (a petabyte: a typo, not a quota, past it). +pub const MAX_QUOTA_GIB: f64 = 1_048_576.0; +const GIB: f64 = 1_073_741_824.0; + +/// Whether the signed-in account is an admin, as the server describes the account (`GET /account`). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum Access { + /// Not asked yet, or the server couldn't be reached: no admin pages. + #[default] + Unknown, + /// The account is a member (or the server refused the admin calls as `forbidden`). + Member, + /// The server says the account is an admin: the admin pages are offered. `two_factor`: whether the account has two-factor + /// sign-in, so that its confirmations ask for a code too. + Admin { two_factor: bool }, +} + +/// An account's role. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum RoleChoice { + #[default] + Member, + Admin, +} + +impl RoleChoice { + /// The server's name for it. + pub fn as_str(self) -> &'static str { + match self { + Self::Member => "member", + Self::Admin => "admin", + } + } + + /// The role the server names (anything but `admin` reads as a member). + pub fn of(role: &str) -> Self { + if role == "admin" { Self::Admin } else { Self::Member } + } + + pub fn label(self) -> &'static str { + match self { + Self::Member => "Member", + Self::Admin => "Admin", + } + } +} + +/// The admin's confirmation of a change: their current password and, when their account has two-factor sign-in, a code from +/// the authenticator app or a recovery code (one field: six digits are a code, anything else a recovery code, as on the Security +/// page). Its `Debug` output hides both. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Confirmation { + pub password: String, + pub code: String, +} + +impl std::fmt::Debug for Confirmation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Confirmation").field("password", &"").field("code", &"").finish() + } +} + +impl Confirmation { + /// Whether enough was typed to send: the password, and the code where the page asks for one. + pub fn ready(&self, needs_code: bool) -> bool { + !self.password.is_empty() && (!needs_code || !self.code.trim().is_empty()) + } + + /// The confirmation as the calls send it, with everything typed cleared from here. + pub fn take(&mut self) -> AdminConfirm { + let password = std::mem::take(&mut self.password); + let code = std::mem::take(&mut self.code); + let code = code.trim(); + let (totp_code, recovery_code) = if code.is_empty() { + (None, None) + } else if is_code(code) { + (Some(digits(code)), None) + } else { + (None, Some(code.to_owned())) + }; + AdminConfirm { password: Some(password), totp_code, recovery_code } + } +} + +/// A change the admin can make, with what its confirmation page says about it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + CreateUser { + username: String, + role: RoleChoice, + }, + /// A new role and/or quota (`quota`: `Some(None)` removes it). + UpdateUser { + id: Uuid, + username: String, + role: Option, + quota: Option>, + }, + Disable { + id: Uuid, + username: String, + }, + Enable { + id: Uuid, + username: String, + }, + Remove { + id: Uuid, + username: String, + }, + IssueInvite { + id: Uuid, + username: String, + }, + RevokeInvite { + id: Uuid, + username: String, + }, + /// A password reset code; `reset_two_factor`: redeeming it also turns the account's two-factor sign-in off. + IssueReset { + id: Uuid, + username: String, + reset_two_factor: bool, + }, + /// Takes the account's two-factor sign-in away and signs its computers out. + ResetTwoFactor { + id: Uuid, + username: String, + }, + RevokeDevice { + id: Uuid, + username: String, + device: DeviceId, + device_name: String, + }, +} + +impl Action { + /// The confirmation page's title. + pub fn title(&self) -> String { + match self { + Self::CreateUser { username, .. } => format!("Create {username}"), + Self::UpdateUser { username, .. } => format!("Change {username}"), + Self::Disable { username, .. } => format!("Disable {username}"), + Self::Enable { username, .. } => format!("Enable {username}"), + Self::Remove { username, .. } => format!("Remove {username}"), + Self::IssueInvite { username, .. } => format!("New invite for {username}"), + Self::RevokeInvite { username, .. } => format!("Revoke the invite of {username}"), + Self::IssueReset { username, .. } => format!("Password reset code for {username}"), + Self::ResetTwoFactor { username, .. } => format!("Turn two-factor sign-in off for {username}"), + Self::RevokeDevice { username, device_name, .. } => format!("Sign {device_name} of {username} out"), + } + } + + /// What the change does, in a sentence or two. + pub fn explain(&self) -> String { + match self { + Self::CreateUser { role, .. } => { + format!("A new {} account without a password. You get an invite code to give them, shown once.", role.as_str()) + } + Self::UpdateUser { role, quota, .. } => { + let mut parts = Vec::new(); + if let Some(role) = role { + parts.push(format!("the role becomes {}", role.as_str())); + } + match quota { + Some(Some(bytes)) => parts.push(format!("the quota becomes {}", format_bytes(*bytes))), + Some(None) => parts.push("the quota is removed".to_owned()), + None => {} + } + sentence(&parts.join(" and ")) + } + Self::Disable { .. } => "Their computers are signed out and they can't sign in until you enable the account again.".into(), + Self::Enable { .. } => "They can sign in again. Computers that were signed out stay signed out.".into(), + Self::Remove { .. } => "The account is marked for removal and its computers are signed out. Nothing it stored is deleted.".into(), + Self::IssueInvite { .. } => "A new invite code for an account that has no password yet, shown once. It replaces the open one.".into(), + Self::RevokeInvite { .. } => "The account's open invite or password reset code stops working.".into(), + Self::IssueReset { reset_two_factor, .. } => { + let mut text = "A code good for an hour that sets a new password. Using it signs every computer of the account out. Shown once.".to_owned(); + if *reset_two_factor { + text.push_str(" Using it also turns the account's two-factor sign-in off."); + } + text + } + Self::ResetTwoFactor { .. } => { + "The account no longer asks for a code at sign-in, and every computer of it is signed out. It can turn it on again on its Security page.".into() + } + Self::RevokeDevice { .. } => "That computer can't sync any more; its files stay there.".into(), + } + } + + /// The label of the confirm button. + pub fn button(&self) -> &'static str { + match self { + Self::CreateUser { .. } => "Create account", + Self::UpdateUser { .. } => "Save changes", + Self::Disable { .. } => "Disable", + Self::Enable { .. } => "Enable", + Self::Remove { .. } => "Remove account", + Self::IssueInvite { .. } => "Make an invite", + Self::RevokeInvite { .. } => "Revoke", + Self::IssueReset { .. } => "Make a reset code", + Self::ResetTwoFactor { .. } => "Turn two-factor off", + Self::RevokeDevice { .. } => "Sign out", + } + } +} + +/// What a change came to. +#[derive(Debug, Clone, PartialEq)] +pub enum Outcome { + Created(AdminCreatedUser), + Updated(AdminUserInfo), + Code(AdminCode), + Done, +} + +/// The password page of one change. +#[derive(Debug, Clone, PartialEq)] +pub struct Dialog { + pub action: Action, + pub confirmation: Confirmation, + /// Whether the page asks for a code too (the admin's account has two-factor sign-in, or the server said so). + pub needs_code: bool, + /// What went wrong with the last try, for people. + pub error: Option, + /// The call is with the server: nothing can be sent twice. + pub busy: bool, +} + +/// What kind of code is shown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CodeKind { + Invite, + PasswordReset, +} + +/// A code the server made, shown once. Its `Debug` output hides the code and the link. +#[derive(Clone, PartialEq, Eq)] +pub struct ShownCode { + pub username: String, + pub kind: CodeKind, + pub code: String, + /// The whole join string, when the server knows its public address. + pub join_url: Option, + pub expires_ms: i64, +} + +impl std::fmt::Debug for ShownCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ShownCode") + .field("username", &self.username) + .field("kind", &self.kind) + .field("code", &"") + .field("join_url", &self.join_url.as_ref().map(|_| "")) + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// The list of accounts: one page, the search that made it, and the cursors back to the first. +#[derive(Debug, Clone, PartialEq)] +pub struct UserList { + /// The search field, as typed. + pub search: String, + /// The search the shown page was asked with. + pub applied: String, + pub users: Vec, + /// How each page up to this one was asked for: `cursors[0]` is `None`, the last is this page's. + pub cursors: Vec>, + /// The cursor of the page after this one, if there is one. + pub next: Option, + pub loading: bool, + pub loaded: bool, + pub error: Option, + /// The number of the call whose answer is wanted; older answers are ignored. + ticket: u64, +} + +impl UserList { + fn new() -> Self { + Self { + search: String::new(), + applied: String::new(), + users: Vec::new(), + cursors: vec![None], + next: None, + loading: false, + loaded: false, + error: None, + ticket: 0, + } + } + + /// The page number shown (from 1). + pub fn page(&self) -> usize { + self.cursors.len() + } + + pub fn has_previous(&self) -> bool { + self.cursors.len() > 1 + } + + /// Asks for the current page. + fn load(&mut self) -> Effect { + self.loading = true; + self.error = None; + self.ticket = fresh(); + let query = UserQuery { + cursor: self.cursors.last().cloned().flatten(), + limit: Some(PAGE_SIZE), + query: (!self.applied.is_empty()).then(|| self.applied.clone()), + }; + Effect::AdminUsers { query, ticket: self.ticket } + } + + fn find(&self, id: Uuid) -> Option<&AdminUserInfo> { + self.users.iter().find(|u| u.id == id) + } +} + +/// The form for a new account. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CreateForm { + pub username: String, + pub role: RoleChoice, + pub error: Option, +} + +/// One account's page: its role and quota to change, and its devices. +#[derive(Debug, Clone, PartialEq)] +pub struct UserView { + pub id: Uuid, + pub username: String, + /// The role field. + pub role: RoleChoice, + /// The quota field, in GiB (empty: none). + pub quota: String, + /// What the quota field showed when the page was opened or last saved, to tell whether it was changed. + pub quota_shown: String, + pub error: Option, + pub devices: Option>, + pub devices_error: Option, + /// The number of the devices call whose answer is wanted. + devices_ticket: u64, + pub usage: Option, + pub usage_error: Option, + pub usage_loading: bool, + usage_ticket: u64, +} + +impl UserView { + fn new(user: &AdminUserInfo) -> Self { + let quota = user.quota_bytes.map(quota_text).unwrap_or_default(); + Self { + id: user.id, + username: user.username.clone(), + role: RoleChoice::of(&user.role), + quota_shown: quota.clone(), + quota, + error: None, + devices: None, + devices_error: None, + devices_ticket: 0, + usage: None, + usage_error: None, + usage_loading: false, + usage_ticket: 0, + } + } +} + +/// Which page of the admin pages is shown. +#[derive(Debug, Clone, PartialEq)] +pub enum View { + List, + Create(CreateForm), + User(Box), + /// The security audit log ([`crate::audit`]). + Audit(Box), +} + +/// The admin pages. +#[derive(Debug, Clone, PartialEq)] +pub struct AdminScreen { + /// The signed-in view the pages came from and go back to. + pub overview: Overview, + pub list: UserList, + /// The server's counters, once they came. + pub server: Option, + pub storage: Option, + pub storage_loading: bool, + pub storage_error: Option, + storage_ticket: u64, + pub view: View, + /// A change waiting for the admin's password. + pub dialog: Option, + /// A code just made, until it is dismissed. + pub code: Option, +} + +/// What happened on the admin pages: clicks and the answers of the admin calls. +#[derive(Debug, Clone, PartialEq)] +pub enum AdminEvent { + /// [`Effect::Account`]'s answer: the role of the signed-in account. + Account { + session: u64, + result: Result, + }, + /// [`Effect::AdminServer`]'s answer: the counters. + Server { + session: u64, + result: Result, + }, + /// [`Effect::AdminStorage`]'s answer. + Storage { + session: u64, + ticket: u64, + result: Result, + }, + /// [`Effect::AdminUsers`]'s answer. + Users { + ticket: u64, + result: Result, + }, + /// [`Effect::AdminDevices`]'s answer. + Devices { + user: Uuid, + ticket: u64, + result: Result, Error>, + }, + /// [`Effect::AdminUserUsage`]'s answer. + Usage { + user: Uuid, + ticket: u64, + result: Result, + }, + /// [`Effect::AdminDo`]'s answer. + Done(Result), + + /// The Administration button. + Open, + /// Back to the signed-in view. + Close, + /// Search for the text in the search field. + Search, + ClearSearch, + NextPage, + PreviousPage, + Reload, + NewUser, + /// Go on from the new account's form to the password. + SubmitCreate, + OpenUser(Uuid), + /// Back to the list from an account's page or the new account's form. + BackToList, + /// Go on from the account's role and quota to the password. + SubmitEdit, + /// Ask for the password of a change. + Ask(Action), + /// Send the change with the password typed. + Confirm, + CancelDialog, + /// The shown code was noted: forget it. + DismissCode, + RefreshDevices, + RefreshUsage, +} + +impl Model { + /// The admin pages, if shown. + pub fn admin_screen(&self) -> Option<&AdminScreen> { + match &self.screen { + Screen::Admin(admin) => Some(admin), + _ => None, + } + } + + /// Whether the admin pages are offered: the server said the signed-in account is an admin. + pub fn is_admin(&self) -> bool { + self.overview().is_some_and(|o| matches!(o.access, Access::Admin { .. })) + } + + pub(crate) fn admin(&mut self, event: AdminEvent) -> Vec { + match event { + AdminEvent::Account { session, result } => self.admin_probed(session, result), + AdminEvent::Open => self.open_admin(), + AdminEvent::Close => { + self.leave_admin(); + Vec::new() + } + event => { + let Screen::Admin(admin) = &mut self.screen else { return Vec::new() }; + admin.update(event) + } + } + } + + /// The account came (an admin or not), or its role can't be told. + fn admin_probed(&mut self, session: u64, result: Result) -> Vec { + // An answer for an earlier sign-in (another account, maybe) says nothing about this one. + if self.overview().map(|o| o.session) != Some(session) { + return Vec::new(); + } + let access = match result { + Ok(account) if account.role == "admin" => Access::Admin { two_factor: account.two_factor_enabled }, + Ok(_) => Access::Member, + Err(Error::Refused { code, .. }) if code == codes::FORBIDDEN => Access::Member, + // Unreachable, or an agent or server that doesn't know the call: asked again at the next sign-in or refresh. + Err(_) => return Vec::new(), + }; + if let Some(overview) = self.overview_mut() { + overview.access = access.clone(); + } + if access == Access::Member { + self.leave_admin(); + } + Vec::new() + } + + fn open_admin(&mut self) -> Vec { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + if !matches!(overview.access, Access::Admin { .. }) { + return Vec::new(); + } + let mut admin = AdminScreen { + overview: overview.clone(), + list: UserList::new(), + server: None, + storage: None, + storage_loading: false, + storage_error: None, + storage_ticket: 0, + view: View::List, + dialog: None, + code: None, + }; + let effects = vec![admin.list.load(), Effect::AdminServer { session: admin.overview.session }, admin.load_storage()]; + self.screen = Screen::Admin(Box::new(admin)); + effects + } + + /// Back to the signed-in view; a code still shown, and anything typed, is dropped. + fn leave_admin(&mut self) { + if let Screen::Admin(admin) = &self.screen { + self.screen = Screen::SignedIn(admin.overview.clone()); + } + } +} + +impl AdminScreen { + fn update(&mut self, event: AdminEvent) -> Vec { + match event { + AdminEvent::Account { .. } | AdminEvent::Open | AdminEvent::Close => Vec::new(), + AdminEvent::Server { session, result } => { + if session != self.overview.session { + return Vec::new(); + } + match result { + Ok(info) => { + self.server = Some(info); + Vec::new() + } + // Not an admin any more, or not told: the account is looked at again. + Err(e) if is_forbidden(&e) => vec![Effect::Account { session: self.overview.session }], + Err(_) => Vec::new(), + } + } + AdminEvent::Storage { session, ticket, result } => { + if session != self.overview.session || ticket != self.storage_ticket { + return Vec::new(); + } + self.storage_loading = false; + match result { + Ok(storage) => { + self.storage = Some(storage); + self.storage_error = None; + Vec::new() + } + Err(error) => { + self.storage_error = Some(error_text(&error)); + if is_forbidden(&error) { vec![Effect::Account { session: self.overview.session }] } else { Vec::new() } + } + } + } + AdminEvent::Users { ticket, result } => self.users_came(ticket, result), + AdminEvent::Devices { user, ticket, result } => { + let mut effects = Vec::new(); + if let View::User(view) = &mut self.view + && view.id == user + && view.devices_ticket == ticket + { + match result { + Ok(list) => { + view.devices = Some(list); + view.devices_error = None; + } + Err(e) => { + view.devices_error = Some(error_text(&e)); + if is_forbidden(&e) { + effects.push(Effect::Account { session: self.overview.session }); + } + } + } + } + effects + } + AdminEvent::Usage { user, ticket, result } => { + let mut effects = Vec::new(); + if let View::User(view) = &mut self.view + && view.id == user + && view.usage_ticket == ticket + { + view.usage_loading = false; + match result { + Ok(usage) => { + view.usage = Some(usage); + view.usage_error = None; + } + Err(error) => { + view.usage_error = Some(error_text(&error)); + if is_forbidden(&error) { + effects.push(Effect::Account { session: self.overview.session }); + } + } + } + } + effects + } + AdminEvent::Done(result) => self.done(result), + AdminEvent::Search => { + self.list.applied = self.list.search.trim().to_owned(); + self.list.search = self.list.applied.clone(); + self.list.cursors = vec![None]; + vec![self.list.load()] + } + AdminEvent::ClearSearch => { + self.list.search.clear(); + self.list.applied.clear(); + self.list.cursors = vec![None]; + vec![self.list.load()] + } + AdminEvent::NextPage => match self.list.next.clone() { + Some(next) if !self.list.loading => { + self.list.cursors.push(Some(next)); + vec![self.list.load()] + } + _ => Vec::new(), + }, + AdminEvent::PreviousPage => { + if self.list.has_previous() && !self.list.loading { + self.list.cursors.pop(); + vec![self.list.load()] + } else { + Vec::new() + } + } + AdminEvent::Reload => { + if self.list.loading { + return Vec::new(); + } + vec![self.list.load(), self.load_storage()] + } + AdminEvent::NewUser => { + if self.dialog.is_none() { + self.view = View::Create(CreateForm::default()); + } + Vec::new() + } + AdminEvent::SubmitCreate => self.submit_create(), + AdminEvent::OpenUser(id) => { + let Some(user) = self.list.find(id) else { return Vec::new() }; + if self.dialog.is_some() { + return Vec::new(); + } + let mut view = UserView::new(user); + view.devices_ticket = fresh(); + view.usage_ticket = fresh(); + view.usage_loading = true; + let effects = vec![ + Effect::AdminDevices { user: id, ticket: view.devices_ticket }, + Effect::AdminUserUsage { user: id, ticket: view.usage_ticket }, + ]; + self.view = View::User(Box::new(view)); + effects + } + AdminEvent::BackToList => { + if self.dialog.is_none() { + self.view = View::List; + } + Vec::new() + } + AdminEvent::SubmitEdit => { + self.submit_edit(); + Vec::new() + } + AdminEvent::Ask(action) => { + if self.dialog.is_none() { + self.dialog = Some(self.dialog(action)); + } + Vec::new() + } + AdminEvent::Confirm => self.confirm(), + AdminEvent::CancelDialog => { + if self.dialog.as_ref().is_some_and(|d| !d.busy) { + self.dialog = None; + } + Vec::new() + } + AdminEvent::DismissCode => { + self.code = None; + Vec::new() + } + AdminEvent::RefreshDevices => match &mut self.view { + View::User(view) => { + view.devices_ticket = fresh(); + vec![Effect::AdminDevices { user: view.id, ticket: view.devices_ticket }] + } + _ => Vec::new(), + }, + AdminEvent::RefreshUsage => match &mut self.view { + View::User(view) => { + view.usage_ticket = fresh(); + view.usage_loading = true; + view.usage_error = None; + vec![Effect::AdminUserUsage { user: view.id, ticket: view.usage_ticket }] + } + _ => Vec::new(), + }, + } + } + + fn load_storage(&mut self) -> Effect { + self.storage_ticket = fresh(); + self.storage_loading = true; + self.storage_error = None; + Effect::AdminStorage { session: self.overview.session, ticket: self.storage_ticket } + } + + fn users_came(&mut self, ticket: u64, result: Result) -> Vec { + if ticket != self.list.ticket { + return Vec::new(); + } + self.list.loading = false; + match result { + Ok(list) => { + self.list.loaded = true; + self.list.error = None; + self.list.users = list.users; + self.list.next = list.next; + Vec::new() + } + Err(e) => { + self.list.error = Some(error_text(&e)); + if is_forbidden(&e) { vec![Effect::Account { session: self.overview.session }] } else { Vec::new() } + } + } + } + + fn submit_create(&mut self) -> Vec { + let View::Create(form) = &mut self.view else { return Vec::new() }; + if self.dialog.is_some() { + return Vec::new(); + } + let username = form.username.trim().to_owned(); + if !valid_username(&username) { + form.error = Some(BAD_USERNAME.to_owned()); + return Vec::new(); + } + form.error = None; + let action = Action::CreateUser { username, role: form.role }; + self.dialog = Some(self.dialog(action)); + Vec::new() + } + + fn submit_edit(&mut self) { + let View::User(view) = &mut self.view else { return }; + if self.dialog.is_some() { + return; + } + let Some(user) = self.list.users.iter().find(|u| u.id == view.id) else { + view.error = Some("That account isn't in the list any more. Go back and look again.".to_owned()); + return; + }; + let role = (view.role.as_str() != user.role).then_some(view.role); + let quota_changed = view.quota.trim() != view.quota_shown.trim(); + let quota = if quota_changed { + match parse_quota(&view.quota) { + Ok(quota) => Some(quota), + Err(why) => { + view.error = Some(why); + return; + } + } + } else { + None + }; + if role.is_none() && quota.is_none() { + view.error = Some("Nothing to change.".to_owned()); + return; + } + view.error = None; + let action = Action::UpdateUser { id: view.id, username: view.username.clone(), role, quota }; + self.dialog = Some(self.dialog(action)); + } + + fn confirm(&mut self) -> Vec { + let Some(dialog) = &mut self.dialog else { return Vec::new() }; + if dialog.busy { + return Vec::new(); + } + if dialog.confirmation.password.is_empty() { + dialog.error = Some(NEEDS_PASSWORD.to_owned()); + return Vec::new(); + } + if !dialog.confirmation.ready(dialog.needs_code) { + dialog.error = Some(NEEDS_CODE.to_owned()); + return Vec::new(); + } + dialog.error = None; + dialog.busy = true; + // The typed password leaves the model here: it travels in the call and nowhere else. + let confirm = dialog.confirmation.take(); + vec![Effect::AdminDo { action: dialog.action.clone(), confirm }] + } + + fn done(&mut self, result: Result) -> Vec { + let Some(dialog) = &mut self.dialog else { return Vec::new() }; + if !dialog.busy { + return Vec::new(); + } + let outcome = match result { + Ok(outcome) => outcome, + Err(e) => { + dialog.busy = false; + dialog.confirmation = Confirmation::default(); + // The server wants a code: ask for it (the account's two-factor state may have changed since sign-in). + if e.code() == Some(codes::TWO_FACTOR_INVALID) { + dialog.needs_code = true; + } + dialog.error = Some(match (&dialog.action, &e) { + (Action::RevokeInvite { .. }, Error::Refused { code, .. }) if code == codes::NOT_FOUND => NO_OPEN_CODE.to_owned(), + _ => error_text(&e), + }); + let none_open = matches!(&dialog.action, Action::RevokeInvite { .. }) && dialog.error.as_deref() == Some(NO_OPEN_CODE); + return if is_forbidden(&e) { + vec![Effect::Account { session: self.overview.session }] + } else if none_open { + // The list said an invite was open, or didn't say: look again. + vec![self.list.load()] + } else { + Vec::new() + }; + } + }; + let action = dialog.action.clone(); + self.dialog = None; + let mut effects = Vec::new(); + match (&action, outcome) { + (Action::CreateUser { username, .. }, Outcome::Created(created)) => { + self.show_code(username, CodeKind::Invite, created.invite); + self.view = View::List; + } + (Action::IssueInvite { username, .. }, Outcome::Code(code)) => self.show_code(username, CodeKind::Invite, code), + (Action::IssueReset { username, .. }, Outcome::Code(code)) => self.show_code(username, CodeKind::PasswordReset, code), + (Action::UpdateUser { .. }, Outcome::Updated(info)) => { + if let View::User(view) = &mut self.view + && view.id == info.id + { + **view = UserView { + devices: view.devices.take(), + devices_error: view.devices_error.take(), + devices_ticket: view.devices_ticket, + ..UserView::new(&info) + }; + view.usage_ticket = fresh(); + view.usage_loading = true; + effects.push(Effect::AdminUserUsage { user: view.id, ticket: view.usage_ticket }); + } + } + (Action::Remove { .. }, _) => self.view = View::List, + (Action::RevokeDevice { id, .. }, _) => { + if let View::User(view) = &mut self.view + && view.id == *id + { + view.devices_ticket = fresh(); + effects.push(Effect::AdminDevices { user: *id, ticket: view.devices_ticket }); + } + } + _ => {} + } + // What changed shows in the list (and the counters). + effects.push(self.list.load()); + effects.push(Effect::AdminServer { session: self.overview.session }); + effects.push(self.load_storage()); + effects + } + + /// A page for the confirmation of `action`: it asks for a code too when the account has two-factor sign-in. + fn dialog(&self, action: Action) -> Dialog { + let needs_code = matches!(self.overview.access, Access::Admin { two_factor: true }); + Dialog { action, confirmation: Confirmation::default(), needs_code, error: None, busy: false } + } + + fn show_code(&mut self, username: &str, kind: CodeKind, code: AdminCode) { + self.code = Some(ShownCode { username: username.to_owned(), kind, code: code.code, join_url: code.join_url, expires_ms: code.expires_ms }); + } +} + +/// The words when a revoke finds no invite or reset code open (the server's `not_found`). +pub const NO_OPEN_CODE: &str = "This account has no open invite or password reset code."; +/// The words the confirmation page says when the code is missing. +pub const NEEDS_CODE: &str = "Type the code from your authenticator app (or a recovery code) to confirm"; +/// The words the confirmation page says when no password was typed. +pub const NEEDS_PASSWORD: &str = "Type your own password to confirm"; +/// The words when a new account's name breaks the server's rules. +pub const BAD_USERNAME: &str = "A name is 1 to 64 letters, digits, dots, underscores or dashes"; + +/// Whether `name` is a username the server takes (1-64 ASCII letters, digits, `.`, `_` or `-`). +pub fn valid_username(name: &str) -> bool { + (1..=64).contains(&name.len()) && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +pub(crate) fn is_forbidden(e: &Error) -> bool { + matches!(e, Error::Refused { code, .. } if code == codes::FORBIDDEN) +} + +/// An admin call's error, for people: the server's refusals the admin can act on in plain words (a wrong password, a wrong or +/// locked second factor and a rate limit as the Security page says them). +pub fn error_text(e: &Error) -> String { + match e.code() { + Some(codes::WRONG_PASSWORD | codes::TWO_FACTOR_INVALID | codes::TWO_FACTOR_LOCKED | codes::RATE_LIMITED) => explain(e), + Some(codes::LAST_ADMIN) => "That would leave the server without an enabled admin. Make another account an admin first.".to_owned(), + Some(codes::ACCOUNT_DISABLED) => "Your account is disabled, so the server refuses this.".to_owned(), + Some(codes::FORBIDDEN) => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => format!("The server doesn't allow this: {}", lower_first(message)), + other => other.to_string(), + }, + _ => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), + other => other.to_string(), + }, + } +} + +fn lower_first(text: &str) -> String { + let mut chars = text.chars(); + chars.next().map(|first| first.to_lowercase().chain(chars).collect()).unwrap_or_default() +} + +/// `text` as a sentence: a capital first, a full stop last (nothing for an empty text). +fn sentence(text: &str) -> String { + let text = text.trim().trim_end_matches('.'); + let mut chars = text.chars(); + match chars.next() { + Some(first) => format!("{}.", first.to_uppercase().chain(chars).collect::()), + None => String::new(), + } +} + +/// A quota typed in GiB (`50`, `0.5`; empty: none). +pub fn parse_quota(text: &str) -> Result, String> { + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let bad = || "A quota is a number of GiB, like 50 or 0.5 (empty: no quota)".to_owned(); + let gib: f64 = text.parse().map_err(|_| bad())?; + if !gib.is_finite() || gib < 0.0 { + return Err(bad()); + } + if gib > MAX_QUOTA_GIB { + return Err(format!("A quota is at most {MAX_QUOTA_GIB:.0} GiB")); + } + // At most 2^20 GiB: the product is far under u64's range and rounds to a whole number of bytes. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = (gib * GIB).round() as u64; + Ok(Some(bytes)) +} + +/// A quota's bytes as the quota field shows them: GiB, to 3 places, without trailing zeros. +pub fn quota_text(bytes: u64) -> String { + let gib = bytes as f64 / GIB; + let text = format!("{gib:.3}"); + text.trim_end_matches('0').trim_end_matches('.').to_owned() +} + +/// A size for people: `512 B`, `1.5 MiB`, `2 GiB`. +pub fn format_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut value = bytes as f64; + let mut unit = 0; + while value >= 1024.0 && unit + 1 < UNITS.len() { + value /= 1024.0; + unit += 1; + } + let number = if unit == 0 || value >= 100.0 { format!("{value:.0}") } else { format!("{value:.1}") }; + let number = if number.contains('.') { number.trim_end_matches('0').trim_end_matches('.').to_owned() } else { number }; + format!("{number} {}", UNITS.get(unit).copied().unwrap_or("B")) +} + +impl From for Event { + fn from(event: AdminEvent) -> Self { + Event::Admin(event) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Arc; + + use super::*; + use crate::agent::fake::{FakeAgent, sample_user}; + use crate::agent::{Starter, call}; + use crate::model::{JoinDefaults, tests::status}; + + /// An agent signed in as `fabio`, an admin (or not), with `others` more accounts after fabio's own. + pub(crate) fn agent(admin: bool, others: usize) -> FakeAgent { + let agent = FakeAgent::new(status(true)); + agent.admin.store(admin, std::sync::atomic::Ordering::SeqCst); + let mut users = vec![AdminUserInfo { role: "admin".into(), ..sample_user("fabio", Uuid::now_v7()) }]; + users.extend((1..=others).map(|n| sample_user(&format!("user{n:02}"), Uuid::now_v7()))); + *agent.users.lock().unwrap() = users; + agent + } + + /// Feeds `events` to the model, each effect answered by the fake until nothing is left (as the link does). + pub(crate) fn run(model: &mut Model, agent: &FakeAgent, events: Vec) { + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err("not started in tests".to_owned())); + for event in events { + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Some(answer) = call(agent, &start, effect) { + pending.extend(model.update(answer)); + } + } + } + } + + pub(crate) fn signed_in(agent: &FakeAgent) -> Model { + let mut model = Model::new(JoinDefaults::default(), true, "run.log".into()); + run(&mut model, agent, vec![Event::Start]); + model + } + + /// The model on the admin pages. + pub(crate) fn on_the_pages(agent: &FakeAgent) -> Model { + let mut model = signed_in(agent); + run(&mut model, agent, vec![Event::Admin(AdminEvent::Open)]); + assert!(model.admin_screen().is_some(), "{:?}", model.screen); + model + } + + pub(crate) fn admin(model: &Model) -> &AdminScreen { + model.admin_screen().unwrap_or_else(|| panic!("not on the admin pages: {:?}", model.screen)) + } + + pub(crate) fn admin_mut(model: &mut Model) -> &mut AdminScreen { + match &mut model.screen { + Screen::Admin(admin) => admin, + other => panic!("not on the admin pages: {other:?}"), + } + } + + pub(crate) fn user_named(model: &Model, name: &str) -> AdminUserInfo { + admin(model).list.users.iter().find(|u| u.username == name).cloned().unwrap_or_else(|| panic!("no {name} in the list")) + } + + pub(crate) fn ev(event: AdminEvent) -> Vec { + vec![Event::Admin(event)] + } + + pub(crate) fn type_password(model: &mut Model, password: &str) { + admin_mut(model).dialog.as_mut().unwrap().confirmation.password = password.to_owned(); + } + + pub(crate) fn refused(code: &str, message: &str) -> Error { + Error::Refused { code: code.into(), message: message.into(), name_taken: None } + } + + /// The pages are for admins: the server's answer to the counters decides, and a member never gets in. + #[test] + fn only_an_admin_is_offered_the_pages() { + let member = agent(false, 3); + let mut model = signed_in(&member); + assert_eq!(model.overview().unwrap().access, Access::Member); + assert!(!model.is_admin()); + assert!(model.update(Event::Admin(AdminEvent::Open)).is_empty()); + assert!(matches!(model.screen, Screen::SignedIn(_))); + assert_eq!(member.asked.lock().unwrap().iter().filter(|a| *a == "account").count(), 1); + + let boss = agent(true, 3); + let mut model = signed_in(&boss); + assert!(model.is_admin()); + assert_eq!(model.overview().unwrap().access, Access::Admin { two_factor: false }); + let effects = model.update(Event::Admin(AdminEvent::Open)); + assert!(matches!(effects.as_slice(), [Effect::AdminUsers { .. }, Effect::AdminServer { .. }, Effect::AdminStorage { .. }]), "{effects:?}"); + assert!(model.admin_screen().is_some()); + // The counters come with the pages, not with the sign-in. + assert!(model.admin_screen().unwrap().server.is_none()); + let mut model = on_the_pages(&boss); + let info = model.admin_screen().unwrap().server.clone().expect("the counters"); + assert_eq!((info.users, info.storage_kind.as_str()), (4, "disk")); + let _ = &mut model; + + // A server that can't be reached says nothing about the role: no pages, asked again at the next refresh. + let mut model = Model::new(JoinDefaults::default(), true, String::new()); + model.update(Event::Start); + model.update(Event::Found(Ok(status(true)))); + let session = model.overview().unwrap().session; + model.update(Event::Admin(AdminEvent::Account { session, result: Err(Error::Io("reset".into())) })); + assert_eq!(model.overview().unwrap().access, Access::Unknown); + assert!(model.update(Event::RefreshDevices).contains(&Effect::Account { session })); + } + + /// An admin who is demoted meanwhile: the server's `forbidden` is checked against the counters, and the pages close. + #[test] + fn a_demoted_admin_is_taken_off_the_pages() { + let boss = agent(true, 2); + let mut model = on_the_pages(&boss); + boss.admin.store(false, std::sync::atomic::Ordering::SeqCst); + run(&mut model, &boss, ev(AdminEvent::Reload)); + assert!(matches!(model.screen, Screen::SignedIn(_)), "{:?}", model.screen); + assert_eq!(model.overview().unwrap().access, Access::Member); + } + + /// Pages of 20, a cursor each way, and a search that starts over from the first page; an old answer is ignored. + #[test] + fn the_list_is_paged_and_searched() { + let boss = agent(true, 44); + let mut model = on_the_pages(&boss); + assert_eq!(admin(&model).list.users.len(), PAGE_SIZE as usize); + assert_eq!((admin(&model).list.page(), admin(&model).list.next.as_deref()), (1, Some("20"))); + assert!(model.update(Event::Admin(AdminEvent::PreviousPage)).is_empty(), "no page before the first"); + run(&mut model, &boss, ev(AdminEvent::NextPage)); + assert_eq!(admin(&model).list.page(), 2); + assert_eq!(admin(&model).list.users[0].username, "user20"); + run(&mut model, &boss, ev(AdminEvent::NextPage)); + assert_eq!((admin(&model).list.users.len(), admin(&model).list.next.clone()), (5, None)); + assert!(model.update(Event::Admin(AdminEvent::NextPage)).is_empty(), "no page after the last"); + run(&mut model, &boss, ev(AdminEvent::PreviousPage)); + assert_eq!((admin(&model).list.page(), admin(&model).list.users[0].username.as_str()), (2, "user20")); + + admin_mut(&mut model).list.search = " USER4 ".into(); + let effects = model.update(Event::Admin(AdminEvent::Search)); + let [Effect::AdminUsers { query, ticket }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((query.cursor.as_deref(), query.query.as_deref(), query.limit), (None, Some("USER4"), Some(PAGE_SIZE))); + let old = ticket - 1; + model.update(Event::Admin(AdminEvent::Users { ticket: old, result: Ok(AdminUserList { users: Vec::new(), next: None }) })); + assert!(admin(&model).list.loading, "an answer to an older call is ignored"); + for effect in effects { + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + if let Some(answer) = call(&boss, &start, effect) { + model.update(answer); + } + } + let names: Vec<&str> = admin(&model).list.users.iter().map(|u| u.username.as_str()).collect(); + assert_eq!(names, ["user40", "user41", "user42", "user43", "user44"]); + assert_eq!(admin(&model).list.page(), 1); + run(&mut model, &boss, ev(AdminEvent::ClearSearch)); + assert_eq!((admin(&model).list.applied.as_str(), admin(&model).list.users.len()), ("", PAGE_SIZE as usize)); + } + + /// Every change asks for the password afresh; it leaves the model when it is sent, and shows in no `Debug` output. + #[test] + fn a_change_sends_the_password_once_and_clears_it() { + let boss = agent(true, 3); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + model.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(admin(&model).dialog.is_some()); + // Nothing typed: nothing is sent. + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_PASSWORD)); + type_password(&mut model, "admin-secret-pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let [Effect::AdminDo { action, confirm }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!(action, &Action::Disable { id: ana.id, username: "user01".into() }); + assert_eq!(confirm.password.as_deref(), Some("admin-secret-pw")); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.busy && dialog.confirmation.password.is_empty(), "{dialog:?}"); + let shown = format!("{model:?} {effects:?}"); + assert!(!shown.contains("admin-secret-pw"), "{shown}"); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty(), "not twice"); + assert!(model.update(Event::Admin(AdminEvent::CancelDialog)).is_empty()); + assert!(admin(&model).dialog.is_some(), "not while it is with the server"); + // The call is made with the password, and done: the dialog goes, the list is asked again. + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + assert_eq!(boss.passwords.lock().unwrap().as_slice(), [Some("admin-secret-pw".to_owned())]); + let effects = model.update(answer); + assert!(admin(&model).dialog.is_none()); + assert!(effects.iter().any(|e| matches!(e, Effect::AdminUsers { .. })), "{effects:?}"); + } + + /// A wrong password is the server's `wrong_password`: the dialog stays, empty, to try again. + #[test] + fn a_wrong_password_keeps_the_dialog_open() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + boss.admin_fails.lock().unwrap().push(refused(codes::WRONG_PASSWORD, "the password is wrong")); + run( + &mut model, + &boss, + vec![ + Event::Admin(AdminEvent::Ask(Action::Remove { id: ana.id, username: "user01".into() })), + Event::Admin(AdminEvent::Confirm), // nothing typed + ], + ); + type_password(&mut model, "not-my-password"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(!dialog.busy && dialog.confirmation.password.is_empty()); + assert_eq!(dialog.error.as_deref(), Some("That password isn't right.")); + type_password(&mut model, "my-password"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.passwords.lock().unwrap().iter().flatten().collect::>(), ["not-my-password", "my-password"]); + } + + #[test] + fn the_servers_refusals_are_said_clearly() { + assert!(error_text(&refused(codes::LAST_ADMIN, "x")).contains("without an enabled admin")); + assert!(error_text(&refused(codes::ACCOUNT_DISABLED, "x")).contains("disabled")); + assert_eq!(error_text(&refused(codes::FORBIDDEN, "Only admins can do this")), "The server doesn't allow this: only admins can do this"); + let slow = Error::RefusedWith { + code: codes::RATE_LIMITED.into(), + message: "too many wrong passwords; wait a few minutes".into(), + two_factor: None, + retry_after_secs: Some(90), + }; + let text = error_text(&slow); + assert!(text.contains("Too many tries") && text.contains("90 seconds"), "{text}"); + let busy = Error::RefusedWith { + code: codes::RATE_LIMITED.into(), + message: "the server is busy".into(), + two_factor: None, + retry_after_secs: Some(600), + }; + assert!(error_text(&busy).contains("10 minutes")); + assert_eq!(error_text(&refused("invalid", "role is admin or member")), "role is admin or member"); + assert!(error_text(&Error::NotRunning("x".into())).contains("isn't running")); + + // In the dialog, as the user reads it. + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + for (fail, wanted) in [ + (refused(codes::LAST_ADMIN, "x"), "without an enabled admin"), + (slow, "Too many tries"), + (refused(codes::ACCOUNT_DISABLED, "x"), "disabled"), + ] { + boss.admin_fails.lock().unwrap().push(fail); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let error = admin(&model).dialog.as_ref().unwrap().error.clone().unwrap(); + assert!(error.contains(wanted), "{error}"); + run(&mut model, &boss, ev(AdminEvent::CancelDialog)); + assert!(admin(&model).dialog.is_none()); + } + // A `forbidden` on a change asks the server again whether this is still an admin. + boss.admin_fails.lock().unwrap().push(refused(codes::FORBIDDEN, "You can't do that to your own account")); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + assert_eq!(model.update(answer), vec![Effect::Account { session: model.overview().unwrap().session }]); + assert!(admin(&model).dialog.as_ref().unwrap().error.as_deref().is_some_and(|e| e.starts_with("The server doesn't allow this"))); + assert!(model.admin_screen().is_some(), "still an admin: the pages stay"); + } + + /// A new account has a name and a role, and no quota (that is a change of its own, on its page); the invite it + /// makes is shown once. + #[test] + fn a_new_account_shows_its_invite_once() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + model.update(Event::Admin(AdminEvent::NewUser)); + assert!(matches!(admin(&model).view, View::Create(_))); + let form = |model: &mut Model| match &mut admin_mut(model).view { + View::Create(form) => form.clone(), + other => panic!("{other:?}"), + }; + let long = "x".repeat(65); + for bad in ["", " ", "a b", "ana/../root", long.as_str()] { + if let View::Create(f) = &mut admin_mut(&mut model).view { + f.username = bad.into(); + } + assert!(model.update(Event::Admin(AdminEvent::SubmitCreate)).is_empty()); + assert_eq!(form(&mut model).error.as_deref(), Some(BAD_USERNAME), "{bad:?}"); + assert!(admin(&model).dialog.is_none()); + } + if let View::Create(f) = &mut admin_mut(&mut model).view { + f.username = " new.user_1 ".into(); + f.role = RoleChoice::Admin; + } + model.update(Event::Admin(AdminEvent::SubmitCreate)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::CreateUser { username: "new.user_1".into(), role: RoleChoice::Admin }); + type_password(&mut model, "admin-secret-pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_create_user").count(), 1); + let screen = admin(&model); + assert!(screen.dialog.is_none() && matches!(screen.view, View::List)); + let code = screen.code.as_ref().unwrap(); + assert_eq!((code.username.as_str(), code.kind, code.code.as_str()), ("new.user_1", CodeKind::Invite, "K7M4Q9TXK7M4Q9TX")); + let shown = format!("{model:?}"); + assert!(!shown.contains("K7M4Q9TX") && !shown.contains("K7M4-Q9TX") && !shown.contains("admin-secret-pw"), "{shown}"); + // Dismissed, it is gone from the model; leaving the pages drops it too. + model.update(Event::Admin(AdminEvent::DismissCode)); + assert!(admin(&model).code.is_none()); + assert_eq!(format!("{:?}", admin(&model).list.users).matches("K7M4").count(), 0); + model.update(Event::Admin(AdminEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + } + + /// Role and quota are one change, with only what differs sent; an empty quota removes it. + #[test] + fn an_account_page_changes_the_role_and_the_quota() { + let boss = agent(true, 2); + let mut model = on_the_pages(&boss); + let ana = AdminUserInfo { quota_bytes: Some(5 * 1_073_741_824), ..user_named(&model, "user01") }; + boss.users.lock().unwrap()[1] = ana.clone(); + run(&mut model, &boss, ev(AdminEvent::Reload)); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + let view = |model: &Model| match &admin(model).view { + View::User(view) => (**view).clone(), + other => panic!("{other:?}"), + }; + let v = view(&model); + assert_eq!((v.role, v.quota.as_str(), v.devices.as_ref().map(Vec::len)), (RoleChoice::Member, "5", Some(1))); + // Nothing changed: nothing to ask. + model.update(Event::Admin(AdminEvent::SubmitEdit)); + assert_eq!(view(&model).error.as_deref(), Some("Nothing to change.")); + let set = |model: &mut Model, role: RoleChoice, quota: &str| { + if let View::User(v) = &mut admin_mut(model).view { + v.role = role; + v.quota = quota.into(); + } + }; + set(&mut model, RoleChoice::Member, "lots"); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + assert!(view(&model).error.unwrap().starts_with("A quota is a number of GiB")); + assert!(admin(&model).dialog.is_none()); + set(&mut model, RoleChoice::Admin, "5"); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::UpdateUser { id: ana.id, username: "user01".into(), role: Some(RoleChoice::Admin), quota: None }); + run(&mut model, &boss, ev(AdminEvent::CancelDialog)); + set(&mut model, RoleChoice::Member, ""); + model.update(Event::Admin(AdminEvent::SubmitEdit)); + let Some(Dialog { action, .. }) = &admin(&model).dialog else { panic!("no dialog") }; + assert_eq!(action, &Action::UpdateUser { id: ana.id, username: "user01".into(), role: None, quota: Some(None) }); + assert!(action.explain().contains("quota is removed")); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let v = view(&model); + assert_eq!((v.quota.as_str(), v.error), ("", None), "the page shows what the server answered"); + assert_eq!(v.devices.map(|d| d.len()), Some(1), "the devices stay"); + } + + /// An invite and a reset code are shown once, each; a revoke and a device revoke refresh what they changed. + #[test] + fn invites_reset_codes_and_devices() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + for (action, kind, code) in [ + (Action::IssueInvite { id: ana.id, username: "user01".into() }, CodeKind::Invite, "ABCDEFGHJKMNPQRS"), + (Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: false }, CodeKind::PasswordReset, "TUVWXYZ234567ABC"), + ] { + run(&mut model, &boss, ev(AdminEvent::Ask(action))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let shown = admin(&model).code.clone().unwrap(); + assert_eq!((shown.kind, shown.code.as_str()), (kind, code)); + assert!(matches!(admin(&model).view, View::User(_)), "still on the account's page"); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + } + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).code.is_none() && admin(&model).dialog.is_none()); + + let View::User(view) = &admin(&model).view else { panic!() }; + let device = view.devices.as_ref().unwrap()[0].clone(); + let before = boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_devices").count(); + let revoke = Action::RevokeDevice { id: ana.id, username: "user01".into(), device: device.id, device_name: device.name.clone() }; + assert!(revoke.title().contains("laptop")); + run(&mut model, &boss, ev(AdminEvent::Ask(revoke))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_devices").count(), before + 1, "the devices are asked again"); + + // Removing the account goes back to the list. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Remove { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(matches!(admin(&model).view, View::List)); + } + + #[test] + fn quotas_are_typed_in_gib_and_shown_in_the_largest_unit() { + assert_eq!(parse_quota(""), Ok(None)); + assert_eq!(parse_quota(" 50 "), Ok(Some(50 * 1_073_741_824))); + assert_eq!(parse_quota("0.5"), Ok(Some(536_870_912))); + assert_eq!(parse_quota("0"), Ok(Some(0))); + for bad in ["-1", "abc", "inf", "NaN", "1e99", "2000000", "5 GB"] { + assert!(parse_quota(bad).is_err(), "{bad}"); + } + assert_eq!(quota_text(5 * 1_073_741_824), "5"); + assert_eq!(quota_text(536_870_912), "0.5"); + assert_eq!(quota_text(0), "0"); + assert_eq!(parse_quota("a_text(1_610_612_736)), Ok(Some(1_610_612_736))); + assert_eq!(format_bytes(0), "0 B"); + assert_eq!(format_bytes(1_536), "1.5 KiB"); + assert_eq!(format_bytes(5_000_000_000), "4.7 GiB"); + assert_eq!(format_bytes(2 * 1_073_741_824), "2 GiB"); + assert!(valid_username("a.b_c-9") && !valid_username("") && !valid_username("a b") && !valid_username("é")); + } + + #[test] + fn nothing_secret_is_in_debug_output() { + let confirmation = Confirmation { password: "admin-secret-pw".into(), code: "918273".into() }; + let shown = format!("{confirmation:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + let code = ShownCode { + username: "ana".into(), + kind: CodeKind::Invite, + code: "ABCDEFGH".into(), + join_url: Some("c.example/ABCD".into()), + expires_ms: 1, + }; + let shown = format!("{code:?}"); + assert!(!shown.contains("ABCDEFGH") && !shown.contains("c.example"), "{shown}"); + } + + fn account(role: &str) -> AccountInfo { + AccountInfo { + user_id: Uuid::nil(), + username: "fabio".into(), + role: role.into(), + created_ms: 1, + password_changed_ms: None, + two_factor_enabled: false, + } + } + + /// An answer that comes late (the pages were left and opened again; the devices were asked for twice) matches + /// nothing: it never overwrites newer data. + #[test] + fn answers_that_come_late_are_ignored() { + let boss = agent(true, 3); + let mut model = signed_in(&boss); + let first = model.update(Event::Admin(AdminEvent::Open)); + let Some(Effect::AdminUsers { ticket: old, .. }) = first.first().cloned() else { panic!("{first:?}") }; + model.update(Event::Admin(AdminEvent::Close)); + run(&mut model, &boss, ev(AdminEvent::Open)); + let current = admin(&model).list.users.clone(); + assert_eq!(current.len(), 4); + let ghost = AdminUserList { users: vec![sample_user("ghost", Uuid::now_v7())], next: None }; + model.update(Event::Admin(AdminEvent::Users { ticket: old, result: Ok(ghost) })); + assert_eq!(admin(&model).list.users, current, "the first page's reply is from before the pages were opened again"); + + // The same for a user's devices. + let ana = user_named(&model, "user01"); + let asked = model.update(Event::Admin(AdminEvent::OpenUser(ana.id))); + let [Effect::AdminDevices { ticket: first_devices, .. }, Effect::AdminUserUsage { ticket: first_usage, .. }] = asked.as_slice() else { + panic!("{asked:?}") + }; + run(&mut model, &boss, ev(AdminEvent::RefreshDevices)); + run(&mut model, &boss, ev(AdminEvent::RefreshUsage)); + let View::User(view) = &admin(&model).view else { panic!() }; + let now = view.devices.clone(); + let usage_now = view.usage.clone(); + assert_eq!(now.as_ref().map(Vec::len), Some(1)); + model.update(Event::Admin(AdminEvent::Devices { user: ana.id, ticket: *first_devices, result: Ok(Vec::new()) })); + let View::User(view) = &admin(&model).view else { panic!() }; + assert_eq!(view.devices, now); + model.update(Event::Admin(AdminEvent::Usage { + user: ana.id, + ticket: *first_usage, + result: Ok(craft_cloud_app::AccountUsage { + stored_bytes: 1, + quota_bytes: None, + trash_bytes: Some(2), + history_bytes: Some(3), + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + }), + })); + let View::User(view) = &admin(&model).view else { panic!() }; + assert_eq!(view.usage, usage_now); + } + + /// The admin check of an earlier sign-in (another account, maybe) says nothing about the current one. + #[test] + fn an_admin_check_for_an_earlier_sign_in_is_ignored() { + let mut model = Model::new(JoinDefaults::default(), true, String::new()); + model.update(Event::Start); + let probe = |effects: &[Effect]| { + effects.iter().find_map(|e| if let Effect::Account { session } = e { Some(*session) } else { None }).expect("an admin check") + }; + let first = probe(&model.update(Event::Found(Ok(status(true))))); + model.update(Event::SignedOut(Ok(status(false)))); + let second = probe(&model.update(Event::Polled(Ok(status(true))))); + assert_ne!(first, second); + // The first account was an admin; its late answer doesn't make this one an admin. + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("admin")) })); + assert_eq!(model.overview().unwrap().access, Access::Unknown); + assert!(!model.is_admin()); + model.update(Event::Admin(AdminEvent::Account { session: second, result: Ok(account("member")) })); + assert_eq!(model.overview().unwrap().access, Access::Member); + // And a late "member" answer doesn't take an admin's pages away. + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("member")) })); + model.update(Event::Admin(AdminEvent::Account { session: second, result: Ok(account("admin")) })); + assert!(model.is_admin()); + model.update(Event::Admin(AdminEvent::Account { session: first, result: Ok(account("member")) })); + assert!(model.is_admin()); + } + + /// A password reset code can be revoked like an invite: the button doesn't depend on `invite_pending` (which + /// doesn't say there is one), and a revoke that finds nothing open says so and looks at the list again. + #[test] + fn a_revoke_that_finds_no_open_code_says_so() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(!ana.invite_pending); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: false }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + // The reset code is open, though the list doesn't say so: it can be revoked. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_revoke_invite").count(), 1); + // Nothing open (the server's `not_found`): said clearly, and the list is asked again. + boss.admin_fails.lock().unwrap().push(refused(codes::NOT_FOUND, "not found")); + run(&mut model, &boss, ev(AdminEvent::Ask(Action::RevokeInvite { id: ana.id, username: "user01".into() }))); + type_password(&mut model, "pw"); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + let effects = model.update(answer); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NO_OPEN_CODE)); + assert!(matches!(effects.as_slice(), [Effect::AdminUsers { .. }]), "{effects:?}"); + } + + /// An admin whose account has two-factor sign-in confirms with a code too: one field, six digits are a code from the app and + /// anything else a recovery code; a missing code is not sent; everything typed leaves the model when it is sent. + #[test] + fn a_confirmation_asks_for_a_code_when_the_account_has_two_factor() { + let boss = agent(true, 1); + boss.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + let mut model = on_the_pages(&boss); + assert_eq!(model.overview().unwrap().access, Access::Admin { two_factor: true }); + let ana = user_named(&model, "user01"); + let type_in = |model: &mut Model, password: &str, code: &str| { + let d = admin_mut(model).dialog.as_mut().unwrap(); + d.confirmation.password = password.into(); + d.confirmation.code = code.into(); + }; + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(admin(&model).dialog.as_ref().unwrap().needs_code); + // A password alone, or a code alone, is not sent. + type_in(&mut model, "admin-secret-pw", ""); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_CODE)); + type_in(&mut model, "", "123456"); + assert!(model.update(Event::Admin(AdminEvent::Confirm)).is_empty()); + assert_eq!(admin(&model).dialog.as_ref().unwrap().error.as_deref(), Some(NEEDS_PASSWORD)); + // Six digits with spaces: a code from the app. + type_in(&mut model, "admin-secret-pw", " 123 456 "); + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let [Effect::AdminDo { confirm, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!( + (confirm.password.as_deref(), confirm.totp_code.as_deref(), confirm.recovery_code.as_deref()), + (Some("admin-secret-pw"), Some("123456"), None) + ); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.busy && dialog.confirmation.password.is_empty() && dialog.confirmation.code.is_empty(), "{dialog:?}"); + let shown = format!("{model:?} {effects:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("123456"), "{shown}"); + run(&mut model, &boss, vec![]); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(boss.factors.lock().unwrap().as_slice(), [(Some("123456".to_owned()), None)]); + // Anything else is a recovery code, in the other field. + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Enable { id: ana.id, username: "user01".into() }))); + type_in(&mut model, "admin-secret-pw", "abcd-efgh-jkmn-pqrs"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(boss.factors.lock().unwrap().last().unwrap(), &(None, Some("abcd-efgh-jkmn-pqrs".to_owned()))); + // The password dialog of an account without it has no code field. + let plain = agent(true, 1); + let mut model = on_the_pages(&plain); + run(&mut model, &plain, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + assert!(!admin(&model).dialog.as_ref().unwrap().needs_code); + } + + /// The server's word on a code: a wrong one (403) is said and the page asks for the code if it didn't; a lock (429) says how + /// long to wait; the typed secrets are gone either way. + #[test] + fn a_wrong_or_locked_code_is_said_and_asked_for_again() { + let boss = agent(true, 1); + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(!{ + run(&mut model, &boss, ev(AdminEvent::Ask(Action::Disable { id: ana.id, username: "user01".into() }))); + admin(&model).dialog.as_ref().unwrap().needs_code + }); + boss.admin_fails.lock().unwrap().push(refused(codes::TWO_FACTOR_INVALID, "wrong code")); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let dialog = admin(&model).dialog.as_ref().unwrap(); + assert!(dialog.needs_code, "the server wants a code: the page asks for it"); + assert!(dialog.error.as_deref().is_some_and(|e| e.contains("code isn't right")), "{:?}", dialog.error); + assert!(!dialog.busy && dialog.confirmation.password.is_empty() && dialog.confirmation.code.is_empty()); + let locked = + Error::RefusedWith { code: codes::TWO_FACTOR_LOCKED.into(), message: "locked".into(), two_factor: None, retry_after_secs: Some(840) }; + boss.admin_fails.lock().unwrap().push(locked.clone()); + type_password(&mut model, "pw"); + admin_mut(&mut model).dialog.as_mut().unwrap().confirmation.code = "123456".into(); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + let error = admin(&model).dialog.as_ref().unwrap().error.clone().unwrap(); + assert!(error.contains("Too many wrong codes") && error.contains("14 minutes"), "{error}"); + assert!(error_text(&locked).contains("14 minutes")); + } + + /// An account's two-factor sign-in is shown, and taken away with a change of its own; a reset code can turn it off when it is + /// used, which the call carries. + #[test] + fn two_factor_is_shown_reset_and_turned_off_by_a_reset_code() { + let boss = agent(true, 1); + boss.users.lock().unwrap()[1].two_factor_enabled = true; + let mut model = on_the_pages(&boss); + let ana = user_named(&model, "user01"); + assert!(ana.two_factor_enabled); + run(&mut model, &boss, ev(AdminEvent::OpenUser(ana.id))); + let reset = Action::ResetTwoFactor { id: ana.id, username: "user01".into() }; + assert!(reset.title().contains("Turn two-factor sign-in off") && reset.explain().contains("signed out")); + run(&mut model, &boss, ev(AdminEvent::Ask(reset))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert!(admin(&model).dialog.is_none()); + assert_eq!(boss.asked.lock().unwrap().iter().filter(|a| *a == "admin_reset_two_factor").count(), 1); + for flag in [false, true] { + run(&mut model, &boss, ev(AdminEvent::Ask(Action::IssueReset { id: ana.id, username: "user01".into(), reset_two_factor: flag }))); + type_password(&mut model, "pw"); + run(&mut model, &boss, ev(AdminEvent::Confirm)); + assert_eq!(admin(&model).code.as_ref().map(|c| c.kind), Some(CodeKind::PasswordReset)); + run(&mut model, &boss, ev(AdminEvent::DismissCode)); + } + assert_eq!(boss.resets.lock().unwrap().as_slice(), [false, true], "the flag goes with the reset code"); + assert!(Action::IssueReset { id: ana.id, username: "x".into(), reset_two_factor: true }.explain().contains("turns the account's two-factor")); + } +} diff --git a/apps/craft-cloud-desktop/src/agent.rs b/apps/craft-cloud-desktop/src/agent.rs index 2ed7185d..5b9288e0 100644 --- a/apps/craft-cloud-desktop/src/agent.rs +++ b/apps/craft-cloud-desktop/src/agent.rs @@ -5,14 +5,23 @@ //! link wakes the UI (`wake`), so the UI thread never waits for the agent; a slow call (a join waits for the server) //! holds up nothing else. A clock thread sends [`Event::Tick`] every 2 s, from which the model asks for the status. +use std::io::Write; use std::path::PathBuf; use std::sync::mpsc::{Receiver, Sender, channel}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use craft_cloud_app::{Client, DeviceId, DeviceInfo, Error, JoinRequest, PairingInfo, PairingStatus, Status, Uuid}; +use craft_cloud_app::{ + AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditPage, AuditQuery, Client, DeviceId, DeviceInfo, Error, ExportOutcome, + HistoryResponse, JoinRequest, NodeId, PairingInfo, PairingStatus, RecoveryCodes, RestoreResponse, RevokedOthers, Status, TrashResponse, + TrashRestoreResponse, TwoFactorProofRequest, TwoFactorSetupResponse, TwoFactorStatus, UserQuery, Uuid, VersionId, +}; +use crate::admin::{Action, AdminEvent, Outcome}; +use crate::audit::AuditMsg; use crate::model::{Effect, Event}; +use crate::security::{SecurityAnswer, SecurityCall}; /// How often the clock ticks (and the status is asked for). pub const TICK: Duration = Duration::from_secs(2); @@ -28,11 +37,46 @@ pub trait Agent: Send + Sync { fn pairing_state(&self, id: Uuid) -> Result; fn cancel_pairing(&self, id: Uuid) -> Result<(), Error>; fn devices(&self) -> Result, Error>; + fn usage(&self) -> Result; fn remove_device(&self, id: DeviceId) -> Result, Error>; fn pause(&self) -> Result; fn resume(&self) -> Result; fn quit(&self) -> Result; fn set_autostart(&self, on: bool) -> Result; + /// The Security page's calls: the account's two-factor sign-in, its password and its other computers. + fn two_factor_status(&self) -> Result; + fn two_factor_setup(&self, password: &str) -> Result; + fn two_factor_confirm(&self, totp_code: &str) -> Result; + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error>; + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result; + fn change_password(&self, current: &str, new: &str) -> Result<(), Error>; + fn revoke_other_devices(&self) -> Result; + // The admin calls: each asks the server, which decides every one. + fn account(&self) -> Result; + /// The account's own activity, and for an admin the whole audit log and its export (which is written to `out` as it comes). + fn account_activity(&self, query: &AuditQuery) -> Result; + fn admin_audit(&self, query: &AuditQuery) -> Result; + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result; + fn admin_server(&self) -> Result; + fn admin_storage(&self) -> Result; + fn admin_users(&self, query: &UserQuery) -> Result; + fn admin_devices(&self, user: Uuid) -> Result; + fn admin_user_usage(&self, user: Uuid) -> Result; + fn admin_create_user(&self, request: &AdminCreateUser) -> Result; + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result; + fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result; + fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result; + fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error>; + fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error>; + fn file_history(&self, path: &str, cursor: Option<&str>) -> Result; + fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result; + fn trash(&self, cursor: Option<&str>) -> Result; + fn restore_from_trash(&self, id: NodeId) -> Result; + fn remove_from_trash(&self, id: NodeId) -> Result<(), Error>; } /// The agent through `craft_cloud_app::Client`. It finds the agent again whenever a call finds it gone (the agent @@ -98,6 +142,9 @@ impl Agent for ClientAgent { fn devices(&self) -> Result, Error> { self.with(Client::devices) } + fn usage(&self) -> Result { + self.with(Client::usage) + } fn remove_device(&self, id: DeviceId) -> Result, Error> { self.with(|c| c.remove_device(id)) } @@ -113,6 +160,99 @@ impl Agent for ClientAgent { fn set_autostart(&self, on: bool) -> Result { self.with(|c| c.set_autostart(on)) } + fn two_factor_status(&self) -> Result { + self.with(Client::two_factor_status) + } + fn two_factor_setup(&self, password: &str) -> Result { + self.with(|c| c.two_factor_setup(password)) + } + fn two_factor_confirm(&self, totp_code: &str) -> Result { + self.with(|c| c.two_factor_confirm(totp_code)) + } + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + self.with(|c| c.two_factor_disable(request)) + } + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + self.with(|c| c.two_factor_recovery_codes(password, totp_code)) + } + fn change_password(&self, current: &str, new: &str) -> Result<(), Error> { + self.with(|c| c.change_password(current, new)) + } + fn revoke_other_devices(&self) -> Result { + self.with(Client::revoke_other_devices) + } + fn account(&self) -> Result { + self.with(Client::account) + } + fn account_activity(&self, query: &AuditQuery) -> Result { + self.with(|c| c.account_activity(query)) + } + fn admin_audit(&self, query: &AuditQuery) -> Result { + self.with(|c| c.admin_audit(query)) + } + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + self.with(|c| c.admin_audit_export(query, out)) + } + fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_reset_two_factor(id, confirm)) + } + fn admin_server(&self) -> Result { + self.with(Client::admin_server) + } + fn admin_storage(&self) -> Result { + self.with(Client::admin_storage) + } + fn admin_users(&self, query: &UserQuery) -> Result { + self.with(|c| c.admin_users(query)) + } + fn admin_devices(&self, user: Uuid) -> Result { + self.with(|c| c.admin_devices(user)) + } + fn admin_user_usage(&self, user: Uuid) -> Result { + self.with(|c| c.admin_user_usage(user)) + } + fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.with(|c| c.admin_create_user(request)) + } + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.with(|c| c.admin_update_user(id, request)) + } + fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_disable_user(id, confirm)) + } + fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_enable_user(id, confirm)) + } + fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_remove_user(id, confirm)) + } + fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result { + self.with(|c| c.admin_issue_invite(id, confirm)) + } + fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_revoke_invite(id, confirm)) + } + fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result { + self.with(|c| c.admin_issue_reset(id, request)) + } + fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.with(|c| c.admin_revoke_device(id, device, confirm)) + } + fn file_history(&self, path: &str, cursor: Option<&str>) -> Result { + self.with(|c| c.file_history(path, cursor, Some(50))) + } + fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result { + self.with(|c| c.restore_file(path, version, expected_version)) + } + fn trash(&self, cursor: Option<&str>) -> Result { + self.with(|c| c.trash(cursor, Some(50))) + } + fn restore_from_trash(&self, id: NodeId) -> Result { + self.with(|c| c.restore_from_trash(id)) + } + fn remove_from_trash(&self, id: NodeId) -> Result<(), Error> { + self.with(|c| c.remove_from_trash(id)) + } } /// Starts the agent and waits for it ([`crate::start_agent::start_and_wait`]), for [`Effect::StartAgent`]. @@ -162,6 +302,7 @@ impl Link { if !is_call(&effect) { return Some(effect); } + let failed_effect = effect.clone(); let (agent, start, wake, tx) = (self.agent.clone(), self.start.clone(), self.wake.clone(), self.tx.clone()); let spawned = std::thread::Builder::new().name("craft-cloud-call".into()).spawn(move || { if let Some(event) = call(agent.as_ref(), &start, effect) { @@ -170,8 +311,15 @@ impl Link { } }); if let Err(e) = spawned { - // No thread to run it on: said as the call's failure. - let _ = self.tx.send(Event::Polled(Err(Error::Io(format!("can't start a thread: {e}"))))); + let error = Error::Io(format!("can't start a thread: {e}")); + let event = match failed_effect { + Effect::Trash { ticket, .. } => Event::TrashPage { ticket, result: Err(error) }, + Effect::RestoreTrash { ticket, .. } => Event::TrashRestored { ticket, result: Err(error) }, + Effect::RemoveTrash { ticket, .. } => Event::TrashRemoved { ticket, result: Err(error) }, + _ => Event::Polled(Err(error)), + }; + let _ = self.tx.send(event); + (self.wake)(); } None } @@ -179,7 +327,10 @@ impl Link { /// Whether `effect` is a call of the agent. fn is_call(effect: &Effect) -> bool { - !matches!(effect, Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_)) + !matches!( + effect, + Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_) | Effect::PickExportFile { .. } + ) } /// Makes the call `effect` names, blocking, and gives its answer as the event the model expects. @@ -198,19 +349,127 @@ pub fn call(agent: &dyn Agent, start: &Starter, effect: Effect) -> Option return None; } Effect::Devices => Event::Devices(agent.devices()), + Effect::Usage(request_id) => Event::Usage { request_id, result: agent.usage() }, Effect::RemoveDevice(id) => Event::Devices(agent.remove_device(id)), + Effect::History { path, cursor, ticket } => Event::History { ticket, result: agent.file_history(&path, cursor.as_deref()) }, + Effect::Restore { path, version, expected_version, ticket } => { + Event::Restored { ticket, result: agent.restore_file(&path, version, expected_version) } + } + Effect::Trash { cursor, ticket } => Event::TrashPage { ticket, result: agent.trash(cursor.as_deref()) }, + Effect::RestoreTrash { id, ticket } => Event::TrashRestored { ticket, result: agent.restore_from_trash(id) }, + Effect::RemoveTrash { id, ticket } => Event::TrashRemoved { ticket, result: agent.remove_from_trash(id) }, Effect::Logout { local } => Event::SignedOut(agent.logout(local)), Effect::Pause => Event::Answered(agent.pause()), Effect::Resume => Event::Answered(agent.resume()), + Effect::Account { session } => Event::Admin(AdminEvent::Account { session, result: agent.account() }), + Effect::AdminServer { session } => Event::Admin(AdminEvent::Server { session, result: agent.admin_server() }), + Effect::AdminStorage { session, ticket } => Event::Admin(AdminEvent::Storage { session, ticket, result: agent.admin_storage() }), + Effect::AdminUsers { query, ticket } => Event::Admin(AdminEvent::Users { ticket, result: agent.admin_users(&query) }), + Effect::AdminDevices { user, ticket } => { + Event::Admin(AdminEvent::Devices { user, ticket, result: agent.admin_devices(user).map(|d| d.devices) }) + } + Effect::AdminUserUsage { user, ticket } => Event::Admin(AdminEvent::Usage { user, ticket, result: agent.admin_user_usage(user) }), + Effect::AuditPage { own, query, ticket } => { + let result = if own { agent.account_activity(&query) } else { agent.admin_audit(&query) }; + Event::Audit(AuditMsg::Page { own, ticket, result }) + } + Effect::AuditExport { query, path, ticket } => Event::Audit(AuditMsg::Exported { ticket, result: export_to_file(agent, &query, &path) }), + Effect::AdminDo { action, confirm } => Event::Admin(AdminEvent::Done(change(agent, &action, confirm))), // Not running is as good as stopped. Effect::QuitAgent => { let _ = agent.quit(); Event::AgentQuit } - Effect::OpenFolder(_) | Effect::QuitApp | Effect::QuitTray | Effect::AppAutostart(_) | Effect::PickFolder(_) => return None, + Effect::Security(call) => Event::SecurityAnswer(match call { + SecurityCall::Status => SecurityAnswer::Status(agent.two_factor_status()), + SecurityCall::Setup { password } => SecurityAnswer::Setup(agent.two_factor_setup(&password)), + SecurityCall::Confirm { code } => SecurityAnswer::Confirm(agent.two_factor_confirm(&code)), + SecurityCall::Disable(request) => SecurityAnswer::Disable(agent.two_factor_disable(&request)), + SecurityCall::RecoveryCodes { password, code } => SecurityAnswer::RecoveryCodes(agent.two_factor_recovery_codes(&password, &code)), + SecurityCall::ChangePassword { current, new } => SecurityAnswer::ChangePassword(agent.change_password(¤t, &new)), + SecurityCall::SignOutOthers => SecurityAnswer::SignOutOthers(agent.revoke_other_devices()), + }), + Effect::OpenFolder(_) + | Effect::QuitApp + | Effect::QuitTray + | Effect::AppAutostart(_) + | Effect::PickFolder(_) + | Effect::PickExportFile { .. } => return None, }) } +/// Writes the audit export to `path`: first to a file of its own in the same folder, created for this export alone (it must not +/// exist yet, and no link in its place is followed), which is moved over `path` only when the stream ended properly (the server's +/// last line, the right count) and the file was written to the end. Anything else removes that file, and only that file, and +/// leaves what is at `path` as it was (a file called `.part` is somebody else's), so a broken export is never mistaken for a +/// whole one. A stream the server cut at its limit is a good file, and says so in the outcome. +fn export_to_file(agent: &dyn Agent, query: &AuditQuery, path: &str) -> Result { + let io = |what: &str, e: std::io::Error| Error::Io(format!("{what}: {e}")); + let target = std::path::Path::new(path); + let name = target.file_name().ok_or_else(|| Error::Io(format!("{path} is not a file name")))?.to_string_lossy().into_owned(); + let dir = target.parent().filter(|d| !d.as_os_str().is_empty()).unwrap_or(std::path::Path::new(".")); + let mut made = None; + for _ in 0..16 { + let part = dir.join(format!(".{name}.{}.{}.part", std::process::id(), crate::admin::fresh())); + match std::fs::OpenOptions::new().write(true).create_new(true).open(&part) { + Ok(file) => { + made = Some((part, file)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(io("can't write the export file", e)), + } + } + let Some((part, file)) = made else { return Err(Error::Io("can't make a file for the export".into())) }; + let mut writer = std::io::BufWriter::with_capacity(64 * 1024, file); + let result = agent.admin_audit_export(query, &mut writer); + let finished = writer.flush().and_then(|()| writer.get_ref().sync_all()); + drop(writer); + let outcome = match (result, finished) { + (Ok(outcome), Ok(())) => outcome, + (Ok(_), Err(e)) => { + let _ = std::fs::remove_file(&part); + return Err(io("can't write the export file", e)); + } + (Err(e), _) => { + let _ = std::fs::remove_file(&part); + return Err(e); + } + }; + if let Err(e) = std::fs::rename(&part, target) { + let _ = std::fs::remove_file(&part); + return Err(io("can't save the export file", e)); + } + Ok(outcome) +} + +/// Makes the change `action` names with the admin's `confirm`ation, blocking. This is the one place where the +/// confirmation goes into a call's body (the create and change calls carry it in their own fields). +fn change(agent: &dyn Agent, action: &Action, confirm: AdminConfirm) -> Result { + let AdminConfirm { password, totp_code, recovery_code } = confirm.clone(); + match action { + Action::CreateUser { username, role } => { + let request = AdminCreateUser { username: username.clone(), role: role.as_str().to_owned(), password, totp_code, recovery_code }; + agent.admin_create_user(&request).map(Outcome::Created) + } + Action::UpdateUser { id, role, quota, .. } => { + let request = AdminUpdateUser { role: role.map(|r| r.as_str().to_owned()), quota_bytes: *quota, password, totp_code, recovery_code }; + agent.admin_update_user(*id, &request).map(Outcome::Updated) + } + Action::Disable { id, .. } => agent.admin_disable_user(*id, &confirm).map(|()| Outcome::Done), + Action::Enable { id, .. } => agent.admin_enable_user(*id, &confirm).map(|()| Outcome::Done), + Action::Remove { id, .. } => agent.admin_remove_user(*id, &confirm).map(|()| Outcome::Done), + Action::IssueInvite { id, .. } => agent.admin_issue_invite(*id, &confirm).map(Outcome::Code), + Action::RevokeInvite { id, .. } => agent.admin_revoke_invite(*id, &confirm).map(|()| Outcome::Done), + Action::IssueReset { id, reset_two_factor, .. } => { + let request = AdminPasswordReset { password, totp_code, recovery_code, reset_two_factor: *reset_two_factor }; + agent.admin_issue_reset(*id, &request).map(Outcome::Code) + } + Action::ResetTwoFactor { id, .. } => agent.admin_reset_two_factor(*id, &confirm).map(|()| Outcome::Done), + Action::RevokeDevice { id, device, .. } => agent.admin_revoke_device(*id, *device, &confirm).map(|()| Outcome::Done), + } +} + #[cfg(test)] pub(crate) mod fake { //! A fake agent: a signed-in or signed-out status, answers set per call, and a switch that makes it go away. @@ -218,17 +477,176 @@ pub(crate) mod fake { use super::*; use std::sync::atomic::{AtomicBool, Ordering}; + /// The password, code and recovery code the fake's account takes. + pub const PASSWORD: &str = "correct horse battery"; + pub const CODE: &str = "123456"; + pub const RECOVERY: &str = "ABCD-EFGH-JKMN-PQRS"; + + /// What the fake agent's server has of the account's two-factor sign-in. + #[derive(Debug, Clone, Default)] + pub struct FakeFactor { + pub enabled: bool, + pub pending: bool, + /// A server from before two-factor sign-in: its calls are `not_found`. + pub unsupported: bool, + /// Every check of a code is refused as locked (840 s to wait). + pub locked: bool, + pub other_computers: u32, + } + pub struct FakeAgent { pub status: Mutex, pub gone: AtomicBool, /// What `join` answers, in turn (then the status). pub joins: Mutex>>, + pub usage_result: Mutex>>, pub asked: Mutex>, + pub factor: Mutex, + /// Whether the account is an admin (else every admin call is refused as `forbidden`). + pub admin: AtomicBool, + /// The accounts the admin calls list. + pub users: Mutex>, + /// What the admin changes answer instead of succeeding, taken from the end. + pub admin_fails: Mutex>, + /// The password each admin change was sent with. + pub passwords: Mutex>>, + /// The code (`totp_code`, `recovery_code`) each admin change was sent with. + pub factors: Mutex, Option)>>, + /// Whether the admin account has two-factor sign-in (what `account` says). + pub two_factor: AtomicBool, + /// The `reset_two_factor` flag of each password reset code asked for. + pub resets: Mutex>, + /// The audit events the audit calls and the activity list (newest first, as given), and what was asked. + pub audit_events: Mutex>, + pub audit_queries: Mutex>, + /// An export that the server cuts at its limit; and failures an export answers instead of succeeding. + pub export_truncated: Mutex, + pub export_fails: Mutex>, + } + + fn refused(code: &str, message: &str) -> Error { + Error::Refused { code: code.into(), message: message.into(), name_taken: None } } impl FakeAgent { pub fn new(status: Status) -> Self { - Self { status: Mutex::new(status), gone: AtomicBool::new(false), joins: Mutex::new(Vec::new()), asked: Mutex::new(Vec::new()) } + Self { + status: Mutex::new(status), + gone: AtomicBool::new(false), + joins: Mutex::new(Vec::new()), + usage_result: Mutex::new(None), + asked: Mutex::new(Vec::new()), + factor: Mutex::new(FakeFactor::default()), + admin: AtomicBool::new(false), + users: Mutex::new(Vec::new()), + admin_fails: Mutex::new(Vec::new()), + passwords: Mutex::new(Vec::new()), + factors: Mutex::new(Vec::new()), + two_factor: AtomicBool::new(false), + resets: Mutex::new(Vec::new()), + audit_events: Mutex::new(Vec::new()), + audit_queries: Mutex::new(Vec::new()), + export_truncated: Mutex::new(false), + export_fails: Mutex::new(Vec::new()), + } + } + + /// A refusal or nothing: whether a call of the account's two-factor sign-in gets through at all. + fn factor_gate(&self, what: &str) -> Result<(), Error> { + self.answer(what, |_| ())?; + if self.factor.lock().unwrap().unsupported { + return Err(refused("not_found", "no such endpoint")); + } + Ok(()) + } + + /// A code is checked: locked, wrong or right. + fn check_code(&self, code: Option<&str>) -> Result<(), Error> { + if self.factor.lock().unwrap().locked { + return Err(Error::RefusedWith { + code: "two_factor_locked".into(), + message: "locked".into(), + two_factor: None, + retry_after_secs: Some(840), + }); + } + if code == Some(CODE) || code == Some(RECOVERY) { Ok(()) } else { Err(refused("two_factor_invalid", "the code is wrong")) } + } + + fn codes() -> RecoveryCodes { + RecoveryCodes { recovery_codes: (0..10).map(|n| format!("AAAA-BBBB-CCCC-{n:04}")).collect() } + } + + /// An admin call: refused as a member's is unless the account is an admin. + fn admin_call(&self, what: &str, ok: impl FnOnce() -> T) -> Result { + self.asked.lock().unwrap().push(what.to_owned()); + if self.gone.load(Ordering::SeqCst) { + return Err(Error::NotRunning("can't reach it: connection refused".into())); + } + if !self.admin.load(Ordering::SeqCst) { + return Err(Error::Refused { code: "forbidden".into(), message: "this needs an administrator".into(), name_taken: None }); + } + Ok(ok()) + } + + /// An admin change: the password it was sent with is kept, and a failure set up is answered first. + fn admin_change( + &self, + what: &str, + password: Option, + factor: (Option, Option), + ok: impl FnOnce() -> T, + ) -> Result { + self.passwords.lock().unwrap().push(password); + self.factors.lock().unwrap().push(factor); + self.admin_call(what, || ())?; + if let Some(e) = self.admin_fails.lock().unwrap().pop() { + return Err(e); + } + Ok(ok()) + } + + /// An event of the audit log, `n` seconds after the epoch. + pub fn sample_event(n: u64, action: &str) -> craft_cloud_proto::api::AuditEvent { + craft_cloud_proto::api::AuditEvent { + id: Uuid::from_u128(u128::from(n) + 1), + at_ms: 1_790_000_000_000 - i64::try_from(n).unwrap_or(0) * 1_000, + actor_user: None, + actor_device: None, + action: action.to_owned(), + target_kind: String::new(), + target_id: None, + outcome: "success".into(), + details: Default::default(), + count: 1, + } + } + + /// The events a query lets through, from the cursor (an index) on: one page, and the cursor of the next. + fn audit_page(&self, query: &AuditQuery) -> AuditPage { + self.audit_queries.lock().unwrap().push(query.clone()); + let all: Vec<_> = self + .audit_events + .lock() + .unwrap() + .iter() + .filter(|e| match query.action.as_deref() { + Some(a) if a.ends_with('.') => e.action.starts_with(a), + Some(a) => e.action == a, + None => true, + }) + .filter(|e| query.user.is_none_or(|u| e.actor_user == Some(u) || e.target_id == Some(u))) + .cloned() + .collect(); + let from = query.cursor.as_deref().and_then(|c| c.parse::().ok()).unwrap_or(0); + let limit = query.limit.unwrap_or(50) as usize; + let events: Vec<_> = all.iter().skip(from).take(limit).cloned().collect(); + let next = (from + limit < all.len()).then(|| (from + limit).to_string()); + AuditPage { events, next } + } + + fn user(&self, id: Uuid) -> AdminUserInfo { + self.users.lock().unwrap().iter().find(|u| u.id == id).cloned().unwrap_or_else(|| sample_user("someone", id)) } fn answer(&self, what: &str, ok: impl FnOnce(&Status) -> T) -> Result { @@ -272,6 +690,19 @@ pub(crate) mod fake { fn devices(&self) -> Result, Error> { self.answer("devices", |_| Vec::new()) } + fn usage(&self) -> Result { + self.answer("usage", |_| ())?; + if let Some(result) = self.usage_result.lock().unwrap().clone() { + return result; + } + Ok(AccountUsage { + stored_bytes: 0, + quota_bytes: None, + trash_bytes: Some(0), + history_bytes: Some(0), + estimates: craft_cloud_proto::api::UsageEstimates { index_complete: true, reason: None }, + }) + } fn remove_device(&self, _id: DeviceId) -> Result, Error> { self.answer("remove_device", |_| Vec::new()) } @@ -287,6 +718,281 @@ pub(crate) mod fake { fn set_autostart(&self, on: bool) -> Result { self.answer("set_autostart", |s| Status { autostart: Some(on), ..s.clone() }) } + fn two_factor_status(&self) -> Result { + self.factor_gate("two_factor_status")?; + let f = self.factor.lock().unwrap(); + Ok(TwoFactorStatus { + enabled: f.enabled, + enabled_ms: f.enabled.then_some(1_000), + recovery_codes_left: if f.enabled { 10 } else { 0 }, + pending: f.pending, + }) + } + fn two_factor_setup(&self, password: &str) -> Result { + self.factor_gate("two_factor_setup")?; + if password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + let mut f = self.factor.lock().unwrap(); + if f.enabled { + return Err(refused("conflict", "on already")); + } + f.pending = true; + Ok(TwoFactorSetupResponse { + secret: "JBSWY3DPEHPK3PXP".into(), + otpauth_uri: "otpauth://totp/Craft%20Cloud:fabio?secret=JBSWY3DPEHPK3PXP&issuer=Craft%20Cloud".into(), + expires_ms: 1_790_000_600_000, + }) + } + fn two_factor_confirm(&self, totp_code: &str) -> Result { + self.factor_gate("two_factor_confirm")?; + if !self.factor.lock().unwrap().pending { + return Err(refused("conflict", "no set-up to confirm")); + } + if totp_code != CODE { + return Err(refused("two_factor_invalid", "the code is wrong")); + } + let mut f = self.factor.lock().unwrap(); + f.enabled = true; + f.pending = false; + Ok(Self::codes()) + } + fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + self.factor_gate("two_factor_disable")?; + if request.password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + self.check_code(request.totp_code.as_deref().or(request.recovery_code.as_deref()))?; + self.factor.lock().unwrap().enabled = false; + Ok(()) + } + fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + self.factor_gate("two_factor_recovery_codes")?; + if password != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + self.check_code(Some(totp_code))?; + Ok(Self::codes()) + } + fn change_password(&self, current: &str, new: &str) -> Result<(), Error> { + self.answer("change_password", |_| ())?; + if current != PASSWORD { + return Err(refused("wrong_password", "wrong")); + } + if new.len() < 8 { + return Err(refused("weak_password", "passwords are 8 to 1024 bytes")); + } + Ok(()) + } + fn revoke_other_devices(&self) -> Result { + self.answer("revoke_other_devices", |_| ())?; + let mut f = self.factor.lock().unwrap(); + let revoked = std::mem::take(&mut f.other_computers); + Ok(RevokedOthers { revoked }) + } + fn account(&self) -> Result { + self.answer("account", |s| AccountInfo { + user_id: Uuid::nil(), + username: s.account.clone().unwrap_or_default(), + role: if self.admin.load(Ordering::SeqCst) { "admin" } else { "member" }.into(), + created_ms: 1_780_000_000_000, + password_changed_ms: None, + two_factor_enabled: self.two_factor.load(Ordering::SeqCst), + }) + } + fn account_activity(&self, query: &AuditQuery) -> Result { + self.answer("account_activity", |_| ())?; + Ok(self.audit_page(&AuditQuery { user: None, action: None, ..query.clone() })) + } + fn admin_audit(&self, query: &AuditQuery) -> Result { + self.admin_call("admin_audit", || self.audit_page(query)) + } + fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + self.admin_call("admin_audit_export", || ())?; + if let Some(e) = self.export_fails.lock().unwrap().pop() { + // Some lines came before the stream broke. + let _ = out.write_all(b"{\"partial\":true}\n"); + return Err(e); + } + let page = self.audit_page(&AuditQuery { cursor: None, limit: Some(u32::MAX), ..query.clone() }); + for event in &page.events { + let _ = out.write_all(format!("{{\"id\":\"{}\",\"action\":\"{}\"}}", event.id, event.action).as_bytes()); + let _ = out.write_all(b"\n"); + } + Ok(ExportOutcome { events: page.events.len() as u64, truncated: *self.export_truncated.lock().unwrap() }) + } + fn admin_reset_two_factor(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_reset_two_factor", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_server(&self) -> Result { + self.admin_call("admin_server", || AdminServerInfo { + version: "0.1.0".into(), + users: self.users.lock().unwrap().len() as u64, + devices_active: 3, + stored_bytes: 5_000_000_000, + live_bytes: 120_000, + storage_kind: "disk".into(), + database_kind: "sqlite".into(), + }) + } + fn admin_storage(&self) -> Result { + self.admin_call("admin_storage", || { + use craft_cloud_app::{AdminManifestIndex, AdminStorageAccount}; + let users = self.users.lock().unwrap(); + let top_accounts = users + .iter() + .take(3) + .map(|user| AdminStorageAccount { + id: user.id, + username: user.username.clone(), + stored_bytes: user.stored_bytes, + quota_bytes: user.quota_bytes.or(Some(10 * 1_073_741_824)), + }) + .collect(); + // The counts follow the fake accounts; the default quota applies where an account has no + // override of its own, and "near" includes accounts at or over their quota, as on the server. + let effective = |user: &AdminUserInfo| user.quota_bytes.unwrap_or(10 * 1_073_741_824); + AdminStorageOverview { + accounts: users.len() as u64, + stored_bytes: 5_000_000_000, + default_quota_bytes: Some(10 * 1_073_741_824), + accounts_with_override: users.iter().filter(|user| user.quota_bytes.is_some()).count() as u64, + accounts_near_quota: users.iter().filter(|user| user.stored_bytes >= effective(user) - effective(user) / 10).count() as u64, + accounts_over_quota: users.iter().filter(|user| user.stored_bytes >= effective(user)).count() as u64, + top_accounts, + manifest_index: AdminManifestIndex { complete: 7, building: 1, not_indexed: 2 }, + ..Default::default() + } + }) + } + fn admin_users(&self, query: &UserQuery) -> Result { + self.admin_call("admin_users", || { + let needle = query.query.as_deref().unwrap_or_default().to_lowercase(); + let all: Vec = + self.users.lock().unwrap().iter().filter(|u| u.username.to_lowercase().contains(&needle)).cloned().collect(); + // The cursor is the index of the page's first account. + let from = query.cursor.as_deref().and_then(|c| c.parse::().ok()).unwrap_or(0); + let limit = query.limit.unwrap_or(50) as usize; + let users: Vec = all.iter().skip(from).take(limit).cloned().collect(); + let next = (from + limit < all.len()).then(|| (from + limit).to_string()); + AdminUserList { users, next } + }) + } + fn admin_devices(&self, user: Uuid) -> Result { + self.admin_call("admin_devices", || AdminDevices { + devices: vec![DeviceInfo { + id: DeviceId::generate(), + name: format!("{} laptop", self.user(user).username), + created_ms: 0, + last_used_ms: 1_789_990_000_000, + revoked: false, + current: false, + client_kind: "agent".into(), + platform: "linux".into(), + app_version: "0.1.0".into(), + revoked_ms: None, + revoke_reason: None, + }], + }) + } + fn admin_user_usage(&self, user: Uuid) -> Result { + self.admin_call("admin_user_usage", || { + let account = self.user(user); + AccountUsage { + stored_bytes: account.stored_bytes, + quota_bytes: account.quota_bytes.or(Some(10 * 1_073_741_824)), + trash_bytes: Some(80_000_000), + history_bytes: Some(240_000_000), + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + } + }) + } + fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.admin_change("admin_create_user", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + AdminCreatedUser { + user: AdminUserInfo { role: request.role.clone(), invite_pending: true, ..sample_user(&request.username, Uuid::now_v7()) }, + invite: AdminCode { + code: "K7M4Q9TXK7M4Q9TX".into(), + join_url: Some("cloud.shadows.ch/K7M4-Q9TX-K7M4-Q9TX".into()), + expires_ms: 1_790_600_000_000, + }, + } + }) + } + fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.admin_change("admin_update_user", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + let mut user = self.user(id); + if let Some(role) = &request.role { + user.role = role.clone(); + } + if let Some(quota) = request.quota_bytes { + user.quota_bytes = quota; + } + user + }) + } + fn admin_disable_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_disable_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_enable_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_enable_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_remove_user(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_remove_user", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_issue_invite(&self, _id: Uuid, confirm: &AdminConfirm) -> Result { + self.admin_change("admin_issue_invite", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || { + AdminCode { + code: "ABCDEFGHJKMNPQRS".into(), + join_url: Some("cloud.shadows.ch/ABCD-EFGH-JKMN-PQRS".into()), + expires_ms: 1_790_600_000_000, + } + }) + } + fn admin_revoke_invite(&self, _id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_revoke_invite", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + fn admin_issue_reset(&self, _id: Uuid, request: &AdminPasswordReset) -> Result { + self.resets.lock().unwrap().push(request.reset_two_factor); + self.admin_change("admin_issue_reset", request.password.clone(), (request.totp_code.clone(), request.recovery_code.clone()), || { + AdminCode { code: "TUVWXYZ234567ABC".into(), join_url: None, expires_ms: 1_790_003_600_000 } + }) + } + fn admin_revoke_device(&self, _id: Uuid, _device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.admin_change("admin_revoke_device", confirm.password.clone(), (confirm.totp_code.clone(), confirm.recovery_code.clone()), || ()) + } + + fn file_history(&self, _path: &str, _cursor: Option<&str>) -> Result { + self.answer("file_history", |_| HistoryResponse { versions: Vec::new(), next: None }) + } + fn restore_file(&self, _path: &str, _version: VersionId, _expected_version: VersionId) -> Result { + Err(Error::Refused { code: "invalid".into(), message: "not configured".into(), name_taken: None }) + } + fn trash(&self, _cursor: Option<&str>) -> Result { + self.answer("trash", |_| TrashResponse { items: Vec::new(), next: None }) + } + fn restore_from_trash(&self, _id: NodeId) -> Result { + Err(Error::Refused { code: "invalid".into(), message: "not configured".into(), name_taken: None }) + } + fn remove_from_trash(&self, _id: NodeId) -> Result<(), Error> { + self.answer("remove_from_trash", |_| ()) + } + } + + /// An enabled member account that stores nothing. + pub fn sample_user(username: &str, id: Uuid) -> AdminUserInfo { + AdminUserInfo { + id, + username: username.to_owned(), + role: "member".into(), + created_ms: 1_780_000_000_000, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 1, + stored_bytes: 0, + quota_bytes: None, + invite_pending: false, + } } } diff --git a/apps/craft-cloud-desktop/src/app.rs b/apps/craft-cloud-desktop/src/app.rs index 391c8d24..b200f7ce 100644 --- a/apps/craft-cloud-desktop/src/app.rs +++ b/apps/craft-cloud-desktop/src/app.rs @@ -36,6 +36,9 @@ pub trait Host { fn set_autostart(&self, on: bool) -> Result<(), String>; /// Shows the OS's folder picker at `start`, off the UI thread; the answer is sent to `answer` (then `wake`). fn pick_folder(&self, start: String, answer: Sender, wake: Arc); + /// Shows the OS's save dialog with `suggested` as the file name, off the UI thread; the answer (the path, or `None` when + /// cancelled) is sent to `answer` as [`crate::audit::AuditMsg::ExportPicked`] (then `wake`). + fn save_file(&self, suggested: String, ticket: u64, answer: Sender, wake: Arc); /// Ends the tray process (Linux, where the tray is a process of its own). fn quit_tray(&self) {} /// Called every frame with the model, to show its state (the tray icon); events it has go to the model. @@ -84,6 +87,10 @@ impl App { let done = self.host.set_autostart(on); self.handle(Event::AppAutostartSet(done), ctx); } + Some(Effect::PickExportFile { suggested, ticket }) => { + let wake_ctx = ctx.clone(); + self.host.save_file(suggested, ticket, self.to_self.clone(), Arc::new(move || wake_ctx.request_repaint())); + } Some(Effect::PickFolder(start)) => { let wake_ctx = ctx.clone(); self.host.pick_folder(start, self.to_self.clone(), Arc::new(move || wake_ctx.request_repaint())); @@ -167,6 +174,15 @@ impl Host for System { crate::autostart::set_for_this_app(on) } + fn save_file(&self, suggested: String, ticket: u64, answer: Sender, wake: Arc) { + let _ = std::thread::Builder::new().name("craft-cloud-saver".into()).spawn(move || { + let dialog = rfd::AsyncFileDialog::new().set_title("Save the audit log").set_file_name(&suggested); + let picked = block_on(dialog.save_file()).map(|handle| handle.path().display().to_string()); + let _ = answer.send(Event::Audit(crate::audit::AuditMsg::ExportPicked { ticket, path: picked })); + wake(); + }); + } + fn pick_folder(&self, start: String, answer: Sender, wake: Arc) { let _ = std::thread::Builder::new().name("craft-cloud-picker".into()).spawn(move || { let mut dialog = rfd::AsyncFileDialog::new().set_title("The folder to sync"); diff --git a/apps/craft-cloud-desktop/src/audit.rs b/apps/craft-cloud-desktop/src/audit.rs new file mode 100644 index 00000000..a6c1b5e3 --- /dev/null +++ b/apps/craft-cloud-desktop/src/audit.rs @@ -0,0 +1,823 @@ +//! The security audit log's pages, as a pure state like [`crate::admin`] (which hosts the admin's view of it as +//! [`View::Audit`]; the account's own activity is [`Screen::Activity`], open to every signed-in account). +//! +//! - **What is shown** is the server's: events newest first, a cursor given back exactly as it came, filters that are only +//! what the server's contract allows (an action from the closed list [`AUDIT_ACTIONS`] or a prefix of it, an account the +//! admin picked on its page). The server decides who reads what; a `forbidden` makes the app ask the account's role again. +//! - **The export** is the agent's stream of the server's NDJSON, written by the app straight to a file the person picked +//! ([`Effect::AuditExport`], which writes to `.part` and moves it into place only when the stream ended properly). It +//! is never held in memory. Whole, or cut by the server at its limit (said so, not as a success), or failed (nothing kept). +//! - Answers that come late (a page asked for before the filter changed) match nothing: the numbers are [`crate::admin::fresh`]. + +use craft_cloud_app::{AuditPage, AuditQuery, Error, ExportOutcome, Uuid}; +use craft_cloud_proto::api::AuditEvent; + +use crate::admin::{AdminScreen, View, error_text, fresh, is_forbidden}; +use crate::model::{Effect, Model, Overview, Screen}; + +/// Events on one page. +pub const PAGE_SIZE: u32 = 25; +/// The file name an export is offered under. +pub const EXPORT_NAME: &str = "craft-cloud-audit.ndjson"; + +/// A page of events with the cursors back to the first, and the call whose answer is wanted. +#[derive(Debug, Clone, PartialEq)] +pub struct AuditList { + pub events: Vec, + /// How each page up to this one was asked for: `cursors[0]` is `None`, the last is this page's. The cursors are the + /// server's, kept as they came. + pub cursors: Vec>, + /// The cursor of the (older) page after this one, if there is one. + pub next: Option, + pub loading: bool, + pub loaded: bool, + pub error: Option, + ticket: u64, +} + +impl AuditList { + pub fn new() -> Self { + Self { events: Vec::new(), cursors: vec![None], next: None, loading: false, loaded: false, error: None, ticket: 0 } + } + + /// The page number shown (from 1). + pub fn page(&self) -> usize { + self.cursors.len() + } + + pub fn has_previous(&self) -> bool { + self.cursors.len() > 1 + } + + /// Asks for the current page with the filters of `filter` (none for the account's own activity). + fn load(&mut self, own: bool, filter: &AuditFilter) -> Effect { + self.loading = true; + self.error = None; + self.ticket = fresh(); + let query = AuditQuery { + cursor: self.cursors.last().cloned().flatten(), + before_ms: None, + limit: Some(PAGE_SIZE), + user: filter.user.as_ref().map(|(id, _)| *id), + action: filter.action.clone(), + }; + Effect::AuditPage { own, query, ticket: self.ticket } + } + + fn came(&mut self, ticket: u64, result: Result) -> Option { + if ticket != self.ticket { + return None; + } + self.loading = false; + match result { + Ok(page) => { + self.loaded = true; + self.error = None; + self.events = page.events; + self.next = page.next; + None + } + Err(e) => { + self.error = Some(error_text(&e)); + Some(e) + } + } + } + + fn start_over(&mut self) { + self.cursors = vec![None]; + } +} + +impl Default for AuditList { + fn default() -> Self { + Self::new() + } +} + +/// What the admin's audit view is narrowed to. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditFilter { + /// An event name from [`AUDIT_ACTIONS`], or a prefix of them ending in a dot (`auth.`); none: every event. + pub action: Option, + /// The account the events are about or by (its ID and name, for the label). + pub user: Option<(Uuid, String)>, +} + +/// Where an export is. +#[derive(Debug, Clone, PartialEq)] +pub enum ExportState { + Idle, + /// The file picker is open (the call `ticket` is the one whose answer is wanted). + Picking { + ticket: u64, + }, + /// The stream is being written to this file (the call `ticket` is the one whose answer is wanted). + Running { + path: String, + ticket: u64, + }, + /// Written: how many events, and whether the server cut it at its limit (then it is not the whole log). + Done { + path: String, + outcome: ExportOutcome, + }, + /// Nothing was kept. + Failed(String), +} + +/// The admin's audit view. +#[derive(Debug, Clone, PartialEq)] +pub struct AuditView { + pub list: AuditList, + pub filter: AuditFilter, + pub export: ExportState, +} + +/// The account's own activity. +#[derive(Debug, Clone, PartialEq)] +pub struct ActivityScreen { + /// The signed-in view the page came from and goes back to. + pub overview: Overview, + pub list: AuditList, +} + +/// What happened on the audit pages: clicks and answers. +#[derive(Debug, Clone, PartialEq)] +pub enum AuditMsg { + /// [`Effect::AuditPage`]'s answer (`own`: the account's own activity). + Page { + own: bool, + ticket: u64, + result: Result, + }, + /// [`Effect::AuditExport`]'s answer. + Exported { + ticket: u64, + result: Result, + }, + + /// The admin's audit view, for one account or all of them. + OpenAudit(Option<(Uuid, String)>), + /// Narrow to an event name or prefix of the closed list; `None`: every event. + SetAction(Option), + /// Every account again. + ClearUser, + Next, + Previous, + Reload, + /// Export what the filters say to a file (the person picks it). + Export, + /// The file picker's answer (`None`: cancelled). + ExportPicked { + ticket: u64, + path: Option, + }, + /// The note about the last export was read. + DismissExport, + + /// The signed-in account's own activity. + OpenActivity, + CloseActivity, + ActivityNext, + ActivityPrevious, + ActivityReload, +} + +impl Model { + /// The account's own activity page, if shown. + pub fn activity_screen(&self) -> Option<&ActivityScreen> { + match &self.screen { + Screen::Activity(a) => Some(a), + _ => None, + } + } + + pub(crate) fn audit(&mut self, msg: AuditMsg) -> Vec { + match msg { + AuditMsg::OpenActivity => { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + let mut screen = ActivityScreen { overview: overview.clone(), list: AuditList::new() }; + let effect = screen.list.load(true, &AuditFilter::default()); + self.screen = Screen::Activity(Box::new(screen)); + vec![effect] + } + AuditMsg::CloseActivity => { + if let Screen::Activity(a) = &self.screen { + self.screen = Screen::SignedIn(a.overview.clone()); + } + Vec::new() + } + AuditMsg::Page { own: true, ticket, result } => { + let Screen::Activity(a) = &mut self.screen else { return Vec::new() }; + a.list.came(ticket, result); + Vec::new() + } + AuditMsg::ActivityNext | AuditMsg::ActivityPrevious | AuditMsg::ActivityReload => { + let Screen::Activity(a) = &mut self.screen else { return Vec::new() }; + let list = &mut a.list; + let own = AuditFilter::default(); + match msg { + AuditMsg::ActivityNext => match list.next.clone() { + Some(next) if !list.loading => { + list.cursors.push(Some(next)); + vec![list.load(true, &own)] + } + _ => Vec::new(), + }, + AuditMsg::ActivityPrevious if list.has_previous() && !list.loading => { + list.cursors.pop(); + vec![list.load(true, &own)] + } + AuditMsg::ActivityReload if !list.loading => vec![list.load(true, &own)], + _ => Vec::new(), + } + } + msg => self.admin_audit(msg), + } + } + + fn admin_audit(&mut self, msg: AuditMsg) -> Vec { + let Screen::Admin(admin) = &mut self.screen else { return Vec::new() }; + admin.audit(msg) + } +} + +impl AdminScreen { + fn audit_view(&mut self) -> Option<&mut AuditView> { + match &mut self.view { + View::Audit(view) => Some(view), + _ => None, + } + } + + pub(crate) fn audit(&mut self, msg: AuditMsg) -> Vec { + let session = self.overview.session; + match msg { + AuditMsg::OpenAudit(user) => { + if self.dialog.is_some() || self.code.is_some() { + return Vec::new(); + } + let mut view = AuditView { list: AuditList::new(), filter: AuditFilter { action: None, user }, export: ExportState::Idle }; + let effect = view.list.load(false, &view.filter); + self.view = View::Audit(Box::new(view)); + vec![effect] + } + AuditMsg::Page { own: false, ticket, result } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + match view.list.came(ticket, result) { + Some(e) if is_forbidden(&e) => vec![Effect::Account { session }], + _ => Vec::new(), + } + } + AuditMsg::SetAction(action) => { + let Some(view) = self.audit_view() else { return Vec::new() }; + // Only what the server's contract allows is ever asked for. + if action.as_deref().is_some_and(|a| !craft_cloud_app::valid_audit_action(a)) { + return Vec::new(); + } + view.filter.action = action; + view.list.start_over(); + vec![view.list.load(false, &view.filter)] + } + AuditMsg::ClearUser => { + let Some(view) = self.audit_view() else { return Vec::new() }; + view.filter.user = None; + view.list.start_over(); + vec![view.list.load(false, &view.filter)] + } + AuditMsg::Next => { + let Some(view) = self.audit_view() else { return Vec::new() }; + match view.list.next.clone() { + Some(next) if !view.list.loading => { + view.list.cursors.push(Some(next)); + vec![view.list.load(false, &view.filter)] + } + _ => Vec::new(), + } + } + AuditMsg::Previous => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.list.has_previous() && !view.list.loading { + view.list.cursors.pop(); + vec![view.list.load(false, &view.filter)] + } else { + Vec::new() + } + } + AuditMsg::Reload => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.list.loading { Vec::new() } else { vec![view.list.load(false, &view.filter)] } + } + AuditMsg::Export => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if matches!(view.export, ExportState::Picking { .. } | ExportState::Running { .. }) { + return Vec::new(); + } + // The answers of the picker and of the export carry this number: one that comes after the pages were left and + // opened again, or for an earlier export, matches nothing. + let ticket = fresh(); + view.export = ExportState::Picking { ticket }; + vec![Effect::PickExportFile { suggested: EXPORT_NAME.to_owned(), ticket }] + } + AuditMsg::ExportPicked { ticket, path: picked } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + if view.export != (ExportState::Picking { ticket }) { + return Vec::new(); + } + match picked { + None => { + view.export = ExportState::Idle; + Vec::new() + } + Some(path) => { + view.export = ExportState::Running { path: path.clone(), ticket }; + // Every event the filters say, not one page: no cursor, no limit. + let query = AuditQuery { + cursor: None, + before_ms: None, + limit: None, + user: view.filter.user.as_ref().map(|(id, _)| *id), + action: view.filter.action.clone(), + }; + vec![Effect::AuditExport { query, path, ticket }] + } + } + } + AuditMsg::Exported { ticket, result } => { + let Some(view) = self.audit_view() else { return Vec::new() }; + let ExportState::Running { path, ticket: running } = &view.export else { return Vec::new() }; + if *running != ticket { + return Vec::new(); + } + let path = path.clone(); + match result { + Ok(outcome) => { + view.export = ExportState::Done { path, outcome }; + Vec::new() + } + Err(e) => { + let forbidden = is_forbidden(&e); + view.export = ExportState::Failed(error_text(&e)); + if forbidden { vec![Effect::Account { session }] } else { Vec::new() } + } + } + } + AuditMsg::DismissExport => { + if let Some(view) = self.audit_view() + && !matches!(view.export, ExportState::Running { .. } | ExportState::Picking { .. }) + { + view.export = ExportState::Idle; + } + Vec::new() + } + // Handled by the model. + AuditMsg::Page { own: true, .. } + | AuditMsg::OpenActivity + | AuditMsg::CloseActivity + | AuditMsg::ActivityNext + | AuditMsg::ActivityPrevious + | AuditMsg::ActivityReload => Vec::new(), + } + } +} + +/// What an event name says, in words; a name that isn't in the list (a newer server's) is shown as it is. +pub fn action_label(action: &str) -> &str { + match action { + "auth.login" => "Signed in", + "auth.login_failed" => "Sign-in failed (wrong password)", + "auth.second_factor_failed" => "Sign-in failed (wrong code)", + "auth.recovery_code_used" => "Recovery code used", + "auth.two_factor_locked" => "Two-factor sign-in locked after wrong codes", + "device.revoked" => "Computer signed out", + "device.revoked_others" => "Other computers signed out", + "account.password_changed" => "Password changed", + "account.two_factor_enabled" => "Two-factor sign-in turned on", + "account.two_factor_disabled" => "Two-factor sign-in turned off", + "account.recovery_codes_regenerated" => "New recovery codes made", + "admin.user_created" => "Account created by an admin", + "admin.user_disabled" => "Account disabled by an admin", + "admin.user_enabled" => "Account enabled by an admin", + "admin.user_removed" => "Account removed by an admin", + "admin.role_changed" => "Role changed by an admin", + "admin.quota_changed" => "Quota changed by an admin", + "admin.invite_issued" => "Invite made by an admin", + "admin.invite_revoked" => "Invite or reset code revoked by an admin", + "admin.password_reset_issued" => "Password reset code made by an admin", + "admin.two_factor_reset" => "Two-factor sign-in turned off by an admin", + "admin.device_revoked" => "Computer signed out by an admin", + "join.invite_redeemed" => "Joined with an invite", + "join.reset_redeemed" => "Joined with a password reset code", + "join.pairing_redeemed" => "Computer added with a code", + "cli.password_reset" => "Password reset on the server", + "cli.two_factor_reset" => "Two-factor sign-in turned off on the server", + "cli.device_revoked" => "Computer signed out on the server", + "cli.devices_revoked_all" => "Every computer signed out on the server", + "cli.user_removed" => "Account removed on the server", + other => other, + } +} + +/// A filter choice's label: a prefix as a group, a name as its words. +pub fn filter_label(filter: &str) -> String { + match filter { + "auth." => "Sign-ins".to_owned(), + "device." => "Computers signed out".to_owned(), + "account." => "Account changes".to_owned(), + "admin." => "Admin changes".to_owned(), + "join." => "Joining".to_owned(), + "cli." => "Server command line".to_owned(), + name => action_label(name).to_owned(), + } +} + +/// The groups of the filter, then every name: the closed list the server filters by. +pub fn filter_choices() -> Vec<&'static str> { + let mut choices = vec!["auth.", "device.", "account.", "admin.", "join.", "cli."]; + choices.extend(craft_cloud_app::AUDIT_ACTIONS); + choices +} + +/// The first eight characters of an ID, to tell accounts and computers apart in a line (the whole ID is on hover). +pub fn short_id(id: impl std::fmt::Display) -> String { + id.to_string().chars().filter(|c| *c != '-').take(8).collect() +} + +/// Who did it and what it was about, in a line, and the same with the whole IDs for hover: the account that acted (or "command +/// line" for the server's own commands, "unknown" for a failed sign-in, where nobody proved who they were), the account or computer +/// it concerned, and the computer a sign-out or admin act names in its details. +pub fn who_line(event: &AuditEvent) -> (String, String) { + let mut line = Vec::new(); + let mut full = Vec::new(); + match (event.actor_user, event.action.starts_with("cli.")) { + (Some(user), _) => { + line.push(format!("by {}", short_id(user))); + full.push(format!("by account {user}")); + } + (None, true) => { + line.push("by the command line".to_owned()); + full.push("by the server's command line".to_owned()); + } + (None, false) => { + line.push("by unknown".to_owned()); + full.push("by unknown: nobody had proved who they were".to_owned()); + } + } + if let Some(device) = event.actor_device { + line.push(format!("on computer {}", short_id(device))); + full.push(format!("on computer {device}")); + } + if let Some(target) = event.target_id { + let kind = if event.target_kind.is_empty() || event.target_kind == "user" { "account" } else { event.target_kind.as_str() }; + line.push(format!("about {kind} {}", short_id(target))); + full.push(format!("about {kind} {target}")); + } + if let Some(device) = event.details.device { + line.push(format!("computer {}", short_id(device))); + full.push(format!("computer {device}")); + } + (line.join(" · "), full.join("\n")) +} + +/// The typed details of an event in a few words (nothing the server doesn't write from closed lists). +pub fn details_line(event: &AuditEvent) -> String { + let d = &event.details; + let mut parts: Vec = Vec::new(); + if let Some(kind) = &d.client_kind { + parts.push(format!("from {kind}")); + } + if let Some(platform) = &d.platform { + parts.push(platform.clone()); + } + if let Some(step) = &d.step { + parts.push(format!("at {step}")); + } + if let (Some(from), Some(to)) = (&d.role_from, &d.role_to) { + parts.push(format!("{from} to {to}")); + } + match &d.quota_bytes { + Some(Some(bytes)) => parts.push(format!("quota {}", crate::admin::format_bytes(*bytes))), + Some(None) => parts.push("quota removed".to_owned()), + None => {} + } + if let Some(n) = d.revoked { + parts.push(format!("{n} computer{} signed out", if n == 1 { "" } else { "s" })); + } + if d.two_factor_removed == Some(true) { + parts.push("two-factor turned off".to_owned()); + } + if let Some(reason) = &d.reason { + parts.push(format!("reason: {reason}")); + } + if event.count > 1 { + parts.push(format!("{} tries", event.count)); + } + if event.outcome == "failure" && !parts.iter().any(|p| p.starts_with("at ")) { + parts.push("failed".to_owned()); + } + parts.join(" · ") +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Arc; + + use craft_cloud_proto::api::AuditDetails; + + use super::*; + use crate::admin::AdminEvent; + use crate::admin::tests as support; + use crate::agent::fake::FakeAgent; + use crate::agent::{Starter, call}; + use crate::model::Event; + + /// The picker's answer for the export that waits for it. + pub(crate) fn picked(model: &mut Model, path: Option<&str>) -> Vec { + let View::Audit(view) = &model.admin_screen().unwrap().view else { panic!("not the audit view") }; + let ExportState::Picking { ticket } = view.export else { panic!("not picking: {:?}", view.export) }; + model.update(Event::Audit(AuditMsg::ExportPicked { ticket, path: path.map(str::to_owned) })) + } + + /// The answer of the export that runs. + pub(crate) fn finished(model: &mut Model, result: Result) { + let View::Audit(view) = &model.admin_screen().unwrap().view else { panic!("not the audit view") }; + let ExportState::Running { ticket, .. } = view.export else { panic!("not running: {:?}", view.export) }; + model.update(Event::Audit(AuditMsg::Exported { ticket, result })); + } + + fn on_audit(agent: &FakeAgent) -> Model { + let mut model = support::on_the_pages(agent); + support::run(&mut model, agent, vec![Event::Audit(AuditMsg::OpenAudit(None))]); + model + } + + fn view(model: &Model) -> &AuditView { + match &model.admin_screen().unwrap().view { + View::Audit(view) => view, + other => panic!("not the audit view: {other:?}"), + } + } + + fn ev(msg: AuditMsg) -> Vec { + vec![Event::Audit(msg)] + } + + #[test] + fn the_admins_audit_view_pages_with_the_servers_cursors() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..60).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + let v = view(&model); + assert_eq!((v.list.page(), v.list.events.len(), v.list.next.as_deref()), (1, PAGE_SIZE as usize, Some("25"))); + assert!(model.update(Event::Audit(AuditMsg::Previous)).is_empty()); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!(view(&model).list.page(), 2); + // The cursor goes back exactly as it came. + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!((asked.cursor.as_deref(), asked.limit), (Some("25"), Some(PAGE_SIZE))); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!((view(&model).list.events.len(), view(&model).list.next.clone()), (10, None)); + assert!(model.update(Event::Audit(AuditMsg::Next)).is_empty(), "no page after the last"); + support::run(&mut model, &boss, ev(AuditMsg::Previous)); + assert_eq!(view(&model).list.page(), 2); + } + + #[test] + fn filters_are_the_closed_list_and_start_the_list_over() { + let boss = support::agent(true, 1); + boss.audit_events + .lock() + .unwrap() + .extend((0..30).map(|n| FakeAgent::sample_event(n, if n % 2 == 0 { "auth.login" } else { "admin.user_created" }))); + let mut model = on_audit(&boss); + support::run(&mut model, &boss, ev(AuditMsg::Next)); + assert_eq!(view(&model).list.page(), 2); + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("admin.".into())))); + assert_eq!(view(&model).list.page(), 1, "a filter starts over from the first page"); + assert!(view(&model).list.events.iter().all(|e| e.action.starts_with("admin."))); + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!((asked.action.as_deref(), asked.cursor), (Some("admin."), None)); + // Anything that is not in the closed list (or a prefix of it) is never asked for. + for bad in ["nope.", "auth.log", "x' OR 1=1", ""] { + let before = boss.audit_queries.lock().unwrap().len(); + assert!(model.update(Event::Audit(AuditMsg::SetAction(Some(bad.into())))).is_empty(), "{bad:?}"); + assert_eq!(boss.audit_queries.lock().unwrap().len(), before); + } + assert_eq!(view(&model).filter.action.as_deref(), Some("admin.")); + support::run(&mut model, &boss, ev(AuditMsg::SetAction(None))); + assert_eq!(view(&model).filter.action, None); + // Every choice the page offers is one the server takes. + assert!(filter_choices().iter().all(|c| craft_cloud_app::valid_audit_action(c))); + assert_eq!(filter_choices().len(), 36); + assert_eq!(action_label("auth.login"), "Signed in"); + assert_eq!(action_label("something.new"), "something.new"); + assert_eq!(filter_label("auth."), "Sign-ins"); + } + + #[test] + fn an_account_filter_comes_from_its_page_and_is_cleared() { + let boss = support::agent(true, 2); + let mut model = support::on_the_pages(&boss); + let ana = support::user_named(&model, "user01"); + support::run(&mut model, &boss, ev(AuditMsg::OpenAudit(Some((ana.id, "user01".into()))))); + let asked = boss.audit_queries.lock().unwrap().last().cloned().unwrap(); + assert_eq!(asked.user, Some(ana.id)); + assert_eq!(view(&model).filter.user.as_ref().map(|(_, n)| n.as_str()), Some("user01")); + support::run(&mut model, &boss, ev(AuditMsg::ClearUser)); + assert_eq!(boss.audit_queries.lock().unwrap().last().unwrap().user, None); + // Not from a password dialog's page. + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::BackToList)]); + let ask = crate::admin::Action::Disable { id: ana.id, username: "user01".into() }; + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::Ask(ask))]); + assert!(model.update(Event::Audit(AuditMsg::OpenAudit(None))).is_empty()); + } + + #[test] + fn a_page_that_comes_late_is_ignored_and_a_refusal_is_said() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..5).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = support::on_the_pages(&boss); + let first = model.update(Event::Audit(AuditMsg::OpenAudit(None))); + let Some(Effect::AuditPage { ticket: old, .. }) = first.first().cloned() else { panic!("{first:?}") }; + // Another filter asks again while the first answer is still on its way. + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("auth.".into())))); + let current = view(&model).list.events.clone(); + assert_eq!(current.len(), 5); + let page = AuditPage { events: vec![FakeAgent::sample_event(99, "auth.login")], next: None }; + model.update(Event::Audit(AuditMsg::Page { own: false, ticket: old, result: Ok(page) })); + assert_eq!(view(&model).list.events, current, "the first call's answer matches nothing"); + // A refusal is said; `forbidden` asks the account's role again. + let forbidden = Error::Refused { code: "forbidden".into(), message: "Only admins".into(), name_taken: None }; + let Some(Effect::AuditPage { ticket, .. }) = model.update(Event::Audit(AuditMsg::Reload)).first().cloned() else { panic!() }; + let effects = model.update(Event::Audit(AuditMsg::Page { own: false, ticket, result: Err(forbidden) })); + assert!(matches!(effects.as_slice(), [Effect::Account { .. }]), "{effects:?}"); + assert!(view(&model).list.error.as_deref().is_some_and(|e| e.starts_with("The server doesn't allow this"))); + } + + #[test] + fn an_export_is_picked_run_and_whole_truncated_or_failed() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..3).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + // Cancelling the picker is nothing. + let asked = model.update(Event::Audit(AuditMsg::Export)); + assert!(matches!(asked.as_slice(), [Effect::PickExportFile { suggested, .. }] if suggested == EXPORT_NAME), "{asked:?}"); + assert!(model.update(Event::Audit(AuditMsg::Export)).is_empty(), "one at a time"); + picked(&mut model, None); + assert_eq!(view(&model).export, ExportState::Idle); + // Whole. + support::run(&mut model, &boss, ev(AuditMsg::SetAction(Some("auth.".into())))); + model.update(Event::Audit(AuditMsg::Export)); + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("out.ndjson").display().to_string(); + let effects = picked(&mut model, Some(&file)); + let [Effect::AuditExport { query, path, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((path.as_str(), query.action.as_deref(), query.cursor.clone(), query.limit), (file.as_str(), Some("auth."), None, None)); + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(view(&model).export, ExportState::Done { path: file.clone(), outcome: ExportOutcome { events: 3, truncated: false } }); + assert_eq!(std::fs::read_to_string(&file).unwrap().lines().count(), 3); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1, "no temporary file is left"); + // Truncated: the file is the newest events and the page says it is not the whole log. + *boss.export_truncated.lock().unwrap() = true; + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert!(matches!(&view(&model).export, ExportState::Done { outcome, .. } if outcome.truncated), "{:?}", view(&model).export); + // Failed (the stream broke): said, and no file of ours is left, and an older one, and a `.part` that is somebody else's, + // are not touched. + *boss.export_truncated.lock().unwrap() = false; + std::fs::write(&file, "old").unwrap(); + let theirs = format!("{file}.part"); + std::fs::write(&theirs, "not ours").unwrap(); + boss.export_fails.lock().unwrap().push(Error::Io("the export broke off: reset".into())); + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert!(matches!(&view(&model).export, ExportState::Failed(m) if m.contains("broke off")), "{:?}", view(&model).export); + assert_eq!(std::fs::read_to_string(&file).unwrap(), "old", "a failed export leaves what was there"); + assert_eq!(std::fs::read_to_string(&theirs).unwrap(), "not ours", "a file called .part is not ours to truncate or delete"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2, "only those two files"); + // A whole export replaces the destination and still leaves the other file alone. + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Export)); + let effects = picked(&mut model, Some(&file)); + let answer = call(&boss, &start, effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); + assert_eq!(std::fs::read_to_string(&file).unwrap().lines().count(), 3); + assert_eq!(std::fs::read_to_string(&theirs).unwrap(), "not ours"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2); + // An answer when no export runs is ignored. + model.update(Event::Audit(AuditMsg::DismissExport)); + model.update(Event::Audit(AuditMsg::Exported { ticket: 1, result: Ok(ExportOutcome { events: 1, truncated: false }) })); + assert_eq!(view(&model).export, ExportState::Idle); + } + + /// An export's answer belongs to the export that asked: leave the pages and open them again, start another, and the first one's + /// answer (or its picker's) finishes nothing of the second. + #[test] + fn an_old_exports_answers_finish_nothing_of_a_newer_one() { + let boss = support::agent(true, 1); + boss.audit_events.lock().unwrap().extend((0..3).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = on_audit(&boss); + model.update(Event::Audit(AuditMsg::Export)); + let a = picked(&mut model, Some("a.ndjson")); + let [Effect::AuditExport { ticket: a_ticket, .. }] = a.as_slice() else { panic!("{a:?}") }; + let a_ticket = *a_ticket; + // Leave and come back, and start B (its picker open, then running). + support::run(&mut model, &boss, vec![Event::Admin(AdminEvent::BackToList)]); + support::run(&mut model, &boss, vec![Event::Audit(AuditMsg::OpenAudit(None))]); + let asked = model.update(Event::Audit(AuditMsg::Export)); + let [Effect::PickExportFile { ticket: b_picker, .. }] = asked.as_slice() else { panic!("{asked:?}") }; + let b_picker = *b_picker; + assert_ne!(a_ticket, b_picker); + // A's answer arrives while B is still picking, and A's old picker answer arrives too. + model.update(Event::Audit(AuditMsg::Exported { ticket: a_ticket, result: Ok(ExportOutcome { events: 3, truncated: false }) })); + assert_eq!(view(&model).export, ExportState::Picking { ticket: b_picker }); + model.update(Event::Audit(AuditMsg::ExportPicked { ticket: a_ticket, path: Some("a.ndjson".into()) })); + assert_eq!(view(&model).export, ExportState::Picking { ticket: b_picker }, "A's picker answer is not B's"); + let b = model.update(Event::Audit(AuditMsg::ExportPicked { ticket: b_picker, path: Some("b.ndjson".into()) })); + assert!(matches!(b.as_slice(), [Effect::AuditExport { path, .. }] if path == "b.ndjson"), "{b:?}"); + // B runs; A's answer comes late and finishes nothing; B's own does. + model.update(Event::Audit(AuditMsg::Exported { ticket: a_ticket, result: Err(Error::Io("the export broke off".into())) })); + assert!(matches!(&view(&model).export, ExportState::Running { path, .. } if path == "b.ndjson"), "{:?}", view(&model).export); + model.update(Event::Audit(AuditMsg::Exported { ticket: b_picker, result: Ok(ExportOutcome { events: 3, truncated: false }) })); + assert!(matches!(&view(&model).export, ExportState::Done { path, .. } if path == "b.ndjson"), "{:?}", view(&model).export); + } + + #[test] + fn who_did_it_and_what_it_was_about_is_in_every_line() { + let user = Uuid::from_u128(0x0123_4567_89ab_cdef_0123_4567_89ab_cdef); + let other = Uuid::from_u128(0xfedc_ba98_7654_3210_fedc_ba98_7654_3210); + let mut e = FakeAgent::sample_event(1, "admin.user_disabled"); + e.actor_user = Some(user); + e.target_kind = "user".into(); + e.target_id = Some(other); + let (line, full) = who_line(&e); + assert_eq!(line, "by 01234567 · about account fedcba98"); + assert!(full.contains(&user.to_string()) && full.contains(&other.to_string()), "{full}"); + // A computer in the details; a computer that acted. + let device = craft_cloud_app::DeviceId::generate(); + e.details.device = Some(device); + e.actor_device = Some(device); + let (line, full) = who_line(&e); + assert!(line.contains(&format!("on computer {}", short_id(device))) && line.contains(&format!("computer {}", short_id(device))), "{line}"); + assert!(full.contains(&device.to_string())); + // Nobody: the command line, or an unproven sign-in. + let cli = FakeAgent::sample_event(2, "cli.password_reset"); + assert_eq!(who_line(&cli).0, "by the command line"); + let failed = FakeAgent::sample_event(3, "auth.login_failed"); + assert_eq!(who_line(&failed).0, "by unknown"); + assert!(who_line(&failed).1.contains("nobody had proved")); + // A target of another kind keeps its kind. + let mut gone = FakeAgent::sample_event(4, "device.revoked"); + gone.actor_user = Some(user); + gone.target_kind = "device".into(); + gone.target_id = Some(other); + assert_eq!(who_line(&gone).0, "by 01234567 · about device fedcba98"); + } + + #[test] + fn every_account_has_its_own_activity() { + // A member: no admin pages, but the activity. + let member = support::agent(false, 1); + member.audit_events.lock().unwrap().extend((0..30).map(|n| FakeAgent::sample_event(n, "auth.login"))); + let mut model = support::signed_in(&member); + assert!(!model.is_admin()); + let effects = model.update(Event::Audit(AuditMsg::OpenActivity)); + let [Effect::AuditPage { own: true, query, .. }] = effects.as_slice() else { panic!("{effects:?}") }; + assert_eq!((query.user, query.action.clone(), query.limit), (None, None, Some(PAGE_SIZE))); + support::run(&mut model, &member, vec![]); + let again = vec![Effect::AuditPage { own: true, query: query.clone(), ticket: model.activity_screen().unwrap().list.ticket }]; + let start: Starter = Arc::new(|_: &dyn crate::agent::Agent| Err(String::new())); + let answer = call(&member, &start, again.into_iter().next().unwrap()).unwrap(); + model.update(answer); + let a = model.activity_screen().unwrap(); + assert_eq!((a.list.events.len(), a.list.next.as_deref()), (PAGE_SIZE as usize, Some("25"))); + // Back to the signed-in view. + model.update(Event::Audit(AuditMsg::CloseActivity)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + // Only from the signed-in view. + let mut out = Model::new(crate::model::JoinDefaults::default(), true, String::new()); + assert!(out.update(Event::Audit(AuditMsg::OpenActivity)).is_empty()); + } + + #[test] + fn events_are_told_in_words_without_anything_the_server_did_not_write() { + let mut e = FakeAgent::sample_event(1, "admin.role_changed"); + e.details = AuditDetails { role_from: Some("member".into()), role_to: Some("admin".into()), ..AuditDetails::default() }; + assert_eq!(details_line(&e), "member to admin"); + e.details = AuditDetails { quota_bytes: Some(None), revoked: Some(2), ..AuditDetails::default() }; + assert_eq!(details_line(&e), "quota removed · 2 computers signed out"); + let mut failed = FakeAgent::sample_event(2, "auth.login_failed"); + failed.outcome = "failure".into(); + failed.count = 4; + assert_eq!(details_line(&failed), "4 tries · failed"); + } +} diff --git a/apps/craft-cloud-desktop/src/lib.rs b/apps/craft-cloud-desktop/src/lib.rs index 8f876736..00f26d6c 100644 --- a/apps/craft-cloud-desktop/src/lib.rs +++ b/apps/craft-cloud-desktop/src/lib.rs @@ -2,7 +2,9 @@ //! another computer, pauses and quits. It never syncs or touches the synced files itself: all of that is the agent's //! (`craft-cloud-agent run`), which it reaches through `craft-cloud-app` and starts when none answers. //! -//! - [`model`]: the state and every way between its screens, pure (unit-tested against a fake agent). +//! - [`model`]: the state and every way between its screens, pure (unit-tested against a fake agent); [`admin`]: the +//! admin pages' part of it. +//! - [`security`]: the Security page and the code a join asks for (two-factor sign-in). //! - [`agent`]: the agent's calls behind a trait, run off the UI thread. //! - [`start_agent`]: starting the agent from this app's folder. //! - [`single_instance`]: one app per user; a second start brings the first window forward. @@ -10,10 +12,13 @@ //! - [`tray`]: the tray icon where there is one; [`autostart`]: the app's own start-at-login entry. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod admin; pub mod agent; pub mod app; +pub mod audit; pub mod autostart; pub mod model; +pub mod security; pub mod single_instance; pub mod start_agent; pub mod tray; diff --git a/apps/craft-cloud-desktop/src/model.rs b/apps/craft-cloud-desktop/src/model.rs index 83c03578..98350cf9 100644 --- a/apps/craft-cloud-desktop/src/model.rs +++ b/apps/craft-cloud-desktop/src/model.rs @@ -7,10 +7,18 @@ use std::path::Path; -use craft_cloud_app::{DeviceId, DeviceInfo, Error, JoinRequest, PairingInfo, PairingState, PairingStatus, RunState, Status, Uuid, codes}; +use craft_cloud_app::{ + AccountUsage, DeviceId, DeviceInfo, Error, HistoryResponse, JoinRequest, NodeId, PairingInfo, PairingState, PairingStatus, RestoreResponse, + RunState, Status, TrashResponse, TrashRestoreResponse, UserQuery, Uuid, VersionId, codes, +}; +use craft_cloud_proto::api::AdminConfirm; use craft_cloud_proto::api::codes as api_codes; use craft_cloud_proto::join::JoinString; +use crate::admin::{Access, Action, AdminEvent, AdminScreen}; +use crate::audit::{ActivityScreen, AuditMsg}; +use crate::security::{SecondFactorForm, SecurityAnswer, SecurityCall, SecurityEvent, SecurityPage}; + /// A code that is wrong, expired or used (the server never says which). pub const INVALID_CODE: &str = "This code doesn't work any more. Make a new one on a signed-in computer."; /// The agent can't be started. @@ -48,10 +56,22 @@ pub enum Screen { Joining { form: JoinForm, password: Option }, /// An invite to a new account: its password, twice. PasswordNeeded(PasswordForm), + /// The account has two-factor sign-in: the join asks for the code ([`SecondFactorForm`]). + SecondFactor(SecondFactorForm), /// Signed in: the account, the sync, the computers. SignedIn(Overview), + /// The Security page ([`SecurityPage`]): two-factor sign-in, the password, the other computers. Boxed: it is large. + Security(Box), + /// File version history for a path inside the sync root. + Versions(Versions), + /// Deleted files and folders awaiting retention expiry. + Trash(Trash), /// Showing an add-a-computer code. AddingComputer(Adding), + /// The admin pages (an admin's only): the users, one user, a new account, the audit log. + Admin(Box), + /// The signed-in account's own activity: its security events ([`crate::audit`]). + Activity(Box), /// The server refused this computer: it was removed from the account. Removed, /// Something the agent can't put right by itself (a damaged account file), or that this app can't work with. @@ -110,8 +130,15 @@ impl JoinForm { folder: Some(folder), device_name: (!name.is_empty()).then(|| name.to_owned()), password, + totp_code: None, + recovery_code: None, } } + + /// [`JoinForm::request`] with the second factor of an account that has two-factor sign-in. + fn request_with_factor(&self, password: Option, factor: (Option, Option)) -> JoinRequest { + JoinRequest { totp_code: factor.0, recovery_code: factor.1, ..self.request(password) } + } } /// The new account's password, typed twice. @@ -136,17 +163,72 @@ pub struct Overview { /// The account's computers, once asked. pub devices: Option>, pub devices_error: Option, + pub usage: Option, + pub usage_loading: bool, + pub usage_error: Option, /// The computer being removed. pub removing: Option, /// What went wrong with the last action (sign out, pause), for people. pub error: Option, /// Signing out failed because the server can't be reached: offer to forget the account here only. pub offer_local_sign_out: bool, + /// Whether the server lets this account use the admin pages (asked at sign-in). + pub access: Access, + /// Which sign-in this view is of: answers to the admin check made for another one are ignored. + pub session: u64, +} + +/// File history and restore state. +#[derive(Debug, Clone, PartialEq)] +pub struct Versions { + pub path: String, + pub page: Option, + pub current_version: Option, + pub loading: bool, + pub restoring: bool, + pub error: Option, + pub notice: Option, + pub confirm: Option, + pub ticket: u64, +} + +/// Trash list and action state. +#[derive(Debug, Clone, PartialEq)] +pub struct Trash { + pub page: Option, + pub loading: bool, + pub action: Option, + pub error: Option, + pub notice: Option, + pub confirm: Option, + pub ticket: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TrashAction { + /// `parent_path` is `None` without a path hint and empty for an item whose hint is a file in the sync root. + Restore { + path: Option, + parent_path: Option, + }, + Remove, } impl Overview { fn new(status: Status) -> Self { - Self { status, devices: None, devices_error: None, removing: None, error: None, offer_local_sign_out: false } + Self { + status, + devices: None, + devices_error: None, + usage: None, + usage_loading: true, + usage_error: None, + removing: None, + error: None, + offer_local_sign_out: false, + access: Access::Unknown, + session: crate::admin::fresh(), + } } } @@ -201,6 +283,30 @@ pub enum Event { PairingMade(Result), PairingPolled(Result), Devices(Result, Error>), + History { + ticket: u64, + result: Result, + }, + Restored { + ticket: u64, + result: Result, + }, + TrashPage { + ticket: u64, + result: Result, + }, + TrashRestored { + ticket: u64, + result: Result, + }, + TrashRemoved { + ticket: u64, + result: Result<(), Error>, + }, + Usage { + request_id: u64, + result: Result, + }, SignedOut(Result), /// [`Effect::Pause`]'s or [`Effect::Resume`]'s answer. Answered(Result), @@ -209,6 +315,16 @@ pub enum Event { /// [`Effect::AppAutostart`]'s outcome. AppAutostartSet(Result<(), String>), + /// The Security button of the signed-in view. + OpenSecurity, + /// A button of the Security page. + Security(SecurityEvent), + /// An answer to one of the page's calls. + SecurityAnswer(SecurityAnswer), + /// The code of a join that asked for one (the button, or Enter). + SubmitSecondFactor, + /// Back from the code field to the join form. + CancelSecondFactor, /// Join (the form's button, or Enter). SubmitJoin, /// The answer to "this folder has files in it: sync them too?". @@ -216,12 +332,29 @@ pub enum Event { SubmitPassword, CancelPassword, OpenFolder, + OpenVersions, + OpenTrash, + VersionsPath(String), + LoadVersions, + NextVersions, + BackVersions, + ConfirmRestore(VersionId), + CancelRestore, + RestoreVersion, + LoadTrash, + NextTrash, + BackTrash, + RestoreTrash(NodeId), + ConfirmRemoveTrash(NodeId), + CancelRemoveTrash, + RemoveTrash, AddComputer, /// Done or Cancel on the add-a-computer view. CloseAdding, RemoveDevice(DeviceId), /// Ask for the computer list again. RefreshDevices, + RefreshUsage, SignOut, /// Sign out on this computer only (the server can't be reached). SignOutHere, @@ -246,6 +379,10 @@ pub enum Event { CloseRequested, /// Linux: the tray process quit Craft Cloud (it stopped the agent): the window ends too. Exit, + /// Something on the admin pages, or the answer to one of their calls ([`crate::admin`]). + Admin(AdminEvent), + /// Something on the audit pages (the admin's log, or the account's own activity), or an answer to one of their calls. + Audit(AuditMsg), } /// What to do. @@ -255,6 +392,8 @@ pub enum Effect { Connect, /// Start the agent hidden from this app's folder and wait up to 5 s for it ([`Event::AgentStarted`]). StartAgent, + /// A call of the Security page ([`Event::SecurityAnswer`]). + Security(SecurityCall), Status, Join(JoinRequest), SetAutostart(bool), @@ -262,6 +401,30 @@ pub enum Effect { PairingState(Uuid), CancelPairing(Uuid), Devices, + History { + path: String, + cursor: Option, + ticket: u64, + }, + Restore { + path: String, + version: VersionId, + expected_version: VersionId, + ticket: u64, + }, + Trash { + cursor: Option, + ticket: u64, + }, + RestoreTrash { + id: NodeId, + ticket: u64, + }, + RemoveTrash { + id: NodeId, + ticket: u64, + }, + Usage(u64), RemoveDevice(DeviceId), Logout { local: bool, @@ -279,6 +442,56 @@ pub enum Effect { QuitApp, /// Write (`true`) or remove the app's own start-at-login entry. AppAutostart(bool), + /// The signed-in account, as the server says it: how the app finds out whether it is an admin ([`AdminEvent::Account`]). + Account { + session: u64, + }, + /// The server's counters, for the admin pages ([`AdminEvent::Server`]). + AdminServer { + session: u64, + }, + /// The server-wide storage overview for the admin pages. + AdminStorage { + session: u64, + ticket: u64, + }, + /// A page of the user list ([`AdminEvent::Users`] with the same `ticket`). + AdminUsers { + query: UserQuery, + ticket: u64, + }, + /// A user's devices ([`AdminEvent::Devices`]). + AdminDevices { + user: Uuid, + ticket: u64, + }, + /// One account's effective quota and overlapping storage estimates. + AdminUserUsage { + user: Uuid, + ticket: u64, + }, + /// A page of the audit log or of the account's own activity ([`AuditMsg::Page`] with the same `ticket`). + AuditPage { + own: bool, + query: craft_cloud_app::AuditQuery, + ticket: u64, + }, + /// The audit export, written to `path` ([`AuditMsg::Exported`]). + AuditExport { + query: craft_cloud_app::AuditQuery, + path: String, + ticket: u64, + }, + /// Ask for the file an export goes to, with the OS's save dialog, starting at this name ([`AuditMsg::ExportPicked`]). + PickExportFile { + suggested: String, + ticket: u64, + }, + /// A change, with the admin's confirmation ([`AdminEvent::Done`]). Its `Debug` output hides the password. + AdminDo { + action: Action, + confirm: AdminConfirm, + }, } /// Whether the app should start at login, in the tray, given the agent's status: while signed in with the agent @@ -319,6 +532,9 @@ pub struct Model { failed_polls: u32, /// The app's start-at-login entry as last asked for. app_autostart: Option, + versions_ticket: u64, + trash_ticket: u64, + usage_request_id: u64, } impl Model { @@ -339,6 +555,9 @@ impl Model { polling_pairing: false, failed_polls: 0, app_autostart: None, + versions_ticket: 0, + trash_ticket: 0, + usage_request_id: 0, } } @@ -346,15 +565,39 @@ impl Model { pub fn overview(&self) -> Option<&Overview> { match &self.screen { Screen::SignedIn(overview) => Some(overview), + Screen::Security(page) => Some(&page.overview), Screen::AddingComputer(adding) => Some(&adding.overview), + Screen::Admin(admin) => Some(&admin.overview), + Screen::Activity(activity) => Some(&activity.overview), _ => None, } } - fn overview_mut(&mut self) -> Option<&mut Overview> { + /// Leaves a page for a fresh overview, which asks again for everything it shows: the computers, the admin + /// check and the storage usage. Without the last two the view would wait for answers nobody asked for. + fn back_to_overview(&mut self) -> Vec { + let Some(status) = self.status.clone().filter(|s| s.signed_in) else { + self.screen = Screen::Starting; + return Vec::new(); + }; + let overview = Overview::new(status); + let session = overview.session; + self.screen = Screen::SignedIn(overview); + vec![Effect::Devices, Effect::Account { session }, self.next_usage_request()] + } + + fn next_usage_request(&mut self) -> Effect { + self.usage_request_id = self.usage_request_id.wrapping_add(1); + Effect::Usage(self.usage_request_id) + } + + pub(crate) fn overview_mut(&mut self) -> Option<&mut Overview> { match &mut self.screen { Screen::SignedIn(overview) => Some(overview), + Screen::Security(page) => Some(&mut page.overview), Screen::AddingComputer(adding) => Some(&mut adding.overview), + Screen::Admin(admin) => Some(&mut admin.overview), + Screen::Activity(activity) => Some(&mut activity.overview), _ => None, } } @@ -432,6 +675,142 @@ impl Model { } Vec::new() } + Event::History { ticket, result } => { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.ticket != ticket { + return Vec::new(); + } + versions.loading = false; + match result { + Ok(page) => { + if versions.current_version.is_none() { + versions.current_version = page.versions.first().map(|row| row.id); + } + versions.page = Some(page); + versions.error = None; + } + Err(e) => versions.error = Some(message(&e)), + } + Vec::new() + } + Event::Restored { ticket, result } => { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.ticket != ticket { + return Vec::new(); + } + versions.restoring = false; + versions.confirm = None; + match result { + Ok(_) => { + versions.notice = Some("Version restored. Sync will update this file.".into()); + versions.current_version = None; + self.request_history(None) + } + Err(Error::Refused { code, .. }) if code == api_codes::CONFLICT => { + versions.notice = Some("The file changed meanwhile. Version list reloaded.".into()); + versions.current_version = None; + self.request_history(None) + } + Err(Error::Refused { code, .. }) if code == api_codes::INVALID => { + versions.error = Some("This file can't be restored: it may be a live document or deleted.".into()); + Vec::new() + } + Err(e) => { + versions.error = Some(message(&e)); + Vec::new() + } + } + } + Event::TrashPage { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + trash.loading = false; + match result { + Ok(page) => { + trash.page = Some(page); + trash.error = None; + } + Err(e) => trash.error = Some(message(&e)), + } + Vec::new() + } + Event::TrashRestored { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + let prior = trash.action.take(); + match result { + Ok(response) => { + let (path, parent_path) = prior + .and_then(|action| match action { + TrashAction::Restore { path, parent_path, .. } => Some((path, parent_path)), + _ => None, + }) + .unwrap_or((None, None)); + trash.notice = Some(if response.renamed { + match parent_path { + Some(parent) if parent.is_empty() => format!("Restored as {} in the sync folder.", response.name), + Some(parent) => format!("Restored as {} in {}.", response.name, parent), + None => format!("Restored as {}. The destination path is unavailable.", response.name), + } + } else { + match path { + Some(path) => format!("Restored to {}.", path), + None => format!("Restored {}. The destination path is unavailable.", response.name), + } + }); + self.request_trash(None) + } + Err(Error::Refused { code, name_taken, .. }) if code == api_codes::CONFLICT => { + trash.error = Some(if name_taken == Some(true) { + "This item couldn't be restored because another item already has that name.".into() + } else { + "This item couldn't be restored because it conflicts with the current folder contents.".into() + }); + Vec::new() + } + Err(e) => { + trash.error = Some(message(&e)); + Vec::new() + } + } + } + Event::TrashRemoved { ticket, result } => { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.ticket != ticket { + return Vec::new(); + } + trash.action = None; + match result { + Ok(()) => { + trash.notice = Some("Removed from trash. It can no longer be restored from the trash.".into()); + self.request_trash(None) + } + Err(e) => { + trash.error = Some(message(&e)); + Vec::new() + } + } + } + Event::Usage { request_id, result } => { + if request_id != self.usage_request_id { + return Vec::new(); + } + if let Some(overview) = self.overview_mut() { + overview.usage_loading = false; + match result { + Ok(usage) => { + overview.usage = Some(usage); + overview.usage_error = None; + } + Err(e) => overview.usage_error = Some(message(&e)), + } + } + Vec::new() + } Event::SignedOut(result) => self.signed_out(result), Event::Answered(result) => { self.busy = self.busy.filter(|b| *b != Busy::Pausing); @@ -451,6 +830,28 @@ impl Model { Vec::new() } + Event::OpenSecurity => { + let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; + let (page, calls) = SecurityPage::open(overview.clone()); + self.screen = Screen::Security(Box::new(page)); + calls.into_iter().map(Effect::Security).collect() + } + Event::Security(event) => self.security(event), + Event::SecurityAnswer(answer) => { + let now_ms = self.now_ms; + let Screen::Security(page) = &mut self.screen else { return Vec::new() }; + page.answer(answer, now_ms).into_iter().map(Effect::Security).collect() + } + Event::SubmitSecondFactor => self.submit_second_factor(), + Event::CancelSecondFactor => { + if let Screen::SecondFactor(form) = &self.screen { + self.screen = match &form.password { + Some(pw) => Screen::PasswordNeeded(pw.clone()), + None => Screen::Join(form.form.clone()), + }; + } + Vec::new() + } Event::SubmitJoin => self.submit_join(), Event::SyncExistingFiles(yes) => { let Screen::Join(form) = &mut self.screen else { return Vec::new() }; @@ -469,6 +870,138 @@ impl Model { Vec::new() } Event::OpenFolder => self.status.as_ref().and_then(|s| s.folder.clone()).map(Effect::OpenFolder).into_iter().collect(), + Event::OpenVersions => { + if !matches!(self.screen, Screen::SignedIn(_)) { + return Vec::new(); + } + self.screen = Screen::Versions(Versions { + path: String::new(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + Vec::new() + } + Event::OpenTrash => { + if !matches!(self.screen, Screen::SignedIn(_)) { + return Vec::new(); + } + self.screen = Screen::Trash(Trash { page: None, loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + self.request_trash(None) + } + Event::LoadTrash => self.request_trash(None), + Event::NextTrash => { + let cursor = match &self.screen { + Screen::Trash(t) if t.action.is_none() && !t.loading => t.page.as_ref().and_then(|p| p.next.clone()), + _ => None, + }; + cursor.map(|cursor| self.request_trash(Some(cursor))).unwrap_or_default() + } + Event::BackTrash => { + if matches!(self.screen, Screen::Trash(_)) { + self.trash_ticket = self.trash_ticket.saturating_add(1); + return self.back_to_overview(); + } + Vec::new() + } + Event::RestoreTrash(id) => { + let Screen::Trash(t) = &mut self.screen else { return Vec::new() }; + if t.loading || t.action.is_some() { + return Vec::new(); + } + let Some(item) = t.page.as_ref().and_then(|p| p.items.iter().find(|item| item.id == id)) else { return Vec::new() }; + let path = item.path.clone(); + let parent_path = path.as_deref().map(|p| p.rsplit_once('/').map(|(parent, _)| parent.to_owned()).unwrap_or_default()); + self.begin_trash_action(id, TrashAction::Restore { path, parent_path }) + } + Event::ConfirmRemoveTrash(id) => { + if let Screen::Trash(t) = &mut self.screen + && !t.loading + && t.action.is_none() + && t.page.as_ref().is_some_and(|p| p.items.iter().any(|i| i.id == id)) + { + t.confirm = Some(id); + } + Vec::new() + } + Event::CancelRemoveTrash => { + if let Screen::Trash(t) = &mut self.screen { + t.confirm = None; + } + Vec::new() + } + Event::RemoveTrash => { + let Screen::Trash(t) = &mut self.screen else { return Vec::new() }; + if t.loading || t.action.is_some() { + return Vec::new(); + } + let Some(id) = t.confirm.take() else { return Vec::new() }; + self.begin_trash_action(id, TrashAction::Remove) + } + Event::VersionsPath(path) => { + if let Screen::Versions(v) = &mut self.screen { + if v.path != path { + self.versions_ticket = self.versions_ticket.saturating_add(1); + v.ticket = self.versions_ticket; + v.page = None; + v.current_version = None; + v.loading = false; + v.restoring = false; + v.confirm = None; + v.notice = None; + } + v.path = path; + v.error = None; + } + Vec::new() + } + Event::LoadVersions => { + if let Screen::Versions(v) = &mut self.screen { + if v.restoring || v.loading { + return Vec::new(); + } + v.notice = None; + v.current_version = None; + } + self.request_history(None) + } + Event::NextVersions => { + let cursor = match &self.screen { + Screen::Versions(v) => v.page.as_ref().and_then(|p| p.next.clone()), + _ => None, + }; + cursor.map(|c| self.request_history(Some(c))).unwrap_or_default() + } + Event::BackVersions => { + if let Screen::Versions(_) = self.screen { + self.versions_ticket = self.versions_ticket.saturating_add(1); + return self.back_to_overview(); + } + Vec::new() + } + Event::ConfirmRestore(version) => { + if let Screen::Versions(v) = &mut self.screen + && !v.restoring + && !v.loading + && Some(version) != v.current_version + && v.page.as_ref().is_some_and(|page| page.versions.iter().any(|row| row.id == version && !row.pruned)) + { + v.confirm = Some(version); + } + Vec::new() + } + Event::CancelRestore => { + if let Screen::Versions(v) = &mut self.screen { + v.confirm = None; + } + Vec::new() + } + Event::RestoreVersion => self.restore_version(), Event::AddComputer => { let Screen::SignedIn(overview) = &self.screen else { return Vec::new() }; self.screen = Screen::AddingComputer(Adding { overview: overview.clone(), info: None, state: AddState::Asking }); @@ -494,7 +1027,23 @@ impl Model { overview.removing = Some(id); vec![Effect::RemoveDevice(id)] } - Event::RefreshDevices => self.overview().map(|_| Effect::Devices).into_iter().collect(), + Event::RefreshDevices => { + let Some(overview) = self.overview() else { return Vec::new() }; + // An admin check that couldn't be made at sign-in is made again. + let again = (overview.access == Access::Unknown).then_some(Effect::Account { session: overview.session }); + std::iter::once(Effect::Devices).chain(again).collect() + } + Event::Admin(event) => self.admin(event), + Event::Audit(msg) => self.audit(msg), + Event::RefreshUsage => { + let Some(overview) = self.overview_mut() else { return Vec::new() }; + if overview.usage_loading { + return Vec::new(); + } + overview.usage_loading = true; + overview.usage_error = None; + vec![self.next_usage_request()] + } Event::SignOut => self.sign_out(false), Event::SignOutHere => self.sign_out(true), Event::Pause | Event::Resume => { @@ -587,6 +1136,12 @@ impl Model { /// A status from the agent: the screen follows it, and the app's start-at-login entry follows the agent's. fn status_came(&mut self, status: Status) -> Vec { + let account_changed = + self.overview().is_some_and(|overview| overview.status.account != status.account || overview.status.server != status.server); + if !status.signed_in { + self.usage_request_id = self.usage_request_id.wrapping_add(1); + } + let new_usage_request = (status.signed_in && account_changed).then(|| self.next_usage_request()); let mut effects = Vec::new(); self.status = Some(status.clone()); if let Some(wanted) = app_autostart_wanted(&status, self.login_entry) @@ -601,25 +1156,51 @@ impl Model { } if status.signed_in { match &mut self.screen { - Screen::SignedIn(overview) => overview.status = status, - Screen::AddingComputer(adding) => adding.overview.status = status, + Screen::SignedIn(overview) => { + overview.status = status; + if account_changed { + overview.usage = None; + overview.usage_loading = true; + overview.usage_error = None; + } + } + Screen::AddingComputer(adding) => { + adding.overview.status = status; + if account_changed { + adding.overview.usage = None; + adding.overview.usage_loading = true; + adding.overview.usage_error = None; + } + } + Screen::Security(page) => page.overview.status = status, + Screen::Admin(admin) => admin.overview.status = status, + Screen::Activity(activity) => activity.overview.status = status, // The join's own answer moves on from there. Screen::Joining { .. } => {} + Screen::Versions(_) | Screen::Trash(_) => {} _ => { - self.screen = Screen::SignedIn(Overview::new(status)); + let overview = Overview::new(status); + let session = overview.session; + self.screen = Screen::SignedIn(overview); effects.push(Effect::Devices); + // Whether the account is an admin, for the admin pages. + effects.push(Effect::Account { session }); + effects.push(self.next_usage_request()); } } + if let Some(effect) = new_usage_request { + effects.push(effect); + } return effects; } match (&status.run_state, &self.screen) { - (RunState::Paused { reason }, _) => { + (RunState::Paused { reason, .. }, _) => { let problem = Problem { line: first_sentence(reason), details: reason.clone(), action: ProblemAction::ForgetAccount }; if !matches!(&self.screen, Screen::Error(p) if *p == problem) { self.screen = Screen::Error(problem); } } - (_, Screen::Join(_) | Screen::Joining { .. } | Screen::PasswordNeeded(_)) => {} + (_, Screen::Join(_) | Screen::Joining { .. } | Screen::PasswordNeeded(_) | Screen::SecondFactor(_)) => {} _ => self.screen = Screen::Join(JoinForm::new(&self.defaults)), } effects @@ -650,7 +1231,7 @@ impl Model { self.version_problem(*v); return; } - Error::Refused { .. } => false, + Error::Refused { .. } | Error::RefusedWith { .. } => false, }; if !gone || matches!(self.screen, Screen::Starting | Screen::NotRunning(_)) || self.busy == Some(Busy::Quitting) { return; @@ -678,6 +1259,34 @@ impl Model { vec![Effect::Join(request)] } + /// The code of a join that asked for one: the same join, with the code, goes to the agent again. + fn submit_second_factor(&mut self) -> Vec { + let Screen::SecondFactor(second) = &mut self.screen else { return Vec::new() }; + let factor = match second.factor() { + Ok(factor) => factor, + Err(why) => { + second.error = Some(why); + return Vec::new(); + } + }; + let request = second.form.request_with_factor(second.password.as_ref().map(|pw| pw.password.clone()), factor); + let (form, password) = (second.form.clone(), second.password.clone()); + self.screen = Screen::Joining { form, password }; + vec![Effect::Join(request)] + } + + /// A button of the Security page. + fn security(&mut self, event: SecurityEvent) -> Vec { + let Screen::Security(page) = &mut self.screen else { return Vec::new() }; + if event == SecurityEvent::Close { + // Whatever the page showed (a secret, recovery codes) goes with it. + let overview = page.overview.clone(); + self.screen = Screen::SignedIn(overview); + return vec![Effect::Devices]; + } + page.update(event).into_iter().map(Effect::Security).collect() + } + fn submit_password(&mut self) -> Vec { let Screen::PasswordNeeded(pw) = &mut self.screen else { return Vec::new() }; pw.error = if pw.password.is_empty() { @@ -704,12 +1313,14 @@ impl Model { let mut effects = self.status_came(status); // Start at login: the agent's entry (the app's follows it). effects.extend([Effect::SetAutostart(true), Effect::Devices]); + effects.extend(self.overview().map(|o| Effect::Account { session: o.session })); + effects.push(self.next_usage_request()); return effects; } Err(e) => e, }; let (code, text) = match &e { - Error::Refused { code, message } => (code.as_str(), message.clone()), + Error::Refused { code, message, .. } | Error::RefusedWith { code, message, .. } => (code.as_str(), message.clone()), Error::NotRunning(_) => { self.status = None; self.screen = Screen::NotRunning(self.not_running(STOPPED, &e.to_string())); @@ -724,6 +1335,11 @@ impl Model { Screen::PasswordNeeded(pw) }; self.screen = match (code, password) { + // The account has two-factor sign-in: the same join goes again with its code (the invite was not used up). + (codes::TWO_FACTOR_REQUIRED, password) => Screen::SecondFactor(SecondFactorForm::new(form, password, None)), + (codes::TWO_FACTOR_INVALID | codes::TWO_FACTOR_LOCKED, password) => { + Screen::SecondFactor(SecondFactorForm::new(form, password, Some(crate::security::explain(&e)))) + } (codes::PASSWORD_REQUIRED, _) => { Screen::PasswordNeeded(PasswordForm { form, password: String::new(), again: String::new(), error: None }) } @@ -785,6 +1401,74 @@ impl Model { vec![Effect::Logout { local }] } + fn request_history(&mut self, cursor: Option) -> Vec { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + if versions.restoring || versions.loading { + return Vec::new(); + } + // The path is sent exactly as typed: a leading or trailing space can be part of a real file name. + let path = versions.path.as_str(); + if path.is_empty() + || Path::new(path).is_absolute() + || Path::new(path) + .components() + .any(|part| matches!(part, std::path::Component::ParentDir | std::path::Component::RootDir | std::path::Component::Prefix(_))) + { + versions.error = Some("Enter a file path inside the sync folder.".into()); + return Vec::new(); + } + self.versions_ticket = self.versions_ticket.saturating_add(1); + versions.ticket = self.versions_ticket; + versions.loading = true; + versions.error = None; + vec![Effect::History { path: path.to_owned(), cursor, ticket: versions.ticket }] + } + + fn request_trash(&mut self, cursor: Option) -> Vec { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.loading || trash.action.is_some() { + return Vec::new(); + } + self.trash_ticket = self.trash_ticket.saturating_add(1); + trash.ticket = self.trash_ticket; + trash.loading = true; + trash.error = None; + vec![Effect::Trash { cursor, ticket: trash.ticket }] + } + + fn begin_trash_action(&mut self, id: NodeId, action: TrashAction) -> Vec { + let Screen::Trash(trash) = &mut self.screen else { return Vec::new() }; + if trash.loading || trash.action.is_some() { + return Vec::new(); + } + self.trash_ticket = self.trash_ticket.saturating_add(1); + trash.ticket = self.trash_ticket; + trash.error = None; + trash.action = Some(action); + match &trash.action { + Some(TrashAction::Restore { .. }) => vec![Effect::RestoreTrash { id, ticket: trash.ticket }], + Some(TrashAction::Remove) => vec![Effect::RemoveTrash { id, ticket: trash.ticket }], + None => Vec::new(), + } + } + + fn restore_version(&mut self) -> Vec { + let Screen::Versions(versions) = &mut self.screen else { return Vec::new() }; + // A restore takes a new ticket, which would discard the answer of a page still loading and leave `loading` set. + if versions.restoring || versions.loading { + return Vec::new(); + } + let Some(version) = versions.confirm else { return Vec::new() }; + let Some(expected_version) = versions.current_version else { + versions.error = Some("Current version unavailable. Reload the version list.".into()); + return Vec::new(); + }; + self.versions_ticket = self.versions_ticket.saturating_add(1); + versions.ticket = self.versions_ticket; + versions.restoring = true; + vec![Effect::Restore { path: versions.path.clone(), version, expected_version, ticket: versions.ticket }] + } + fn signed_out(&mut self, result: Result) -> Vec { self.busy = self.busy.filter(|b| *b != Busy::SigningOut); match result { @@ -820,7 +1504,7 @@ impl Model { /// An error's message, for people. fn message(e: &Error) -> String { match e { - Error::Refused { message, .. } => message.clone(), + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), other => other.to_string(), } } @@ -852,6 +1536,40 @@ pub(crate) mod tests { Model::new(defaults(), true, "/home/ana/.local/share/craft-cloud-agent/run.log".into()) } + fn history_page() -> HistoryResponse { + HistoryResponse { + versions: vec![craft_cloud_app::HistoryVersion { + id: VersionId::generate(), + parent_version: None, + epoch: craft_cloud_proto::Epoch::generate(), + seq: 3, + size: 12, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }], + next: Some("opaque-cursor".into()), + } + } + + fn trash_page() -> TrashResponse { + TrashResponse { + items: vec![craft_cloud_app::TrashItem { + id: NodeId::generate(), + parent: NodeId::generate(), + name: "painting.png".into(), + is_dir: false, + size: Some(42_000), + deleted_ms: 1_790_000_000_000, + purge_after_ms: 1_792_000_000_000, + path: Some("Projects/painting.png".into()), + }], + next: Some("trash-cursor".into()), + } + } + pub(crate) fn status(signed_in: bool) -> Status { Status { api: 1, @@ -868,8 +1586,527 @@ pub(crate) mod tests { } } + #[test] + fn versions_load_page_and_restore_with_ticket_guard() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenVersions); + m.update(Event::VersionsPath("art/file.png".into())); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: "art/file.png".into(), cursor: None, ticket: 2 }]); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let mut page = history_page(); + let current = page.versions[0].id; + let old = VersionId::generate(); + page.versions.push(craft_cloud_app::HistoryVersion { + id: old, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + assert!(m.update(Event::History { ticket: 2, result: Ok(page.clone()) }).is_empty()); + assert_eq!( + m.update(Event::NextVersions), + vec![Effect::History { path: "art/file.png".into(), cursor: Some("opaque-cursor".into()), ticket: 3 }] + ); + m.update(Event::History { ticket: 3, result: Ok(page.clone()) }); + m.update(Event::ConfirmRestore(old)); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "art/file.png".into(), version: old, expected_version: current, ticket: 4 }] + ); + assert!(m.update(Event::LoadVersions).is_empty()); + assert!(m.update(Event::NextVersions).is_empty()); + assert!( + m.update(Event::Restored { + ticket: 3, + result: Ok(craft_cloud_app::RestoreResponse { + version: old, + parent_version: current, + seq: 5, + epoch: craft_cloud_proto::Epoch::generate() + }) + }) + .is_empty() + ); + assert!( + m.update(Event::Restored { + ticket: 4, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: None }) + }) + .iter() + .any(|e| matches!(e, Effect::History { cursor: None, .. })) + ); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert_eq!(v.notice.as_deref(), Some("The file changed meanwhile. Version list reloaded.")); + } + + #[test] + fn trash_pages_restore_remove_confirmation_and_stale_tickets() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + assert_eq!(m.update(Event::OpenTrash), vec![Effect::Trash { cursor: None, ticket: 1 }]); + let page = trash_page(); + let id = page.items[0].id; + assert!(m.update(Event::TrashPage { ticket: 0, result: Ok(page.clone()) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.loading && t.page.is_none())); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 2 }]); + assert!(m.update(Event::RestoreTrash(id)).is_empty()); // page load excludes actions + assert!(m.update(Event::ConfirmRemoveTrash(id)).is_empty()); + assert!(m.update(Event::RemoveTrash).is_empty()); + m.update(Event::TrashPage { ticket: 2, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::RestoreTrash(id)), vec![Effect::RestoreTrash { id, ticket: 3 }]); + assert!(m.update(Event::LoadTrash).is_empty()); // action excludes page loads + assert!(m.update(Event::TrashRestored { ticket: 2, result: Err(Error::Io("stale".into())) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.action.is_some() && !t.loading)); + let restored = TrashRestoreResponse { + node: id, + parent: page.items[0].parent, + name: "painting (1).png".into(), + renamed: true, + version: None, + seq: 2, + epoch: craft_cloud_proto::Epoch::generate(), + }; + assert_eq!(m.update(Event::TrashRestored { ticket: 3, result: Ok(restored) }), vec![Effect::Trash { cursor: None, ticket: 4 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png in Projects.")); + m.update(Event::TrashPage { ticket: 4, result: Ok(page.clone()) }); + assert!(m.update(Event::ConfirmRemoveTrash(id)).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.confirm == Some(id))); + assert_eq!(m.update(Event::RemoveTrash), vec![Effect::RemoveTrash { id, ticket: 5 }]); + assert!(m.update(Event::LoadTrash).is_empty()); // removal excludes page loads too + assert!(m.update(Event::TrashRemoved { ticket: 4, result: Ok(()) }).is_empty()); + assert_eq!(m.update(Event::TrashRemoved { ticket: 5, result: Ok(()) }), vec![Effect::Trash { cursor: None, ticket: 6 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Removed from trash. It can no longer be restored from the trash.")); + } + + #[test] + fn trash_restore_name_taken_is_clear_and_returns_to_idle_without_reload() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(m.update(Event::RemoveTrash).is_empty()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert!( + m.update(Event::TrashRestored { + ticket, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: Some(true) }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.as_deref().unwrap_or_default().contains("another item already has that name")); + assert!(!t.loading); + assert!(t.action.is_none()); + } + + #[test] + fn trash_restore_without_path_does_not_invent_a_destination() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = None; + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert_eq!( + m.update(Event::TrashRestored { + ticket, + result: Ok(TrashRestoreResponse { + node: id, + parent: NodeId::generate(), + name: "painting.png".into(), + renamed: false, + version: None, + seq: 1, + epoch: craft_cloud_proto::Epoch::generate(), + }) + }), + vec![Effect::Trash { cursor: None, ticket: ticket + 1 }] + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored painting.png. The destination path is unavailable.")); + } + + #[test] + fn other_trash_conflicts_are_generic_and_do_not_reload() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + }; + assert!( + m.update(Event::TrashRestored { + ticket, + result: Err(Error::Refused { code: api_codes::CONFLICT.into(), message: "conflict".into(), name_taken: Some(false) }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.as_deref().unwrap_or_default().contains("conflicts with the current folder contents")); + assert!(!t.loading); + assert!(t.action.is_none()); + } + + fn trash_restore_response(node: NodeId, name: &str, renamed: bool) -> TrashRestoreResponse { + TrashRestoreResponse { + node, + parent: NodeId::generate(), + name: name.into(), + renamed, + version: None, + seq: 1, + epoch: craft_cloud_proto::Epoch::generate(), + } + } + + fn trash_ticket(m: &Model) -> u64 { + match &m.screen { + Screen::Trash(t) => t.ticket, + _ => 0, + } + } + + #[test] + fn trash_restore_at_the_sync_root_says_so_even_when_renamed() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = Some("painting.png".into()); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + let ticket = trash_ticket(&m); + m.update(Event::TrashRestored { ticket, result: Ok(trash_restore_response(id, "painting (1).png", true)) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png in the sync folder.")); + } + + #[test] + fn trash_restore_renamed_without_path_keeps_the_unavailable_notice() { + let mut m = model(); + let mut page = trash_page(); + page.items[0].path = None; + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + m.update(Event::RestoreTrash(id)); + let ticket = trash_ticket(&m); + m.update(Event::TrashRestored { ticket, result: Ok(trash_restore_response(id, "painting (1).png", true)) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert_eq!(t.notice.as_deref(), Some("Restored as painting (1).png. The destination path is unavailable.")); + } + + #[test] + fn back_from_trash_or_versions_asks_again_for_usage_and_admin_access() { + for (open, back) in [(Event::OpenTrash, Event::BackTrash), (Event::OpenVersions, Event::BackVersions)] { + let mut m = model(); + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(open); + let effects = m.update(back); + let overview = m.overview().expect("back on the overview"); + assert!(overview.usage_loading, "{effects:?}"); + let session = overview.session; + assert!(effects.contains(&Effect::Devices), "{effects:?}"); + assert!(effects.contains(&Effect::Account { session }), "{effects:?}"); + let Some(&Effect::Usage(id)) = effects.iter().find(|e| matches!(e, Effect::Usage(_))) else { panic!("{effects:?}") }; + let usage = craft_cloud_app::AccountUsage { + stored_bytes: 1, + quota_bytes: None, + trash_bytes: None, + history_bytes: None, + estimates: craft_cloud_app::UsageEstimates { index_complete: true, reason: None }, + }; + m.update(Event::Usage { request_id: id, result: Ok(usage) }); + let overview = m.overview().unwrap(); + assert!(!overview.usage_loading && overview.usage.is_some(), "the overview doesn't stay loading"); + } + } + + #[test] + fn trash_late_page_reply_after_back_and_reopen_is_dropped() { + let mut m = model(); + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + assert_eq!(m.update(Event::OpenTrash), vec![Effect::Trash { cursor: None, ticket: 1 }]); + m.update(Event::BackTrash); + assert!(matches!(m.screen, Screen::SignedIn(_))); + let reopened = m.update(Event::OpenTrash); + let Some(Effect::Trash { ticket: fresh, .. }) = reopened.first().cloned() else { panic!("{reopened:?}") }; + assert_ne!(fresh, 1); + assert!(m.update(Event::TrashPage { ticket: 1, result: Ok(trash_page()) }).is_empty()); + assert!(matches!(&m.screen, Screen::Trash(t) if t.loading && t.page.is_none())); + m.update(Event::TrashPage { ticket: fresh, result: Ok(trash_page()) }); + assert!(matches!(&m.screen, Screen::Trash(t) if !t.loading && t.page.is_some() && t.error.is_none())); + } + + #[test] + fn trash_late_action_reply_after_back_and_reopen_is_dropped() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.status = Some(status(true)); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenTrash); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::RestoreTrash(id)), vec![Effect::RestoreTrash { id, ticket: 2 }]); + m.update(Event::BackTrash); + let reopened = m.update(Event::OpenTrash); + let Some(Effect::Trash { ticket: fresh, .. }) = reopened.first().cloned() else { panic!("{reopened:?}") }; + assert!(m.update(Event::TrashRestored { ticket: 2, result: Ok(trash_restore_response(id, "painting.png", false)) }).is_empty()); + assert!(m.update(Event::TrashRemoved { ticket: 2, result: Ok(()) }).is_empty()); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.loading && t.notice.is_none()); + m.update(Event::TrashPage { ticket: fresh, result: Ok(page) }); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(!t.loading && t.action.is_none() && t.page.is_some() && t.error.is_none()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + } + + #[test] + fn trash_remove_failure_shows_the_error_and_frees_the_buttons() { + let mut m = model(); + let page = trash_page(); + let id = page.items[0].id; + m.screen = Screen::Trash(Trash { page: Some(page), loading: false, action: None, error: None, notice: None, confirm: None, ticket: 0 }); + m.update(Event::ConfirmRemoveTrash(id)); + assert!(matches!(m.update(Event::RemoveTrash).first(), Some(Effect::RemoveTrash { .. }))); + let ticket = trash_ticket(&m); + assert!( + m.update(Event::TrashRemoved { + ticket, + result: Err(Error::Refused { code: api_codes::INTERNAL.into(), message: "server broke".into(), name_taken: None }) + }) + .is_empty() + ); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.is_some()); + assert!(t.action.is_none() && !t.loading && t.confirm.is_none()); + assert!(t.page.is_some()); + assert!(matches!(m.update(Event::RestoreTrash(id)).first(), Some(Effect::RestoreTrash { .. }))); + } + + #[test] + fn trash_failed_next_page_keeps_rows_clears_loading_and_can_retry() { + let mut m = model(); + m.screen = Screen::SignedIn(Overview::new(status(true))); + m.update(Event::OpenTrash); + let page = trash_page(); + m.update(Event::TrashPage { ticket: 1, result: Ok(page.clone()) }); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 2 }]); + assert!(m.update(Event::TrashPage { ticket: 2, result: Err(Error::Io("offline".into())) }).is_empty()); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.error.is_some()); + assert!(!t.loading); + assert_eq!(t.page.as_ref(), Some(&page)); + assert_eq!(m.update(Event::NextTrash), vec![Effect::Trash { cursor: Some("trash-cursor".into()), ticket: 3 }]); + let Screen::Trash(t) = &m.screen else { panic!("expected trash screen") }; + assert!(t.loading && t.error.is_none()); + } + + #[test] + fn versions_refuse_unsafe_paths_and_explain_invalid_restore() { + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: "../secret".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert!(m.update(Event::LoadVersions).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.error.as_deref().unwrap_or_default().contains("inside the sync folder")); + let mut page = history_page(); + let current_version = page.versions.first().map(|version| version.id); + let old_version = VersionId::generate(); + if let Some(current) = current_version { + page.versions.push(craft_cloud_app::HistoryVersion { + id: old_version, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + } + m.screen = Screen::Versions(Versions { + path: "file.txt".into(), + page: Some(page), + current_version, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: Some(old_version), + ticket: 0, + }); + let effect = m.update(Event::RestoreVersion); + let ticket = match effect.first() { + Some(Effect::Restore { ticket, .. }) => *ticket, + other => panic!("{other:?}"), + }; + m.update(Event::Restored { + ticket, + result: Err(Error::Refused { code: api_codes::INVALID.into(), message: "invalid".into(), name_taken: None }), + }); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.error.as_deref().unwrap_or_default().contains("live document or deleted")); + } + + #[test] + fn versions_path_change_discards_answers_for_previous_path() { + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: "old.txt".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: "old.txt".into(), cursor: None, ticket: 1 }]); + m.update(Event::VersionsPath("new.txt".into())); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert_eq!(v.path, "new.txt"); + assert!(v.page.is_none()); + assert!(!v.loading); + } + + /// A versions screen for `path` with two loaded versions, the older one awaiting restore confirmation. + fn versions_ready_to_restore(path: &str) -> (Model, VersionId, VersionId) { + let mut page = history_page(); + let current = page.versions[0].id; + let old = VersionId::generate(); + page.versions.push(craft_cloud_app::HistoryVersion { + id: old, + parent_version: Some(current), + epoch: craft_cloud_proto::Epoch::generate(), + seq: 2, + size: 10, + mtime_ms: 0, + device: craft_cloud_app::DeviceId::generate(), + created_ms: 0, + save: false, + pruned: false, + }); + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: path.into(), + page: Some(page), + current_version: Some(current), + loading: false, + restoring: false, + error: None, + notice: None, + confirm: Some(old), + ticket: 0, + }); + (m, current, old) + } + + #[test] + fn versions_restore_is_refused_while_a_page_loads_and_never_sticks() { + let (mut m, current, old) = versions_ready_to_restore("file.txt"); + let more = m.update(Event::NextVersions); + assert_eq!(more, vec![Effect::History { path: "file.txt".into(), cursor: Some("opaque-cursor".into()), ticket: 1 }]); + assert!(m.update(Event::RestoreVersion).is_empty(), "restore must not start while a page loads"); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.loading && !v.restoring); + assert_eq!(v.ticket, 1, "a refused restore must not take a ticket"); + assert!(m.update(Event::History { ticket: 1, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(!v.loading); + // Not stuck: the confirmation is still open and the restore now goes through. + assert_eq!(v.confirm, Some(old)); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "file.txt".into(), version: old, expected_version: current, ticket: 2 }] + ); + } + + #[test] + fn versions_page_request_during_restore_is_ignored_and_reload_follows() { + let (mut m, current, old) = versions_ready_to_restore("file.txt"); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: "file.txt".into(), version: old, expected_version: current, ticket: 1 }] + ); + assert!(m.update(Event::NextVersions).is_empty()); + assert!(m.update(Event::LoadVersions).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.restoring && !v.loading); + let restored = + craft_cloud_app::RestoreResponse { version: old, parent_version: current, seq: 5, epoch: craft_cloud_proto::Epoch::generate() }; + let effects = m.update(Event::Restored { ticket: 1, result: Ok(restored) }); + assert_eq!(effects, vec![Effect::History { path: "file.txt".into(), cursor: None, ticket: 2 }]); + assert!(m.update(Event::History { ticket: 2, result: Ok(history_page()) }).is_empty()); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(!v.loading && !v.restoring); + } + + #[test] + fn versions_send_the_path_exactly_as_typed() { + let (mut m, current, old) = versions_ready_to_restore(" Projects/file.txt "); + assert_eq!( + m.update(Event::RestoreVersion), + vec![Effect::Restore { path: " Projects/file.txt ".into(), version: old, expected_version: current, ticket: 1 }] + ); + m.update(Event::Restored { ticket: 1, result: Err(refused(api_codes::CONFLICT, "conflict")) }); + let Screen::Versions(v) = &m.screen else { panic!("expected versions screen") }; + assert!(v.loading, "the conflict reload is under way"); + let mut m = model(); + m.screen = Screen::Versions(Versions { + path: " Projects/file.txt".into(), + page: None, + current_version: None, + loading: false, + restoring: false, + error: None, + notice: None, + confirm: None, + ticket: 0, + }); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: " Projects/file.txt".into(), cursor: None, ticket: 1 }]); + m.update(Event::VersionsPath(" ".into())); + assert_eq!(m.update(Event::LoadVersions), vec![Effect::History { path: " ".into(), cursor: None, ticket: 3 }]); + m.update(Event::VersionsPath(String::new())); + assert!(m.update(Event::LoadVersions).is_empty()); + } + fn refused(code: &str, message: &str) -> Error { - Error::Refused { code: code.into(), message: message.into() } + Error::Refused { code: code.into(), message: message.into(), name_taken: None } } /// A model past the start, on the join form. @@ -1105,8 +2342,32 @@ pub(crate) mod tests { let Screen::SignedIn(o) = &m.screen else { panic!() }; assert_eq!(o.status.account.as_deref(), Some("fabio")); let devices = vec![ - DeviceInfo { id: DeviceId::generate(), name: "lptp1".into(), created_ms: 1, last_used_ms: 2, revoked: false, current: true }, - DeviceInfo { id: DeviceId::generate(), name: "BEAST".into(), created_ms: 1, last_used_ms: 3, revoked: false, current: false }, + DeviceInfo { + id: DeviceId::generate(), + name: "lptp1".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: true, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, + DeviceInfo { + id: DeviceId::generate(), + name: "BEAST".into(), + created_ms: 1, + last_used_ms: 3, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, ]; m.update(Event::Devices(Ok(devices.clone()))); // This computer can't be removed from here; another one can, once at a time. @@ -1180,8 +2441,11 @@ pub(crate) mod tests { #[test] fn a_removed_computer_says_so_and_joins_again() { let mut m = signed_in(); - let removed = - Status { removed: true, run_state: RunState::Paused { reason: "the server no longer accepts this device".into() }, ..status(true) }; + let removed = Status { + removed: true, + run_state: RunState::Paused { reason: "the server no longer accepts this device".into(), reason_code: None }, + ..status(true) + }; m.update(Event::Polled(Ok(removed))); assert_eq!(m.screen, Screen::Removed); assert_eq!(m.update(Event::ForgetAccount), vec![Effect::Logout { local: true }]); @@ -1198,7 +2462,7 @@ pub(crate) mod tests { let mut m = model(); m.update(Event::Start); let reason = "/home/ana/.local/share/craft-cloud-agent/account.json is damaged (expected value at line 1); move it away. Then sign in again"; - m.update(Event::Found(Ok(Status { run_state: RunState::Paused { reason: reason.into() }, ..status(false) }))); + m.update(Event::Found(Ok(Status { run_state: RunState::Paused { reason: reason.into(), reason_code: None }, ..status(false) }))); let Screen::Error(problem) = &m.screen else { panic!("{:?}", m.screen) }; assert_eq!(problem.line, "/home/ana/.local/share/craft-cloud-agent/account.json is damaged (expected value at line 1); move it away."); assert_eq!(problem.details, reason); @@ -1308,7 +2572,10 @@ pub(crate) mod tests { // At each start, matched once; again only when it changes. let mut m = model(); m.update(Event::Start); - assert_eq!(m.update(Event::Found(Ok(on.clone()))), vec![Effect::AppAutostart(true), Effect::Devices]); + assert_eq!( + m.update(Event::Found(Ok(on.clone()))), + vec![Effect::AppAutostart(true), Effect::Devices, Effect::Account { session: m.overview().unwrap().session }, Effect::Usage(1)] + ); assert!(m.update(Event::Polled(Ok(on.clone()))).is_empty()); assert_eq!(m.update(Event::Polled(Ok(Status { autostart: Some(false), ..on }))), vec![Effect::AppAutostart(false)]); } diff --git a/apps/craft-cloud-desktop/src/security.rs b/apps/craft-cloud-desktop/src/security.rs new file mode 100644 index 00000000..c97e2be4 --- /dev/null +++ b/apps/craft-cloud-desktop/src/security.rs @@ -0,0 +1,532 @@ +//! The Security page and the second step of a join: the signed-in account's two-factor sign-in (status, setting it up with a +//! secret and a QR code, the recovery codes shown once, turning it off, new recovery codes), the password change and signing +//! the other computers out; and the code field a join asks for when the account has two-factor sign-in. +//! +//! Pure like the rest of the model: [`SecurityPage::update`] takes an event and returns the calls to make, answers come back +//! as [`SecurityAnswer`]s. **Nothing secret is kept longer than it is shown**: the password a form held is moved into its call +//! and cleared, the secret and the recovery codes live in the page's step only, and leaving the page (or a signed-out or +//! removed screen) drops them. Every type that holds one hides it in `Debug`. + +use craft_cloud_app::{Error, RecoveryCodes, RevokedOthers, TwoFactorProofRequest, TwoFactorSetupResponse, TwoFactorStatus, codes}; +use craft_cloud_proto::api::codes as api_codes; + +use crate::model::Overview; + +/// The server doesn't know two-factor sign-in (it is older than this app), or the agent doesn't pass it on. +pub const UNSUPPORTED: &str = "This server doesn't support two-factor sign-in yet"; + +/// What the page knows of the account's two-factor sign-in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TwoFactorInfo { + /// Asking the server. + Loading, + /// The server doesn't have it ([`UNSUPPORTED`]). + Unsupported, + Known(TwoFactorStatus), + /// The status couldn't be read (the message). + Failed(String), +} + +/// Why recovery codes are shown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CodesWhy { + /// Two-factor sign-in was just turned on. + TurnedOn, + /// New codes were made; the old ones don't work. + New, +} + +/// What the page is doing. +#[derive(Clone, PartialEq, Eq)] +pub enum Step { + /// The status and the actions. + Home, + /// Set-up, first: the password. + SetupPassword { password: String }, + /// Set-up, second: the secret to put into the authenticator app, and the first code it shows. + SetupCode { secret: String, uri: String, expires_ms: i64, code: String }, + /// The recovery codes, shown once. + Codes { codes: Vec, why: CodesWhy }, + /// Turning it off: the password, and a code or a recovery code. + Disable { password: String, code: String, recovery: bool }, + /// New recovery codes: the password and a code. + Regenerate { password: String, code: String }, + /// A new password: the current one, the new one twice. + Password { current: String, new: String, again: String }, + /// Signing the other computers out: asked once more. + SignOutOthers, +} + +impl std::fmt::Debug for Step { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Which step, never what was typed or shown in it. + f.write_str(match self { + Step::Home => "Home", + Step::SetupPassword { .. } => "SetupPassword()", + Step::SetupCode { .. } => "SetupCode()", + Step::Codes { .. } => "Codes()", + Step::Disable { .. } => "Disable()", + Step::Regenerate { .. } => "Regenerate()", + Step::Password { .. } => "Password()", + Step::SignOutOthers => "SignOutOthers", + }) + } +} + +/// The Security page. +#[derive(Debug, Clone, PartialEq)] +pub struct SecurityPage { + /// The signed-in view the page was opened from and returns to. + pub overview: Overview, + pub info: TwoFactorInfo, + pub step: Step, + /// What went wrong with the last try, for people. + pub error: Option, + /// What the last action did ("Password changed"). + pub note: Option, + /// A call is under way: nothing can be sent twice. + pub busy: bool, +} + +/// What a button of the page asks for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecurityEvent { + /// Back to the signed-in view; whatever the page showed is dropped. + Close, + /// Back to the page's start (the recovery codes shown are dropped). + Home, + StartSetup, + StartDisable, + StartRegenerate, + StartPassword, + StartSignOutOthers, + /// The current step's button: continue, turn on, turn off, make, change, sign out. + Submit, + /// Disable's switch between a code and a recovery code. + ToggleRecovery, + /// Ask the server for the status again. + Refresh, +} + +/// A call of the agent the page makes. +#[derive(Clone, PartialEq, Eq)] +pub enum SecurityCall { + Status, + Setup { password: String }, + Confirm { code: String }, + Disable(TwoFactorProofRequest), + RecoveryCodes { password: String, code: String }, + ChangePassword { current: String, new: String }, + SignOutOthers, +} + +impl std::fmt::Debug for SecurityCall { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SecurityCall::Status => "Status", + SecurityCall::Setup { .. } => "Setup()", + SecurityCall::Confirm { .. } => "Confirm()", + SecurityCall::Disable(_) => "Disable()", + SecurityCall::RecoveryCodes { .. } => "RecoveryCodes()", + SecurityCall::ChangePassword { .. } => "ChangePassword()", + SecurityCall::SignOutOthers => "SignOutOthers", + }) + } +} + +/// An agent's answer to a [`SecurityCall`]. +#[derive(Clone, PartialEq)] +pub enum SecurityAnswer { + Status(Result), + Setup(Result), + Confirm(Result), + Disable(Result<(), Error>), + RecoveryCodes(Result), + ChangePassword(Result<(), Error>), + SignOutOthers(Result), +} + +impl std::fmt::Debug for SecurityAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // An answer may hold a secret or recovery codes: only whether it worked. + let (name, ok) = match self { + SecurityAnswer::Status(r) => ("Status", r.is_ok()), + SecurityAnswer::Setup(r) => ("Setup", r.is_ok()), + SecurityAnswer::Confirm(r) => ("Confirm", r.is_ok()), + SecurityAnswer::Disable(r) => ("Disable", r.is_ok()), + SecurityAnswer::RecoveryCodes(r) => ("RecoveryCodes", r.is_ok()), + SecurityAnswer::ChangePassword(r) => ("ChangePassword", r.is_ok()), + SecurityAnswer::SignOutOthers(r) => ("SignOutOthers", r.is_ok()), + }; + write!(f, "{name}({})", if ok { "ok" } else { "refused" }) + } +} + +/// Six digits (spaces allowed), the shape of a code from the authenticator app. +pub fn is_code(text: &str) -> bool { + let digits: String = text.chars().filter(|c| !c.is_whitespace()).collect(); + digits.len() == 6 && digits.bytes().all(|b| b.is_ascii_digit()) +} + +pub fn digits(text: &str) -> String { + text.chars().filter(|c| !c.is_whitespace()).collect() +} + +/// A wait in words. +fn wait_in_words(secs: u64) -> String { + match secs { + 0..=90 => format!("{secs} seconds"), + 91..=5_400 => format!("{} minutes", secs.div_ceil(60)), + _ => format!("{} hours", secs.div_ceil(3_600)), + } +} + +/// What a refusal says, for people, and what to do about it. +pub fn explain(e: &Error) -> String { + let wait = e.retry_after_secs().map(wait_in_words); + match e.code() { + Some(codes::WRONG_PASSWORD) => "That password isn't right.".into(), + Some(codes::TWO_FACTOR_INVALID) => "That code isn't right, or it was used already. Wait for the app's next code and try again.".into(), + Some(codes::TWO_FACTOR_LOCKED) => match wait { + Some(wait) => format!("Too many wrong codes. Try again in {wait}."), + None => "Too many wrong codes. Try again later.".into(), + }, + Some(codes::TWO_FACTOR_UNAVAILABLE) => "This server can't set up two-factor sign-in yet: its admin has to give it a key first.".into(), + Some(codes::TWO_FACTOR_PENDING) => "Setting it up is open on another computer. Finish it there, or wait ten minutes.".into(), + Some(codes::OFFLINE) => "Can't reach the server. Try again when this computer is online.".into(), + Some(craft_cloud_app::codes::NEEDS_YOU) => "This computer can't use the account any more. Sign out, then join again.".into(), + Some("rate_limited") => match wait { + Some(wait) => format!("Too many tries. Try again in {wait}."), + None => "Too many tries. Try again in a few minutes.".into(), + }, + _ => match e { + Error::Refused { message, .. } | Error::RefusedWith { message, .. } => message.clone(), + other => other.to_string(), + }, + } +} + +impl SecurityPage { + /// The page, opened from `overview`; its first call asks the server for the status. + pub fn open(overview: Overview) -> (Self, Vec) { + (Self { overview, info: TwoFactorInfo::Loading, step: Step::Home, error: None, note: None, busy: false }, vec![SecurityCall::Status]) + } + + /// Whether two-factor sign-in is known to be on. + pub fn is_on(&self) -> bool { + matches!(&self.info, TwoFactorInfo::Known(s) if s.enabled) + } + + fn start(&mut self, step: Step) { + self.step = step; + self.error = None; + self.note = None; + } + + /// A button of the page. Returns the calls to make; the page is `busy` while one is under way. + pub fn update(&mut self, event: SecurityEvent) -> Vec { + match event { + SecurityEvent::Close => Vec::new(), + SecurityEvent::Home => { + if !self.busy { + self.start(Step::Home); + } + Vec::new() + } + SecurityEvent::Refresh => { + self.info = TwoFactorInfo::Loading; + vec![SecurityCall::Status] + } + SecurityEvent::StartSetup => { + if !self.busy && !self.is_on() && matches!(self.info, TwoFactorInfo::Known(_)) { + self.start(Step::SetupPassword { password: String::new() }); + } + Vec::new() + } + SecurityEvent::StartDisable => { + if !self.busy && self.is_on() { + self.start(Step::Disable { password: String::new(), code: String::new(), recovery: false }); + } + Vec::new() + } + SecurityEvent::StartRegenerate => { + if !self.busy && self.is_on() { + self.start(Step::Regenerate { password: String::new(), code: String::new() }); + } + Vec::new() + } + SecurityEvent::StartPassword => { + if !self.busy { + self.start(Step::Password { current: String::new(), new: String::new(), again: String::new() }); + } + Vec::new() + } + SecurityEvent::StartSignOutOthers => { + if !self.busy { + self.start(Step::SignOutOthers); + } + Vec::new() + } + SecurityEvent::ToggleRecovery => { + if let Step::Disable { code, recovery, .. } = &mut self.step { + *recovery = !*recovery; + code.clear(); + self.error = None; + } + Vec::new() + } + SecurityEvent::Submit => self.submit(), + } + } + + /// The current step's call. A step that isn't filled in says what is missing and calls nothing; the typed password moves into + /// the call and is cleared from the form. + fn submit(&mut self) -> Vec { + if self.busy { + return Vec::new(); + } + self.error = None; + let call = match &mut self.step { + Step::SetupPassword { password } => { + if password.is_empty() { + self.error = Some("Type your password to start.".into()); + return Vec::new(); + } + SecurityCall::Setup { password: std::mem::take(password) } + } + Step::SetupCode { code, .. } => { + if !is_code(code) { + self.error = Some("Type the six digits the app shows.".into()); + return Vec::new(); + } + SecurityCall::Confirm { code: digits(&std::mem::take(code)) } + } + Step::Disable { password, code, recovery } => { + if password.is_empty() || code.trim().is_empty() { + self.error = Some("Type your password and a code (or a recovery code).".into()); + return Vec::new(); + } + if !*recovery && !is_code(code) { + self.error = Some("A code from the app is six digits.".into()); + return Vec::new(); + } + let typed = std::mem::take(code); + let password = std::mem::take(password); + if *recovery { + SecurityCall::Disable(TwoFactorProofRequest { password, totp_code: None, recovery_code: Some(typed.trim().to_owned()) }) + } else { + SecurityCall::Disable(TwoFactorProofRequest { password, totp_code: Some(digits(&typed)), recovery_code: None }) + } + } + Step::Regenerate { password, code } => { + if password.is_empty() || !is_code(code) { + self.error = Some("Type your password and the six digits the app shows.".into()); + return Vec::new(); + } + SecurityCall::RecoveryCodes { password: std::mem::take(password), code: digits(&std::mem::take(code)) } + } + Step::Password { current, new, again } => { + if current.is_empty() || new.is_empty() { + self.error = Some("Type your current password and the new one.".into()); + return Vec::new(); + } + if new != again { + self.error = Some("The two new passwords aren't the same.".into()); + return Vec::new(); + } + SecurityCall::ChangePassword { current: std::mem::take(current), new: std::mem::take(new) } + } + Step::SignOutOthers => SecurityCall::SignOutOthers, + Step::Home | Step::Codes { .. } => return Vec::new(), + }; + // A form that goes on with a password or a code is emptied for good: the typed ones live in the call now. + if let Step::Password { again, .. } = &mut self.step { + again.clear(); + } + self.busy = true; + vec![call] + } + + /// An answer. May ask for more (the status, after a change). + pub fn answer(&mut self, answer: SecurityAnswer, now_ms: i64) -> Vec { + match answer { + SecurityAnswer::Status(result) => { + self.info = match result { + Ok(status) => TwoFactorInfo::Known(status), + Err(e) if e.code() == Some(api_codes::NOT_FOUND) || e.code() == Some(codes::AGENT_TOO_OLD) => TwoFactorInfo::Unsupported, + Err(e) => TwoFactorInfo::Failed(explain(&e)), + }; + Vec::new() + } + SecurityAnswer::Setup(result) => { + self.busy = false; + match result { + Ok(setup) => { + self.step = + Step::SetupCode { secret: setup.secret, uri: setup.otpauth_uri, expires_ms: setup.expires_ms, code: String::new() }; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::Confirm(result) => { + self.busy = false; + match result { + Ok(codes) => { + self.step = Step::Codes { codes: codes.recovery_codes, why: CodesWhy::TurnedOn }; + self.note = Some("Two-factor sign-in is on. Your other computers were signed out.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => { + // A set-up that ran out or was used up (five wrong codes) has to start again. + if e.code() == Some(codes::CONFLICT) { + self.step = Step::Home; + } + self.error = Some(explain(&e)); + // Past its ten minutes: the secret is of no use. + if matches!(&self.step, Step::SetupCode { expires_ms, .. } if now_ms >= *expires_ms) { + self.step = Step::Home; + } + } + } + Vec::new() + } + SecurityAnswer::Disable(result) => { + self.busy = false; + match result { + Ok(()) => { + self.step = Step::Home; + self.note = Some("Two-factor sign-in is off. Your other computers were signed out.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::RecoveryCodes(result) => { + self.busy = false; + match result { + Ok(codes) => { + self.step = Step::Codes { codes: codes.recovery_codes, why: CodesWhy::New }; + self.note = Some("New recovery codes. The old ones don't work any more.".into()); + return vec![SecurityCall::Status]; + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::ChangePassword(result) => { + self.busy = false; + match result { + Ok(()) => { + self.step = Step::Home; + self.note = Some("Password changed. Your other computers were signed out.".into()); + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + SecurityAnswer::SignOutOthers(result) => { + self.busy = false; + match result { + Ok(done) => { + self.step = Step::Home; + self.note = Some(match done.revoked { + 0 => "No other computer was signed in.".to_owned(), + 1 => "Signed 1 other computer out.".to_owned(), + n => format!("Signed {n} other computers out."), + }); + } + Err(e) => self.error = Some(explain(&e)), + } + Vec::new() + } + } + } +} + +/// The second step of a join: the account has two-factor sign-in, so the code comes with the same join. +#[derive(Clone, PartialEq, Eq)] +pub struct SecondFactorForm { + pub form: crate::model::JoinForm, + /// The new account's password, when the join came by an invite that needed one (an account that has two-factor sign-in has + /// one already, so this is only ever `None` for it). + pub password: Option, + /// What was typed: a code from the app, or a recovery code. + pub code: String, + /// The field takes a recovery code instead. + pub recovery: bool, + /// What went wrong with the last try, for people. + pub error: Option, +} + +impl std::fmt::Debug for SecondFactorForm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorForm") + .field("form", &self.form) + .field("recovery", &self.recovery) + .field("error", &self.error) + .finish_non_exhaustive() + } +} + +impl SecondFactorForm { + pub fn new(form: crate::model::JoinForm, password: Option, error: Option) -> Self { + Self { form, password, code: String::new(), recovery: false, error } + } + + /// The code as the join takes it: `(totp_code, recovery_code)`, or what is wrong with what was typed. + pub fn factor(&self) -> Result<(Option, Option), String> { + if self.recovery { + let typed = self.code.trim(); + if typed.is_empty() { + return Err("Type a recovery code.".into()); + } + return Ok((None, Some(typed.to_owned()))); + } + if !is_code(&self.code) { + return Err("Type the six digits the app shows.".into()); + } + Ok((Some(digits(&self.code)), None)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_code_is_six_digits_with_spaces_allowed() { + assert!(is_code("123456") && is_code(" 123 456 ")); + for bad in ["12345", "1234567", "12345a", "", "١٢٣٤٥٦"] { + assert!(!is_code(bad), "{bad:?}"); + } + } + + #[test] + fn refusals_are_explained_for_people() { + let refused = |code: &str| Error::Refused { code: code.into(), message: "m".into(), name_taken: None }; + assert!(explain(&refused("wrong_password")).contains("password isn't right")); + assert!(explain(&refused("two_factor_invalid")).contains("next code")); + let locked = Error::RefusedWith { code: "two_factor_locked".into(), message: "m".into(), two_factor: None, retry_after_secs: Some(840) }; + assert_eq!(explain(&locked), "Too many wrong codes. Try again in 14 minutes."); + assert!(explain(&refused("two_factor_unavailable")).contains("admin has to give it a key")); + assert_eq!(explain(&refused("weak_password")), "m", "an unknown code says what the server said"); + } + + #[test] + fn secrets_stay_out_of_debug_output() { + let step = Step::SetupCode { secret: "JBSWY3DPEHPK3PXP".into(), uri: "otpauth://totp/x".into(), expires_ms: 1, code: "918273".into() }; + let codes = Step::Codes { codes: vec!["ABCD-EFGH-JKMN-PQRS".into()], why: CodesWhy::TurnedOn }; + let password = Step::Disable { password: "pw-secret-1".into(), code: "918273".into(), recovery: false }; + for shown in [format!("{step:?}"), format!("{codes:?}"), format!("{password:?}")] { + assert!(!shown.contains("JBSWY") && !shown.contains("ABCD") && !shown.contains("pw-secret") && !shown.contains("918273"), "{shown}"); + } + let call = SecurityCall::Setup { password: "pw-secret-1".into() }; + assert!(!format!("{call:?}").contains("pw-secret")); + let answer = SecurityAnswer::Confirm(Ok(RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into()] })); + assert!(!format!("{answer:?}").contains("ABCD")); + let form = SecondFactorForm { form: Default::default(), password: None, code: "918273".into(), recovery: false, error: None }; + assert!(!format!("{form:?}").contains("918273")); + } +} diff --git a/apps/craft-cloud-desktop/src/tray/mod.rs b/apps/craft-cloud-desktop/src/tray/mod.rs index e98c499c..5ba955fc 100644 --- a/apps/craft-cloud-desktop/src/tray/mod.rs +++ b/apps/craft-cloud-desktop/src/tray/mod.rs @@ -187,7 +187,7 @@ mod tests { let retrying = Status { online: false, run_state: RunState::Retrying { message: "can't reach it".into() }, ..status() }; assert_eq!(state_of(Some(&retrying)), TrayState::Offline); assert_eq!(state_of(Some(&Status { run_state: RunState::PausedByUser, ..status() })), TrayState::Paused); - let stuck = Status { run_state: RunState::Paused { reason: "the folder is missing".into() }, ..status() }; + let stuck = Status { run_state: RunState::Paused { reason: "the folder is missing".into(), reason_code: None }, ..status() }; assert_eq!(state_of(Some(&stuck)), TrayState::NeedsAttention); assert_eq!(state_of(Some(&Status { removed: true, ..stuck })), TrayState::NeedsAttention); assert_eq!(state_of(Some(&crate::model::tests::status(false))), TrayState::NeedsAttention, "not joined yet"); diff --git a/apps/craft-cloud-desktop/src/ui/add_computer.rs b/apps/craft-cloud-desktop/src/ui/add_computer.rs index 5e4a3c23..230fb672 100644 --- a/apps/craft-cloud-desktop/src/ui/add_computer.rs +++ b/apps/craft-cloud-desktop/src/ui/add_computer.rs @@ -65,7 +65,7 @@ pub(super) fn show(ui: &mut egui::Ui, adding: &Adding, now_ms: i64, events: &mut /// `text` as a QR code `side` points wide: dark modules on white with the standard quiet zone, whatever the theme /// (cameras read it that way round). -fn qr(ui: &mut egui::Ui, text: &str, side: f32) { +pub(super) fn qr(ui: &mut egui::Ui, text: &str, side: f32) { let Ok(code) = QrCode::encode_text(text, QrCodeEcc::Medium) else { return; }; diff --git a/apps/craft-cloud-desktop/src/ui/admin.rs b/apps/craft-cloud-desktop/src/ui/admin.rs new file mode 100644 index 00000000..898e763e --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/admin.rs @@ -0,0 +1,502 @@ +//! The admin pages: the users (search, pages), a new account, one account (role, quota, disable, invites, devices), +//! the password a change asks for, and the code a change made, shown once. + +use egui::RichText; + +use crate::admin::{Action, AdminEvent, AdminScreen, CodeKind, CreateForm, Dialog, RoleChoice, ShownCode, UserList, UserView, View, format_bytes}; +use craft_cloud_app::AdminUserInfo; + +use crate::model::Event; + +struct StorageState<'a> { + overview: Option<&'a craft_cloud_app::AdminStorageOverview>, + loading: bool, + error: Option<&'a str>, +} + +/// The name of the confirmation page's password field, for [`super::security::secret_id`] and the forgetting of its text. +pub(super) const PASSWORD_FIELD: &str = "admin-password"; +/// The name of its field for a code (from the authenticator app, or a recovery code). +pub(super) const CODE_FIELD: &str = "admin-code"; + +pub(super) fn show(ui: &mut egui::Ui, admin: &mut AdminScreen, now_ms: i64, events: &mut Vec) { + let me = admin.overview.status.account.clone().unwrap_or_default(); + // A code just made comes first, then the password a change asks for, then the page. + if let Some(code) = &admin.code { + return shown_code(ui, code, now_ms, events); + } + if let Some(dialog) = &mut admin.dialog { + return confirmation(ui, dialog, events); + } + let server = admin.server.clone(); + match &mut admin.view { + View::List => users( + ui, + server.as_ref(), + StorageState { overview: admin.storage.as_ref(), loading: admin.storage_loading, error: admin.storage_error.as_deref() }, + &mut admin.list, + &me, + events, + ), + View::Create(form) => create(ui, form, events), + View::Audit(view) => super::audit::log(ui, view, now_ms, events), + View::User(view) => { + let user = admin.list.users.iter().find(|u| u.id == view.id).cloned(); + account(ui, view, user.as_ref(), &me, now_ms, events); + } + } +} + +fn send(events: &mut Vec, event: AdminEvent) { + events.push(Event::Admin(event)); +} + +/// "Enter" in the field just edited. +fn entered(response: &egui::Response, ui: &egui::Ui) -> bool { + response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)) +} + +/// What a user is, in a line: role, computers, what they store, and the state of the account. +fn describe(user: &AdminUserInfo) -> String { + let mut parts = vec![if user.role == "admin" { "Admin".to_owned() } else { "Member".to_owned() }]; + parts.push(match user.devices_active { + 1 => "1 computer".to_owned(), + n => format!("{n} computers"), + }); + parts.push(match user.quota_bytes { + Some(quota) => format!("{} of {}", format_bytes(user.stored_bytes), format_bytes(quota)), + None => format_bytes(user.stored_bytes), + }); + if user.disabled_ms.is_some() { + parts.push("disabled".to_owned()); + } + if user.invite_pending { + parts.push("invite open".to_owned()); + } + if user.two_factor_enabled { + parts.push("two-factor on".to_owned()); + } + parts.join(" · ") +} + +fn users( + ui: &mut egui::Ui, + server: Option<&craft_cloud_app::AdminServerInfo>, + storage: StorageState<'_>, + list: &mut UserList, + me: &str, + events: &mut Vec, +) { + super::heading(ui, "Administration"); + if let Some(info) = server { + ui.label( + RichText::new(format!( + "Craft Cloud server {} · {} accounts · {} computers · {} stored ({}, {})", + info.version, + info.users, + info.devices_active, + format_bytes(info.stored_bytes), + info.storage_kind, + info.database_kind + )) + .small() + .weak(), + ); + ui.add_space(8.0); + } + storage_overview(ui, storage); + ui.add_space(8.0); + ui.horizontal(|ui| { + if ui.button("Back").clicked() { + send(events, AdminEvent::Close); + } + if ui.button("New account").clicked() { + send(events, AdminEvent::NewUser); + } + if ui.add_enabled(!list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AdminEvent::Reload); + } + if ui.button("Audit log").on_hover_text("Sign-ins, changes and what admins did").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenAudit(None))); + } + }); + ui.add_space(8.0); + + ui.horizontal(|ui| { + let width = (ui.available_width() - 130.0).max(120.0); + let field = ui.add(egui::TextEdit::singleline(&mut list.search).hint_text("Search accounts").desired_width(width)); + if ui.button("Search").clicked() || entered(&field, ui) { + send(events, AdminEvent::Search); + } + if !list.applied.is_empty() && ui.button("Clear").clicked() { + send(events, AdminEvent::ClearSearch); + } + }); + ui.add_space(8.0); + + if let Some(error) = &list.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if list.loading && !list.loaded { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } else if list.users.is_empty() && list.loaded && list.error.is_none() { + ui.label(if list.applied.is_empty() { "No accounts." } else { "No account has that in its name." }); + } + for user in list.users.clone() { + ui.push_id(user.id, |ui| { + ui.horizontal(|ui| { + ui.vertical(|ui| { + let name = if user.username == me { format!("{} (you)", user.username) } else { user.username.clone() }; + ui.label(RichText::new(name).strong()); + ui.add(egui::Label::new(RichText::new(describe(&user)).small().weak()).wrap()); + }); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if ui.button("Manage").clicked() { + send(events, AdminEvent::OpenUser(user.id)); + } + }); + }); + ui.separator(); + }); + } + if list.has_previous() || list.next.is_some() { + ui.horizontal(|ui| { + if ui.add_enabled(list.has_previous() && !list.loading, egui::Button::new("Previous")).clicked() { + send(events, AdminEvent::PreviousPage); + } + ui.label(format!("Page {}", list.page())); + if ui.add_enabled(list.next.is_some() && !list.loading, egui::Button::new("Next")).clicked() { + send(events, AdminEvent::NextPage); + } + }); + } +} + +fn storage_overview(ui: &mut egui::Ui, state: StorageState<'_>) { + ui.label(RichText::new("Storage").strong()); + if let Some(error) = state.error { + super::error_line(ui, error); + return; + } + let Some(storage) = state.overview else { + if state.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + return; + }; + ui.label(format!( + "{} stored · {} accounts · default quota {}", + format_bytes(storage.stored_bytes), + storage.accounts, + storage.default_quota_bytes.map_or_else(|| "none".to_owned(), format_bytes) + )); + ui.label(format!( + "{} near quota · {} over quota · {} with a quota of their own", + storage.accounts_near_quota, storage.accounts_over_quota, storage.accounts_with_override + )); + ui.label( + RichText::new(format!( + "Manifest index: {} complete · {} building · {} not indexed", + storage.manifest_index.complete, storage.manifest_index.building, storage.manifest_index.not_indexed + )) + .small() + .weak(), + ); + if storage.manifest_index.building > 0 || storage.manifest_index.not_indexed > 0 { + ui.label(RichText::new("Storage estimates are incomplete; run `craft-cloud-server index build`. ").small().weak()); + } + if !storage.top_accounts.is_empty() { + ui.label(RichText::new("Top accounts").small().strong()); + egui::Grid::new("admin-storage-top").num_columns(3).striped(true).spacing([12.0, 4.0]).show(ui, |ui| { + ui.label(RichText::new("Account").small().weak()); + ui.label(RichText::new("Stored").small().weak()); + ui.label(RichText::new("Effective quota").small().weak()); + ui.end_row(); + for account in &storage.top_accounts { + ui.label(&account.username); + ui.label(format_bytes(account.stored_bytes)); + ui.label(account.quota_bytes.map_or_else(|| "none".to_owned(), format_bytes)); + ui.end_row(); + } + }); + } + ui.separator(); +} + +fn role_choice(ui: &mut egui::Ui, role: &mut RoleChoice) { + ui.horizontal(|ui| { + ui.radio_value(role, RoleChoice::Member, RoleChoice::Member.label()); + ui.radio_value(role, RoleChoice::Admin, RoleChoice::Admin.label()); + }); +} + +fn create(ui: &mut egui::Ui, form: &mut CreateForm, events: &mut Vec) { + super::heading(ui, "New account"); + ui.label("Name"); + let name = ui.add(egui::TextEdit::singleline(&mut form.username).hint_text("ana").desired_width(f32::INFINITY)); + let enter = entered(&name, ui); + ui.add_space(8.0); + ui.label("Role"); + role_choice(ui, &mut form.role); + ui.add_space(6.0); + ui.add( + egui::Label::new( + RichText::new( + "The account starts without a password: you get an invite code to give them. Set a quota afterwards, on the account's page.", + ) + .small() + .weak(), + ) + .wrap(), + ); + ui.add_space(10.0); + if let Some(error) = &form.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if super::wide_button(ui, true, "Continue") || enter { + send(events, AdminEvent::SubmitCreate); + } + ui.add_space(4.0); + if ui.button("Cancel").clicked() { + send(events, AdminEvent::BackToList); + } +} + +fn account(ui: &mut egui::Ui, view: &mut UserView, user: Option<&AdminUserInfo>, me: &str, now_ms: i64, events: &mut Vec) { + super::heading(ui, &view.username); + if ui.button("Back to the accounts").clicked() { + send(events, AdminEvent::BackToList); + } + ui.add_space(8.0); + let Some(user) = user else { + super::error_line(ui, "This account isn't in the list any more."); + return; + }; + ui.label(RichText::new(describe(user)).small().weak()); + ui.add_space(10.0); + let is_me = user.username == me; + + ui.horizontal(|ui| { + ui.label(RichText::new("Storage usage").strong()); + if ui.small_button("Refresh").clicked() { + send(events, AdminEvent::RefreshUsage); + } + }); + if let Some(error) = &view.usage_error { + super::error_line(ui, error); + } else if let Some(usage) = &view.usage { + ui.label(format!("Stored: {}", format_bytes(usage.stored_bytes))); + ui.label(format!("Effective quota: {}", usage.quota_bytes.map_or_else(|| "none".to_owned(), format_bytes))); + if usage.estimates.index_complete { + ui.label(format!("Trash estimate (overlapping): {}", usage.trash_bytes.map_or_else(|| "not available".to_owned(), format_bytes))); + ui.label(format!("History estimate (overlapping): {}", usage.history_bytes.map_or_else(|| "not available".to_owned(), format_bytes))); + } else { + ui.label("estimates not available yet"); + } + } else if view.usage_loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + ui.add_space(8.0); + + if ui.button("Activity of this account").on_hover_text("The audit log, for this account").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenAudit(Some((user.id, user.username.clone()))))); + } + ui.add_space(8.0); + + ui.label(RichText::new("Role and quota").strong()); + role_choice(ui, &mut view.role); + ui.horizontal(|ui| { + ui.label("Quota (GiB)"); + ui.add(egui::TextEdit::singleline(&mut view.quota).hint_text("none").desired_width(100.0)); + }); + if let Some(error) = &view.error { + super::error_line(ui, error); + } + if ui.button("Save changes").clicked() { + send(events, AdminEvent::SubmitEdit); + } + ui.add_space(12.0); + + ui.label(RichText::new("Account").strong()); + let id = user.id; + let username = user.username.clone(); + ui.horizontal_wrapped(|ui| { + if is_me { + ui.label(RichText::new("You can't disable or remove your own account here.").small().weak()); + } else if user.disabled_ms.is_some() { + if ui.button("Enable").clicked() { + send(events, AdminEvent::Ask(Action::Enable { id, username: username.clone() })); + } + } else if ui.button("Disable").clicked() { + send(events, AdminEvent::Ask(Action::Disable { id, username: username.clone() })); + } + if !is_me && ui.button("Remove").clicked() { + send(events, AdminEvent::Ask(Action::Remove { id, username: username.clone() })); + } + if !is_me + && user.two_factor_enabled + && ui.button("Turn two-factor off").on_hover_text("They sign in with their password alone; their computers are signed out").clicked() + { + send(events, AdminEvent::Ask(Action::ResetTwoFactor { id, username: username.clone() })); + } + }); + ui.add_space(12.0); + + ui.label(RichText::new("Invites and codes").strong()); + ui.horizontal_wrapped(|ui| { + if ui.button("New invite").on_hover_text("For an account that has no password yet").clicked() { + send(events, AdminEvent::Ask(Action::IssueInvite { id, username: username.clone() })); + } + if ui.button("Password reset code").on_hover_text("Good for an hour; using it signs all their computers out").clicked() { + send(events, AdminEvent::Ask(Action::IssueReset { id, username: username.clone(), reset_two_factor: false })); + } + if user.two_factor_enabled + && ui.button("Reset code, two-factor off").on_hover_text("As a reset code, and using it turns their two-factor sign-in off").clicked() + { + send(events, AdminEvent::Ask(Action::IssueReset { id, username: username.clone(), reset_two_factor: true })); + } + // Always offered: the list says an invite is open (`invite_pending`), but not a password reset code. The server + // answers `not_found` when none is. + if ui.button("Revoke open code").on_hover_text("Ends the account's open invite or password reset code").clicked() { + send(events, AdminEvent::Ask(Action::RevokeInvite { id, username: username.clone() })); + } + }); + ui.add_space(12.0); + + ui.horizontal(|ui| { + ui.label(RichText::new("Computers").strong()); + if ui.small_button("Refresh").clicked() { + send(events, AdminEvent::RefreshDevices); + } + }); + if let Some(error) = &view.devices_error { + super::error_line(ui, error); + } + let Some(devices) = &view.devices else { + if view.devices_error.is_none() { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + return; + }; + if devices.iter().all(|d| d.revoked) { + ui.label(RichText::new("No computers signed in.").weak()); + } + egui::Grid::new("admin-devices").num_columns(3).striped(true).spacing([12.0, 6.0]).show(ui, |ui| { + for device in devices.iter().filter(|d| !d.revoked) { + ui.label(&device.name); + ui.label(RichText::new(format!("last seen {}", super::ago(now_ms, device.last_used_ms))).weak()); + if ui.button("Revoke").on_hover_text("It can't sync any more; its files stay there").clicked() { + let action = Action::RevokeDevice { id, username: username.clone(), device: device.id, device_name: device.name.clone() }; + send(events, AdminEvent::Ask(action)); + } + ui.end_row(); + } + }); +} + +/// The password a change asks for. +fn confirmation(ui: &mut egui::Ui, dialog: &mut Dialog, events: &mut Vec) { + super::heading(ui, &dialog.action.title()); + ui.add(egui::Label::new(dialog.action.explain()).wrap()); + ui.add_space(10.0); + ui.label("Your password"); + let field = ui.add_enabled( + !dialog.busy, + egui::TextEdit::singleline(&mut dialog.confirmation.password) + .id(super::security::secret_id(PASSWORD_FIELD)) + .password(true) + .hint_text("your own current password") + .desired_width(f32::INFINITY), + ); + let mut enter = entered(&field, ui); + if dialog.needs_code { + ui.add_space(6.0); + ui.label("Code from your authenticator app, or a recovery code"); + let code = ui.add_enabled( + !dialog.busy, + egui::TextEdit::singleline(&mut dialog.confirmation.code) + .id(super::security::secret_id(CODE_FIELD)) + .password(true) + .hint_text("123456") + .desired_width(f32::INFINITY), + ); + enter |= entered(&code, ui); + } + ui.add_space(10.0); + if let Some(error) = &dialog.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if dialog.busy { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + return; + } + if super::wide_button(ui, true, dialog.action.button()) || enter { + send(events, AdminEvent::Confirm); + } + ui.add_space(4.0); + if ui.button("Cancel").clicked() { + send(events, AdminEvent::CancelDialog); + } +} + +/// How long until `at_ms`, from `now_ms`: "in 7 days", "in 59 min". +fn until(now_ms: i64, at_ms: i64) -> String { + let secs = at_ms.saturating_sub(now_ms).max(0) / 1000; + match secs { + 0..60 => "in under a minute".into(), + 60..3_600 => format!("in {} min", secs / 60), + 3_600..86_400 => format!("in {} h", secs / 3_600), + _ => format!("in {} days", secs / 86_400), + } +} + +/// A code the server made: shown once, to copy and hand over. +fn shown_code(ui: &mut egui::Ui, code: &ShownCode, now_ms: i64, events: &mut Vec) { + let title = match code.kind { + CodeKind::Invite => format!("Invite for {}", code.username), + CodeKind::PasswordReset => format!("Password reset code for {}", code.username), + }; + super::heading(ui, &title); + super::error_line(ui, "Note it now: it is shown once and can't be shown again."); + ui.add_space(8.0); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(&code.code).monospace().size(18.0).strong()).selectable(true)); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(code.code.clone()); + } + }); + if let Some(url) = &code.join_url { + ui.add_space(8.0); + ui.label("Or the whole join string:"); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(url).monospace()).selectable(true).wrap()); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(url.clone()); + } + }); + } + ui.add_space(8.0); + ui.label(RichText::new(format!("Stops working {}.", until(now_ms, code.expires_ms))).weak()); + ui.add_space(14.0); + if super::wide_button(ui, true, "Done") { + send(events, AdminEvent::DismissCode); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/audit.rs b/apps/craft-cloud-desktop/src/ui/audit.rs new file mode 100644 index 00000000..7416ffb8 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/audit.rs @@ -0,0 +1,175 @@ +//! The audit log's pages: the admin's view (a filter by event and by account, pages, an export to a file) and the account's own +//! activity. Both show events as words, with the typed details the server writes and nothing else. + +use egui::RichText; + +use crate::admin::AdminEvent; +use crate::audit::{ActivityScreen, AuditList, AuditMsg, AuditView, ExportState, action_label, details_line, filter_choices, filter_label, who_line}; +use crate::model::Event; +use craft_cloud_proto::api::AuditEvent; + +fn send(events: &mut Vec, msg: AuditMsg) { + events.push(Event::Audit(msg)); +} + +/// The events of a page: what happened, how long ago, and its details. +fn rows(ui: &mut egui::Ui, list: &AuditList, now_ms: i64) { + if let Some(error) = &list.error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if list.loading && !list.loaded { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } else if list.events.is_empty() && list.loaded && list.error.is_none() { + ui.label("Nothing was recorded."); + } + for event in &list.events { + event_row(ui, event, now_ms); + } +} + +fn event_row(ui: &mut egui::Ui, event: &AuditEvent, now_ms: i64) { + ui.push_id(event.id, |ui| { + let label = RichText::new(action_label(&event.action)).strong(); + ui.add(egui::Label::new(if event.outcome == "failure" { label.color(ui.visuals().error_fg_color) } else { label }).wrap()); + let mut line = super::ago(now_ms, event.at_ms); + let details = details_line(event); + if !details.is_empty() { + line.push_str(" · "); + line.push_str(&details); + } + ui.add(egui::Label::new(RichText::new(line).small().weak()).wrap()); + // Who did it and what it was about, with the whole IDs on hover. + let (who, full) = who_line(event); + ui.add(egui::Label::new(RichText::new(who).small().weak()).wrap().sense(egui::Sense::hover())).on_hover_text(full); + ui.separator(); + }); +} + +/// "Previous / Page n / Next", sending `previous` and `next`. +fn paging(ui: &mut egui::Ui, list: &AuditList, previous: AuditMsg, next: AuditMsg, events: &mut Vec) { + if !list.has_previous() && list.next.is_none() { + return; + } + ui.horizontal(|ui| { + if ui.add_enabled(list.has_previous() && !list.loading, egui::Button::new("Previous")).clicked() { + send(events, previous); + } + ui.label(format!("Page {}", list.page())); + if ui.add_enabled(list.next.is_some() && !list.loading, egui::Button::new("Next")).clicked() { + send(events, next); + } + }); +} + +/// The admin's audit log. +pub(super) fn log(ui: &mut egui::Ui, view: &mut AuditView, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Audit log"); + ui.horizontal(|ui| { + if ui.button("Back to the accounts").clicked() { + events.push(Event::Admin(AdminEvent::BackToList)); + } + if ui.add_enabled(!view.list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AuditMsg::Reload); + } + }); + ui.add_space(8.0); + + // The filters: an event (the server's closed list, by group or by name) and the account the page was opened for. + ui.horizontal(|ui| { + ui.label("Show"); + let shown = view.filter.action.as_deref().map_or_else(|| "All events".to_owned(), filter_label); + egui::ComboBox::from_id_salt("audit-action").selected_text(shown).width(220.0).show_ui(ui, |ui| { + if ui.selectable_label(view.filter.action.is_none(), "All events").clicked() && view.filter.action.is_some() { + send(events, AuditMsg::SetAction(None)); + } + for choice in filter_choices() { + let selected = view.filter.action.as_deref() == Some(choice); + let text = if choice.ends_with('.') { format!("{} (all)", filter_label(choice)) } else { filter_label(choice) }; + if ui.selectable_label(selected, text).clicked() && !selected { + send(events, AuditMsg::SetAction(Some(choice.to_owned()))); + } + } + }); + }); + if let Some((_, name)) = &view.filter.user { + ui.horizontal(|ui| { + ui.label(format!("Only events about {name}")); + if ui.small_button("Show everyone").clicked() { + send(events, AuditMsg::ClearUser); + } + }); + } + ui.add_space(6.0); + + // The export. + ui.horizontal(|ui| { + let idle = !matches!(view.export, ExportState::Picking { .. } | ExportState::Running { .. }); + if ui.add_enabled(idle, egui::Button::new("Export…")).on_hover_text("Save what these filters show, all of it, as a file").clicked() { + send(events, AuditMsg::Export); + } + match &view.export { + ExportState::Idle => {} + ExportState::Picking { .. } => { + ui.label(RichText::new("Choose where to save it…").weak()); + } + ExportState::Running { .. } => { + ui.spinner(); + ui.label("Saving…"); + } + ExportState::Done { .. } | ExportState::Failed(_) => { + if ui.small_button("OK").clicked() { + send(events, AuditMsg::DismissExport); + } + } + } + }); + match &view.export { + ExportState::Done { path, outcome } if outcome.truncated => { + super::error_line( + ui, + &format!( + "Only the newest {} events were saved to {path}: the log holds more than an export can. Narrow the filters to get the rest.", + outcome.events + ), + ); + } + ExportState::Done { path, outcome } => { + ui.label(RichText::new(format!("Saved {} events to {path}.", outcome.events)).color(super::good(ui))); + } + ExportState::Failed(why) => super::error_line(ui, &format!("The export failed, and no file was kept: {why}")), + _ => {} + } + ui.add_space(8.0); + + rows(ui, &view.list, now_ms); + paging(ui, &view.list, AuditMsg::Previous, AuditMsg::Next, events); +} + +/// The signed-in account's own activity. +pub(super) fn activity(ui: &mut egui::Ui, screen: &mut ActivityScreen, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Activity"); + ui.add( + egui::Label::new( + RichText::new("What the server recorded for your account: sign-ins, password and two-factor changes, computers signed out.") + .small() + .weak(), + ) + .wrap(), + ); + ui.add_space(6.0); + ui.horizontal(|ui| { + if ui.button("Back").clicked() { + send(events, AuditMsg::CloseActivity); + } + if ui.add_enabled(!screen.list.loading, egui::Button::new("Refresh")).clicked() { + send(events, AuditMsg::ActivityReload); + } + }); + ui.add_space(8.0); + rows(ui, &screen.list, now_ms); + paging(ui, &screen.list, AuditMsg::ActivityPrevious, AuditMsg::ActivityNext, events); +} diff --git a/apps/craft-cloud-desktop/src/ui/mod.rs b/apps/craft-cloud-desktop/src/ui/mod.rs index 6bd7b2cb..499b9302 100644 --- a/apps/craft-cloud-desktop/src/ui/mod.rs +++ b/apps/craft-cloud-desktop/src/ui/mod.rs @@ -3,10 +3,16 @@ //! I/O, so kittest draws every screen against a fake agent. mod add_computer; +mod admin; +mod audit; mod computers; mod join; mod overview; mod problem; +mod second_factor; +mod security; +mod trash; +mod versions; use egui::{Color32, RichText}; @@ -20,6 +26,7 @@ pub const TITLE: &str = "Craft Cloud"; /// Draws the window for `model` at `now_ms`, and returns what its buttons asked for. pub fn show(ui: &mut egui::Ui, model: &mut Model, now_ms: i64) -> Vec { let mut events = Vec::new(); + security::forget_secrets(ui.ctx(), &model.screen); egui::CentralPanel::default().show(ui, |ui| { egui::ScrollArea::vertical().auto_shrink([false, false]).show(ui, |ui| { ui.add_space(8.0); @@ -31,8 +38,14 @@ pub fn show(ui: &mut egui::Ui, model: &mut Model, now_ms: i64) -> Vec { Screen::Join(form) => join::form(ui, form, false, &mut events), Screen::Joining { form, .. } => join::form(ui, form, true, &mut events), Screen::PasswordNeeded(pw) => join::password(ui, pw, &mut events), + Screen::SecondFactor(form) => second_factor::show(ui, form, &mut events), + Screen::Security(page) => security::show(ui, page, &mut events), Screen::SignedIn(o) => overview::show(ui, o, busy, now_ms, autostart_problem.as_deref(), &mut events), Screen::AddingComputer(adding) => add_computer::show(ui, adding, now_ms, &mut events), + Screen::Admin(a) => admin::show(ui, a, now_ms, &mut events), + Screen::Activity(a) => audit::activity(ui, a, now_ms, &mut events), + Screen::Versions(versions) => self::versions::show(ui, versions, now_ms, &mut events), + Screen::Trash(trash) => self::trash::show(ui, trash, now_ms, &mut events), Screen::Removed => problem::removed(ui, busy, &mut events), } if let Some(note) = note { @@ -99,3 +112,6 @@ pub fn ago(now_ms: i64, then_ms: i64) -> String { #[cfg(test)] mod tests; + +#[cfg(test)] +mod tests_security; diff --git a/apps/craft-cloud-desktop/src/ui/overview.rs b/apps/craft-cloud-desktop/src/ui/overview.rs index 92aa3da3..f4214a63 100644 --- a/apps/craft-cloud-desktop/src/ui/overview.rs +++ b/apps/craft-cloud-desktop/src/ui/overview.rs @@ -1,9 +1,10 @@ //! Signed in: "fabio on cloud.shadows.ch", online or offline and the last sync, what the agent is doing, Open folder, //! Pause, Add a computer, the computers, Sign out and Quit. -use craft_cloud_app::{RunState, Status}; +use craft_cloud_app::{AccountUsage, RunState, Status}; use egui::RichText; +use crate::admin::{Access, AdminEvent}; use crate::model::{Busy, Event, Overview}; /// What the sync is doing, in one line, and the Details when it is a problem. @@ -14,7 +15,7 @@ pub fn sync_line(status: &Status) -> (String, Option) { RunState::Syncing => ("Syncing…".into(), None), RunState::Retrying { message } if !status.online => (format!("Can't reach {server} — trying again"), Some(message.clone())), RunState::Retrying { message } => ("Syncing failed — trying again".into(), Some(message.clone())), - RunState::Paused { reason } => (super::problem_line(reason), Some(reason.clone())), + RunState::Paused { reason, .. } => (super::problem_line(reason), Some(reason.clone())), RunState::PausedByUser => ("Paused: nothing syncs until you resume".into(), None), RunState::Stopping => ("Stopping…".into(), None), RunState::SignedOut => ("Not signed in".into(), None), @@ -53,6 +54,7 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: events.push(Event::RetryAutostart); } } + storage(ui, o, events); ui.add_space(12.0); ui.horizontal(|ui| { @@ -69,6 +71,28 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: if super::wide_button(ui, true, "Add a computer") { events.push(Event::AddComputer); } + ui.add_space(4.0); + ui.horizontal(|ui| { + if ui.button("Security…").on_hover_text("Two-factor sign-in, your password, your other computers").clicked() { + events.push(Event::OpenSecurity); + } + if ui.button("Activity").on_hover_text("What happened to your account: sign-ins, password changes, computers signed out").clicked() { + events.push(Event::Audit(crate::audit::AuditMsg::OpenActivity)); + } + if matches!(o.access, Access::Admin { .. }) + && ui.button("Administration").on_hover_text("Accounts, invites, computers and the audit log").clicked() + { + events.push(Event::Admin(AdminEvent::Open)); + } + }); + ui.horizontal(|ui| { + if ui.button("File versions…").clicked() { + events.push(Event::OpenVersions); + } + if ui.button("Trash…").clicked() { + events.push(Event::OpenTrash); + } + }); ui.add_space(14.0); super::computers::list(ui, o, now_ms, events); @@ -91,3 +115,79 @@ pub(super) fn show(ui: &mut egui::Ui, o: &Overview, busy: Option, now_ms: } }); } + +fn storage(ui: &mut egui::Ui, o: &Overview, events: &mut Vec) { + ui.group(|ui| { + ui.horizontal(|ui| { + ui.strong("Storage"); + if ui.add_enabled(!o.usage_loading, egui::Button::new("Refresh")).clicked() { + events.push(Event::RefreshUsage); + } + }); + if o.usage_loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading storage usage…"); + }); + } else if let Some(error) = &o.usage_error { + super::error_line(ui, "Storage unavailable"); + ui.label(egui::RichText::new("Details").small().weak()).on_hover_text(error); + } else if let Some(usage) = &o.usage { + usage_lines(ui, usage); + } + }); +} + +fn usage_lines(ui: &mut egui::Ui, usage: &AccountUsage) { + match usage.quota_bytes { + Some(quota) => { + ui.label(format!("{} of {} used", bytes(displayed_total_bytes(usage)), bytes(quota))); + ui.add( + egui::ProgressBar::new(if quota == 0 { 0.0 } else { (displayed_total_bytes(usage) as f64 / quota as f64).min(1.0) as f32 }) + .show_percentage(), + ); + } + None => { + ui.label(format!("{} used · no limit", bytes(displayed_total_bytes(usage)))); + } + }; + if let (true, Some(trash), Some(history)) = (usage.estimates.index_complete, usage.trash_bytes, usage.history_bytes) { + ui.label(format!("about {} in trash", bytes(trash))); + ui.label(format!("about {} in old versions", bytes(history))) + .on_hover_text("Trash and old-version estimates overlap; they are not added to stored bytes."); + } else { + ui.label("Estimates not available yet"); + } +} + +fn displayed_total_bytes(usage: &AccountUsage) -> u64 { + usage.stored_bytes +} + +fn bytes(value: u64) -> String { + const UNITS: [&str; 4] = ["B", "KiB", "MiB", "GiB"]; + let mut amount = value as f64; + let mut unit = 0; + while amount >= 1024.0 && unit < UNITS.len() - 1 { + amount /= 1024.0; + unit += 1; + } + if unit == 0 { format!("{value} B") } else { format!("{amount:.1} {}", UNITS[unit]) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trash_and_history_are_never_added_to_the_stored_total() { + let usage = AccountUsage { + stored_bytes: 11, + quota_bytes: None, + trash_bytes: Some(20), + history_bytes: Some(30), + estimates: craft_cloud_proto::api::UsageEstimates { index_complete: true, reason: None }, + }; + assert_eq!(displayed_total_bytes(&usage), 11); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/second_factor.rs b/apps/craft-cloud-desktop/src/ui/second_factor.rs new file mode 100644 index 00000000..b77d9ccc --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/second_factor.rs @@ -0,0 +1,38 @@ +//! The code a join asks for when the account has two-factor sign-in: the six digits from the authenticator app, or a recovery +//! code instead. + +use egui::RichText; + +use crate::model::Event; +use crate::security::SecondFactorForm; + +pub(super) fn show(ui: &mut egui::Ui, form: &mut SecondFactorForm, events: &mut Vec) { + super::heading(ui, "Enter your code"); + ui.label("This account has two-factor sign-in."); + ui.add_space(8.0); + ui.label(if form.recovery { "Recovery code" } else { "Code from the authenticator app" }); + let hint = if form.recovery { "ABCD-EFGH-JKMN-PQRS" } else { "123456" }; + let response = ui.add( + egui::TextEdit::singleline(&mut form.code).id(super::security::secret_id("second-factor-code")).hint_text(hint).desired_width(f32::INFINITY), + ); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(4.0); + let toggle = if form.recovery { "Use a code from the app" } else { "Use a recovery code" }; + if ui.checkbox(&mut form.recovery, toggle).changed() { + form.code.clear(); + form.error = None; + } + ui.label(RichText::new("A recovery code works once. Your invite or code isn't used up until this works.").small().weak()); + ui.add_space(10.0); + if let Some(error) = &form.error { + super::error_line(ui, error); + ui.add_space(8.0); + } + if super::wide_button(ui, true, "Join") || enter { + events.push(Event::SubmitSecondFactor); + } + ui.add_space(4.0); + if ui.button("Back").clicked() { + events.push(Event::CancelSecondFactor); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/security.rs b/apps/craft-cloud-desktop/src/ui/security.rs new file mode 100644 index 00000000..6a5ee327 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/security.rs @@ -0,0 +1,269 @@ +//! Security: two-factor sign-in (turn it on with a secret and a QR code, the recovery codes shown once, turn it off, new +//! recovery codes), the password, and signing the other computers out. + +use egui::RichText; + +use crate::model::Event; +use crate::security::{CodesWhy, SecurityEvent, SecurityPage, Step, TwoFactorInfo}; + +fn send(events: &mut Vec, event: SecurityEvent) { + events.push(Event::Security(event)); +} + +/// The id of a secret field of the page (a password or a code), by name. +pub(super) fn secret_id(name: &str) -> egui::Id { + egui::Id::new(("security-secret", name)) +} + +/// Every secret field of the page and the second step of a join: its name, and the text the model holds for it in `screen` +/// (empty when the field isn't there). +fn secrets(screen: &crate::model::Screen) -> Vec<(&'static str, &str)> { + use crate::model::Screen; + match screen { + Screen::Security(page) => match &page.step { + Step::SetupPassword { password } => vec![("setup-password", password)], + Step::SetupCode { code, .. } => vec![("setup-code", code)], + Step::Disable { password, code, .. } => vec![("disable-password", password), ("disable-code", code)], + Step::Regenerate { password, code } => vec![("regenerate-password", password), ("regenerate-code", code)], + Step::Password { current, new, again } => vec![("password-current", current), ("password-new", new), ("password-again", again)], + Step::Home | Step::Codes { .. } | Step::SignOutOthers => Vec::new(), + }, + Screen::SecondFactor(form) => vec![("second-factor-code", &form.code)], + // The admin's confirmation: only while it is open to be typed in. + Screen::Admin(a) if a.code.is_none() => match &a.dialog { + Some(d) if !d.busy => { + vec![(super::admin::PASSWORD_FIELD, &d.confirmation.password), (super::admin::CODE_FIELD, &d.confirmation.code)] + } + _ => Vec::new(), + }, + _ => Vec::new(), + } +} + +const SECRET_NAMES: [&str; 12] = [ + "setup-password", + "setup-code", + "disable-password", + "disable-code", + "regenerate-password", + "regenerate-code", + "password-current", + "password-new", + "password-again", + "second-factor-code", + super::admin::PASSWORD_FIELD, + super::admin::CODE_FIELD, +]; + +/// Drops what egui remembers of every secret field (its cursor and, above all, its undo history, which holds the text as it +/// was typed) unless the model holds something typed in that field right now. So once a password or code was sent, a step was +/// cancelled or the page left, Undo can't bring it back; called every frame, before the fields are drawn. egui keeps this state +/// in memory only (this app doesn't enable eframe's persistence, and the undo history is not serialized either). +pub(super) fn forget_secrets(ctx: &egui::Context, screen: &crate::model::Screen) { + let typed = secrets(screen); + for name in SECRET_NAMES { + if !typed.iter().any(|(n, text)| *n == name && !text.is_empty()) { + ctx.data_mut(|d| d.remove::(secret_id(name))); + } + } +} + +/// A password field. +fn password_field(ui: &mut egui::Ui, name: &str, label: &str, text: &mut String) -> egui::Response { + ui.label(label); + let response = ui.add(egui::TextEdit::singleline(text).id(secret_id(name)).password(true).desired_width(f32::INFINITY)); + ui.add_space(6.0); + response +} + +pub(super) fn show(ui: &mut egui::Ui, page: &mut SecurityPage, events: &mut Vec) { + super::heading(ui, "Security"); + let (busy, error, note) = (page.busy, page.error.clone(), page.note.clone()); + let info = page.info.clone(); + match &mut page.step { + Step::Home => home(ui, &info, busy, note.as_deref(), error.as_deref(), events), + Step::SetupPassword { password } => { + ui.label( + "Two-factor sign-in asks for a code from an app on your phone (Aegis, 1Password, Google Authenticator…) as well as your password.", + ); + ui.add_space(8.0); + let response = password_field(ui, "setup-password", "Your password", password); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + footer(ui, busy, error.as_deref(), "Continue", enter, events); + } + Step::SetupCode { secret, uri, expires_ms: _, code } => { + ui.label("1. Scan this with your authenticator app, or type the secret into it."); + ui.add_space(6.0); + super::add_computer::qr(ui, uri, 180.0); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.add(egui::Label::new(RichText::new(secret.as_str()).monospace().size(16.0).strong()).selectable(true)); + if ui.small_button("Copy").clicked() { + ui.ctx().copy_text(secret.clone()); + } + }); + ui.collapsing("The setup link", |ui| { + ui.add(egui::Label::new(RichText::new(uri.as_str()).monospace().small()).selectable(true).wrap()); + if ui.small_button("Copy link").clicked() { + ui.ctx().copy_text(uri.clone()); + } + }); + ui.add_space(10.0); + ui.label("2. Type the six digits the app shows now."); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("setup-code")).hint_text("123456").desired_width(160.0)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Turn on", enter, events); + } + Step::Codes { codes, why } => { + let (line, button) = match why { + CodesWhy::TurnedOn => ("Two-factor sign-in is on.", "I have saved them"), + CodesWhy::New => ("These are your new recovery codes.", "I have saved them"), + }; + ui.label(RichText::new(line).strong().color(super::good(ui))); + ui.add_space(6.0); + ui.label("Save these recovery codes somewhere safe now. Each works once, if you lose your phone, and they are shown only this once."); + ui.add_space(8.0); + egui::Grid::new("recovery-codes").num_columns(2).spacing([16.0, 4.0]).show(ui, |ui| { + for (n, code) in codes.iter().enumerate() { + ui.add(egui::Label::new(RichText::new(code.as_str()).monospace().size(15.0)).selectable(true)); + if n % 2 == 1 { + ui.end_row(); + } + } + }); + ui.add_space(8.0); + if ui.button("Copy all").clicked() { + ui.ctx().copy_text(codes.join("\n")); + } + ui.add_space(10.0); + if super::wide_button(ui, true, button) { + send(events, SecurityEvent::Home); + } + } + Step::Disable { password, code, recovery } => { + ui.label("Turning it off signs your other computers out. Type your password and a code from the app."); + ui.add_space(8.0); + password_field(ui, "disable-password", "Your password", password); + ui.label(if *recovery { "A recovery code" } else { "The six digits from the app" }); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("disable-code")).desired_width(f32::INFINITY)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + if ui.small_button(if *recovery { "Use a code from the app" } else { "Use a recovery code instead" }).clicked() { + send(events, SecurityEvent::ToggleRecovery); + } + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Turn off", enter, events); + } + Step::Regenerate { password, code } => { + ui.label("New recovery codes replace the old ones at once. Type your password and a code from the app."); + ui.add_space(8.0); + password_field(ui, "regenerate-password", "Your password", password); + ui.label("The six digits from the app"); + let response = ui.add(egui::TextEdit::singleline(code).id(secret_id("regenerate-code")).desired_width(160.0)); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.add_space(6.0); + footer(ui, busy, error.as_deref(), "Make new codes", enter, events); + } + Step::Password { current, new, again } => { + ui.label("Changing your password signs your other computers out."); + ui.add_space(8.0); + password_field(ui, "password-current", "Current password", current); + password_field(ui, "password-new", "New password", new); + let response = password_field(ui, "password-again", "The new password again", again); + let enter = response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + footer(ui, busy, error.as_deref(), "Change password", enter, events); + } + Step::SignOutOthers => { + ui.label("Every other computer of your account is signed out. Their files stay there; they join again to sync."); + ui.add_space(8.0); + footer(ui, busy, error.as_deref(), "Sign them out", false, events); + } + } +} + +/// The step's error, its main button and the way back. +fn footer(ui: &mut egui::Ui, busy: bool, error: Option<&str>, button: &str, enter: bool, events: &mut Vec) { + if let Some(error) = error { + super::error_line(ui, error); + ui.add_space(6.0); + } + if busy { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Working…"); + }); + } else if super::wide_button(ui, true, button) || enter { + send(events, SecurityEvent::Submit); + } + ui.add_space(4.0); + if ui.add_enabled(!busy, egui::Button::new("Cancel")).clicked() { + send(events, SecurityEvent::Home); + } +} + +fn home(ui: &mut egui::Ui, info: &TwoFactorInfo, busy: bool, note: Option<&str>, error: Option<&str>, events: &mut Vec) { + if let Some(note) = note { + ui.label(RichText::new(note).color(super::good(ui))); + ui.add_space(8.0); + } + if let Some(error) = error { + super::error_line(ui, error); + ui.add_space(8.0); + } + ui.label(RichText::new("Two-factor sign-in").strong()); + match info { + TwoFactorInfo::Loading => { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Asking the server…"); + }); + } + TwoFactorInfo::Unsupported => { + ui.label(crate::security::UNSUPPORTED); + ui.label(RichText::new("Ask whoever runs your Craft Cloud server to update it.").small().weak()); + } + TwoFactorInfo::Failed(why) => { + super::error_line(ui, why); + if ui.button("Try again").clicked() { + send(events, SecurityEvent::Refresh); + } + } + TwoFactorInfo::Known(status) if status.enabled => { + ui.label(format!("On. {} recovery codes left.", status.recovery_codes_left)); + ui.add_space(4.0); + ui.horizontal(|ui| { + if ui.add_enabled(!busy, egui::Button::new("Turn off…")).clicked() { + send(events, SecurityEvent::StartDisable); + } + if ui.add_enabled(!busy, egui::Button::new("New recovery codes…")).clicked() { + send(events, SecurityEvent::StartRegenerate); + } + }); + } + TwoFactorInfo::Known(status) => { + ui.label("Off. Anyone who knows your password can sign in."); + if status.pending { + ui.label(RichText::new("A set-up was started and isn't finished.").small().weak()); + } + ui.add_space(4.0); + if ui.add_enabled(!busy, egui::Button::new("Set up two-factor sign-in…")).clicked() { + send(events, SecurityEvent::StartSetup); + } + } + } + ui.add_space(14.0); + ui.separator(); + ui.add_space(6.0); + ui.horizontal(|ui| { + if ui.button("Change password…").clicked() { + send(events, SecurityEvent::StartPassword); + } + if ui.button("Sign out other computers…").clicked() { + send(events, SecurityEvent::StartSignOutOthers); + } + }); + ui.add_space(14.0); + if super::wide_button(ui, true, "Back") { + send(events, SecurityEvent::Close); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/tests.rs b/apps/craft-cloud-desktop/src/ui/tests.rs index e059f6be..73145a87 100644 --- a/apps/craft-cloud-desktop/src/ui/tests.rs +++ b/apps/craft-cloud-desktop/src/ui/tests.rs @@ -6,7 +6,9 @@ //! [`TOLERANCE`] in a channel. use std::path::PathBuf; +use std::sync::OnceLock; +use craft_cloud_app::{AccountUsage, Error}; use craft_cloud_app::{DeviceId, DeviceInfo, PairingInfo, RunState, Status, Uuid}; use egui_kittest::Harness; use egui_kittest::kittest::Queryable; @@ -77,7 +79,19 @@ fn after(agent: &FakeAgent, events: Vec) -> Model { fn devices() -> Vec { vec![ - DeviceInfo { id: DeviceId::generate(), name: "lptp1".into(), created_ms: 0, last_used_ms: NOW - 5_000, revoked: false, current: true }, + DeviceInfo { + id: DeviceId::generate(), + name: "lptp1".into(), + created_ms: 0, + last_used_ms: NOW - 5_000, + revoked: false, + current: true, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }, DeviceInfo { id: DeviceId::generate(), name: "BEAST".into(), @@ -85,13 +99,37 @@ fn devices() -> Vec { last_used_ms: NOW - 3 * 3_600_000, revoked: false, current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, }, ] } +/// The one wgpu instance, adapter, device and queue every test renders with. Tests run on parallel threads, and +/// creating and dropping a D3D12 device (WARP) per harness at the same time crashed the test process with an access +/// violation. Each harness still gets a renderer of its own on the shared device. It is never dropped, so no device is +/// torn down while other threads render. Every kittest harness of the app, in any test module, must render with it. +pub(super) fn shared_wgpu() -> eframe::egui_wgpu::WgpuSetup { + static SHARED: OnceLock = OnceLock::new(); + let shared = SHARED.get_or_init(|| { + let state = + egui_kittest::wgpu::create_render_state(egui_kittest::wgpu::default_wgpu_setup(), eframe::egui_wgpu::RendererOptions::PREDICTABLE); + eframe::egui_wgpu::WgpuSetupExisting { + instance: state.instance.clone(), + adapter: state.adapter.clone(), + device: state.device.clone(), + queue: state.queue.clone(), + } + }); + eframe::egui_wgpu::WgpuSetup::Existing(shared.clone()) +} + /// The harness: the window drawn for the model, its events kept. fn harness(model: Model, theme: egui::Theme) -> Harness<'static, (Model, Vec)> { - Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu().build_ui_state( + Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu_setup(shared_wgpu()).build_ui_state( |ui, (model, events): &mut (Model, Vec)| { events.extend(show(ui, model, NOW)); }, @@ -107,6 +145,9 @@ fn snapshot(model: Model, theme: egui::Theme, name: &str) { let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("snapshots"); let path = dir.join(format!("{name}.png")); if std::env::var_os("UPDATE_SNAPSHOTS").is_some() { + if std::env::var("UPDATE_SNAPSHOT_ONLY").ok().is_some_and(|names| !names.split(',').any(|allowed| allowed == name)) { + return; + } std::fs::create_dir_all(&dir).unwrap(); image.save(&path).unwrap(); return; @@ -155,11 +196,15 @@ fn snapshot_join() { snapshot(model.clone(), egui::Theme::Light, "join"); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX"); snapshot(model.clone(), egui::Theme::Light, "joining"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "folder_has_files".into(), message: "has files".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { + code: "folder_has_files".into(), + message: "has files".into(), + name_taken: None, + }))); snapshot(model.clone(), egui::Theme::Light, "join-folder-has-files"); model.update(Event::SyncExistingFiles(false)); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "invalid_code".into(), message: "x".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "invalid_code".into(), message: "x".into(), name_taken: None }))); snapshot(model, egui::Theme::Dark, "join-invalid-code-dark"); } @@ -167,7 +212,7 @@ fn snapshot_join() { fn snapshot_password_needed() { let mut model = after(&FakeAgent::new(signed_out()), Vec::new()); sent(&mut model, "cloud.shadows.ch/K7M4-Q9TX-K7M4-Q9TX"); - model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "password_required".into(), message: "x".into() }))); + model.update(Event::Joined(Err(craft_cloud_app::Error::Refused { code: "password_required".into(), message: "x".into(), name_taken: None }))); if let Screen::PasswordNeeded(pw) = &mut model.screen { pw.password = "correct horse".into(); pw.again = "correct house".into(); @@ -182,6 +227,119 @@ fn signed_in_model(status: Status) -> Model { model } +fn usage(stored_bytes: u64, quota_bytes: Option, complete: bool) -> AccountUsage { + AccountUsage { + stored_bytes, + quota_bytes, + trash_bytes: complete.then_some(20), + history_bytes: complete.then_some(30), + estimates: craft_cloud_proto::api::UsageEstimates { + index_complete: complete, + reason: (!complete).then_some(craft_cloud_proto::api::UsageEstimateReason::IndexIncomplete), + }, + } +} + +#[test] +fn snapshot_storage_states() { + let with_quota = signed_in_with_usage(Ok(usage(11, Some(100), true))); + snapshot(with_quota, egui::Theme::Light, "storage-with-quota"); + + let no_quota = signed_in_with_usage(Ok(usage(11, None, true))); + snapshot(no_quota, egui::Theme::Light, "storage-no-quota"); + + let estimates_unavailable = signed_in_with_usage(Ok(usage(11, Some(100), false))); + snapshot(estimates_unavailable, egui::Theme::Light, "storage-estimates-unavailable"); + + let error = signed_in_with_usage(Err(Error::Refused { code: "offline".into(), message: "can't reach the server".into(), name_taken: None })); + snapshot(error, egui::Theme::Light, "storage-error"); +} + +fn signed_in_with_usage(result: Result) -> Model { + let agent = FakeAgent::new(signed_in()); + *agent.usage_result.lock().unwrap() = Some(result); + after(&agent, Vec::new()) +} + +#[test] +fn storage_refresh_requests_a_fresh_server_answer() { + let mut model = signed_in_model(signed_in()); + assert_eq!(model.update(Event::RefreshUsage), vec![Effect::Usage(2)]); + let Screen::SignedIn(overview) = model.screen else { panic!("signed-in screen expected") }; + assert!(overview.usage_loading); + assert_eq!(overview.usage_error, None); +} + +#[test] +fn storage_refresh_click_sends_a_request_and_loading_blocks_duplicates() { + let mut h = harness(signed_in_model(signed_in()), egui::Theme::Light); + h.run_steps(2); + h.query_all_by_label("Refresh").next().unwrap().click(); + h.run_steps(2); + let (model, events) = h.state_mut(); + assert!(events.contains(&Event::RefreshUsage)); + assert_eq!(model.update(Event::RefreshUsage), vec![Effect::Usage(2)]); + assert!(model.update(Event::RefreshUsage).is_empty(), "already loading blocks repeat request"); +} + +#[test] +fn stale_refresh_answer_cannot_replace_a_newer_answer() { + let mut model = Model::new(defaults(), true, String::new()); + let first = model + .update(Event::Found(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("signed-in overview asks for usage"); + model.update(Event::Usage { request_id: first, result: Ok(usage(11, None, true)) }); + let second = model + .update(Event::RefreshUsage) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("refresh asks again"); + model.update(Event::Usage { request_id: second, result: Ok(usage(25, None, true)) }); + model.update(Event::Usage { + request_id: first, + result: Err(Error::Refused { code: "offline".into(), message: "old error".into(), name_taken: None }), + }); + let Screen::SignedIn(overview) = &model.screen else { panic!("signed-in screen expected") }; + assert!(!overview.usage_loading); + assert_eq!(overview.usage_error, None); + assert_eq!(overview.usage.as_ref().map(|value| value.stored_bytes), Some(25)); +} + +#[test] +fn usage_answer_from_previous_sign_in_cannot_replace_new_account_state() { + let mut model = Model::new(defaults(), true, String::new()); + let old_request = model + .update(Event::Found(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("signed-in overview asks for usage"); + model.update(Event::Polled(Ok(signed_out()))); + let new_request = model + .update(Event::Polled(Ok(signed_in()))) + .into_iter() + .find_map(|effect| match effect { + Effect::Usage(id) => Some(id), + _ => None, + }) + .expect("new sign-in asks for usage"); + model.update(Event::Usage { request_id: new_request, result: Ok(usage(20, None, true)) }); + model.update(Event::Usage { request_id: old_request, result: Ok(usage(999, None, true)) }); + let Screen::SignedIn(overview) = &model.screen else { panic!("signed-in screen expected") }; + assert!(!overview.usage_loading); + assert_eq!(overview.usage.as_ref().map(|value| value.stored_bytes), Some(20)); +} + #[test] fn snapshot_signed_in() { snapshot(signed_in_model(signed_in()), egui::Theme::Light, "signed-in"); @@ -193,6 +351,147 @@ fn snapshot_signed_in() { snapshot(signed_in_model(paused), egui::Theme::Light, "signed-in-paused"); } +fn versions_model(loaded: bool) -> Model { + let mut model = signed_in_model(signed_in()); + model.update(Event::OpenVersions); + model.update(Event::VersionsPath("Projects/painting.png".into())); + if loaded { + let versions = vec![ + craft_cloud_app::HistoryVersion { + id: craft_cloud_app::VersionId::from_uuid(Uuid::nil()), + parent_version: None, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + seq: 9, + size: 42_000, + mtime_ms: NOW, + device: DeviceId::from_uuid(Uuid::nil()), + created_ms: NOW - 30_000, + save: false, + pruned: false, + }, + craft_cloud_app::HistoryVersion { + id: craft_cloud_app::VersionId::from_uuid(Uuid::from_u128(1)), + parent_version: None, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + seq: 8, + size: 38_000, + mtime_ms: NOW - 60_000, + device: DeviceId::from_uuid(Uuid::from_u128(1)), + created_ms: NOW - 60_000, + save: true, + pruned: false, + }, + ]; + model.update(Event::History { ticket: 1, result: Ok(craft_cloud_app::HistoryResponse { versions, next: Some("next-page".into()) }) }); + } + model +} + +#[test] +fn snapshot_versions() { + snapshot(versions_model(false), egui::Theme::Light, "versions-empty"); + snapshot(versions_model(true), egui::Theme::Light, "versions-loaded"); + let mut model = versions_model(true); + if let Screen::Versions(versions) = &mut model.screen { + versions.confirm = versions.page.as_ref().and_then(|page| page.versions.get(1)).map(|version| version.id); + } + snapshot(model, egui::Theme::Light, "versions-confirm"); +} + +fn trash_model(loaded: bool) -> Model { + let mut model = signed_in_model(signed_in()); + let effects = model.update(Event::OpenTrash); + let ticket = match effects.first() { + Some(Effect::Trash { ticket, .. }) => *ticket, + _ => 0, + }; + let items = if loaded { + vec![ + craft_cloud_app::TrashItem { + id: craft_cloud_app::NodeId::from_uuid(Uuid::nil()), + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(1)), + name: "painting.png".into(), + is_dir: false, + size: Some(42_000), + deleted_ms: NOW - 30_000, + purge_after_ms: NOW + 14 * 86_400_000, + path: Some("Projects/painting.png".into()), + }, + craft_cloud_app::TrashItem { + id: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(2)), + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(3)), + name: "Old exports".into(), + is_dir: true, + size: None, + deleted_ms: NOW - 86_400_000, + purge_after_ms: NOW + 13 * 86_400_000, + path: None, + }, + ] + } else { + Vec::new() + }; + model.update(Event::TrashPage { ticket, result: Ok(craft_cloud_app::TrashResponse { items, next: loaded.then(|| "trash-next".into()) }) }); + model +} + +#[test] +fn snapshot_trash() { + snapshot(trash_model(false), egui::Theme::Light, "trash-empty"); + snapshot(trash_model(true), egui::Theme::Light, "trash-loaded"); + let mut model = trash_model(true); + let id = match &model.screen { + Screen::Trash(t) => t.page.as_ref().and_then(|p| p.items.first()).map(|i| i.id), + _ => None, + }; + if let Some(id) = id { + model.update(Event::ConfirmRemoveTrash(id)); + snapshot(model.clone(), egui::Theme::Light, "trash-confirm"); + model.update(Event::CancelRemoveTrash); + let effect = model.update(Event::RestoreTrash(id)); + let ticket = match effect.first() { + Some(Effect::RestoreTrash { ticket, .. }) => *ticket, + _ => 0, + }; + model.update(Event::TrashRestored { + ticket, + result: Err(craft_cloud_app::Error::Refused { code: "conflict".into(), message: "name taken".into(), name_taken: Some(true) }), + }); + snapshot(model.clone(), egui::Theme::Light, "trash-name-taken"); + let restored = craft_cloud_app::TrashRestoreResponse { + node: id, + parent: craft_cloud_app::NodeId::from_uuid(Uuid::from_u128(1)), + name: "painting (1).png".into(), + renamed: true, + version: None, + seq: 2, + epoch: craft_cloud_proto::Epoch::from_uuid(Uuid::nil()), + }; + let effect = model.update(Event::RestoreTrash(id)); + let ticket = match effect.first() { + Some(Effect::RestoreTrash { ticket, .. }) => *ticket, + _ => 0, + }; + model.update(Event::TrashRestored { ticket, result: Ok(restored) }); + snapshot(model, egui::Theme::Light, "trash-restored"); + } else { + snapshot(model, egui::Theme::Light, "trash-confirm"); + } +} + +#[test] +fn trash_buttons_send_restore_and_confirmation_events() { + let mut h = harness(trash_model(true), egui::Theme::Light); + h.run_steps(2); + h.query_all_by_label("Restore").next().unwrap().click(); + h.run_steps(2); + h.query_all_by_label("Remove from trash").next().unwrap().click(); + h.run_steps(2); + let events = std::mem::take(&mut h.state_mut().1); + assert!(events.iter().any(|e| matches!(e, Event::RestoreTrash(_))), "{events:?}"); + assert!(events.contains(&Event::ConfirmRemoveTrash(craft_cloud_app::NodeId::from_uuid(Uuid::nil()))), "{events:?}"); +} + #[test] fn snapshot_adding_a_computer() { let mut model = signed_in_model(signed_in()); @@ -214,12 +513,16 @@ fn snapshot_adding_a_computer() { #[test] fn snapshot_removed_and_error() { - let removed = Status { removed: true, run_state: RunState::Paused { reason: "the server no longer accepts this device".into() }, ..signed_in() }; + let removed = Status { + removed: true, + run_state: RunState::Paused { reason: "the server no longer accepts this device".into(), reason_code: None }, + ..signed_in() + }; let model = after(&FakeAgent::new(removed), Vec::new()); assert_eq!(model.screen, Screen::Removed); snapshot(model, egui::Theme::Light, "removed"); let damaged = Status { - run_state: RunState::Paused { reason: "/home/fabio/.local/share/craft-cloud-agent/account.json is damaged (expected value at line 1 column 2); move it away. Then sign in again".into() }, + run_state: RunState::Paused { reason: "/home/fabio/.local/share/craft-cloud-agent/account.json is damaged (expected value at line 1 column 2); move it away. Then sign in again".into(), reason_code: None }, ..signed_out() }; snapshot(after(&FakeAgent::new(damaged), Vec::new()), egui::Theme::Light, "error"); @@ -266,3 +569,541 @@ fn times_and_addresses_read_as_people_say_them() { let (line, _) = overview::sync_line(&Status { online: false, run_state: RunState::Retrying { message: "x".into() }, ..signed_in() }); assert_eq!(line, "Can't reach cloud.shadows.ch — trying again"); } + +// ---- the admin pages ---- + +use crate::admin::{Action, AdminEvent, CodeKind, RoleChoice, View}; +use crate::agent::fake::sample_user; +use craft_cloud_app::AdminUserInfo; + +/// An agent signed in as `fabio`, an admin, with a few more accounts. +fn admin_agent() -> FakeAgent { + let agent = FakeAgent::new(signed_in()); + agent.admin.store(true, std::sync::atomic::Ordering::SeqCst); + let account = |name: &str, f: &dyn Fn(&mut AdminUserInfo)| { + let mut user = sample_user(name, Uuid::now_v7()); + f(&mut user); + user + }; + *agent.users.lock().unwrap() = vec![ + account("fabio", &|u| { + u.role = "admin".into(); + u.devices_active = 2; + u.stored_bytes = 3_400_000_000; + }), + account("ana", &|u| { + u.devices_active = 1; + u.stored_bytes = 640_000_000; + u.quota_bytes = Some(5 * 1_073_741_824); + }), + account("ben", &|u| { + u.devices_active = 0; + u.disabled_ms = Some(1_789_000_000_000); + }), + account("cleo.m", &|u| { + u.devices_active = 0; + u.invite_pending = true; + }), + ]; + agent +} + +fn on_admin_pages(agent: &FakeAgent, more: Vec) -> Model { + let mut events = vec![Event::Devices(Ok(devices())), Event::Admin(AdminEvent::Open)]; + events.extend(more); + after(agent, events) +} + +fn id_of(model: &Model, name: &str) -> Uuid { + model.admin_screen().unwrap().list.users.iter().find(|u| u.username == name).unwrap().id +} + +fn password_typed(model: &mut Model, password: &str) { + let Screen::Admin(admin) = &mut model.screen else { panic!("{:?}", model.screen) }; + admin.dialog.as_mut().unwrap().confirmation.password = password.to_owned(); +} + +/// The model after the change asked for was sent and the fake answered it. +fn confirm_sent(model: &mut Model, agent: &FakeAgent) { + let effects = model.update(Event::Admin(AdminEvent::Confirm)); + let answer = call(agent, &no_start(), effects.into_iter().next().unwrap()).unwrap(); + model.update(answer); +} + +#[test] +fn snapshot_admin_pages() { + let agent = admin_agent(); + // The signed-in view offers the pages to an admin only. + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + assert!(model.is_admin()); + snapshot(model, egui::Theme::Light, "admin-signed-in"); + + let mut model = on_admin_pages(&agent, Vec::new()); + snapshot(model.clone(), egui::Theme::Light, "admin-users"); + snapshot(model.clone(), egui::Theme::Dark, "admin-users-dark"); + snapshot(model.clone(), egui::Theme::Light, "admin-storage"); + + model.update(Event::Admin(AdminEvent::NewUser)); + if let Screen::Admin(admin) = &mut model.screen + && let View::Create(form) = &mut admin.view + { + form.username = "dora".into(); + form.role = RoleChoice::Member; + } + snapshot(model, egui::Theme::Light, "admin-new-account"); + + let ana = id_of(&on_admin_pages(&agent, Vec::new()), "ana"); + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + snapshot(model.clone(), egui::Theme::Light, "admin-account"); + snapshot(model.clone(), egui::Theme::Light, "admin-account-usage"); + if let Screen::Admin(admin) = &mut model.screen + && let View::User(view) = &mut admin.view + { + view.usage = Some(craft_cloud_app::AccountUsage { + stored_bytes: 640_000_000, + quota_bytes: Some(5 * 1_073_741_824), + trash_bytes: None, + history_bytes: None, + estimates: craft_cloud_app::UsageEstimates { index_complete: false, reason: Some(craft_cloud_app::UsageEstimateReason::IndexIncomplete) }, + }); + view.usage_loading = false; + } + snapshot(model.clone(), egui::Theme::Light, "admin-account-usage-unavailable"); + + // A change asks for the password; the wrong one is said in the words of the server's refusal. + model.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + password_typed(&mut model, "not-my-password"); + snapshot(model.clone(), egui::Theme::Light, "admin-confirm"); + let error = craft_cloud_app::Error::Refused { code: "wrong_password".into(), message: "the password is wrong".into(), name_taken: None }; + agent.admin_fails.lock().unwrap().push(error); + confirm_sent(&mut model, &agent); + snapshot(model, egui::Theme::Light, "admin-confirm-wrong-password"); + + // The invite a new account got, shown once. + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::NewUser)]); + if let Screen::Admin(admin) = &mut model.screen + && let View::Create(form) = &mut admin.view + { + form.username = "dora".into(); + } + model.update(Event::Admin(AdminEvent::SubmitCreate)); + password_typed(&mut model, "admin-secret-pw"); + confirm_sent(&mut model, &agent); + let code = model.admin_screen().unwrap().code.clone().unwrap(); + assert_eq!((code.username.as_str(), code.kind), ("dora", CodeKind::Invite)); + snapshot(model, egui::Theme::Light, "admin-invite-code"); +} + +#[test] +fn account_estimates_are_not_added_to_stored_total() { + let agent = admin_agent(); + let ana = id_of(&on_admin_pages(&agent, Vec::new()), "ana"); + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(4); + assert_eq!(h.query_all_by_label("Stored: 610 MiB").count(), 1, "estimates stay separate from stored bytes"); + assert_eq!(h.query_all_by_label("Trash estimate (overlapping): 76.3 MiB").count(), 1); + assert_eq!(h.query_all_by_label("History estimate (overlapping): 229 MiB").count(), 1); +} + +/// A member never sees the way in. +#[test] +fn a_member_has_no_administration_button() { + let agent = admin_agent(); + agent.admin.store(false, std::sync::atomic::Ordering::SeqCst); + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + assert!(!model.is_admin()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Administration").count(), 0); +} + +/// The admin pages' buttons send the events they stand for. +#[test] +fn admin_buttons_send_their_events() { + let agent = admin_agent(); + let model = after(&agent, vec![Event::Devices(Ok(devices()))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Administration").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::Open)]); + + let model = on_admin_pages(&agent, Vec::new()); + let ana = id_of(&model, "ana"); + let fabio = id_of(&model, "fabio"); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("New account").click(); + h.run_steps(2); + h.get_by_label("Search").click(); + h.run_steps(2); + h.get_by_label("Refresh").click(); + h.run_steps(2); + // One Manage per account. + assert_eq!(h.query_all_by_label("Manage").count(), 4); + h.query_all_by_label("Manage").nth(1).unwrap().click(); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + let events = std::mem::take(&mut h.state_mut().1); + assert_eq!( + events, + [ + Event::Admin(AdminEvent::NewUser), + Event::Admin(AdminEvent::Search), + Event::Admin(AdminEvent::Reload), + Event::Admin(AdminEvent::OpenUser(ana)), + Event::Admin(AdminEvent::Close), + ] + ); + + // Ana is enabled and has a computer: the buttons ask for the password of their change. + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + for label in ["Save changes", "Disable", "Remove", "New invite", "Password reset code", "Revoke open code", "Revoke"] { + h.get_by_label(label).click(); + h.run_steps(2); + } + let events = std::mem::take(&mut h.state_mut().1); + let asked: Vec<&Action> = events.iter().filter_map(|e| if let Event::Admin(AdminEvent::Ask(a)) = e { Some(a) } else { None }).collect(); + assert!( + matches!( + asked.as_slice(), + [ + Action::Disable { .. }, + Action::Remove { .. }, + Action::IssueInvite { .. }, + Action::IssueReset { .. }, + Action::RevokeInvite { .. }, + Action::RevokeDevice { .. } + ] + ), + "{events:?}" + ); + assert!(events.contains(&Event::Admin(AdminEvent::SubmitEdit))); + // Your own account has no Disable or Remove. + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(fabio))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Disable").count() + h.query_all_by_label("Remove").count(), 0); + + // The password page: the confirm button and Cancel. + let mut model = on_admin_pages(&agent, Vec::new()); + model.update(Event::Admin(AdminEvent::Ask(Action::Enable { id: ana, username: "ana".into() }))); + password_typed(&mut model, "pw"); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Enable").click(); + h.run_steps(2); + h.get_by_label("Cancel").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::Confirm), Event::Admin(AdminEvent::CancelDialog)]); + + // A code shown: Done forgets it. + let mut model = on_admin_pages(&agent, Vec::new()); + model.update(Event::Admin(AdminEvent::Ask(Action::IssueReset { id: ana, username: "ana".into(), reset_two_factor: false }))); + password_typed(&mut model, "pw"); + confirm_sent(&mut model, &agent); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Done").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Admin(AdminEvent::DismissCode)]); +} + +/// egui keeps a text field's undo history, which holds what was typed: for the password field it is dropped once the +/// password was sent, or the page cancelled or left, so Undo can't bring a password back to a field opened later. +#[test] +fn a_typed_password_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + use egui::text_edit::TextEditState; + + let agent = admin_agent(); + let model = on_admin_pages(&agent, Vec::new()); + let ana = id_of(&model, "ana"); + let id = super::security::secret_id(super::admin::PASSWORD_FIELD); + let ask = |h: &mut Harness<'static, (Model, Vec)>| { + h.state_mut().0.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + h.run_steps(3); + }; + // Whether egui could still bring typed text back: its state for the field has an undo history for an empty field. + let remembered = |h: &Harness<'static, (Model, Vec)>| { + TextEditState::load(&h.ctx, id).is_some_and(|state| state.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()) + }; + let password = + |h: &Harness<'static, (Model, Vec)>| h.state().0.admin_screen().unwrap().dialog.as_ref().unwrap().confirmation.password.clone(); + let undo = |h: &mut Harness<'static, (Model, Vec)>| { + h.get_by_role(Role::PasswordInput).focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); + }; + + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert!(!remembered(&h)); + ask(&mut h); + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("hunter2-secret"); + // Long enough for egui to settle the typing into an undo point. + h.run_steps(8); + assert_eq!(password(&h), "hunter2-secret"); + assert!(remembered(&h), "while it is typed, egui has the field's state (the test sees what it guards against)"); + + // Sent, and refused as a wrong password: the dialog is open again, empty, and egui has forgotten the text. + let effects = h.state_mut().0.update(Event::Admin(AdminEvent::Confirm)); + assert!(matches!(effects.as_slice(), [Effect::AdminDo { .. }])); + h.run_steps(2); + assert!(!remembered(&h), "once sent"); + let refused = craft_cloud_app::Error::Refused { code: "wrong_password".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::Admin(AdminEvent::Done(Err(refused)))); + h.run_steps(3); + undo(&mut h); + assert_eq!(password(&h), "", "Undo brings nothing back"); + + // Typed again and cancelled; the same change asked for again: Undo still brings nothing back. + h.get_by_role(Role::PasswordInput).type_text("another-secret"); + h.run_steps(8); + assert!(remembered(&h)); + h.state_mut().0.update(Event::Admin(AdminEvent::CancelDialog)); + h.run_steps(2); + assert!(!remembered(&h), "once cancelled"); + ask(&mut h); + undo(&mut h); + assert_eq!(password(&h), ""); + + // Left with the password typed: the same. + h.get_by_role(Role::PasswordInput).type_text("third-secret"); + h.run_steps(3); + h.state_mut().0.update(Event::Admin(AdminEvent::Close)); + h.run_steps(2); + assert!(!remembered(&h), "once the pages are left"); +} + +/// The same for the confirmation's code field (a code from the app, or a recovery code), beside the password's: both are +/// forgotten once the confirmation was sent, cancelled or left, so Undo can't bring either back into a field opened later. +#[test] +fn a_typed_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + use egui::text_edit::TextEditState; + + let agent = admin_agent(); + agent.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + let model = on_admin_pages(&agent, Vec::new()); + assert!(matches!(model.overview().unwrap().access, crate::admin::Access::Admin { two_factor: true })); + let ana = id_of(&model, "ana"); + let ids = [super::security::secret_id(super::admin::PASSWORD_FIELD), super::security::secret_id(super::admin::CODE_FIELD)]; + let ask = |h: &mut Harness<'static, (Model, Vec)>| { + h.state_mut().0.update(Event::Admin(AdminEvent::Ask(Action::Disable { id: ana, username: "ana".into() }))); + h.run_steps(3); + }; + let remembered = |h: &Harness<'static, (Model, Vec)>, which: usize| { + TextEditState::load(&h.ctx, ids[which]) + .is_some_and(|state| state.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()) + }; + let typed = |h: &Harness<'static, (Model, Vec)>| { + let c = &h.state().0.admin_screen().unwrap().dialog.as_ref().unwrap().confirmation; + (c.password.clone(), c.code.clone()) + }; + let undo = |h: &mut Harness<'static, (Model, Vec)>, which: usize| { + h.get_all_by_role(Role::PasswordInput).nth(which).unwrap().focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); + }; + + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + ask(&mut h); + assert_eq!(h.query_all_by_role(Role::PasswordInput).count(), 2, "a password field and a code field"); + h.get_all_by_role(Role::PasswordInput).next().unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).next().unwrap().type_text("hunter2-secret"); + h.run_steps(2); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().type_text("918273"); + h.run_steps(8); + assert_eq!(typed(&h), ("hunter2-secret".to_owned(), "918273".to_owned())); + assert!(TextEditState::load(&h.ctx, ids[0]).is_some() && remembered(&h, 1), "while they are typed, egui has their state"); + + // Sent and refused (a wrong code): both fields are empty and egui has forgotten both texts; Undo brings nothing back. + let effects = h.state_mut().0.update(Event::Admin(AdminEvent::Confirm)); + assert!(matches!(effects.as_slice(), [Effect::AdminDo { .. }])); + h.run_steps(2); + assert!(!remembered(&h, 0) && !remembered(&h, 1), "once sent"); + let refused = craft_cloud_app::Error::Refused { code: "two_factor_invalid".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::Admin(AdminEvent::Done(Err(refused)))); + h.run_steps(3); + for which in [0, 1] { + undo(&mut h, which); + assert_eq!(typed(&h), (String::new(), String::new()), "Undo of field {which} brings nothing back"); + } + + // Typed again and cancelled: forgotten; the same change asked for again: nothing comes back. + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().focus(); + h.get_all_by_role(Role::PasswordInput).nth(1).unwrap().type_text("ABCD-EFGH-JKMN-PQRS"); + h.run_steps(8); + assert!(remembered(&h, 1)); + h.state_mut().0.update(Event::Admin(AdminEvent::CancelDialog)); + h.run_steps(2); + assert!(!remembered(&h, 1), "once cancelled"); + ask(&mut h); + undo(&mut h, 1); + assert_eq!(typed(&h).1, ""); +} + +// ---- the audit log, the account's own activity and the second factor's confirmation ---- + +use crate::audit::{AuditMsg, ExportState}; +use craft_cloud_proto::api::{AuditDetails, AuditEvent}; + +/// Events as the server writes them, newest first, with the variety the page tells in words. +fn audit_events() -> Vec { + let event = |n: u64, action: &str, outcome: &str, count: u64, details: AuditDetails| AuditEvent { + outcome: outcome.into(), + count, + details, + at_ms: NOW - (n as i64) * 3_600_000 - 5_000, + actor_user: (!action.starts_with("cli.") && !action.ends_with("login_failed")) + .then(|| Uuid::from_u128(0x0123_4567_89ab_cdef_0000_0000_0000_0000 + u128::from(n))), + target_kind: if action.starts_with("admin.") { "user".into() } else { String::new() }, + target_id: action.starts_with("admin.").then(|| Uuid::from_u128(0xfedc_ba98_7654_3210_0000_0000_0000_0000 + u128::from(n))), + ..FakeAgent::sample_event(n, action) + }; + vec![ + event( + 0, + "admin.role_changed", + "success", + 1, + AuditDetails { role_from: Some("member".into()), role_to: Some("admin".into()), ..Default::default() }, + ), + event(1, "auth.login_failed", "failure", 4, AuditDetails { step: Some("password".into()), ..Default::default() }), + event( + 2, + "auth.login", + "success", + 1, + AuditDetails { client_kind: Some("desktop".into()), platform: Some("windows".into()), ..Default::default() }, + ), + event(3, "device.revoked_others", "success", 1, AuditDetails { revoked: Some(2), ..Default::default() }), + event(4, "admin.quota_changed", "success", 1, AuditDetails { quota_bytes: Some(Some(5_368_709_120)), ..Default::default() }), + event(5, "account.two_factor_enabled", "success", 1, AuditDetails::default()), + ] +} + +fn audit_model(agent: &FakeAgent, more: Vec) -> Model { + *agent.audit_events.lock().unwrap() = audit_events(); + let mut events = vec![Event::Devices(Ok(devices())), Event::Admin(AdminEvent::Open), Event::Audit(AuditMsg::OpenAudit(None))]; + events.extend(more); + after(agent, events) +} + +fn view_of(model: &Model) -> &crate::audit::AuditView { + match &model.admin_screen().unwrap().view { + View::Audit(view) => view, + other => panic!("{other:?}"), + } +} + +#[test] +fn snapshot_audit_and_activity_pages() { + let agent = admin_agent(); + let mut model = audit_model(&agent, Vec::new()); + snapshot(model.clone(), egui::Theme::Light, "audit-log"); + snapshot(model.clone(), egui::Theme::Dark, "audit-log-dark"); + + // Narrowed to one account, with an export that the server cut at its limit, and one that failed. + let ana = id_of(&model, "ana"); + let mut narrowed = audit_model(&agent, vec![Event::Audit(AuditMsg::OpenAudit(Some((ana, "ana".into()))))]); + narrowed.update(Event::Audit(AuditMsg::Export)); + crate::audit::tests::picked(&mut narrowed, Some("C:/Users/fabio/craft-cloud-audit.ndjson")); + snapshot(narrowed.clone(), egui::Theme::Light, "audit-log-exporting"); + crate::audit::tests::finished(&mut narrowed, Ok(craft_cloud_app::ExportOutcome { events: 100_000, truncated: true })); + assert!(matches!(view_of(&narrowed).export, ExportState::Done { .. })); + snapshot(narrowed.clone(), egui::Theme::Light, "audit-log-export-truncated"); + narrowed.update(Event::Audit(AuditMsg::DismissExport)); + narrowed.update(Event::Audit(AuditMsg::Export)); + crate::audit::tests::picked(&mut narrowed, Some("out.ndjson")); + crate::audit::tests::finished(&mut narrowed, Err(craft_cloud_app::Error::Io("the export broke off: connection reset".into()))); + snapshot(narrowed, egui::Theme::Light, "audit-log-export-failed"); + + // The account's own activity, for a member too. + model.update(Event::Admin(AdminEvent::Close)); + *agent.audit_events.lock().unwrap() = audit_events(); + let member = FakeAgent::new(signed_in()); + *member.audit_events.lock().unwrap() = audit_events(); + let activity = after(&member, vec![Event::Devices(Ok(devices())), Event::Audit(AuditMsg::OpenActivity)]); + assert!(activity.activity_screen().is_some()); + snapshot(activity, egui::Theme::Light, "activity"); +} + +/// An account with two-factor sign-in: its page says so and offers to turn it off; confirming asks for a code too. +#[test] +fn snapshot_two_factor_in_the_admin_pages() { + let agent = admin_agent(); + agent.two_factor.store(true, std::sync::atomic::Ordering::SeqCst); + agent.users.lock().unwrap()[1].two_factor_enabled = true; + let ana = { + let model = on_admin_pages(&agent, Vec::new()); + assert!(matches!(model.overview().unwrap().access, crate::admin::Access::Admin { two_factor: true })); + id_of(&model, "ana") + }; + let mut model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + snapshot(model.clone(), egui::Theme::Light, "admin-account-two-factor"); + model.update(Event::Admin(AdminEvent::Ask(Action::ResetTwoFactor { id: ana, username: "ana".into() }))); + password_typed(&mut model, "not-my-password"); + if let Screen::Admin(admin) = &mut model.screen { + admin.dialog.as_mut().unwrap().confirmation.code = "123456".into(); + } + snapshot(model, egui::Theme::Light, "admin-confirm-two-factor"); +} + +/// The buttons of the audit and activity pages send what they stand for. +#[test] +fn audit_buttons_send_their_events() { + let agent = admin_agent(); + let model = audit_model(&agent, Vec::new()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Refresh").click(); + h.run_steps(2); + h.get_by_label("Export…").click(); + h.run_steps(2); + h.get_by_label("Back to the accounts").click(); + h.run_steps(2); + assert_eq!( + std::mem::take(&mut h.state_mut().1), + [Event::Audit(AuditMsg::Reload), Event::Audit(AuditMsg::Export), Event::Admin(AdminEvent::BackToList)] + ); + // A page of the account list opens the log, and an account's page opens its own part of it. + let model = on_admin_pages(&agent, Vec::new()); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Audit log").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenAudit(None))]); + let ana = id_of(&h.state().0, "ana"); + let model = on_admin_pages(&agent, vec![Event::Admin(AdminEvent::OpenUser(ana))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Activity of this account").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenAudit(Some((ana, "ana".into()))))]); + // The signed-in view has an Activity button for everyone; the activity page goes back. + let member = FakeAgent::new(signed_in()); + let model = after(&member, vec![Event::Devices(Ok(devices()))]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Administration").count(), 0); + h.get_by_label("Activity").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::OpenActivity)]); + let model = after(&member, vec![Event::Devices(Ok(devices())), Event::Audit(AuditMsg::OpenActivity)]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + assert_eq!(std::mem::take(&mut h.state_mut().1), [Event::Audit(AuditMsg::CloseActivity)]); +} diff --git a/apps/craft-cloud-desktop/src/ui/tests_security.rs b/apps/craft-cloud-desktop/src/ui/tests_security.rs new file mode 100644 index 00000000..df3ec244 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/tests_security.rs @@ -0,0 +1,629 @@ +//! The Security page and the join's code field: every step against the fake agent (the model, effect by effect), the screens as +//! kittest draws them (snapshots `second-factor*` and `security-*` in `tests/snapshots/`, written with `UPDATE_SNAPSHOTS=1`), +//! and that nothing secret outlives the screen that showed it. + +use std::path::PathBuf; + +use craft_cloud_app::{Error, JoinRequest, RunState, Status}; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; + +use super::*; +use crate::agent::fake::{CODE, FakeAgent, PASSWORD, RECOVERY}; +use crate::agent::{Agent, Starter, call}; +use crate::model::{Effect, JoinDefaults, Model, Screen}; +use crate::security::{SecurityEvent, Step, TwoFactorInfo}; + +const NOW: i64 = 1_790_000_000_000; +const TOLERANCE: u8 = 48; +const MAX_DIFFERENT: usize = 4; + +fn defaults() -> JoinDefaults { + JoinDefaults { folder: "/home/fabio/Craft Cloud".into(), device_name: "lptp1".into() } +} + +fn signed_in() -> Status { + Status { + api: 1, + signed_in: true, + account: Some("fabio".into()), + server: Some("https://cloud.shadows.ch".into()), + folder: Some("/home/fabio/Craft Cloud".into()), + machine: Some("lptp1".into()), + online: true, + run_state: RunState::Idle, + autostart: Some(true), + last_sync_ms: Some(NOW - 120_000), + removed: false, + } +} + +fn signed_out() -> Status { + Status { + signed_in: false, + account: None, + server: None, + folder: None, + machine: None, + online: false, + run_state: RunState::SignedOut, + ..signed_in() + } +} + +fn no_start() -> Starter { + std::sync::Arc::new(|_: &dyn Agent| Err("no agent in tests".to_owned())) +} + +/// Feeds `event` to the model and answers every effect with the fake, as the link does, until nothing is left. +fn feed(model: &mut Model, agent: &FakeAgent, event: Event) { + let start = no_start(); + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Some(answer) = call(agent, &start, effect) { + pending.extend(model.update(answer)); + } + } +} + +fn model_after(agent: &FakeAgent, events: Vec) -> Model { + let mut model = Model::new(defaults(), true, "run.log".into()); + for event in std::iter::once(Event::Start).chain(events) { + feed(&mut model, agent, event); + } + model +} + +fn on_security(agent: &FakeAgent) -> Model { + model_after(agent, vec![Event::OpenSecurity]) +} + +/// A folder that is a full path on every platform (the join form refuses anything else). +fn absolute_folder() -> String { + std::env::temp_dir().to_string_lossy().into_owned() +} + +fn page(model: &mut Model) -> &mut crate::security::SecurityPage { + match &mut model.screen { + Screen::Security(page) => page, + other => panic!("not the Security page: {other:?}"), + } +} + +fn type_into(model: &mut Model, fill: impl FnOnce(&mut Step)) { + fill(&mut page(model).step); +} + +// ---- the model ---- + +#[test] +fn the_page_asks_the_server_and_shows_what_it_says() { + let agent = FakeAgent::new(signed_in()); + let mut model = on_security(&agent); + assert_eq!( + page(&mut model).info, + TwoFactorInfo::Known(craft_cloud_app::TwoFactorStatus { enabled: false, enabled_ms: None, recovery_codes_left: 0, pending: false }) + ); + assert!(agent.asked.lock().unwrap().iter().any(|a| a == "two_factor_status")); + // A server from before two-factor sign-in: said so, the page doesn't break. + let old = FakeAgent::new(signed_in()); + old.factor.lock().unwrap().unsupported = true; + let mut model = on_security(&old); + assert_eq!(page(&mut model).info, TwoFactorInfo::Unsupported); + feed(&mut model, &old, Event::Security(SecurityEvent::StartSetup)); + assert_eq!(page(&mut model).step, Step::Home, "nothing to set up"); + // Back to the signed-in view. + feed(&mut model, &old, Event::Security(SecurityEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); +} + +#[test] +fn setting_it_up_turns_it_on_and_shows_the_recovery_codes_once() { + let agent = FakeAgent::new(signed_in()); + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartSetup)); + // Nothing typed: nothing is asked of the server. + let asked = agent.asked.lock().unwrap().len(); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.is_some()); + assert_eq!(agent.asked.lock().unwrap().len(), asked); + // A wrong password stays on the step, with the password gone from the field. + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = "wrong password!".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("That password isn't right.")); + assert!(matches!(&page(&mut model).step, Step::SetupPassword { password } if password.is_empty())); + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = PASSWORD.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + let Step::SetupCode { secret, uri, .. } = &page(&mut model).step else { panic!("{:?}", page(&mut model).step) }; + assert_eq!(secret, "JBSWY3DPEHPK3PXP"); + assert!(uri.starts_with("otpauth://totp/"), "{uri}"); + // A code that isn't six digits is told so; a wrong one is refused by the server and the secret stays. + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = "12345".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("Type the six digits the app shows.")); + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = "654 321".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.as_deref().is_some_and(|e| e.contains("isn't right")), "{:?}", page(&mut model).error); + assert!(matches!(&page(&mut model).step, Step::SetupCode { .. })); + type_into(&mut model, |step| { + if let Step::SetupCode { code, .. } = step { + *code = " 123 456 ".into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + let Step::Codes { codes, .. } = &page(&mut model).step else { panic!("{:?}", page(&mut model).step) }; + assert_eq!(codes.len(), 10); + assert!(page(&mut model).is_on(), "the status was asked again"); + // Done: the codes are dropped, and so is everything secret of the page. + feed(&mut model, &agent, Event::Security(SecurityEvent::Home)); + assert_eq!(page(&mut model).step, Step::Home); + let shown = format!("{:?}", model.screen); + assert!(!shown.contains("AAAA-BBBB") && !shown.contains("JBSWY"), "{shown}"); +} + +#[test] +fn leaving_the_page_drops_the_recovery_codes() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartRegenerate)); + type_into(&mut model, |step| { + if let Step::Regenerate { password, code } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::Codes { .. })); + feed(&mut model, &agent, Event::Security(SecurityEvent::Close)); + assert!(matches!(model.screen, Screen::SignedIn(_))); + let shown = format!("{:?}", model.screen); + assert!(!shown.contains("AAAA"), "{shown}"); + // Opened again, the page starts at its start with nothing shown. + feed(&mut model, &agent, Event::OpenSecurity); + assert_eq!(page(&mut model).step, Step::Home); +} + +#[test] +fn turning_it_off_takes_the_password_and_a_code_or_a_recovery_code() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartDisable)); + let set = |model: &mut Model, pw: &str, code: &str| { + type_into(model, |step| { + if let Step::Disable { password, code: c, .. } = step { + *password = pw.into(); + *c = code.into(); + } + }); + }; + set(&mut model, PASSWORD, "12"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("A code from the app is six digits.")); + set(&mut model, "wrong password!", CODE); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("That password isn't right.")); + // A lock says how long to wait, and nothing changes. + agent.factor.lock().unwrap().locked = true; + set(&mut model, PASSWORD, CODE); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("Too many wrong codes. Try again in 14 minutes.")); + assert!(agent.factor.lock().unwrap().enabled); + agent.factor.lock().unwrap().locked = false; + // A recovery code, once the field is switched. + feed(&mut model, &agent, Event::Security(SecurityEvent::ToggleRecovery)); + set(&mut model, PASSWORD, RECOVERY); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).step, Step::Home); + assert!(!page(&mut model).is_on()); + assert!(page(&mut model).note.as_deref().is_some_and(|n| n.contains("is off"))); + // Passwords never stay in the page. + assert!(!format!("{:?}", model.screen).contains(PASSWORD)); +} + +#[test] +fn the_password_and_the_other_computers() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().other_computers = 2; + let mut model = on_security(&agent); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartPassword)); + let set = |model: &mut Model, cur: &str, new: &str, again: &str| { + type_into(model, |step| { + if let Step::Password { current, new: n, again: a } = step { + *current = cur.into(); + *n = new.into(); + *a = again.into(); + } + }); + }; + set(&mut model, PASSWORD, "a brand new password", "another one"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("The two new passwords aren't the same.")); + set(&mut model, PASSWORD, "short", "short"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).error.as_deref(), Some("passwords are 8 to 1024 bytes")); + set(&mut model, PASSWORD, "a brand new password", "a brand new password"); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).note.as_deref().is_some_and(|n| n.starts_with("Password changed"))); + feed(&mut model, &agent, Event::Security(SecurityEvent::StartSignOutOthers)); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert_eq!(page(&mut model).note.as_deref(), Some("Signed 2 other computers out.")); + assert!(!format!("{:?}", model.screen).contains("brand new")); +} + +#[test] +fn a_join_for_an_account_with_two_factor_asks_for_the_code_and_keeps_the_invite() { + let agent = FakeAgent::new(signed_out()); + let required = Error::RefusedWith { + code: "two_factor_required".into(), + message: "needs a code".into(), + two_factor: Some(craft_cloud_app::TwoFactorChallenge { challenge: String::new(), expires_ms: 0 }), + retry_after_secs: None, + }; + agent.joins.lock().unwrap().push(Ok(signed_in())); + agent.joins.lock().unwrap().push(Err(Error::Refused { code: "two_factor_invalid".into(), message: "wrong".into(), name_taken: None })); + agent.joins.lock().unwrap().push(Err(required)); + let mut model = model_after(&agent, vec![]); + let Screen::Join(form) = &mut model.screen else { panic!("{:?}", model.screen) }; + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + let mut sent = Vec::new(); + let start = no_start(); + let run = |model: &mut Model, event: Event, sent: &mut Vec| { + let mut pending = model.update(event); + while let Some(effect) = pending.pop() { + if let Effect::Join(request) = &effect { + sent.push(request.clone()); + } + if let Some(answer) = call(&agent, &start, effect) { + pending.extend(model.update(answer)); + } + } + }; + run(&mut model, Event::SubmitJoin, &mut sent); + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert_eq!(second.error, None); + assert_eq!((sent[0].totp_code.clone(), sent[0].recovery_code.clone()), (None, None), "the first join carries no code"); + // Six digits are needed; the same join goes again with them. + second.code = "12".into(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert_eq!(second.error.as_deref(), Some("Type the six digits the app shows.")); + assert_eq!(sent.len(), 1); + second.code = "654 321".into(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + assert_eq!(sent.len(), 2); + assert_eq!( + (sent[1].join.as_str(), sent[1].totp_code.as_deref(), sent[1].recovery_code.as_deref()), + (sent[0].join.as_str(), Some("654321"), None) + ); + // The server said it was wrong: back at the field, with the reason, the code cleared. + let Screen::SecondFactor(second) = &mut model.screen else { panic!("{:?}", model.screen) }; + assert!(second.error.as_deref().is_some_and(|e| e.contains("isn't right")), "{:?}", second.error); + assert!(second.code.is_empty()); + // A recovery code instead. + second.recovery = true; + second.code = RECOVERY.into(); + *agent.status.lock().unwrap() = signed_in(); + run(&mut model, Event::SubmitSecondFactor, &mut sent); + assert_eq!(sent[2].recovery_code.as_deref(), Some(RECOVERY)); + assert!(matches!(model.screen, Screen::SignedIn(_)), "{:?}", model.screen); + // Back from the field goes to the join form, which keeps what was typed. + *agent.status.lock().unwrap() = signed_out(); + let mut again = model_after(&agent, vec![]); + if let Screen::Join(form) = &mut again.screen { + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + } + agent.joins.lock().unwrap().push(Err(Error::RefusedWith { + code: "two_factor_required".into(), + message: "m".into(), + two_factor: None, + retry_after_secs: None, + })); + run(&mut again, Event::SubmitJoin, &mut sent); + run(&mut again, Event::CancelSecondFactor, &mut sent); + assert!(matches!(&again.screen, Screen::Join(form) if form.join == "cloud.shadows.ch/K7M4-Q9TX")); +} + +// ---- the screens ---- + +fn harness(model: Model, theme: egui::Theme) -> Harness<'static, (Model, Vec)> { + Harness::builder().with_size(egui::Vec2::from(WINDOW_SIZE)).with_theme(theme).wgpu_setup(super::tests::shared_wgpu()).build_ui_state( + |ui, (model, events): &mut (Model, Vec)| { + events.extend(show(ui, model, NOW)); + }, + (model, Vec::new()), + ) +} + +fn snapshot(model: Model, name: &str) { + let mut h = harness(model, egui::Theme::Light); + h.run_steps(4); + let image = h.render().unwrap(); + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("snapshots"); + let path = dir.join(format!("{name}.png")); + if std::env::var_os("UPDATE_SNAPSHOTS").is_some() { + std::fs::create_dir_all(&dir).unwrap(); + image.save(&path).unwrap(); + return; + } + let new = dir.join(format!("{name}.new.png")); + let Ok(old) = image::open(&path) else { + image.save(&new).unwrap(); + panic!("no snapshot {} (UPDATE_SNAPSHOTS=1 writes it; this run's is {})", path.display(), new.display()); + }; + let old = old.into_rgba8(); + let same_size = old.dimensions() == image.dimensions(); + let different = old.pixels().zip(image.pixels()).filter(|(a, b)| a.0.iter().zip(b.0.iter()).any(|(x, y)| x.abs_diff(*y) > TOLERANCE)).count(); + let allowed = (image.width() * image.height()) as usize * MAX_DIFFERENT / 1000; + if !same_size || different > allowed { + image.save(&new).unwrap(); + panic!("{name}: {different} pixels differ (at most {allowed}); this run's is {}", new.display()); + } + let _ = std::fs::remove_file(new); +} + +#[test] +fn snapshot_second_factor() { + let agent = FakeAgent::new(signed_out()); + agent.joins.lock().unwrap().push(Err(Error::RefusedWith { + code: "two_factor_required".into(), + message: "m".into(), + two_factor: None, + retry_after_secs: None, + })); + let mut model = model_after(&agent, vec![]); + if let Screen::Join(form) = &mut model.screen { + form.join = "cloud.shadows.ch/K7M4-Q9TX".into(); + form.folder = absolute_folder(); + } + feed(&mut model, &agent, Event::SubmitJoin); + assert!(matches!(model.screen, Screen::SecondFactor(_))); + snapshot(model, "second-factor"); +} + +#[test] +fn snapshot_second_factor_with_a_wrong_code() { + let form = crate::security::SecondFactorForm::new( + crate::model::JoinForm { join: "cloud.shadows.ch/K7M4-Q9TX".into(), folder: "/home/fabio/Craft Cloud".into(), ..Default::default() }, + None, + Some("That code isn't right, or it was used already. Wait for the app's next code and try again.".into()), + ); + let mut model = Model::new(defaults(), true, String::new()); + model.screen = Screen::SecondFactor(form); + snapshot(model, "second-factor-wrong-code"); +} + +fn security_model(agent: &FakeAgent, events: Vec) -> Model { + let mut model = on_security(agent); + for event in events { + feed(&mut model, agent, event); + } + model +} + +#[test] +fn snapshot_security_off() { + snapshot(security_model(&FakeAgent::new(signed_in()), vec![]), "security-off"); +} + +#[test] +fn snapshot_security_on() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + snapshot(security_model(&agent, vec![]), "security-on"); +} + +#[test] +fn snapshot_security_unsupported() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().unsupported = true; + snapshot(security_model(&agent, vec![]), "security-unsupported"); +} + +#[test] +fn snapshot_security_setup_with_its_qr_code() { + let agent = FakeAgent::new(signed_in()); + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartSetup)]); + type_into(&mut model, |step| { + if let Step::SetupPassword { password } = step { + *password = PASSWORD.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::SetupCode { .. })); + snapshot(model, "security-setup"); +} + +#[test] +fn snapshot_security_recovery_codes() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartRegenerate)]); + type_into(&mut model, |step| { + if let Step::Regenerate { password, code } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(matches!(&page(&mut model).step, Step::Codes { .. })); + snapshot(model, "security-recovery-codes"); +} + +#[test] +fn snapshot_security_turning_it_off_when_locked() { + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let mut model = security_model(&agent, vec![Event::Security(SecurityEvent::StartDisable)]); + agent.factor.lock().unwrap().locked = true; + type_into(&mut model, |step| { + if let Step::Disable { password, code, .. } = step { + *password = PASSWORD.into(); + *code = CODE.into(); + } + }); + feed(&mut model, &agent, Event::Security(SecurityEvent::Submit)); + assert!(page(&mut model).error.is_some()); + snapshot(model, "security-disable-locked"); +} + +// ---- the buttons ---- + +#[test] +fn the_overview_has_a_security_button_and_the_page_its_buttons() { + let model = model_after(&FakeAgent::new(signed_in()), vec![]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Security…").click(); + h.run_steps(2); + assert!(h.state().1.contains(&Event::OpenSecurity), "{:?}", h.state().1); + + let agent = FakeAgent::new(signed_in()); + let mut h = harness(security_model(&agent, vec![]), egui::Theme::Light); + h.run_steps(2); + h.get_by_label("Set up two-factor sign-in…").click(); + h.run_steps(2); + h.get_by_label("Change password…").click(); + h.run_steps(2); + h.get_by_label("Back").click(); + h.run_steps(2); + let events = &h.state().1; + assert!(events.contains(&Event::Security(SecurityEvent::StartSetup)), "{events:?}"); + assert!(events.contains(&Event::Security(SecurityEvent::StartPassword)), "{events:?}"); + assert!(events.contains(&Event::Security(SecurityEvent::Close)), "{events:?}"); +} + +// ---- what egui remembers of what was typed ---- + +type H = Harness<'static, (Model, Vec)>; + +/// Whether egui still has state for the field: and whether Undo could bring typed text back into an empty field. +fn remembered(h: &H, name: &str) -> (bool, bool) { + use egui::text_edit::TextEditState; + let state = TextEditState::load(&h.ctx, super::security::secret_id(name)); + let undoable = state.as_ref().is_some_and(|s| s.undoer().undo(&(egui::text::CCursorRange::default(), String::new())).is_some()); + (state.is_some(), undoable) +} + +fn undo(h: &mut H, role: egui::accesskit::Role) { + h.get_by_role(role).focus(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Z); + h.run_steps(3); +} + +#[test] +fn a_typed_password_or_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + let agent = FakeAgent::new(signed_in()); + agent.factor.lock().unwrap().enabled = true; + let model = security_model(&agent, vec![Event::Security(SecurityEvent::StartDisable)]); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(3); + let typed = |h: &H| match &h.state().0.screen { + Screen::Security(page) => match &page.step { + Step::Disable { password, code, .. } => (password.clone(), code.clone()), + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + }; + + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("hunter2-secret"); + h.run_steps(8); + assert!(remembered(&h, "disable-password").1, "while typed, egui has the field's history (the test sees what it guards against)"); + h.get_by_role(Role::TextInput).focus(); + h.get_by_role(Role::TextInput).type_text("123456"); + h.run_steps(8); + assert_eq!(typed(&h), ("hunter2-secret".into(), "123456".into())); + assert!(remembered(&h, "disable-code").1, "and the code's"); + + // Sent: the model moved the typed text into the call and cleared the fields. + let effects = h.state_mut().0.update(Event::Security(SecurityEvent::Submit)); + assert!(matches!(effects.as_slice(), [Effect::Security(_)]), "{effects:?}"); + h.run_steps(2); + assert!(!remembered(&h, "disable-password").1, "once sent: no history (a field still drawn has an empty state)"); + assert!(!remembered(&h, "disable-code").1, "once sent: no history (a field still drawn has an empty state)"); + let refused = Error::Refused { code: "wrong_password".into(), message: "no".into(), name_taken: None }; + h.state_mut().0.update(Event::SecurityAnswer(crate::security::SecurityAnswer::Disable(Err(refused)))); + h.run_steps(3); + undo(&mut h, Role::PasswordInput); + undo(&mut h, Role::TextInput); + assert_eq!(typed(&h), (String::new(), String::new()), "Undo brings nothing back"); + + // Typed again and cancelled, then the step opened again. + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("another-secret"); + h.run_steps(8); + assert!(remembered(&h, "disable-password").1); + h.state_mut().0.update(Event::Security(SecurityEvent::Home)); + h.run_steps(2); + assert_eq!(remembered(&h, "disable-password"), (false, false), "once cancelled"); + h.state_mut().0.update(Event::Security(SecurityEvent::StartDisable)); + h.run_steps(3); + undo(&mut h, Role::PasswordInput); + assert_eq!(typed(&h).0, ""); + + // Left with something typed: the same. + h.get_by_role(Role::PasswordInput).focus(); + h.get_by_role(Role::PasswordInput).type_text("third-secret"); + h.run_steps(3); + h.state_mut().0.update(Event::Security(SecurityEvent::Close)); + h.run_steps(2); + assert_eq!(remembered(&h, "disable-password"), (false, false), "once the page is left"); +} + +#[test] +fn a_typed_join_code_is_not_left_in_eguis_undo_history() { + use egui::accesskit::Role; + let form = crate::security::SecondFactorForm::new( + crate::model::JoinForm { join: "cloud.shadows.ch/K7M4-Q9TX".into(), folder: absolute_folder(), ..Default::default() }, + None, + None, + ); + let mut model = Model::new(defaults(), true, String::new()); + model.screen = Screen::SecondFactor(form); + let mut h = harness(model, egui::Theme::Light); + h.run_steps(3); + h.get_by_role(Role::TextInput).focus(); + h.get_by_role(Role::TextInput).type_text("654321"); + h.run_steps(8); + assert!(remembered(&h, "second-factor-code").1, "while typed, egui has its history"); + // Back to the join form, then the field again. + h.state_mut().0.update(Event::CancelSecondFactor); + h.run_steps(2); + assert_eq!(remembered(&h, "second-factor-code"), (false, false), "once cancelled"); + let form = crate::security::SecondFactorForm::new(crate::model::JoinForm::default(), None, None); + h.state_mut().0.screen = Screen::SecondFactor(form); + h.run_steps(3); + undo(&mut h, Role::TextInput); + match &h.state().0.screen { + Screen::SecondFactor(form) => assert_eq!(form.code, "", "Undo brings nothing back"), + other => panic!("{other:?}"), + } + // A code sent (the model clears or leaves the screen): forgotten as well. + h.get_by_role(Role::TextInput).type_text("111111"); + h.run_steps(8); + h.state_mut().0.screen = Screen::Join(Default::default()); + h.run_steps(2); + assert_eq!(remembered(&h, "second-factor-code"), (false, false), "once the screen is left"); +} diff --git a/apps/craft-cloud-desktop/src/ui/trash.rs b/apps/craft-cloud-desktop/src/ui/trash.rs new file mode 100644 index 00000000..1f2ac02b --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/trash.rs @@ -0,0 +1,102 @@ +//! Browse deleted files and restore or remove them from the trash. + +use egui::RichText; + +use crate::model::{Event, Trash}; + +pub(super) fn show(ui: &mut egui::Ui, trash: &mut Trash, now_ms: i64, events: &mut Vec) { + super::heading(ui, "Trash"); + if ui.button("Back").clicked() { + events.push(Event::BackTrash); + } + if super::wide_button(ui, !trash.loading && trash.action.is_none(), "Refresh") { + events.push(Event::LoadTrash); + } + if trash.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading trash…"); + }); + } + if let Some(error) = &trash.error { + super::error_line(ui, error); + } + if let Some(notice) = &trash.notice { + ui.label(RichText::new(notice).color(super::good(ui))); + } + if let Some(page) = &trash.page { + for item in &page.items { + ui.group(|ui| { + ui.label(RichText::new(&item.name).strong()); + if let Some(path) = &item.path { + ui.label(path); + } + let kind = if item.is_dir { "Folder" } else { "File" }; + ui.label(match item.size { + Some(size) => format!("{kind} · {size} bytes"), + None => format!("{kind} · size unavailable"), + }); + ui.label(format!("Deleted {} · {}", super::ago(now_ms, item.deleted_ms), date(item.deleted_ms))); + ui.label(format!("Removed for good after {}", date(item.purge_after_ms))); + ui.horizontal(|ui| { + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Restore")).clicked() { + events.push(Event::RestoreTrash(item.id)); + } + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Remove from trash")).clicked() { + events.push(Event::ConfirmRemoveTrash(item.id)); + } + }); + }); + ui.add_space(4.0); + } + if page.items.is_empty() { + ui.label("Trash is empty."); + } + if page.next.is_some() && super::wide_button(ui, !trash.loading && trash.action.is_none(), "More") { + events.push(Event::NextTrash); + } + } + if trash.action.is_some() { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Updating trash…"); + }); + } + if trash.confirm.is_some() { + egui::Window::new("Remove from trash?").collapsible(false).resizable(false).show(ui.ctx(), |ui| { + ui.label("This item can no longer be restored from the trash."); + ui.horizontal(|ui| { + if ui.add_enabled(!trash.loading && trash.action.is_none(), egui::Button::new("Remove from trash")).clicked() { + events.push(Event::RemoveTrash); + } + if ui.button("Cancel").clicked() { + events.push(Event::CancelRemoveTrash); + } + }); + }); + } +} + +fn date(timestamp_ms: i64) -> String { + let days = timestamp_ms.div_euclid(86_400_000) + 719_468; + let era = if days >= 0 { days } else { days - 146_096 } / 146_097; + let day_of_era = days - era * 146_097; + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_part = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_part + 2) / 5 + 1; + let month = month_part + if month_part < 10 { 3 } else { -9 }; + let year = year + if month <= 2 { 1 } else { 0 }; + format!("{year:04}-{month:02}-{day:02}") +} + +#[cfg(test)] +mod tests { + #[test] + fn dates_use_utc_calendar_days() { + assert_eq!(super::date(0), "1970-01-01"); + assert_eq!(super::date(1_790_000_000_000), "2026-09-21"); + assert_eq!(super::date(-1), "1969-12-31"); + } +} diff --git a/apps/craft-cloud-desktop/src/ui/versions.rs b/apps/craft-cloud-desktop/src/ui/versions.rs new file mode 100644 index 00000000..dce5c7c5 --- /dev/null +++ b/apps/craft-cloud-desktop/src/ui/versions.rs @@ -0,0 +1,94 @@ +//! Browse and restore versions of a file inside the sync folder. + +use egui::RichText; + +use crate::model::{Event, Versions}; + +pub(super) fn show(ui: &mut egui::Ui, versions: &mut Versions, now_ms: i64, events: &mut Vec) { + super::heading(ui, "File versions"); + if ui.button("Back").clicked() { + events.push(Event::BackVersions); + } + ui.add_space(8.0); + ui.label("Path inside the sync folder"); + let mut path = versions.path.clone(); + let response = ui.add_enabled_ui(!versions.restoring, |ui| ui.add(egui::TextEdit::singleline(&mut path).hint_text("Projects/file.txt"))).inner; + if response.changed() { + events.push(Event::VersionsPath(path)); + } + if super::wide_button(ui, !versions.loading && !versions.restoring, "Load versions") { + events.push(Event::LoadVersions); + } + if versions.loading { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Loading versions…"); + }); + } + if let Some(error) = &versions.error { + super::error_line(ui, error); + } + if let Some(notice) = &versions.notice { + ui.label(RichText::new(notice).color(super::good(ui))); + } + if let Some(page) = &versions.page { + ui.add_space(8.0); + for version in &page.versions { + let current = Some(version.id) == versions.current_version; + ui.group(|ui| { + ui.label(RichText::new(if current { "Current version" } else { "Saved version" }).strong()); + ui.label(format!("{} · {} · {} bytes", date(version.created_ms), super::ago(now_ms, version.created_ms), version.size)); + ui.label(format!("Device {}", version.device)); + if version.pruned { + ui.label("This version can no longer be restored"); + } else if !current && ui.add_enabled(!versions.loading && !versions.restoring, egui::Button::new("Restore this version")).clicked() { + events.push(Event::ConfirmRestore(version.id)); + } + }); + ui.add_space(4.0); + } + if page.versions.is_empty() { + ui.label("No versions found."); + } + if page.next.is_some() && super::wide_button(ui, !versions.loading && !versions.restoring, "More versions") { + events.push(Event::NextVersions); + } + } + if let Some(version) = versions.confirm { + egui::Window::new("Restore version?").collapsible(false).resizable(false).show(ui.ctx(), |ui| { + ui.label("This makes the selected version the current file. Sync will update this computer."); + ui.horizontal(|ui| { + if ui.add_enabled(!versions.restoring && !versions.loading, egui::Button::new("Restore this version")).clicked() { + events.push(Event::RestoreVersion); + } + if ui.button("Cancel").clicked() { + events.push(Event::CancelRestore); + } + }); + let _ = version; + }); + } +} + +fn date(timestamp_ms: i64) -> String { + let days = timestamp_ms.div_euclid(86_400_000) + 719_468; + let era = if days >= 0 { days } else { days - 146_096 } / 146_097; + let day_of_era = days - era * 146_097; + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_part = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_part + 2) / 5 + 1; + let month = month_part + if month_part < 10 { 3 } else { -9 }; + let year = year + if month <= 2 { 1 } else { 0 }; + format!("{year:04}-{month:02}-{day:02}") +} + +#[cfg(test)] +mod tests { + #[test] + fn dates_use_utc_calendar_days() { + assert_eq!(super::date(0), "1970-01-01"); + assert_eq!(super::date(1_790_000_000_000), "2026-09-21"); + } +} diff --git a/apps/craft-cloud-desktop/tests/snapshots/activity.png b/apps/craft-cloud-desktop/tests/snapshots/activity.png new file mode 100644 index 00000000..7af536ac Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/activity.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png new file mode 100644 index 00000000..d9e5541f Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-two-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png new file mode 100644 index 00000000..bb61c09b Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage-unavailable.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png new file mode 100644 index 00000000..108f0e3d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account-usage.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-account.png b/apps/craft-cloud-desktop/tests/snapshots/admin-account.png new file mode 100644 index 00000000..108f0e3d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-account.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png new file mode 100644 index 00000000..8b561717 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-two-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png new file mode 100644 index 00000000..2910cddc Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm-wrong-password.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png new file mode 100644 index 00000000..914390c9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png b/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png new file mode 100644 index 00000000..a582dbdc Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-invite-code.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png b/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png new file mode 100644 index 00000000..9eb8a4d6 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-new-account.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png b/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png new file mode 100644 index 00000000..7031f82c Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-signed-in.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png b/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png new file mode 100644 index 00000000..899aedb9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-storage.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png b/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png new file mode 100644 index 00000000..6cd92ef5 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-users-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/admin-users.png b/apps/craft-cloud-desktop/tests/snapshots/admin-users.png new file mode 100644 index 00000000..899aedb9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/admin-users.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png new file mode 100644 index 00000000..51a0258c Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png new file mode 100644 index 00000000..517b829d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-failed.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png new file mode 100644 index 00000000..0894fd55 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-export-truncated.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png new file mode 100644 index 00000000..7c256de8 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log-exporting.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/audit-log.png b/apps/craft-cloud-desktop/tests/snapshots/audit-log.png new file mode 100644 index 00000000..d0d491d1 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/audit-log.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png b/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png new file mode 100644 index 00000000..ab5e2747 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/second-factor-wrong-code.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/second-factor.png b/apps/craft-cloud-desktop/tests/snapshots/second-factor.png new file mode 100644 index 00000000..1857d806 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/second-factor.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png b/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png new file mode 100644 index 00000000..cce9701e Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-disable-locked.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-off.png b/apps/craft-cloud-desktop/tests/snapshots/security-off.png new file mode 100644 index 00000000..4283d7dd Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-off.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-on.png b/apps/craft-cloud-desktop/tests/snapshots/security-on.png new file mode 100644 index 00000000..1399264e Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-on.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png b/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png new file mode 100644 index 00000000..a9215832 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-recovery-codes.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-setup.png b/apps/craft-cloud-desktop/tests/snapshots/security-setup.png new file mode 100644 index 00000000..4e4b7082 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-setup.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png b/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png new file mode 100644 index 00000000..ed75428d Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/security-unsupported.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png index b443f0bb..371c5428 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-dark.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png index 8a80cb23..8812a766 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-offline.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png index b6365788..0fccbc93 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in-paused.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/signed-in.png b/apps/craft-cloud-desktop/tests/snapshots/signed-in.png index aef3f674..8fd3c3eb 100644 Binary files a/apps/craft-cloud-desktop/tests/snapshots/signed-in.png and b/apps/craft-cloud-desktop/tests/snapshots/signed-in.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-error.png b/apps/craft-cloud-desktop/tests/snapshots/storage-error.png new file mode 100644 index 00000000..fdc3a154 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-error.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png b/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png new file mode 100644 index 00000000..bd504fdf Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-estimates-unavailable.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png b/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png new file mode 100644 index 00000000..3d395ff9 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-no-quota.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png b/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png new file mode 100644 index 00000000..92613785 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/storage-with-quota.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png new file mode 100644 index 00000000..7057fc91 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png b/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png new file mode 100644 index 00000000..388a63e0 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-empty.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png b/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png new file mode 100644 index 00000000..e4b47462 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-loaded.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png b/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png new file mode 100644 index 00000000..c78a5fd8 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-name-taken.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png b/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png new file mode 100644 index 00000000..35dba9e1 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/trash-restored.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png b/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png new file mode 100644 index 00000000..5ca7a743 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-confirm.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png b/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png new file mode 100644 index 00000000..be3b1e82 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-empty.png differ diff --git a/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png b/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png new file mode 100644 index 00000000..2a943990 Binary files /dev/null and b/apps/craft-cloud-desktop/tests/snapshots/versions-loaded.png differ diff --git a/apps/craft-cloud-server/Cargo.toml b/apps/craft-cloud-server/Cargo.toml index 4c1c3ef6..3d73a4fb 100644 --- a/apps/craft-cloud-server/Cargo.toml +++ b/apps/craft-cloud-server/Cargo.toml @@ -12,6 +12,7 @@ publish = false test-support = [] [dependencies] +aes-gcm.workspace = true argon2.workspace = true axum.workspace = true base64.workspace = true @@ -21,20 +22,29 @@ clap.workspace = true craft-cloud-meta.workspace = true craft-cloud-proto.workspace = true craft-cloud-store.workspace = true +fs4.workspace = true +# The audit export's body is a stream of pages. +futures-util.workspace = true getrandom.workspace = true +hmac.workspace = true +percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true +sha1.workspace = true +subtle.workspace = true thiserror.workspace = true tokio.workspace = true toml.workspace = true tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true +zeroize.workspace = true [dev-dependencies] +# The tests move the server's clock (`AppState::with_clock`): the feature is on for them only. +craft-cloud-server = { path = ".", features = ["test-support"] } reqwest.workspace = true tokio-tungstenite.workspace = true -futures-util.workspace = true serde_json.workspace = true sqlx.workspace = true tempfile.workspace = true diff --git a/apps/craft-cloud-server/src/account.rs b/apps/craft-cloud-server/src/account.rs new file mode 100644 index 00000000..6af4d177 --- /dev/null +++ b/apps/craft-cloud-server/src/account.rs @@ -0,0 +1,170 @@ +//! The signed-in account: `GET /api/v1/account`, `POST /api/v1/account/password`, and the device routes that change +//! a device (`PATCH /api/v1/devices/{id}`, `POST /api/v1/devices/revoke-others`). +//! +//! - **A wrong current password is `403 wrong_password`, never 401**: on a signed-in route the agent reads every 401 +//! as "this device was revoked" and signs out. The same holds for every later route that asks for a password. +//! - **Changing the password** reads the body (at most [`TINY_JSON_BYTES`]), counts the try against the account +//! ([`PASSWORD_CHANGES_PER_USER`] in 15 minutes, wrong passwords included), takes one of the server-wide +//! password-check slots (every slot busy is `429 rate_limited` and the try is given back), checks the current +//! password and the new one's rules, and hashes the new one, all in that one slot. Then one transaction sets the +//! hash, revokes every other device and deletes the account's open invite and pairings +//! ([`craft_cloud_meta::Meta::change_password`]); afterwards each revoked device's live connections end at once. +//! The device that asked stays signed in. +//! - **Nothing secret is logged**: not a password, a hash or a token. The count of revoked devices is logged at +//! `info`, the username at most at `debug`. + +use std::sync::Arc; + +use axum::Json; +use axum::extract::rejection::PathRejection; +use axum::extract::{Extension, Path, Request, State}; +use axum::http::StatusCode; +use craft_cloud_meta::{DeviceAuth, RevokeReason, UserId}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{AccountInfo, ChangePasswordRequest, DeviceInfo, RenameDeviceRequest, RevokedOthers, codes}; + +use crate::api::{PASSWORD_RETRY_SECS, TINY_JSON_BYTES, device_info, read_json}; +use crate::auth::{MAX_PASSWORD_BYTES, check_password_rules, hash_password, verify_password}; +use crate::error::ApiError; +use crate::state::AppState; + +/// Password change tries one account may make in [`AppState::password_changes`]'s window (15 minutes). +pub const PASSWORD_CHANGES_PER_USER: u32 = 5; + +type Shared = Arc; + +/// `403 wrong_password`: the current password was wrong. Never 401, see the module's notes. +pub(crate) fn wrong_password() -> ApiError { + ApiError::new(StatusCode::FORBIDDEN, codes::WRONG_PASSWORD, "the current password is wrong") +} + +/// Checks the signed-in account's own password (`user`, named `username`: a device's account, or a browser session's) for a +/// route that asks for one again (a change of the two-factor settings, an admin's confirmation) and returns the password hash it was verified against, which the change then compares with the +/// stored one under the account's lock. Wrong passwords are counted per account ([`PASSWORD_CHANGES_PER_USER`] in 15 +/// minutes, the window of the password change) and a right one is given back, so only a wrong password stays counted; +/// every check holds one of the server-wide password-check slots, and all busy is `429 rate_limited` that costs no try. A +/// wrong password is `403 wrong_password`, never 401. +pub(crate) async fn confirm_password(state: &AppState, user_id: UserId, username: &str, password: String) -> Result { + if password.len() > MAX_PASSWORD_BYTES { + return Err(ApiError::invalid("passwords are at most 1024 bytes")); + } + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{username}"); + let Some(counted) = state.password_changes.count(&key, PASSWORD_CHANGES_PER_USER, now) else { + return Err(ApiError::rate_limited("too many wrong passwords; wait a few minutes", None)); + }; + // Each check costs 19 MiB and real CPU time: all slots busy is told to come back, not queued, and costs no try. + let Ok(permit) = state.password_checks.clone().try_acquire_owned() else { + state.password_changes.refund(&key, counted); + return Err(ApiError::rate_limited("the server is busy checking other passwords; try again in a moment", Some(PASSWORD_RETRY_SECS))); + }; + let user = match state.meta.user_by_name(username).await { + Ok(Some(user)) if user.id == user_id => user, + other => { + state.password_changes.refund(&key, counted); + other?; + return Err(ApiError::unauthorized()); + } + }; + let hash = user.password_hash.clone(); + let ok = tokio::task::spawn_blocking(move || { + let _permit = permit; + verify_password(&password, &hash) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "password check task failed"); + ApiError::internal() + })?; + if !ok { + return Err(wrong_password()); + } + state.password_changes.refund(&key, counted); + Ok(user.password_hash) +} + +/// `GET /api/v1/account`. +pub async fn get_account(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let account = state.meta.account(auth.user).await?; + Ok(Json(AccountInfo { + user_id: account.id.0, + username: account.username, + role: account.role.as_str().to_owned(), + created_ms: account.created_ms, + password_changed_ms: account.password_changed_ms, + two_factor_enabled: account.two_factor_enabled, + })) +} + +/// `POST /api/v1/account/password`: `204` once the password is changed and the account's other devices are signed out. +pub async fn change_password(State(state): State, Extension(auth): Extension, request: Request) -> Result { + let req: ChangePasswordRequest = read_json(&state, request, TINY_JSON_BYTES).await?; + if req.current_password.len() > MAX_PASSWORD_BYTES || req.new_password.len() > MAX_PASSWORD_BYTES { + return Err(ApiError::invalid("passwords are at most 1024 bytes")); + } + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{}", auth.username); + let Some(counted) = state.password_changes.count(&key, PASSWORD_CHANGES_PER_USER, now) else { + return Err(ApiError::rate_limited("too many password change attempts; wait a few minutes", None)); + }; + // Each check costs 19 MiB and real CPU time (see the sign-in): all slots busy is told to come back, not queued, + // and costs the account no try, since nothing was checked. + let Ok(permit) = state.password_checks.clone().try_acquire_owned() else { + state.password_changes.refund(&key, counted); + return Err(ApiError::rate_limited("the server is busy checking other passwords; try again in a moment", Some(PASSWORD_RETRY_SECS))); + }; + let user = state.meta.user_by_name(&auth.username).await?.ok_or_else(ApiError::unauthorized)?; + let current_hash = user.password_hash.clone(); + let ChangePasswordRequest { current_password, new_password } = req; + let new_hash = tokio::task::spawn_blocking(move || { + let _permit = permit; + if !verify_password(¤t_password, ¤t_hash) { + return Err(wrong_password()); + } + check_password_rules(&new_password).map_err(|e| ApiError::new(StatusCode::BAD_REQUEST, codes::WEAK_PASSWORD, e.message))?; + if new_password == current_password { + return Err(ApiError::new(StatusCode::BAD_REQUEST, codes::WEAK_PASSWORD, "the new password must differ from the current one")); + } + hash_password(&new_password) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "password change task failed"); + ApiError::internal() + })??; + let revoked = state.meta.change_password(auth.user, &user.password_hash, &new_hash, auth.device, state.now_ms()).await?; + for device in &revoked { + state.live.revoke(*device); + } + tracing::debug!(user = %auth.username, "password changed"); + tracing::info!(device = %auth.device, revoked = revoked.len(), "password changed"); + Ok(StatusCode::NO_CONTENT) +} + +/// `PATCH /api/v1/devices/{id}`: renames one of the account's own devices that isn't revoked. Anything else is +/// `404 not_found`, so the answer never says whether another account has the device. +pub async fn rename_device( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let Path(id) = id?; + let req: RenameDeviceRequest = read_json(&state, request, TINY_JSON_BYTES).await?; + crate::api::check_device_name(&req.name)?; + let device = state.meta.rename_device(auth.user, id, &req.name).await?; + Ok(Json(device_info(device, auth.device))) +} + +/// `POST /api/v1/devices/revoke-others`: signs out every other device of the account. +pub async fn revoke_others(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let revoked = state.meta.revoke_other_devices(auth.user, auth.device, RevokeReason::Others, state.now_ms()).await?; + // Their live connections end now, not at their next request. + for device in &revoked { + state.live.revoke(*device); + } + tracing::info!(device = %auth.device, revoked = revoked.len(), "other devices signed out"); + Ok(Json(RevokedOthers { revoked: u32::try_from(revoked.len()).unwrap_or(u32::MAX) })) +} diff --git a/apps/craft-cloud-server/src/admin.rs b/apps/craft-cloud-server/src/admin.rs new file mode 100644 index 00000000..8307a323 --- /dev/null +++ b/apps/craft-cloud-server/src/admin.rs @@ -0,0 +1,462 @@ +//! The admin API, under `/api/v1/admin`: server counters, accounts (list, create, change role and quota, disable, +//! enable, mark for removal), invites, password reset codes, and a user's devices. +//! +//! - **Every route is behind [`require_admin`](crate::auth::require_admin)** (after `require_device`): a member gets `403 +//! forbidden`, a bad token `401`. The role is read from the database for each request, so a demoted admin is out with +//! the next one. +//! - **Every change needs a fresh confirmation in its body**: the acting admin's current `password`. A device token never +//! expires, so a stolen admin token alone must not be enough to make an account an admin's or to sign others out. A +//! missing or mistyped field is `400 invalid`, a wrong password `403 wrong_password` (never 401: on a signed-in route +//! the agent reads 401 as "this device was revoked"). Tries are counted per account in +//! [`AppState::password_changes`], the same window the password change uses, but only a wrong password stays counted; +//! every check holds a server-wide password-check slot. An admin whose account has two-factor sign-in adds `totp_code` +//! (or `recovery_code`): missing or wrong is `403 two_factor_invalid`, too many wrong ones `429 two_factor_locked`. The +//! code is used up by the confirmation (one time step serves one change, as at sign-in), and the check shares its +//! counters and limiter with sign-in. +//! - **Bodies are tiny** ([`TINY_JSON_BYTES`]), the `DELETE` routes carry one too, and are read under the same deadline +//! as every JSON body. +//! - **Nothing secret is logged**: not a password, a code or a join link. Events are logged at `info` with the kind and +//! the target's ID, never a username. +//! - **Live connections** of devices an admin signs out end at once in this process (after the commit); another server +//! process, or the command line, finds out at the next ping (about 30 seconds). + +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::rejection::{PathRejection, QueryRejection}; +use axum::extract::{DefaultBodyLimit, Extension, FromRequest, Path, Query, Request, State}; +use axum::http::StatusCode; +use axum::routing::{get, post}; +use axum::{Router, middleware}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{AdminActor, AdminUser, AdminVia, DeviceAuth, Role, UserId, valid_username}; +use craft_cloud_proto::DeviceId; +use craft_cloud_proto::api::{ + AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminManifestIndex, AdminPasswordReset, AdminServerInfo, + AdminStorageAccount, AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, UsageEstimateReason, UsageEstimates, +}; +use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::account::confirm_password; +use crate::api::{TINY_JSON_BYTES, body_too_slow, device_info}; +use crate::auth::require_admin; +use crate::error::ApiError; +use crate::state::AppState; + +type Shared = Arc; + +/// Accounts one page of `GET /admin/users` holds when `limit` is not given. +pub(crate) const DEFAULT_PAGE: u32 = 50; +/// The most accounts a page may hold. +pub(crate) const MAX_PAGE: u32 = 200; +/// The longest `query` of the user list, in bytes. +pub(crate) const MAX_QUERY_BYTES: usize = craft_cloud_meta::MAX_SEARCH_BYTES; + +/// The admin routes, all behind `require_admin`. `router` merges them into the signed-in routes, so `require_device` +/// wraps them too and runs first. +pub fn routes() -> Router { + Router::new() + .route("/admin/server", get(server)) + .route("/admin/storage", get(storage)) + .route("/admin/users", get(list_users).post(create_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}", axum::routing::patch(update_user).delete(remove_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/usage", get(user_usage)) + .route("/admin/users/{id}/disable", post(disable_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/enable", post(enable_user).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/invite", post(issue_invite).delete(revoke_invite).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/password-reset", post(issue_reset).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/reset-two-factor", post(reset_two_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/admin/users/{id}/devices", get(list_devices)) + .route("/admin/users/{id}/devices/{device_id}", axum::routing::delete(revoke_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route_layer(middleware::from_fn(require_admin)) + .route_layer(middleware::from_fn(crate::web::reject_mixed_credentials)) +} + +// ---- helpers ---- + +/// Reads a JSON body of at most [`TINY_JSON_BYTES`] under the usual deadline. Whatever is wrong with its shape (not JSON, +/// a missing or mistyped field, an unknown role) is `400 invalid`; too long is `413 too_large`. +async fn read_body(state: &AppState, request: Request) -> Result { + let bytes = match tokio::time::timeout(state.limits.body_time(TINY_JSON_BYTES), Bytes::from_request(request, &())).await { + Ok(bytes) => bytes?, + Err(_) => return Err(body_too_slow()), + }; + serde_json::from_slice(&bytes) + .map_err(|_| ApiError::invalid("the body must be a JSON object with your current password in `password`, and the fields of this call")) +} + +/// What an admin's body gives for a confirmation: the password, and the second factor when the account has one. +pub(crate) struct Confirmation { + password: Option, + totp_code: Option, + recovery_code: Option, +} + +impl Confirmation { + pub(crate) fn new(password: Option, totp_code: Option, recovery_code: Option) -> Self { + Self { password, totp_code, recovery_code } + } +} + +/// Checks the acting admin's confirmation and returns the actor to hand to the change: the password ([`confirm_password`]: +/// one wrong password stays counted against the account, a right one is given back), then, when the admin's account has +/// two-factor sign-in, the second factor (`totp_code`, or `recovery_code`), which is used up here: the time step or the +/// recovery code can't serve a second change, and wrong ones are counted and can lock ([`crate::two_factor`]). A password +/// that is wrong is told before anything about the second factor. +async fn confirm(state: &AppState, auth: &DeviceAuth, given: Confirmation) -> Result { + confirm_actor(state, auth.user, &auth.username, AdminVia::Device(auth.device), given).await +} + +/// [`confirm`] for any admin who acts: `via` says whether a device or a browser session does (the web pages call this with +/// the same confirmation, the same counters and the same statuses as the JSON routes above). +pub(crate) async fn confirm_actor( + state: &AppState, + user: UserId, + username: &str, + via: AdminVia, + given: Confirmation, +) -> Result { + let password = given.password.ok_or_else(|| ApiError::invalid("this change needs your current password in `password`"))?; + // The shape of the second factor is checked before anything is counted. + let proof = crate::two_factor::parse_proof(given.totp_code, given.recovery_code)?; + let password_hash = confirm_password(state, user, username, password).await?; + let two_factor = crate::two_factor::confirm_admin_second_factor(state, user, username, proof).await?; + Ok(AdminActor { user, via, password_hash, two_factor }) +} + +fn user_id(id: Result, PathRejection>) -> Result { + let Path(id) = id?; + Ok(UserId(id)) +} + +pub(crate) fn role_of(name: &str) -> Result { + match name { + "admin" => Ok(Role::Admin), + "member" => Ok(Role::Member), + _ => Err(ApiError::invalid("role is admin or member")), + } +} + +fn user_info(u: AdminUser) -> AdminUserInfo { + AdminUserInfo { + id: u.id.0, + username: u.username, + role: u.role.as_str().to_owned(), + created_ms: u.created_ms, + disabled_ms: u.disabled_ms, + two_factor_enabled: u.two_factor_enabled, + devices_active: u.devices_active, + stored_bytes: u.stored_bytes, + quota_bytes: u.quota_bytes, + invite_pending: u.invite_pending, + } +} + +/// A fresh invite or reset code, as the admin gets it: shown once, never logged. +pub(crate) fn new_code() -> Result { + Code::generate(INVITE_CHARS).map_err(|e| { + tracing::error!(error = %e, "the system random source failed"); + ApiError::internal() + }) +} + +pub(crate) fn admin_code(state: &AppState, code: Code, expires_ms: i64) -> AdminCode { + let shown = code.as_str().to_owned(); + let join_url = state.public_url.as_ref().map(|server| JoinString::new(server.clone(), code).link()); + AdminCode { code: shown, join_url, expires_ms } +} + +pub(crate) fn end_live_connections(state: &AppState, devices: &[DeviceId]) { + for device in devices { + state.live.revoke(*device); + } +} + +// ---- reads ---- + +/// `GET /admin/server`: counters and kinds only. +async fn server(State(state): State) -> Result, ApiError> { + let counts = state.meta.admin_server_counts().await?; + Ok(Json(AdminServerInfo { + version: env!("CARGO_PKG_VERSION").to_owned(), + users: counts.users, + devices_active: counts.devices_active, + stored_bytes: counts.stored_bytes, + live_bytes: counts.live_bytes, + storage_kind: state.store.kind().to_owned(), + database_kind: state.meta.database_kind().to_owned(), + })) +} + +/// `GET /admin/storage`: bounded account and manifest-index counts, plus the ten largest accounts, and the room the +/// server has (filesystem numbers, the database's size and the configured capacity). +async fn storage(State(state): State) -> Result, ApiError> { + let overview = state.meta.admin_storage_overview().await?; + let info = state.storage_info.clone(); + // Filesystem queries block. If the task fails, the numbers are left out. + let measured = tokio::task::spawn_blocking(move || info.measure()).await.unwrap_or_default(); + Ok(Json(AdminStorageOverview { + accounts: overview.accounts, + stored_bytes: overview.stored_bytes, + default_quota_bytes: overview.default_quota_bytes, + accounts_with_override: overview.accounts_with_override, + accounts_near_quota: overview.accounts_near_quota, + accounts_over_quota: overview.accounts_over_quota, + top_accounts: overview + .top_accounts + .into_iter() + .map(|account| AdminStorageAccount { + id: account.id.0, + username: account.username, + stored_bytes: account.stored_bytes, + quota_bytes: account.quota_bytes, + }) + .collect(), + manifest_index: AdminManifestIndex { + complete: overview.manifest_index.complete, + building: overview.manifest_index.building, + not_indexed: overview.manifest_index.not_indexed, + }, + store_kind: Some(state.store.kind().to_owned()), + capacity_bytes: state.storage_info.capacity(), + store_fs_total_bytes: measured.store_fs_total, + store_fs_free_bytes: measured.store_fs_free, + server_fs_total_bytes: measured.server_fs_total, + server_fs_free_bytes: measured.server_fs_free, + database_bytes: measured.database_bytes, + })) +} + +/// `GET /admin/users/{id}/usage`: the account's stored bytes and labeled estimates. +async fn user_usage(State(state): State, id: Result, PathRejection>) -> Result, ApiError> { + let estimate = state.meta.admin_storage_estimate(user_id(id)?).await?; + let index_complete = estimate.reason.is_none(); + Ok(Json(AccountUsage { + stored_bytes: estimate.stored_bytes, + quota_bytes: estimate.quota_bytes, + trash_bytes: estimate.trash_bytes, + history_bytes: estimate.history_bytes, + estimates: UsageEstimates { index_complete, reason: estimate.reason.map(|_| UsageEstimateReason::IndexIncomplete) }, + })) +} + +#[derive(Debug, Deserialize)] +struct ListQuery { + cursor: Option, + limit: Option, + query: Option, +} + +/// The username a cursor stands for. A cursor is the last username of a page, in base64url: opaque to clients, and +/// checked like any username, since it is hostile input. +pub(crate) fn decode_cursor(cursor: &str) -> Result { + let bad = || ApiError::invalid("the cursor is not one this server gave"); + let bytes = URL_SAFE_NO_PAD.decode(cursor).map_err(|_| bad())?; + let name = String::from_utf8(bytes).map_err(|_| bad())?; + if valid_username(&name) { Ok(name) } else { Err(bad()) } +} + +/// `GET /admin/users?cursor&limit&query`: accounts by username, 50 a page (at most 200). `query` finds a username +/// containing it (case does not matter, at most 64 bytes of printable characters). +async fn list_users(State(state): State, q: Result, QueryRejection>) -> Result, ApiError> { + let Query(q) = q?; + let limit = q.limit.unwrap_or(DEFAULT_PAGE); + if !(1..=MAX_PAGE).contains(&limit) { + return Err(ApiError::invalid(format!("limit is 1 to {MAX_PAGE}"))); + } + let after = q.cursor.as_deref().map(decode_cursor).transpose()?; + let search = q.query.as_deref().filter(|s| !s.is_empty()); + if let Some(search) = search + && (search.len() > MAX_QUERY_BYTES || search.chars().any(char::is_control)) + { + return Err(ApiError::invalid(format!("query is at most {MAX_QUERY_BYTES} bytes of printable characters"))); + } + let page = state.meta.admin_list_users(after.as_deref(), search, limit, state.now_ms()).await?; + Ok(Json(AdminUserList { + users: page.users.into_iter().map(user_info).collect(), + next: page.next_after.map(|name| URL_SAFE_NO_PAD.encode(name.as_bytes())), + })) +} + +/// `GET /admin/users/{id}/devices`: every device of the account, revoked ones with when and why. +async fn list_devices( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let devices = state.meta.admin_devices(user_id(id)?).await?; + Ok(Json(AdminDevices { devices: devices.into_iter().map(|d| device_info(d, auth.device)).collect() })) +} + +// ---- changes ---- + +/// `POST /admin/users`: a new account without a password and its invite. +async fn create_user( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: AdminCreateUser = read_body(&state, request).await?; + if !valid_username(&req.username) { + return Err(ApiError::invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); + } + let role = role_of(&req.role)?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let now = state.now_ms(); + let (user, issued) = state.meta.admin_create_user(&actor, &req.username, role, &code.hash(), now, INVITE_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %user.id.0, "admin: account created"); + Ok(Json(AdminCreatedUser { user: user_info(user), invite: admin_code(&state, code, issued.expires_ms) })) +} + +/// `PATCH /admin/users/{id}`: a new role and/or quota. +async fn update_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminUpdateUser = read_body(&state, request).await?; + let role = req.role.as_deref().map(role_of).transpose()?; + if role.is_none() && req.quota_bytes.is_none() { + return Err(ApiError::invalid("give a role or a quota_bytes to change")); + } + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let user = state.meta.admin_update_user(&actor, target, role, req.quota_bytes, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, role_changed = role.is_some(), quota_changed = req.quota_bytes.is_some(), "admin: account changed"); + Ok(Json(user_info(user))) +} + +/// `POST /admin/users/{id}/disable`: 204. +async fn disable_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_disable_user(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: account disabled"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /admin/users/{id}/enable`: 204. Devices that were signed out stay signed out. +async fn enable_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_enable_user(&actor, target, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: account enabled"); + Ok(StatusCode::NO_CONTENT) +} + +/// `DELETE /admin/users/{id}`: 202. Marks the account for removal; nothing it stored is deleted. +async fn remove_user( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_remove_user(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: account marked for removal"); + Ok(StatusCode::ACCEPTED) +} + +/// `POST /admin/users/{id}/invite`: a new invite for an account that has no password yet. +async fn issue_invite( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let issued = state.meta.admin_issue_invite(&actor, target, &code.hash(), state.now_ms(), INVITE_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: invite issued"); + Ok(Json(admin_code(&state, code, issued.expires_ms))) +} + +/// `DELETE /admin/users/{id}/invite`: 204. Ends the account's open invite or reset code. +async fn revoke_invite( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_revoke_invite(&actor, target, state.now_ms()).await?; + tracing::info!(actor = %auth.device, target = %target.0, "admin: invite revoked"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /admin/users/{id}/password-reset`: a code good for an hour. Redeemed at `POST /join`, it sets a new password, +/// signs every device of the account out, and signs the redeeming computer in. +async fn issue_reset( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let target = user_id(id)?; + let req: AdminPasswordReset = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let code = new_code()?; + let issued = + state.meta.admin_issue_reset(&actor, target, &code.hash(), req.reset_two_factor, state.now_ms(), craft_cloud_meta::RESET_TTL_MS).await?; + tracing::info!(actor = %auth.device, target = %target.0, reset_two_factor = req.reset_two_factor, "admin: password reset code issued"); + Ok(Json(admin_code(&state, code, issued.expires_ms))) +} + +/// `POST /admin/users/{id}/reset-two-factor`: 204. Takes the account's two-factor sign-in away and signs every device of +/// it out (for someone who lost their authenticator and their recovery codes). +async fn reset_two_factor( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result { + let target = user_id(id)?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + let revoked = state.meta.admin_reset_two_factor(&actor, target, state.now_ms()).await?; + end_live_connections(&state, &revoked); + tracing::info!(actor = %auth.device, target = %target.0, revoked = revoked.len(), "admin: two-factor sign-in reset"); + Ok(StatusCode::NO_CONTENT) +} + +/// `DELETE /admin/users/{id}/devices/{device_id}`: 204. +async fn revoke_device( + State(state): State, + Extension(auth): Extension, + ids: Result, PathRejection>, + request: Request, +) -> Result { + let Path((target, device)) = ids?; + let req: AdminConfirm = read_body(&state, request).await?; + let actor = confirm(&state, &auth, Confirmation::new(req.password, req.totp_code, req.recovery_code)).await?; + state.meta.admin_revoke_device(&actor, UserId(target), device, state.now_ms()).await?; + state.live.revoke(device); + tracing::info!(actor = %auth.device, target = %target, %device, "admin: device revoked"); + Ok(StatusCode::NO_CONTENT) +} diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index efc5392f..346b09e2 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -1,4 +1,6 @@ -//! The HTTP API, under `/api/v1`. Everything but `info`, `auth/login` and `join` needs a device token. `GET /live` +//! The HTTP API, under `/api/v1`. Everything but `info`, `auth/login`, `auth/login/second-factor` and `join` needs a device +//! token. The account and its devices' routes (`account`, `account/password`, renaming and signing out devices) are in +//! [`crate::account`], two-factor sign-in's in [`crate::two_factor`]. `GET /live` //! becomes the live service's WebSocket ([`crate::live`]); join codes (`pairings`, `join`, and the `/join/{code}` page //! outside the API) are in [`crate::join`]. @@ -13,33 +15,39 @@ use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post, put}; use axum::{Json, Router, middleware}; -use craft_cloud_meta::{BlobKind, DeviceAuth, valid_username}; +use craft_cloud_meta::{BlobKind, DeviceAuth, DeviceClient, DeviceRecord, NewChallenge, Origin, RevokeReason, SignIn, valid_username}; use craft_cloud_proto::api::{ - CommitRequest, CommitResponse, DeviceInfo, InfoResponse, LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, - PROTOCOL_MAX, PROTOCOL_MIN, SpaceResponse, codes, + AccountUsage, CommitRequest, CommitResponse, DeviceInfo, HISTORY_DEFAULT_LIMIT, HistoryCursor, HistoryResponse, InfoResponse, LoginRequest, + LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, RestoreRequest, RestoreResponse, SpaceResponse, + TRASH_DEFAULT_LIMIT, TrashCursor, TrashItem, TrashResponse, TrashRestoreResponse, UsageEstimateReason, UsageEstimates, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_JSON_BYTES, MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, Manifest, ManifestId, NodeId, PageCursor}; use craft_cloud_store::{BlobKey, StoreError}; use serde::Deserialize; +use crate::account; +use crate::audit; use crate::auth::{new_token, require_device, verify_password}; use crate::error::ApiError; use crate::join; use crate::ratelimit::{client_ip, ip_key}; use crate::state::AppState; +use crate::two_factor; /// Chunker parameter sets the server accepts. pub const CHUNKER_PARAMS: &[&str] = &["fastcdc-v1"]; /// Login attempts per account and per client address in five minutes. -const LOGINS_PER_USER: u32 = 10; -const LOGINS_PER_IP: u32 = 30; +pub(crate) const LOGINS_PER_USER: u32 = 10; +pub(crate) const LOGINS_PER_IP: u32 = 30; /// Longest change-feed long-poll. const MAX_WAIT_SECS: u64 = 60; /// `Retry-After` for a refused long-poll. const WAITERS_RETRY_SECS: u32 = 5; /// Small JSON bodies (login, join and missing-chunk queries). pub(crate) const SMALL_JSON_BYTES: usize = 1024 * 1024; +/// Tiny JSON bodies (a password change, a device's new name): a few short strings, so far less than a small body. +pub(crate) const TINY_JSON_BYTES: usize = 4 * 1024; /// `Retry-After` for a sign-in refused because every password check slot is busy. pub(crate) const PASSWORD_RETRY_SECS: u32 = 1; @@ -47,8 +55,17 @@ type Shared = Arc; pub fn router(state: Shared) -> Router { let signed_in = Router::new() + .route("/account", get(account::get_account)) + .route("/account/password", post(account::change_password).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/activity", get(audit::account_activity)) + .route("/account/two-factor", get(two_factor::get_status)) + .route("/account/two-factor/setup", post(two_factor::setup).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/confirm", post(two_factor::confirm).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/disable", post(two_factor::disable).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) + .route("/account/two-factor/recovery-codes", post(two_factor::recovery_codes).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/devices", get(list_devices)) - .route("/devices/{id}", delete(revoke_device)) + .route("/devices/revoke-others", post(account::revoke_others)) + .route("/devices/{id}", delete(revoke_device).patch(account::rename_device).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/pairings", post(join::create_pairing)) .route("/pairings/{id}", get(join::pairing_status).delete(join::delete_pairing)) .route("/chunks/missing", post(missing_chunks).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) @@ -56,24 +73,37 @@ pub fn router(state: Shared) -> Router { .route("/manifests/{id}", put(put_manifest).get(get_manifest).layer(DefaultBodyLimit::max(MAX_MANIFEST_BYTES))) .route("/commit", post(commit).layer(DefaultBodyLimit::max(MAX_BATCH_JSON_BYTES))) .route("/saves", post(attach_save).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/nodes/{id}/history", get(history)) + .route("/nodes/{id}/restore", post(restore).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/trash", get(trash)) + .route("/trash/{id}/restore", post(restore_from_trash)) + .route("/trash/{id}", delete(remove_from_trash)) .route("/space", get(space)) + .route("/account/usage", get(account_usage)) .route("/tree", get(tree)) .route("/changes", get(changes)) .route("/live", get(crate::live::connect)) + // The admin routes carry `require_admin` themselves; `require_device` below wraps them too and runs first. + .merge(crate::admin::routes()) + .merge(audit::admin_routes()) .route_layer(middleware::from_fn_with_state(state.clone(), require_device)); let public = Router::new() .route("/info", get(info)) .route("/auth/login", post(login).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/auth/login/second-factor", post(two_factor::second_factor).layer(DefaultBodyLimit::max(TINY_JSON_BYTES))) .route("/join", post(join::join).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))); - Router::new() + let app = Router::new() .nest("/api/v1", public.merge(signed_in)) // Version 1 isn't known yet when a client first asks what the server speaks. .route("/api/info", get(info)) // A join link opened in a browser. .route("/join/{code}", get(join::join_page)) .fallback(no_such_endpoint) - .method_not_allowed_fallback(wrong_method) - .with_state(state) + .method_not_allowed_fallback(wrong_method); + // The admin pages exist only on a server that was set up for them. + let app = + if state.web.is_some() { app.nest("/web", crate::web::routes(state.clone())).merge(crate::web::trailing_slash(state.clone())) } else { app }; + app.with_state(state) } async fn no_such_endpoint() -> ApiError { @@ -109,6 +139,7 @@ async fn login( return Err(ApiError::invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); } check_device_name(&req.device_name)?; + let client = device_client(req.client_kind.as_deref(), req.platform.as_deref(), req.app_version.as_deref())?; // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. let now = tokio::time::Instant::now().into_std(); let ip = client_ip(peer, &headers, &state.trusted_proxies); @@ -131,7 +162,10 @@ async fn login( return Err(ApiError::invalid("passwords are at most 1024 bytes")); } let user = state.meta.user_by_name(&username).await?; - let hash = user.as_ref().map_or(state.dummy_hash.as_str(), |u| u.password_hash.as_str()).to_owned(); + let known = user.as_ref().map(|u| u.id); + // An account without a password (an open invite) costs the same Argon2id check as a name nobody has, and never signs in. + let usable = user.as_ref().is_some_and(|u| crate::auth::usable_hash(&u.password_hash)); + let hash = user.as_ref().filter(|_| usable).map_or(state.dummy_hash.as_str(), |u| u.password_hash.as_str()).to_owned(); let password = req.password.clone(); let ok = tokio::task::spawn_blocking(move || { let _permit = permit; @@ -142,11 +176,29 @@ async fn login( tracing::error!(error = %e, "password check task failed"); ApiError::internal() })?; - let (Some(user), true) = (user, ok) else { + let (Some(user), true) = (user, ok && usable) else { + // Written after the answer is decided and in a transaction of its own: a log that cannot be written changes + // neither the answer nor the limits above. A name no account has is never put in the log. + audit::login_failed(&state, known).await; return Err(ApiError::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "wrong username or password")); }; let (token, token_hash) = new_token()?; - let device = state.meta.create_device(user.id, &req.device_name, &token_hash, state.now_ms()).await?; + // A challenge is made only if the account needs a second factor; the token the client gets for it is a secret of its own. + let (challenge_token, challenge_hash) = new_token()?; + let challenge = NewChallenge { id_hash: challenge_hash, client_key: ip_key(ip) }; + // Created only if the password just checked is still the account's: one changed meanwhile (and the devices + // signed out with it) must not be followed by a device that the old password let in. Answered as a wrong password. The + // account's two-factor state is read under the same lock: with it on, there is a challenge and no device. + let signed_in = + state.meta.sign_in(user.id, (&user.password_hash, user.auth_gen), &req.device_name, &client, &token_hash, &challenge, state.now_ms()).await?; + let device = match signed_in { + None => return Err(ApiError::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "wrong username or password")), + Some(SignIn::Device(device)) => device, + Some(SignIn::Challenge { expires_ms }) => { + tracing::info!("sign-in needs a second factor"); + return Err(ApiError::two_factor_required(challenge_token, expires_ms)); + } + }; // The username is the only personal identifier the server could log here: kept to debug. tracing::debug!(user = %user.username, "sign-in"); tracing::info!(%device, "device signed in"); @@ -162,26 +214,49 @@ pub(crate) fn check_device_name(name: &str) -> Result<(), ApiError> { Ok(()) } +/// The client fields of a sign-in or join, checked before anything is counted: an unknown `client_kind`, a field over +/// 32 bytes or a control character is `400 invalid`. The fields are only shown in the device list. +pub(crate) fn device_client(kind: Option<&str>, platform: Option<&str>, app_version: Option<&str>) -> Result { + Ok(DeviceClient::new(kind, platform, app_version)?) +} + +/// A device as the API shows it; `current` is the device making the request. +pub(crate) fn device_info(d: DeviceRecord, current: DeviceId) -> DeviceInfo { + DeviceInfo { + current: d.id == current, + id: d.id, + name: d.name, + created_ms: d.created_ms, + last_used_ms: d.last_used_ms, + revoked: d.revoked, + client_kind: d.client.kind, + platform: d.client.platform, + app_version: d.client.app_version, + revoked_ms: d.revoked_ms, + revoke_reason: d.revoke_reason.map(|r| r.as_str().to_owned()), + } +} + async fn space(State(state): State, Extension(auth): Extension) -> Result, ApiError> { let info = state.meta.space_info(auth.space).await?; Ok(Json(SpaceResponse { space: auth.space, root: info.root, epoch: info.epoch, seq: info.seq, username: auth.username })) } +async fn account_usage(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let estimate = state.meta.storage_estimate(auth.space).await?; + let index_complete = estimate.reason.is_none(); + Ok(Json(AccountUsage { + stored_bytes: estimate.stored_bytes, + quota_bytes: estimate.quota_bytes, + trash_bytes: estimate.trash_bytes, + history_bytes: estimate.history_bytes, + estimates: UsageEstimates { index_complete, reason: estimate.reason.map(|_| UsageEstimateReason::IndexIncomplete) }, + })) +} + async fn list_devices(State(state): State, Extension(auth): Extension) -> Result>, ApiError> { let devices = state.meta.list_devices(auth.user).await?; - Ok(Json( - devices - .into_iter() - .map(|d| DeviceInfo { - current: d.id == auth.device, - id: d.id, - name: d.name, - created_ms: d.created_ms, - last_used_ms: d.last_used_ms, - revoked: d.revoked, - }) - .collect(), - )) + Ok(Json(devices.into_iter().map(|d| device_info(d, auth.device)).collect())) } async fn revoke_device( @@ -190,7 +265,7 @@ async fn revoke_device( id: Result, PathRejection>, ) -> Result { let Path(id) = id?; - state.meta.revoke_device(auth.user, id).await?; + state.meta.revoke_device_by(auth.user, id, RevokeReason::User, Origin::Device(auth.device), state.now_ms()).await?; // Its live connections end now, not at their next request. state.live.revoke(id); Ok(StatusCode::NO_CONTENT) @@ -252,7 +327,12 @@ pub(crate) async fn read_json(state: &AppState, } } -fn body_too_slow() -> ApiError { +/// The body size a request declares (`Content-Length`), 0 when it declares none. +fn declared_len(request: &Request) -> u64 { + request.headers().get(axum::http::header::CONTENT_LENGTH).and_then(|v| v.to_str().ok()).and_then(|v| v.parse().ok()).unwrap_or(0) +} + +pub(crate) fn body_too_slow() -> ApiError { ApiError::new(StatusCode::REQUEST_TIMEOUT, codes::TIMEOUT, "the request body took too long to arrive") } @@ -266,15 +346,32 @@ async fn put_chunk( let id = parse_chunk_id(&id)?; // Held until the chunk is stored: the user's buffered bodies are bounded. let _body = state.bodies.acquire(auth.user).await?; + // A body over the route's limit is refused before anything else is asked of the server, by what the request declares. + let declared = declared_len(&request); + if declared > u64::from(MAX_CHUNK_BYTES) { + return Err(ApiError::too_large("the chunk is over the size limit")); + } + // A chunk the space already has costs the server nothing (as for `check_room`), so a full server still takes it. + let known = state.meta.missing_chunks(auth.space, &[id]).await?.is_empty(); + // A full server says so before the body is read, by what the request declares, and again with the real size. + if !known { + state.check_server_room(declared).await?; + } let body = read_body(&state, request, MAX_CHUNK_BYTES as usize).await?; if body.is_empty() { return Err(ApiError::invalid("a chunk can't be empty")); } + if !known { + state.check_server_room(body.len() as u64 + 1).await?; + } // A space at its cap doesn't get to store bytes it would then be refused for. state.meta.check_room(auth.space, BlobKind::Chunk, id.as_bytes(), body.len() as u64 + 1).await?; // Bytes first, record second: the database never points at a blob that isn't stored. state.store.put(BlobKey::chunk(&id), &body).await?; state.meta.add_chunk(auth.space, &id, body.len() as u64 + 1, state.now_ms()).await?; + if !known { + state.note_stored(body.len() as u64 + 1); + } Ok(StatusCode::NO_CONTENT) } @@ -304,6 +401,15 @@ async fn put_manifest( // bounded; then, with the body in, one of the user's manifest slots for the real work (decoding and checking a // big manifest). A body that is slow to arrive holds only its own user's body slot, and only until its time is up. let _body = state.bodies.acquire(auth.user).await?; + let declared = declared_len(&request); + if declared > MAX_MANIFEST_BYTES as u64 { + return Err(ApiError::too_large("the manifest is over the size limit")); + } + // A manifest the space already has costs the server nothing, so a full server still takes it again. + let known = state.meta.has_manifest(auth.space, &id).await?; + if !known { + state.check_server_room(declared).await?; + } let body = read_body(&state, request, MAX_MANIFEST_BYTES).await?; let _work = state.manifests.acquire(auth.user).await?; let manifest = Manifest::decode(&body).map_err(|e| ApiError::invalid(format!("bad manifest: {e}")))?; @@ -325,9 +431,18 @@ async fn put_manifest( } return Err(err); } + if !known { + state.check_server_room(body.len() as u64 + 1).await?; + } state.meta.check_room(auth.space, BlobKind::Manifest, id.as_bytes(), body.len() as u64 + 1).await?; state.store.put(BlobKey::manifest(&id), &body).await?; state.meta.add_manifest(auth.space, &manifest, body.len() as u64 + 1, state.now_ms()).await?; + if !known { + state.note_stored(body.len() as u64 + 1); + } + // The index of its chunks is complete before the upload is acknowledged: a manifest of up to the index budget was indexed + // in the transaction above, a longer one is finished here in batches (and by `index build` if this is cut short). + state.meta.index_complete(&manifest).await?; Ok(StatusCode::NO_CONTENT) } @@ -385,6 +500,129 @@ async fn attach_save( Ok(Json(CommitResponse { seq: outcome.seq, epoch: outcome.epoch })) } +#[derive(Debug, Deserialize)] +struct HistoryQuery { + cursor: Option, + limit: Option, +} + +/// `GET /nodes/{id}/history`: the versions of a file of the caller's own space, newest first. The node is looked +/// up in the space of the device token, so another account's node is `not_found`. +async fn history( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad history cursor"))?; + let page = state.meta.history(auth.space, node, after, q.limit.unwrap_or(HISTORY_DEFAULT_LIMIT)).await?; + Ok(Json(HistoryResponse { versions: page.versions, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /nodes/{id}/restore`: a new version with the content of an old one, as an ordinary update of the file. +async fn restore( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let Path(node) = id?; + let req: RestoreRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let outcome = tokio::spawn(async move { + let outcome = task_state.meta.restore_version(auth.space, auth.device, node, req.version, req.expected_version, task_state.now_ms()).await?; + task_state.notifier.publish(auth.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "restore task failed"); + ApiError::internal() + })??; + Ok(Json(RestoreResponse { version: outcome.version, parent_version: outcome.parent_version, seq: outcome.seq, epoch: outcome.epoch })) +} + +const MS_PER_DAY: i64 = 86_400_000; + +#[derive(Debug, Deserialize)] +struct TrashQuery { + cursor: Option, + limit: Option, +} + +/// `GET /trash`: the deleted nodes of the caller's own space, newest deletion first. +async fn trash( + State(state): State, + Extension(auth): Extension, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad trash cursor"))?; + let page = state.meta.trash(auth.space, after, q.limit.unwrap_or(TRASH_DEFAULT_LIMIT), state.now_ms()).await?; + if page.repaired > 0 { + tracing::warn!(count = page.repaired, "deleted nodes without a deletion time were stamped with the current time"); + } + let keep = i64::from(state.trash_days).saturating_mul(MS_PER_DAY); + let items = page + .entries + .into_iter() + .map(|e| TrashItem { + id: e.id, + parent: e.parent, + name: e.name, + is_dir: e.is_dir, + size: e.size, + deleted_ms: e.deleted_ms, + purge_after_ms: e.deleted_ms.saturating_add(keep), + }) + .collect(); + Ok(Json(TrashResponse { items, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /trash/{id}/restore`: a deleted node of the caller's space comes back, in one change. +async fn restore_from_trash( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let outcome = tokio::spawn(async move { + let outcome = task_state.meta.restore_from_trash(auth.space, auth.device, node, task_state.now_ms()).await?; + task_state.notifier.publish(auth.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "trash restore task failed"); + ApiError::internal() + })??; + Ok(Json(TrashRestoreResponse { + node: outcome.node, + parent: outcome.parent, + name: outcome.name, + renamed: outcome.renamed, + version: outcome.version, + seq: outcome.seq, + epoch: outcome.epoch, + })) +} + +/// `DELETE /trash/{id}`: takes a deleted node out of the trash. Only marks it: nothing is freed. +async fn remove_from_trash( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result { + let Path(node) = id?; + state.meta.request_purge(auth.space, node, state.now_ms()).await?; + Ok(StatusCode::NO_CONTENT) +} + fn cursor(after_seq: Option, after_node: Option) -> Result, ApiError> { match (after_seq, after_node) { (Some(seq), Some(node)) => Ok(Some(PageCursor { seq, node })), @@ -409,6 +647,19 @@ async fn tree( Ok(Json(state.meta.tree(auth.space, cursor(q.after_seq, q.after_node)?, q.limit.unwrap_or(MAX_PAGE_NODES)).await?)) } +/// Resolves when `device` is revoked (or some revocations were missed, so that it must be looked up). +async fn revoked(revocations: &mut tokio::sync::broadcast::Receiver, device: DeviceId) { + use tokio::sync::broadcast::error::RecvError; + loop { + match revocations.recv().await { + Ok(revoked) if revoked == device => return, + Ok(_) => {} + Err(RecvError::Lagged(_)) => return, + Err(RecvError::Closed) => std::future::pending::<()>().await, + } + } +} + // The paging fields are written out here rather than flattened from `PageQuery`: serde's `flatten` buffers // values as strings, so numbers in a query string would never parse. #[derive(Debug, Deserialize)] @@ -441,14 +692,29 @@ async fn changes( // Subscribe before reading the sequence number, so a commit in between still wakes us. let mut updates = state.notifier.subscribe(auth.space); let mut stopping = state.notifier.shutdown_signal(); + // Listening for revocations first, then looking the device up: a revoke before the lookup is seen by it, one + // after it ends the wait, so none is missed between the sign-in check and the wait. + let mut revocations = state.live.revocations(); + if !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } if state.meta.space_info(auth.space).await?.seq == q.since { let deadline = Duration::from_secs(q.wait.min(MAX_WAIT_SECS)); - // Whichever comes first: a commit, the deadline, or the server shutting down. + // Whichever comes first: a commit, the deadline, the server shutting down, or this device signed out. tokio::select! { _ = tokio::time::timeout(deadline, updates.wait_for(|seq| *seq > q.since)) => {} _ = stopping.wait_for(|stop| *stop) => {} + () = revoked(&mut revocations, auth.device) => {} } } } - Ok(Json(state.meta.changes(auth.space, q.since, cursor, q.limit.unwrap_or(MAX_PAGE_NODES)).await?)) + let changes = state.meta.changes(auth.space, q.since, cursor, q.limit.unwrap_or(MAX_PAGE_NODES)).await?; + // A long-poll looks the device up once more after the fetch, the last thing before the answer: a device signed + // out while it waited or while the change set was read (by the API, or by the command line in another process, + // which wakes nothing) gets no change feed. A revoke that commits after this lookup is a sign-out that happened + // after the answer was decided. + if slot.is_some() && !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } + Ok(Json(changes)) } diff --git a/apps/craft-cloud-server/src/audit.rs b/apps/craft-cloud-server/src/audit.rs new file mode 100644 index 00000000..5571b9a3 --- /dev/null +++ b/apps/craft-cloud-server/src/audit.rs @@ -0,0 +1,486 @@ +//! The security audit log over HTTP: `GET /api/v1/account/activity` (the signed-in account's own events), and for +//! administrators `GET /api/v1/admin/audit` (the whole log, filtered) and `GET /api/v1/admin/audit/export` (the same as +//! NDJSON, streamed). +//! +//! - **Events are written by the changes they record**, in the transaction of the change (`craft_cloud_meta::audit`). +//! This module only reads, and writes the one event that is not part of a change: a failed sign-in +//! ([`login_failed`]), after the answer is decided, whose failure to be written never changes the answer. +//! - **Pages** are newest first in the order `(at_ms, id)` descending. `cursor` is opaque (base64url of that pair) and +//! exclusive: the next page starts strictly behind it, so events with one timestamp are neither repeated nor lost. +//! A cursor this server did not make is `400 invalid`. `cursor` wins over `before_ms`, which only limits a listing's +//! first call. `limit` over the route's most is `400 invalid` rather than quietly smaller. +//! - **The account's own activity** holds the events the account acted in or was the target of, and shows the device that +//! acted only when it is the account's own. +//! - **The export** is bounded in every way: pages of [`AuditExportLimits::page`] rows per query, a channel of +//! [`AuditExportLimits::channel`] pages between the database and the connection (so a slow client slows the reading +//! and does not fill memory), the end of the log fixed when it starts, at most [`AuditExportLimits::max_rows`] event +//! lines. The last line says how it ended: `{"complete":true,...}`, or `{"truncated":true,...}` when there were more +//! events than it holds. A database error in the middle ends the connection with an error and no last line, and a +//! client that goes away stops the reading at once. A client counts an export as whole only with the `complete` line. +//! - **Nothing is logged of the events.** A failure to write or read one is logged without its content. + +use std::future::Future; +use std::sync::Arc; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::rejection::QueryRejection; +use axum::extract::{Extension, Query, State}; +use axum::http::{StatusCode, header}; +use axum::response::Response; +use axum::routing::get; +use axum::{Json, middleware}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{ActionFilter, AuditCursor, AuditEvent, AuditFailure, AuditQuery, DeviceAuth, MetaError, Step, UserId}; +use craft_cloud_proto::api::{AuditDetails, AuditExportEnd, AuditPage}; +use serde::Deserialize; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::auth::require_admin; +use crate::error::ApiError; +use crate::state::AppState; + +type Shared = Arc; + +/// Events one page of the account's activity holds when `limit` is not given, and the most it may hold. +const DEFAULT_PAGE: u32 = 50; +const ACCOUNT_MAX_PAGE: u32 = 100; +/// The same for the admin's list. +const ADMIN_MAX_PAGE: u32 = 200; +/// How an export is offered: always under this name, whatever the filters. +const EXPORT_DISPOSITION: &str = "attachment; filename=\"craft-cloud-audit.ndjson\""; + +/// How an export is cut. The defaults are the contract in `docs/design/sync.md`; tests make them smaller. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditExportLimits { + /// Rows one database query reads. + pub page: u32, + /// Pages the channel to the connection holds. + pub channel: usize, + /// Event lines in an export; more events than that make it `truncated`. + pub max_rows: u64, +} + +impl Default for AuditExportLimits { + fn default() -> Self { + Self { page: 500, channel: 2, max_rows: 100_000 } + } +} + +/// The admin routes, behind `require_admin`; `crate::api::router` merges them into the signed-in routes, so +/// `require_device` runs first. +pub fn admin_routes() -> Router { + Router::new() + .route("/admin/audit", get(admin_list)) + .route("/admin/audit/export", get(admin_export)) + .route_layer(middleware::from_fn(require_admin)) + .route_layer(middleware::from_fn(crate::web::reject_mixed_credentials)) +} + +/// Writes the event of a sign-in that failed at its password, in a transaction of its own. `target` is the account the +/// guess was about, `None` for a name no account has (the log never holds a name that was typed). A failure is logged and +/// otherwise ignored: the answer to the client and the limits around it do not depend on the log. +pub(crate) async fn login_failed(state: &AppState, target: Option) { + record_login_failure(state, target, false).await; +} + +pub(crate) async fn web_login_failed(state: &AppState, target: Option) { + record_login_failure(state, target, true).await; +} + +async fn record_login_failure(state: &AppState, target: Option, web: bool) { + let failure = AuditFailure { action: craft_cloud_meta::AuditAction::AuthLoginFailed, target, step: Step::Password }; + let result = if web { + state.meta.audit_record_web_failure(failure, state.now_ms()).await + } else { + state.meta.audit_record_failure(failure, state.now_ms()).await + }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } +} + +/// Writes the event of a sign-in whose challenge was not good (unknown, expired, used, or out of tries), as +/// [`login_failed`] does. `target` is the account when the challenge was found, `None` when it was not (nothing a client sent +/// is stored). The wrong second factors themselves are written by the store, where the account is known. +pub(crate) async fn challenge_failed(state: &AppState, target: Option) { + record_challenge_failure(state, target, false).await; +} + +pub(crate) async fn web_challenge_failed(state: &AppState, target: Option) { + record_challenge_failure(state, target, true).await; +} + +async fn record_challenge_failure(state: &AppState, target: Option, web: bool) { + let failure = AuditFailure { action: craft_cloud_meta::AuditAction::AuthSecondFactorFailed, target, step: Step::ChallengeExpired }; + let result = if web { + state.meta.audit_record_web_failure(failure, state.now_ms()).await + } else { + state.meta.audit_record_failure(failure, state.now_ms()).await + }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } +} + +// ---- reading parameters ---- + +/// The query of the two lists and the export. Unknown parameters are ignored, as everywhere in the API. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct Params { + pub(crate) cursor: Option, + before_ms: Option, + limit: Option, + pub(crate) user: Option, + pub(crate) action: Option, +} + +impl Params { + /// The same filters with the empty ones taken out: a form that was sent with a field left blank asks for no filter. + pub(crate) fn nonempty(&self) -> Params { + let keep = |field: &Option| field.clone().filter(|text| !text.is_empty()); + Params { cursor: keep(&self.cursor), before_ms: self.before_ms, limit: self.limit, user: keep(&self.user), action: keep(&self.action) } + } +} + +fn bad_cursor() -> ApiError { + ApiError::invalid("the cursor is not one this server gave") +} + +pub(crate) fn encode_cursor(cursor: AuditCursor) -> String { + URL_SAFE_NO_PAD.encode(cursor.to_bytes()) +} + +pub(crate) fn decode_cursor(text: &str) -> Result { + let bytes = URL_SAFE_NO_PAD.decode(text).map_err(|_| bad_cursor())?; + AuditCursor::from_bytes(&bytes).ok_or_else(bad_cursor) +} + +/// The limit of a list: 1 to `max`, `DEFAULT_PAGE` when not given. +fn page_limit(limit: Option, max: u32) -> Result { + let limit = limit.unwrap_or(DEFAULT_PAGE); + if (1..=max).contains(&limit) { Ok(limit) } else { Err(ApiError::invalid(format!("limit is 1 to {max}"))) } +} + +/// The `user` and `action` filters. A `user` is an account ID, an `action` one of the names or a prefix of them ending +/// in a dot like `auth.`; anything else is `400 invalid`, so nothing typed reaches a query. +pub(crate) fn filters(params: &Params) -> Result<(Option, Option), ApiError> { + let user = + params.user.as_deref().map(|text| Uuid::parse_str(text).map(UserId).map_err(|_| ApiError::invalid("user is an account ID"))).transpose()?; + let action = params + .action + .as_deref() + .map(|text| ActionFilter::parse(text).ok_or_else(|| ApiError::invalid("action is an event name like auth.login, or a prefix like auth."))) + .transpose()?; + Ok((user, action)) +} + +/// The event as the API shows it. `own`: an account whose own activity this is, which sees the device only of its own acts, and +/// the details of an event about another account (a former administrator's past act on someone else) not at all. +fn wire(event: AuditEvent, own: Option) -> craft_cloud_proto::api::AuditEvent { + let device_visible = own.is_none_or(|own| event.actor_user == Some(own)); + let about_another = own.is_some_and(|own| event.target_kind == "user" && event.target_id.is_some_and(|t| t != own.0)); + craft_cloud_proto::api::AuditEvent { + id: event.id, + at_ms: event.at_ms, + actor_user: event.actor_user.map(|u| u.0), + actor_device: if device_visible { event.actor_device } else { None }, + action: event.action.as_str().to_owned(), + target_kind: event.target_kind, + target_id: event.target_id, + outcome: event.outcome.as_str().to_owned(), + // Written by this code from closed lists; a row that does not read as such shows no details rather than failing the page. + details: if about_another { AuditDetails::default() } else { serde_json::from_str::(&event.details).unwrap_or_default() }, + count: event.count, + } +} + +/// One page: asks for one more event than `limit` to know whether there is a next one. +async fn page(state: &AppState, params: &Params, max: u32, restrict_to: Option) -> Result, ApiError> { + let limit = page_limit(params.limit, max)?; + let (user, action) = filters(params)?; + if params.before_ms.is_some_and(|before| before < 0) { + return Err(ApiError::invalid("before_ms is a time in milliseconds")); + } + let after = params.cursor.as_deref().map(decode_cursor).transpose()?; + let query = AuditQuery { + // An account sees nobody's events but its own, whatever the request names. + user: restrict_to.or(user), + action, + // The cursor is a position and wins over the time. + before_ms: if after.is_some() { None } else { params.before_ms }, + until_ms: None, + after, + limit: limit.saturating_add(1), + }; + let mut events = state.meta.audit_list(&query).await?; + let more = events.len() > limit as usize; + events.truncate(limit as usize); + let next = if more { events.last().map(|event| encode_cursor(event.cursor())) } else { None }; + Ok(Json(AuditPage { events: events.into_iter().map(|event| wire(event, restrict_to)).collect(), next })) +} + +/// `GET /api/v1/account/activity?cursor&before_ms&limit`: the signed-in account's own security events. +pub(crate) async fn account_activity( + State(state): State, + Extension(auth): Extension, + params: Result, QueryRejection>, +) -> Result, ApiError> { + let Query(params) = params?; + // The account sees events about itself; `user` and `action` are the admin's filters and are not read here. + let own = Params { user: None, action: None, ..params }; + page(&state, &own, ACCOUNT_MAX_PAGE, Some(auth.user)).await +} + +/// `GET /api/v1/admin/audit?cursor&before_ms&limit&user&action`. +async fn admin_list(State(state): State, params: Result, QueryRejection>) -> Result, ApiError> { + let Query(params) = params?; + page(&state, ¶ms, ADMIN_MAX_PAGE, None).await +} + +// ---- the export ---- + +/// One page of the export's reading: the events strictly behind `after`, at most `rows`. +type Fetch<'a> = + Box, u32) -> std::pin::Pin, MetaError>> + Send + 'a>> + Send + 'a>; + +/// Reads the log in pages with `fetch` and sends NDJSON to `tx`, a page at a time, ending with the line that says how it +/// ended. Returns without a last line when `fetch` fails (the error is sent, so the connection ends in error) or when the +/// receiver is gone (the reading stops there and then: it also stops a query that is under way). +async fn export_pages(mut fetch: Fetch<'_>, limits: AuditExportLimits, tx: mpsc::Sender>) { + let page = limits.page.max(1); + let mut after = None; + let mut exported = 0u64; + loop { + let rows = tokio::select! { + () = tx.closed() => return, + rows = fetch(after, page) => rows, + }; + let rows = match rows { + Ok(rows) => rows, + Err(e) => { + // The content of the error is the database's; the client only learns that the stream broke. + tracing::warn!(error = %e, "the audit export broke off"); + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + } + }; + let more_in_log = rows.len() >= page as usize; + let room = limits.max_rows.saturating_sub(exported); + let take = usize::try_from(room).unwrap_or(usize::MAX).min(rows.len()); + let mut chunk = Vec::new(); + for event in rows.iter().take(take) { + // Serializing these plain structs cannot fail; if it did, the stream breaks like any other error. + let Ok(line) = serde_json::to_vec(&wire(event.clone(), None)) else { + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + }; + chunk.extend_from_slice(&line); + chunk.push(b'\n'); + } + exported = exported.saturating_add(take as u64); + let cut = take < rows.len(); + if let Some(last) = take.checked_sub(1).and_then(|i| rows.get(i)) { + after = Some(last.cursor()); + } + // At the limit with a full page behind it there may be more, which one more row tells. + let at_limit = !cut && exported >= limits.max_rows && more_in_log; + let ended = if cut { + Some(false) + } else if !more_in_log { + Some(true) + } else if at_limit { + let probe = tokio::select! { + () = tx.closed() => return, + probe = fetch(after, 1) => probe, + }; + match probe { + Ok(rows) => Some(rows.is_empty()), + Err(e) => { + tracing::warn!(error = %e, "the audit export broke off"); + let _ = tx.send(Err(std::io::Error::other("the audit export broke off"))).await; + return; + } + } + } else { + None + }; + if let Some(complete) = ended { + let end = AuditExportEnd { complete: complete.then_some(true), truncated: (!complete).then_some(true), exported }; + if let Ok(line) = serde_json::to_vec(&end) { + chunk.extend_from_slice(&line); + chunk.push(b'\n'); + } + } + if !chunk.is_empty() && tx.send(Ok(Bytes::from(chunk))).await.is_err() { + return; + } + if ended.is_some() { + return; + } + } +} + +/// `GET /api/v1/admin/audit/export?user&action&before_ms`: the log as NDJSON (`before_ms` also limits it, besides the fixed end). +async fn admin_export(State(state): State, params: Result, QueryRejection>) -> Result { + let Query(params) = params?; + export_response(&state, params).await +} + +/// The export as a response, for the JSON route and the web pages' download alike: the same bounds, the same last line. +pub(crate) async fn export_response(state: &Shared, params: Params) -> Result { + let (user, action) = filters(¶ms)?; + if params.before_ms.is_some_and(|before| before < 0) { + return Err(ApiError::invalid("before_ms is a time in milliseconds")); + } + let before_ms = params.before_ms; + let limits = state.audit_export; + // The end of the log is fixed now: events that arrive while the export runs are not in it. + let until_ms = state.now_ms(); + let (tx, rx) = mpsc::channel(limits.channel.max(1)); + let reader = state.clone(); + let fetch: Fetch<'static> = Box::new(move |after, rows| { + let meta = reader.meta.clone(); + Box::pin(async move { meta.audit_list(&AuditQuery { user, action, before_ms, until_ms: Some(until_ms), after, limit: rows }).await }) + }); + tokio::spawn(export_pages(fetch, limits, tx)); + let body = Body::from_stream(futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|item| (item, rx)) })); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::OK; + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, header::HeaderValue::from_static("application/x-ndjson")); + headers.insert(header::CONTENT_DISPOSITION, header::HeaderValue::from_static(EXPORT_DISPOSITION)); + headers.insert(header::CACHE_CONTROL, header::HeaderValue::from_static("no-store")); + Ok(response) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use craft_cloud_meta::{AuditAction, AuditOutcome}; + + use super::*; + + fn event(n: i64) -> AuditEvent { + AuditEvent { + id: Uuid::from_u128(n as u128), + at_ms: 1000 - n, + actor_user: None, + actor_device: None, + action: AuditAction::AuthLogin, + target_kind: String::new(), + target_id: None, + outcome: AuditOutcome::Success, + details: "{}".to_owned(), + count: 1, + } + } + + /// A log of `total` events, newest first, read `rows` at a time; `fail_at` makes that call (counting from 0) fail. + fn fake_log(total: i64, calls: Arc, fail_at: Option) -> Fetch<'static> { + Box::new(move |after, rows| { + let call = calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + if fail_at == Some(call) { + return Err(MetaError::Corrupt("a test failure")); + } + let start = after.map_or(0, |c| 1000 - c.at_ms); + Ok((start + 1..=total).take(rows as usize).map(event).collect()) + }) + }) + } + + async fn run(fetch: Fetch<'static>, limits: AuditExportLimits) -> (Vec, bool) { + let (tx, mut rx) = mpsc::channel(limits.channel); + tokio::spawn(export_pages(fetch, limits, tx)); + let mut lines = Vec::new(); + let mut broke = false; + while let Some(item) = rx.recv().await { + match item { + Ok(bytes) => lines.extend(String::from_utf8(bytes.to_vec()).unwrap().lines().map(str::to_owned)), + Err(_) => broke = true, + } + } + (lines, broke) + } + + fn limits(page: u32, max_rows: u64) -> AuditExportLimits { + AuditExportLimits { page, channel: 1, max_rows } + } + + #[tokio::test] + async fn an_export_ends_with_complete_when_it_holds_every_event() { + for total in [0, 1, 4, 5, 6, 12] { + let (lines, broke) = run(fake_log(total, Arc::default(), None), limits(5, 100)).await; + assert!(!broke); + assert_eq!(lines.len() as i64, total + 1, "total {total}"); + assert_eq!(lines.last().unwrap(), &format!("{{\"complete\":true,\"exported\":{total}}}")); + for line in &lines[..lines.len() - 1] { + serde_json::from_str::(line).unwrap(); + } + } + } + + #[tokio::test] + async fn an_export_over_the_limit_ends_with_truncated_and_counts_only_events() { + // 12 events, at most 7 lines of events: truncated, in the middle of a page. + let (lines, broke) = run(fake_log(12, Arc::default(), None), limits(5, 7)).await; + assert!(!broke); + assert_eq!(lines.len(), 8); + assert_eq!(lines.last().unwrap(), r#"{"truncated":true,"exported":7}"#); + // Exactly at the limit with events behind it: truncated, found by the probe after a full page. + let (lines, _) = run(fake_log(12, Arc::default(), None), limits(5, 10)).await; + assert_eq!(lines.len(), 11); + assert_eq!(lines.last().unwrap(), r#"{"truncated":true,"exported":10}"#); + // Exactly the limit and nothing behind it: complete, not truncated (a full page, then the probe finds nothing). + let (lines, _) = run(fake_log(10, Arc::default(), None), limits(5, 10)).await; + assert_eq!(lines.last().unwrap(), r#"{"complete":true,"exported":10}"#); + // The limit is a multiple of nothing: a page ending short of it. + let (lines, _) = run(fake_log(7, Arc::default(), None), limits(5, 7)).await; + assert_eq!(lines.last().unwrap(), r#"{"complete":true,"exported":7}"#); + } + + #[tokio::test] + async fn a_database_error_in_the_middle_ends_the_stream_without_a_last_line() { + let (lines, broke) = run(fake_log(20, Arc::default(), Some(2)), limits(5, 100)).await; + assert!(broke); + assert_eq!(lines.len(), 10, "the two pages before the error arrived"); + assert!(lines.iter().all(|line| !line.contains("complete") && !line.contains("truncated"))); + } + + #[tokio::test] + async fn a_client_that_goes_away_stops_the_reading() { + let calls = Arc::new(AtomicUsize::new(0)); + let (tx, mut rx) = mpsc::channel(1); + let task = tokio::spawn(export_pages(fake_log(1_000_000, calls.clone(), None), limits(5, u64::MAX), tx)); + // The first page is read, and the client leaves. + assert!(rx.recv().await.is_some()); + drop(rx); + tokio::time::timeout(std::time::Duration::from_secs(10), task).await.unwrap().unwrap(); + let after_leaving = calls.load(Ordering::SeqCst); + // Reads in flight when it left may finish, the channel's depth at most; nothing goes on after that. + assert!(after_leaving <= 4, "{after_leaving}"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!(calls.load(Ordering::SeqCst), after_leaving); + } + + #[test] + fn cursors_are_opaque_and_checked() { + let cursor = AuditCursor { at_ms: 12, id: Uuid::now_v7() }; + assert_eq!(decode_cursor(&encode_cursor(cursor)).unwrap(), cursor); + for bad in ["", "!", "AAAA", &URL_SAFE_NO_PAD.encode([0u8; 23]), &URL_SAFE_NO_PAD.encode([0xffu8; 24]), &URL_SAFE_NO_PAD.encode([0u8; 25])] { + assert!(decode_cursor(bad).is_err(), "{bad}"); + } + } + + #[test] + fn a_limit_is_one_to_the_most_of_the_route() { + assert_eq!(page_limit(None, 100).unwrap(), 50); + assert_eq!(page_limit(Some(100), 100).unwrap(), 100); + assert!(page_limit(Some(101), 100).is_err()); + assert!(page_limit(Some(0), 100).is_err()); + } +} diff --git a/apps/craft-cloud-server/src/auth.rs b/apps/craft-cloud-server/src/auth.rs index 719e9cb3..b7525fa6 100644 --- a/apps/craft-cloud-server/src/auth.rs +++ b/apps/craft-cloud-server/src/auth.rs @@ -15,6 +15,7 @@ use axum::middleware::Next; use axum::response::Response; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{DeviceAuth, Role}; use craft_cloud_proto::api::EPOCH_HEADER; use crate::error::ApiError; @@ -44,6 +45,13 @@ pub fn verify_password(password: &str, hash: &str) -> bool { } } +/// Whether a stored password hash can be verified at all. An account made by an invite has no password yet (an empty hash): a +/// sign-in must still spend a full Argon2id check on it (against the dummy hash, see [`AppState::dummy_hash`]) and then fail +/// whatever was typed, or the answer's timing tells a pending account from an unknown name. +pub fn usable_hash(hash: &str) -> bool { + PasswordHash::new(hash).is_ok() +} + /// A new device token and the hash to store for it. pub fn new_token() -> Result<(String, [u8; 32]), ApiError> { let mut raw = [0u8; 32]; @@ -72,10 +80,17 @@ fn bearer(request: &Request) -> Option<&str> { pub async fn require_device(State(state): State>, mut request: Request, next: Next) -> Result { let token = bearer(&request).ok_or_else(ApiError::unauthorized)?; let device = state.meta.device_by_token(&token_hash(token)).await?.ok_or_else(ApiError::unauthorized)?; - // Off the request's path: the write may wait for the SQLite write lock, and the request needn't wait with it. - let touching = state.clone(); - let (id, now) = (device.device, state.now_ms()); - tokio::spawn(async move { touching.touch(id, now).await }); + // Usage reads and the storage overview are fingerprinted as read-only; do not update last-used timestamps for them. + let path = request.uri().path(); + let admin_usage = ["/api/v1/admin/users/", "/admin/users/"] + .iter() + .any(|prefix| path.strip_prefix(prefix).and_then(|id| id.strip_suffix("/usage")).is_some_and(|id| !id.is_empty() && !id.contains('/'))); + if !matches!(path, "/api/v1/account/usage" | "/account/usage" | "/api/v1/admin/storage" | "/admin/storage") && !admin_usage { + // Off the request's path: the write may wait for the SQLite write lock, and the request needn't wait with it. + let touching = state.clone(); + let (id, now) = (device.device, state.now_ms()); + tokio::spawn(async move { touching.touch(id, now).await }); + } let space = device.space; let epoch = state.meta.space_info(space).await?.epoch; request.extensions_mut().insert(device); @@ -86,6 +101,19 @@ pub async fn require_device(State(state): State>, mut request: Req Ok(response) } +/// Lets only an admin through, after [`require_device`] (it reads the device that layer put in the request extensions). +/// The role is the one `device_by_token` read from the database for this request, never cached, so a demoted admin +/// loses access with the next request. It is only the first check: every admin change checks the role again inside +/// its transaction. +pub async fn require_admin(request: Request, next: Next) -> Result { + match request.extensions().get::() { + Some(device) if device.role == Role::Admin => Ok(next.run(request).await), + Some(_) => Err(ApiError::forbidden("this needs an administrator account")), + // Not reachable behind `require_device`; a request without a device is not signed in. + None => Err(ApiError::unauthorized()), + } +} + impl AppState { /// Updates the device's `last used` time, at most once a minute (every write takes the SQLite write lock). async fn touch(&self, device: craft_cloud_proto::DeviceId, now: i64) { @@ -118,6 +146,13 @@ mod tests { assert!(!verify_password("correct horse", "not a hash")); } + #[test] + fn only_a_parseable_hash_is_usable() { + assert!(usable_hash(&hash_password("correct horse").unwrap())); + assert!(!usable_hash(""), "an invited account has no password yet"); + assert!(!usable_hash("not a hash")); + } + #[test] fn password_rules() { assert!(check_password_rules("short").is_err()); diff --git a/apps/craft-cloud-server/src/cli.rs b/apps/craft-cloud-server/src/cli.rs index a9b4908e..7974d8ce 100644 --- a/apps/craft-cloud-server/src/cli.rs +++ b/apps/craft-cloud-server/src/cli.rs @@ -2,11 +2,12 @@ //! appear in the process list or shell history: `printf '%s\n' "$PASSWORD" | craft-cloud-server user add …`. use std::io::{BufRead, Read, Write}; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; use clap::{Parser, Subcommand}; -use craft_cloud_meta::{Meta, Role, valid_username}; +use craft_cloud_meta::{ActionFilter, AuditQuery, Meta, Origin, RevokeReason, Role, valid_username}; use craft_cloud_proto::DeviceId; use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, JoinString}; use craft_cloud_store::Store; @@ -15,6 +16,7 @@ use craft_cloud_store::s3::S3Backend; use crate::auth::{check_password_rules, hash_password}; use crate::config::{Config, StorageConfig}; +use crate::secretbox::TwoFactorKey; use crate::state::now_ms; #[derive(Debug, Parser)] @@ -43,6 +45,11 @@ pub enum Command { /// The reverse proxy's address, whose `X-Forwarded-For` is believed (repeat for several proxies). #[arg(long = "trusted-proxy")] trusted_proxies: Vec, + /// What the blob store may hold, like "1 TiB" or "500 GB" (B, KB, MB, GB, TB, KiB, MiB, GiB, TiB). Written to the + /// config as `[storage] capacity`; the admin's storage overview shows it. Without it none is set (a disk store's + /// overview still shows its filesystem's size). `init` makes a disk store; for S3, set `capacity` in the config. + #[arg(long)] + capacity: Option, }, /// Run the server. Serve { @@ -77,6 +84,61 @@ pub enum Command { #[arg(long)] all: bool, }, + /// Read and prune the security audit log. + Audit { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: AuditCommand, + }, + /// Check the database and the blob store against each other and report: the reference index of every manifest, the + /// manifests and chunks the database names against the objects in the store, and what names what. Changes nothing; + /// exits with an error when it finds a problem. Reads every chunk object (`--skip-chunk-reads` leaves that out). + Verify { + /// Path to config.toml. + #[arg(long)] + config: Option, + /// Print the report as JSON. + #[arg(long)] + json: bool, + /// Check chunks against the database only; do not read their objects from the store. + #[arg(long)] + skip_chunk_reads: bool, + }, + /// Inventory the store and report possible collection candidates without changing anything. + Gc { + #[arg(long)] + config: Option, + #[arg(long)] + dry_run: bool, + #[arg(long)] + json: bool, + }, + /// Clean up the admin pages' browser sessions. + WebSessions { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: WebSessionsCommand, + }, + /// Manage the key for two-factor secrets. + TwoFactor { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: TwoFactorAction, + }, + /// The reference index of what each manifest names. + Index { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: IndexAction, + }, /// Manage live documents. Live { /// Path to config.toml. May come before or after the action. @@ -85,6 +147,71 @@ pub enum Command { #[command(subcommand)] action: LiveAction, }, + /// Ask a running server whether it answers (`GET /api/info`). Prints nothing and exits 0 if it does; otherwise + /// exits 1 with one line on standard error. For a container's health check, which has no shell and no `curl`. + Healthcheck { + /// The address the server listens on, as seen from here. + #[arg(long, default_value = "127.0.0.1:8080")] + addr: String, + }, +} + +#[derive(Debug, Subcommand)] +pub enum AuditCommand { + /// Print the newest security events, newest first, as text. An event names accounts by ID, never by name. + List { + /// Only events this account acted in or was the target of. + #[arg(long)] + user: Option, + /// Only this event (like auth.login) or a group of them (like auth.). + #[arg(long)] + action: Option, + /// How many events at most (1 to 200). + #[arg(long, default_value_t = 50)] + limit: u32, + }, + /// Delete the events older than `[audit] retention_days` and say how many. Run it now and then (daily, from cron or + /// the task scheduler): the server also prunes when it writes an event, but not while no one signs in or changes anything. + Prune { + /// Say how many events would go and delete none. + #[arg(long)] + dry_run: bool, + }, +} + +#[derive(Debug, Subcommand)] +pub enum WebSessionsCommand { + /// Delete the browser sessions that are over (logged out, past `[web_admin] absolute_hours`, or unused for longer than + /// `idle_minutes`) and say how many. The server also deletes a few at each sign-in, but not while no one signs in; run this + /// now and then (daily, from cron or the task scheduler) to empty the table. A session that is still valid is never touched. + Prune { + /// Say how many sessions would go and delete none. + #[arg(long)] + dry_run: bool, + }, +} + +#[derive(Debug, Subcommand)] +pub enum TwoFactorAction { + /// Rewrite every two-factor secret that is sealed with the older key (`key_file_old`) under the current one + /// (`key_file`). To rotate the key: put the new key in `key_file` and the old one in `key_file_old`, run this, then + /// remove `key_file_old`. It can be run again, changes nothing when it is done, and refuses (changing nothing) when a + /// secret is sealed with a key that is neither. Prints counts, never a secret or a key. Restart every server instance on + /// the database with the new `key_file` and `key_file_old` first (a running server holds the keys it started with), and + /// remove `key_file_old` only after this reports that every secret is under the current key. + Rekey, +} + +#[derive(Debug, Subcommand)] +pub enum IndexAction { + /// Complete the index of every manifest whose index is not complete: reads each manifest from the store, checks it + /// against its ID, and writes its edges in batches. Writes only that derived index. Safe to stop and run again, and to + /// run beside the server. + Build { + /// Edges written per database transaction (default 5000). + #[arg(long)] + batch_rows: Option, + }, } #[derive(Debug, Subcommand)] @@ -103,11 +230,26 @@ pub enum UserAction { #[arg(long)] admin: bool, }, - /// Set a new password, read from standard input. - ResetPassword { username: String }, + /// Set a new password, read from standard input. Every device of the account is signed out and its open invite + /// and add-a-computer codes are deleted, so a stolen device or code does not outlive the old password; with + /// `--keep-devices` the devices stay signed in. A device's live connection ends at its next ping (about 30 + /// seconds) and a change-feed long-poll within 60 seconds, since the server runs in another process. + ResetPassword { + username: String, + /// Leave the account's devices signed in. + #[arg(long)] + keep_devices: bool, + }, + /// Take away an account's two-factor sign-in, for someone who lost their authenticator and their recovery codes: its + /// secret, recovery codes and open sign-in challenges are deleted, the lock after wrong codes goes, **every device of the + /// account is signed out** and its open invite and add-a-computer codes are deleted. Prints how many devices were signed + /// out. The person signs in with their password again and can set it up anew. A device's live connection ends at its next + /// ping (about 30 seconds), since the server runs in another process. + #[command(name = "reset-2fa")] + Reset2fa { username: String }, /// Print an invite code for an account, good once for 7 days, to join a computer with in Craft Cloud. Creates the /// account (without a password, chosen when the invite is redeemed) if there is none, and replaces its earlier - /// invite. + /// invite or password reset code. Invite { username: String, /// Revoke the account's invite instead. @@ -118,8 +260,18 @@ pub enum UserAction { #[derive(Debug, Subcommand)] pub enum DeviceAction { - List { username: String }, - Revoke { username: String, device: DeviceId }, + List { + username: String, + }, + Revoke { + username: String, + device: DeviceId, + }, + /// Sign out every device of an account. Live connections end at their next ping (about 30 seconds), change-feed + /// long-polls within 60 seconds. + RevokeAll { + username: String, + }, } #[derive(Debug, thiserror::Error)] @@ -130,19 +282,44 @@ pub enum CliError { Meta(#[from] craft_cloud_meta::MetaError), #[error(transparent)] Store(#[from] craft_cloud_store::StoreError), + #[error(transparent)] + Gc(#[from] crate::gc::GcError), #[error("{0}")] Usage(String), + /// A check found something wrong (its report was written first). + #[error("{0}")] + Inconsistent(String), #[error("I/O error: {0}")] Io(#[from] std::io::Error), } /// Opens the database and blob storage a config names. pub async fn open(config: &Config) -> Result<(Meta, Store), CliError> { - let meta = - Meta::open(&config.database.url).await?.with_stored_bytes_cap(config.stored_bytes_per_user).with_live_bytes_cap(config.live_bytes_per_user); + let meta = Meta::open(&config.database.url) + .await? + .with_stored_bytes_cap(config.stored_bytes_per_user) + .with_live_bytes_cap(config.live_bytes_per_user) + .with_audit_retention_days(config.audit.retention_days); + let store = match (&config.storage, config.storage.s3()?) { + (_, Some(s3)) => Store::new(S3Backend::new(&s3)?), + (StorageConfig::Disk { path, .. }, None) => Store::new(DiskBackend::new(path)?), + (StorageConfig::S3 { .. }, None) => return Err(CliError::Usage("S3 storage without settings".into())), + }; + Ok((meta, store)) +} + +/// Like [`open`], for a command that only reads: the database and the storage must exist already, and nothing is +/// created or upgraded. A config that points at the wrong place is an error, not a new empty server that has no problems. +pub async fn open_existing(config: &Config) -> Result<(Meta, Store), CliError> { + let meta = Meta::open_existing(&config.database.url).await?; let store = match (&config.storage, config.storage.s3()?) { (_, Some(s3)) => Store::new(S3Backend::new(&s3)?), - (StorageConfig::Disk { path }, None) => Store::new(DiskBackend::new(path)?), + (StorageConfig::Disk { path, .. }, None) => { + if !path.is_dir() { + return Err(CliError::Usage(format!("the storage directory {} does not exist (is the config's path right?)", path.display()))); + } + Store::new(DiskBackend::new(path)?) + } (StorageConfig::S3 { .. }, None) => return Err(CliError::Usage("S3 storage without settings".into())), }; Ok((meta, store)) @@ -175,10 +352,15 @@ async fn user_id(meta: &Meta, username: &str) -> Result Result<(), CliError> { match command { - Command::Init { dir, admin, listen, behind_proxy, trusted_proxies } => { - init(&dir, &admin, &listen, (behind_proxy, &trusted_proxies), input, out).await + Command::Init { dir, admin, listen, behind_proxy, trusted_proxies, capacity } => { + init(&dir, &admin, &listen, (behind_proxy, &trusted_proxies), capacity.as_deref(), input, out).await } Command::Serve { .. } => Err(CliError::Usage("serve is run by main".into())), + Command::Healthcheck { addr } => { + let addr: SocketAddr = addr.parse().map_err(|_| CliError::Usage(format!("bad address {addr}")))?; + // Blocking for at most `HEALTH_TIMEOUT`: this process does nothing else. + check_health(addr, HEALTH_TIMEOUT) + } Command::User { config, action } => { let config = Config::load(&config_path(config)?)?; let (meta, _) = open(&config).await?; @@ -190,6 +372,12 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ writeln!(out, "revoked the invite for {}", username.to_ascii_lowercase())?; } UserAction::Invite { username, revoke: false } => invite(&meta, &config, &username, out).await?, + UserAction::Reset2fa { username } => { + let user = user_id(&meta, &username).await?; + let revoked = meta.reset_two_factor(user.id, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "two-factor sign-in reset"); + writeln!(out, "two-factor sign-in of {} is off; signed out {} device(s)", user.username, revoked.len())?; + } UserAction::Add { username, admin } => { let password = read_password(input)?; let role = if admin { Role::Admin } else { Role::Member }; @@ -197,12 +385,17 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ let user = meta.create_user(&username, &hash, role, now_ms()).await?; writeln!(out, "added {} ({})", user.username, if admin { "admin" } else { "member" })?; } - UserAction::ResetPassword { username } => { + UserAction::ResetPassword { username, keep_devices } => { let user = user_id(&meta, &username).await?; let password = read_password(input)?; let hash = hash_password(&password).map_err(|e| CliError::Usage(e.message))?; - meta.set_password_hash(user.id, &hash).await?; - writeln!(out, "password changed for {}", user.username)?; + let revoked = meta.reset_password(user.id, &hash, keep_devices, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "password reset"); + if keep_devices { + writeln!(out, "password changed for {}; the devices stay signed in", user.username)?; + } else { + writeln!(out, "password changed for {}; signed out {} device(s)", user.username, revoked.len())?; + } } } Ok(()) @@ -218,12 +411,28 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } DeviceAction::Revoke { username, device } => { let user = user_id(&meta, &username).await?; - meta.revoke_device(user.id, device).await?; + meta.revoke_device_by(user.id, device, RevokeReason::Admin, Origin::Cli, now_ms()).await?; writeln!(out, "revoked {device}")?; } + DeviceAction::RevokeAll { username } => { + let user = user_id(&meta, &username).await?; + let revoked = meta.revoke_all_devices(user.id, RevokeReason::Admin, now_ms()).await?; + tracing::info!(revoked = revoked.len(), "all devices revoked"); + writeln!(out, "revoked {} device(s) of {}", revoked.len(), user.username)?; + } } Ok(()) } + Command::Audit { config, action } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + audit(&meta, &config, action, out).await + } + Command::WebSessions { config, action } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + web_sessions(&meta, &config, action, out).await + } Command::ResetEpoch { config, username, all } => { let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; if all { @@ -241,6 +450,70 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } Ok(()) } + Command::TwoFactor { config, action: TwoFactorAction::Rekey } => { + let config = Config::load(&config_path(config)?)?; + let (meta, _) = open(&config).await?; + let keys = crate::two_factor::load_keys(&config).map_err(|e| CliError::Usage(e.to_string()))?; + let crate::two_factor::KeyState::Loaded(secret_box) = keys else { + return Err(CliError::Usage( + "there is no key: set [two_factor] key_file (and key_file_old for the key to rotate away from) in the config".into(), + )); + }; + let done = crate::two_factor::rekey(&meta, &secret_box).await.map_err(CliError::Usage)?; + writeln!(out, "rewrote {} secret(s) under the current key; {} were under it already", done.rewritten, done.already_current)?; + if secret_box.old_id().is_some() { + writeln!( + out, + "every secret is under the current key now. key_file_old can be removed once every server instance on this database has been \ + restarted with key_file (the new key) and key_file_old, as it must have been before this command: an instance still running with \ + only the old key would seal new secrets with it" + )?; + } + Ok(()) + } + Command::Verify { config, json, skip_chunk_reads } => { + let (meta, store) = open_existing(&Config::load(&config_path(config)?)?).await?; + let report = crate::verify::verify(&meta, &store, crate::verify::VerifyOptions { read_chunks: !skip_chunk_reads }).await?; + if json { + writeln!(out, "{}", report.to_json())?; + } else { + write!(out, "{}", report.to_text())?; + } + match report.problem_count() { + 0 => Ok(()), + n => Err(CliError::Inconsistent(format!("verify found {n} problem(s)"))), + } + } + Command::Gc { dry_run: false, .. } => Err(CliError::Usage("deleting is not implemented yet; run with --dry-run".into())), + Command::Gc { config, dry_run: true, json } => { + let config = Config::load(&config_path(config)?)?; + let (meta, store) = open_existing(&config).await?; + let report = crate::gc::dry_run(&meta, &store).await?; + if json { + writeln!(out, "{}", report.to_json())?; + } else { + write!(out, "{}", report.to_text())?; + } + if report.has_problems() { Err(CliError::Inconsistent("gc dry-run found problems".into())) } else { Ok(()) } + } + Command::Index { config, action: IndexAction::Build { batch_rows } } => { + let (meta, store) = open(&Config::load(&config_path(config)?)?).await?; + let meta = match batch_rows { + Some(0) => return Err(CliError::Usage("--batch-rows must be at least 1".into())), + Some(rows) => meta.with_index_batch_rows(rows), + None => meta, + }; + let report = crate::verify::build_index(&meta, &store).await?; + writeln!(out, "indexed {} manifest(s)", report.indexed)?; + for (id, why) in &report.unreadable { + writeln!(out, "could not index manifest {id}: {why}")?; + } + if report.unreadable.is_empty() { + Ok(()) + } else { + Err(CliError::Inconsistent(format!("{} manifest(s) could not be indexed", report.unreadable.len()))) + } + } Command::Live { config, action: LiveAction::ResetDoc { username, app } } => { let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; let user = user_id(&meta, &username).await?; @@ -259,6 +532,125 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } } +/// How long `healthcheck` waits for the whole exchange: connecting, sending and reading the status line. +pub const HEALTH_TIMEOUT: Duration = Duration::from_secs(3); + +/// Sends `GET /api/info` to `addr` and succeeds only if the status line says 200. The whole exchange must finish +/// within `timeout`. The body is never read, and nothing the server sends is printed. +pub fn check_health(addr: SocketAddr, timeout: Duration) -> Result<(), CliError> { + let fail = |what: &str| CliError::Usage(format!("{addr} is not healthy: {what}")); + let deadline = Instant::now() + timeout; + let left = || deadline.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()); + let mut stream = TcpStream::connect_timeout(&addr, timeout).map_err(|e| fail(&format!("can't connect: {e}")))?; + let request = format!("GET /api/info HTTP/1.0\r\nHost: {addr}\r\nConnection: close\r\n\r\n"); + stream.set_write_timeout(left()).map_err(|e| fail(&e.to_string()))?; + stream.write_all(request.as_bytes()).map_err(|e| fail(&format!("can't send the request: {e}")))?; + // The status line fits in 64 bytes. It counts only when it is complete (ends in CRLF): an answer that stops inside + // it, or goes on for 64 bytes without a line end, is not a healthy server. + let mut head = [0u8; 64]; + let mut len = 0; + let line_end = loop { + if let Some(end) = head.get(..len).and_then(|read| read.windows(2).position(|pair| pair == b"\r\n")) { + break end; + } + let wait = left().ok_or_else(|| fail("no answer in time"))?; + stream.set_read_timeout(Some(wait)).map_err(|e| fail(&e.to_string()))?; + let Some(room) = head.get_mut(len..).filter(|room| !room.is_empty()) else { return Err(fail("not an HTTP answer")) }; + match stream.read(room) { + Ok(0) => return Err(fail("the answer ended inside the status line")), + Ok(n) => len += n, + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => return Err(fail("no answer in time")), + Err(e) => return Err(fail(&format!("can't read the answer: {e}"))), + } + }; + let line = String::from_utf8_lossy(head.get(..line_end).unwrap_or_default()); + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some(version), Some("200")) if version.starts_with("HTTP/1.") => Ok(()), + (Some(version), Some(status)) if version.starts_with("HTTP/") => Err(fail(&format!("status {}", status.chars().take(3).collect::()))), + _ => Err(fail("not an HTTP answer")), + } +} + +/// Sessions one batch of `web-sessions prune` deletes. +const WEB_SESSION_PRUNE_BATCH: u32 = 1000; + +/// `web-sessions prune`: the batches go on until none is left, each in a transaction of its own. +async fn web_sessions(meta: &Meta, config: &Config, action: WebSessionsCommand, out: &mut impl Write) -> Result<(), CliError> { + let idle_ms = config.web_admin.idle_ms(); + let WebSessionsCommand::Prune { dry_run } = action; + if dry_run { + let due = meta.web_sessions_count_over(now_ms(), idle_ms).await?; + writeln!(out, "{due} web session(s) are over; dry run, nothing deleted")?; + return Ok(()); + } + let mut deleted = 0u64; + loop { + let batch = meta.web_sessions_prune(now_ms(), idle_ms, WEB_SESSION_PRUNE_BATCH).await?; + deleted = deleted.saturating_add(batch); + if batch < u64::from(WEB_SESSION_PRUNE_BATCH) { + break; + } + } + tracing::info!(deleted, "web sessions pruned"); + writeln!(out, "deleted {deleted} web session(s) that were over")?; + Ok(()) +} + +/// `audit list` and `audit prune`. +async fn audit(meta: &Meta, config: &Config, action: AuditCommand, out: &mut impl Write) -> Result<(), CliError> { + match action { + AuditCommand::List { user, action, limit } => { + if !(1..=200).contains(&limit) { + return Err(CliError::Usage("--limit is 1 to 200".into())); + } + let user = match user { + Some(name) => Some(user_id(meta, &name).await?.id), + None => None, + }; + let action = match action { + Some(text) => Some( + ActionFilter::parse(&text) + .ok_or_else(|| CliError::Usage("--action is an event name like auth.login, or a group like auth.".into()))?, + ), + None => None, + }; + let events = meta.audit_list(&AuditQuery { user, action, limit, ..AuditQuery::default() }).await?; + let id = |id: Option| id.map_or_else(|| "-".to_owned(), |id| id.to_string()); + for e in &events { + // Only closed words, IDs and numbers: nothing here is a secret or a name anyone typed. + writeln!( + out, + "{} {} {} actor={} device={} target={}:{} count={} {}", + e.at_ms, + e.action.as_str(), + e.outcome.as_str(), + id(e.actor_user.map(|u| u.0)), + id(e.actor_device.map(|d| *d.as_uuid())), + if e.target_kind.is_empty() { "-" } else { e.target_kind.as_str() }, + id(e.target_id), + e.count, + e.details, + )?; + } + writeln!(out, "{} event(s)", events.len())?; + } + AuditCommand::Prune { dry_run } => { + let days = config.audit.retention_days; + let before = now_ms().saturating_sub(i64::from(days).saturating_mul(24 * 60 * 60 * 1000)); + if dry_run { + let due = meta.audit_count_before(before).await?; + writeln!(out, "{due} audit event(s) are older than {days} days; dry run, nothing deleted")?; + } else { + let deleted = meta.audit_prune(before).await?; + tracing::info!(deleted, "audit events pruned"); + writeln!(out, "deleted {deleted} audit event(s) older than {days} days")?; + } + } + } + Ok(()) +} + /// `user invite`: makes the invite and prints how to use it. The code is printed only here, never logged. async fn invite(meta: &Meta, config: &Config, username: &str, out: &mut impl Write) -> Result<(), CliError> { let code = Code::generate(INVITE_CHARS).map_err(|e| CliError::Usage(format!("the system random source failed: {e}")))?; @@ -300,6 +692,7 @@ async fn init( admin: &str, listen: &str, proxy: (bool, &[IpAddr]), + capacity: Option<&str>, input: &mut impl BufRead, out: &mut impl Write, ) -> Result<(), CliError> { @@ -318,12 +711,22 @@ async fn init( let dir = std::path::absolute(dir)?; let db = without_verbatim_prefix(&dir.join("meta.db").display().to_string()).to_owned(); let blobs = without_verbatim_prefix(&dir.join("blobs").display().to_string()).to_owned(); + // Checked here so a bad size is named as `--capacity`, then written as the admin typed it. + let capacity_line = match capacity { + Some(text) => { + crate::config::Capacity::parse(text).map_err(|e| CliError::Usage(e.replace("storage.capacity", "--capacity")))?; + format!("capacity = {}\n", toml_string(text.trim())) + } + None => String::new(), + }; let trusted: Vec = trusted_proxies.iter().map(|ip| format!("\"{ip}\"")).collect(); let config_text = format!( - "listen = \"{listen}\"\nbehind_proxy = {behind_proxy}\ntrusted_proxies = [{}]\n\n[database]\nurl = {}\n\n[storage]\nkind = \"disk\"\npath = {}\n", + "listen = \"{listen}\"\nbehind_proxy = {behind_proxy}\ntrusted_proxies = [{}]\n\n[database]\nurl = {}\n\n[storage]\nkind = \"disk\"\npath = {}\n{}\n\ + [two_factor]\nkey_file = \"{KEY_FILE_NAME}\"\n", trusted.join(", "), toml_string(&format!("sqlite:{db}")), toml_string(&blobs), + capacity_line, ); let config = Config::parse(&config_text)?; let password = read_password(input)?; @@ -335,12 +738,18 @@ async fn init( return Err(e); } writeln!(out, "created {}\nrun: craft-cloud-server serve --config {}", dir.display(), dir.join("config.toml").display())?; + writeln!( + out, + "two-factor key: {}. Keep a copy apart from your database backups: without it the two-factor secrets in a backup can't be opened.", + dir.join(KEY_FILE_NAME).display() + )?; Ok(()) } async fn create_files(dir: &Path, config_text: &str, config: &Config, admin: &str, hash: &str) -> Result<(), CliError> { create_data_dir(dir)?; write_config(&dir.join("config.toml"), config_text)?; + write_key_file(&dir.join(KEY_FILE_NAME))?; let (meta, _) = open(config).await?; meta.create_user(admin, hash, Role::Admin, now_ms()).await?; for name in ["meta.db", "meta.db-wal", "meta.db-shm"] { @@ -406,10 +815,28 @@ fn write_config(path: &Path, text: &str) -> Result<(), CliError> { Ok(()) } +/// The key file `init` makes, next to `config.toml`. +const KEY_FILE_NAME: &str = "two-factor.key"; + +/// Writes a new random key for two-factor secrets: 32 bytes, owner-only on Unix, and never over a file that is there. +fn write_key_file(path: &Path) -> Result<(), CliError> { + let key = TwoFactorKey::generate().map_err(|e| CliError::Usage(format!("the system random source failed: {e}")))?; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(key.to_file_bytes().as_slice())?; + Ok(()) +} + /// Removes what a failed `init` made. The directory was empty (or absent) when `init` started, so /// everything in it is `init`'s. Best effort: the original error is what the admin needs to see. fn remove_made(dir: &Path, created_dir: bool) { - for name in ["config.toml", "meta.db", "meta.db-wal", "meta.db-shm"] { + for name in ["config.toml", KEY_FILE_NAME, "meta.db", "meta.db-wal", "meta.db-shm"] { let _ = std::fs::remove_file(dir.join(name)); } let _ = std::fs::remove_dir_all(dir.join("blobs")); @@ -464,7 +891,7 @@ mod tests { let parent = tempfile::tempdir().unwrap(); let dir = parent.path().join("data"); std::fs::create_dir_all(dir.join("blobs")).unwrap(); - for name in ["config.toml", "meta.db", "meta.db-wal", "blobs/chunk"] { + for name in ["config.toml", "two-factor.key", "meta.db", "meta.db-wal", "blobs/chunk"] { std::fs::write(dir.join(name), b"x").unwrap(); } remove_made(&dir, true); @@ -513,6 +940,14 @@ mod tests { } } + #[test] + fn healthcheck_parses_with_and_without_an_address() { + let cli = Cli::try_parse_from(["craft-cloud-server", "healthcheck"]).unwrap(); + assert!(matches!(&cli.command, Command::Healthcheck { addr } if addr == "127.0.0.1:8080")); + let cli = Cli::try_parse_from(["craft-cloud-server", "healthcheck", "--addr", "127.0.0.1:9000"]).unwrap(); + assert!(matches!(&cli.command, Command::Healthcheck { addr } if addr == "127.0.0.1:9000")); + } + #[test] fn reset_epoch_takes_a_username_or_all_but_not_both_or_neither() { assert!(Cli::try_parse_from(["craft-cloud-server", "reset-epoch", "--config", "c.toml", "--all"]).is_ok()); diff --git a/apps/craft-cloud-server/src/config.rs b/apps/craft-cloud-server/src/config.rs index 725e9a48..1aff275e 100644 --- a/apps/craft-cloud-server/src/config.rs +++ b/apps/craft-cloud-server/src/config.rs @@ -9,13 +9,30 @@ //! # live_bytes_per_user = 1_073_741_824 # cap on what one account's live documents hold, in bytes (1 GiB) //! # public_url = "https://cloud.example.com" # where computers reach the server; `user invite` prints it //! +//! # [audit] +//! # retention_days = 365 # how long security events are kept (1 to 3650); `audit prune` deletes older ones +//! +//! # [web_admin] # the admin pages under /web, off unless enabled; needs behind_proxy and an https public_url +//! # enabled = true +//! # idle_minutes = 480 # a browser session ends after this long unused (5 to 1440) +//! # absolute_hours = 168 # and at the latest after this long (1 to 720) +//! # [retention] +//! # trash_days = 30 # how long a deleted file stays in the trash before the server may remove it (1 to 3650) +//! //! [database] //! url = "sqlite:/var/lib/craft-cloud/meta.db" # or postgres://user:pass@host/db //! //! [storage] //! kind = "disk" //! path = "/var/lib/craft-cloud/blobs" -//! # kind = "s3", bucket, prefix, region, endpoint, access_key_id, secret_access_key, allow_http +//! # capacity = "1 TiB" # what the store may hold, for the admin's storage overview (needed for S3 to show a size) +//! # kind = "s3", bucket, prefix, region, endpoint, access_key_id, secret_access_key, allow_http, capacity +//! # refuse_when_full = true # refuse new uploads when the disk is down to min_free, or the store holds `capacity` (default off) +//! # min_free = "1 GiB" # disk only, with refuse_when_full: the free space to keep (default 1 GiB) +//! +//! [two_factor] # optional: without it, two-factor sign-in can't be set up +//! key_file = "two-factor.key" # 32 random bytes, made by `init`; relative to this file +//! # key_file_old = "old.key" # while rotating the key, see `two-factor rekey` //! ``` //! //! The server speaks plain HTTP. It refuses to listen anywhere but loopback unless `behind_proxy = true`, @@ -55,10 +72,61 @@ pub enum ConfigError { agent could not tell a full account from a file that is too big. Raise it, or remove it for no cap" )] StoredBytesCapTooSmall(u64, u32), + #[error("retention.trash_days = {0} is not between {MIN_TRASH_DAYS} and {MAX_TRASH_DAYS}")] + TrashDays(u32), #[error("public_url = {0:?} is not a server address like https://cloud.example.com (a host, an optional port and path, no query)")] PublicUrl(String), #[error("storage.{0} is missing (set it in the config or the {1} environment variable)")] MissingS3Credential(&'static str, &'static str), + #[error("audit.retention_days = {0} is out of range: keep security events for {1} to {2} days")] + AuditRetention(u32, u32, u32), + #[error( + "web_admin.enabled = true needs public_url = \"https://host[:port]\" with no path: the browser's cookie is only sent over \ + HTTPS, and the pages check the Origin of every form against that address" + )] + WebAdminPublicUrl, + #[error( + "web_admin.enabled = true needs behind_proxy = true: the server speaks plain HTTP, and the session cookie is marked Secure, \ + so a TLS reverse proxy must stand in front of it" + )] + WebAdminNeedsProxy, + #[error("web_admin.{0} = {1} is out of range: {2} to {3}")] + WebAdminRange(&'static str, u32, u32, u32), + #[error("storage.min_free only goes with storage.refuse_when_full = true: without it nothing is refused, so the reserve would mean nothing")] + MinFreeWithoutRefusing, + #[error( + "storage.min_free is for a disk store: an S3 bucket has no free space to measure. Remove it, and set storage.capacity to bound the bucket" + )] + MinFreeOnS3, + #[error( + "storage.refuse_when_full = true on an S3 store needs storage.capacity: the bucket has no free space to measure, so the capacity is the only limit there is" + )] + RefuseOnS3WithoutCapacity, +} + +/// Fewest and most days a deleted node waits in the trash. +pub const MIN_TRASH_DAYS: u32 = 1; +pub const MAX_TRASH_DAYS: u32 = 3650; +/// Days in the trash when `retention.trash_days` is not set. +pub const DEFAULT_TRASH_DAYS: u32 = 30; + +/// `[retention]`: how long things are kept. Only the trash today; nothing is removed by it yet. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RetentionConfig { + /// Days a deleted node stays in the trash before the server may remove it for good (shown as `purge_after_ms`). + #[serde(default = "default_trash_days")] + pub trash_days: u32, +} + +impl Default for RetentionConfig { + fn default() -> Self { + Self { trash_days: DEFAULT_TRASH_DAYS } + } +} + +fn default_trash_days() -> u32 { + DEFAULT_TRASH_DAYS } #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -90,14 +158,107 @@ pub struct Config { /// trailing slash. Unset: `user invite` prints the bare code and says to put the address in front. #[serde(default)] pub public_url: Option, + /// The security audit log. The section is optional and its defaults are safe. + #[serde(default)] + pub audit: AuditConfig, + /// The admin pages in the browser (`/web`). Off unless the section says `enabled = true`. + #[serde(default)] + pub web_admin: WebAdminConfig, + #[serde(default)] + pub retention: RetentionConfig, pub database: DatabaseConfig, pub storage: StorageConfig, + /// The key for the secrets of two-factor sign-in. Without it two-factor sign-in can't be set up (`503 + /// two_factor_unavailable`), and a server whose database already holds secrets refuses to start. `init` makes the + /// key file and writes this section. The server never makes the key itself: a second server on the same database, + /// or a restore without the key, would otherwise seal secrets nobody can open. + #[serde(default)] + pub two_factor: Option, + /// The directory the config file is in, which `key_file` paths are relative to. Set by [`Config::load`], never read + /// from the file. + #[serde(skip)] + pub dir: Option, +} + +/// `[two_factor]`: where the key for two-factor secrets is. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TwoFactorConfig { + /// A file of 32 random bytes (mode 0600; Windows has no equivalent here, keep it out of everyone's reach). A relative + /// path is relative to the directory of `config.toml`. + pub key_file: PathBuf, + /// The key the secrets were sealed with before a rotation: new secrets are sealed with `key_file`, old ones still + /// open with this one until `two-factor rekey` has rewritten them. Remove it after that. + #[serde(default)] + pub key_file_old: Option, } fn default_live_bytes() -> u64 { craft_cloud_meta::DEFAULT_LIVE_BYTES_PER_SPACE } +/// `[audit]`: how the security audit log is kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuditConfig { + /// Days a security event is kept: `audit prune` deletes older ones, and a write does so now and then (at most once an + /// hour). 1 to 3650, default 365. A server that no one writes to keeps expired events until `audit prune` runs. + #[serde(default = "default_retention_days")] + pub retention_days: u32, +} + +impl Default for AuditConfig { + fn default() -> Self { + Self { retention_days: default_retention_days() } + } +} + +fn default_retention_days() -> u32 { + 365 +} + +/// `[web_admin]`: the admin pages in the browser and the lifetimes of their sessions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WebAdminConfig { + /// Serve the admin pages under `/web`. Default off: a server that was not told to speak to browsers does not. Needs + /// `behind_proxy = true` and an `https://` `public_url` without a path. + #[serde(default)] + pub enabled: bool, + /// A session ends when it was not used for this many minutes: 5 to 1440, default 480 (8 hours). + #[serde(default = "default_idle_minutes")] + pub idle_minutes: u32, + /// A session ends this many hours after sign-in, however busy it is: 1 to 720, default 168 (7 days). + #[serde(default = "default_absolute_hours")] + pub absolute_hours: u32, +} + +impl Default for WebAdminConfig { + fn default() -> Self { + Self { enabled: false, idle_minutes: default_idle_minutes(), absolute_hours: default_absolute_hours() } + } +} + +fn default_idle_minutes() -> u32 { + 8 * 60 +} + +fn default_absolute_hours() -> u32 { + 7 * 24 +} + +impl WebAdminConfig { + /// The idle lifetime of a session in milliseconds. + pub fn idle_ms(&self) -> i64 { + i64::from(self.idle_minutes) * 60_000 + } + + /// The absolute lifetime of a session in milliseconds. + pub fn absolute_ms(&self) -> i64 { + i64::from(self.absolute_hours) * 3_600_000 + } +} + #[derive(Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct DatabaseConfig { @@ -128,8 +289,27 @@ pub fn redact_url(url: &str) -> String { pub enum StorageConfig { Disk { path: PathBuf, + /// How much the store may hold, for the admin's storage overview. Unset: none is shown (the overview has the disk's own size besides). + #[serde(default)] + capacity: Option, + /// Refuse new uploads (`507 server_full`) when the filesystem would be left with less than `min_free`, or the store + /// would hold more than `capacity`. Default off: the numbers are information only. + #[serde(default)] + refuse_when_full: bool, + /// The free space to keep on the store's filesystem while `refuse_when_full` is on. Default 1 GiB. + #[serde(default, deserialize_with = "min_free_size")] + min_free: Option, }, S3 { + /// How much the bucket may hold, for the admin's storage overview. S3 has no size to ask for, so unset: none is shown. + #[serde(default)] + capacity: Option, + /// Refuse new uploads (`507 server_full`) when the store would hold more than `capacity` (which it then needs). + #[serde(default)] + refuse_when_full: bool, + /// Not for S3; read only to say so. + #[serde(default, deserialize_with = "min_free_size")] + min_free: Option, bucket: String, #[serde(default)] prefix: String, @@ -146,6 +326,101 @@ pub enum StorageConfig { }, } +/// A size in bytes written as `"1 TiB"`: a whole number or one with one decimal, then B, KB, MB, GB, TB (powers of 1000) +/// or KiB, MiB, GiB, TiB (powers of 1024), in any case, with or without a space. Or a plain positive integer, in bytes. +/// At least one byte. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Capacity(pub u64); + +impl Capacity { + /// Reads `text` as a size, or says what is wrong with it. + pub fn parse(text: &str) -> Result { + Self::parse_as(text, "storage.capacity") + } + + /// [`Capacity::parse`] for the setting `field`, which the messages name. + pub fn parse_as(text: &str, field: &str) -> Result { + let bad = || { + format!( + "{field} = {text:?} is not a size like \"1 TiB\" or \"500 GB\" (a number with at most one decimal, then B, KB, MB, GB, TB, KiB, MiB, GiB or TiB)" + ) + }; + let trimmed = text.trim(); + // A plain whole number is bytes. + if !trimmed.is_empty() && trimmed.bytes().all(|b| b.is_ascii_digit()) { + let bytes: u64 = trimmed.parse().map_err(|_| format!("{field} = {text:?} is too big (the most is about 16 EiB)"))?; + return if bytes == 0 { Err(format!("{field} = {text:?} is empty: give a size like \"1 TiB\"")) } else { Ok(Self(bytes)) }; + } + let split = trimmed.find(|c: char| !(c.is_ascii_digit() || c == '.')).ok_or_else(bad)?; + let (number, unit) = trimmed.split_at(split); + let multiplier: u64 = match unit.trim().to_ascii_lowercase().as_str() { + "b" => 1, + "kb" => 1_000, + "mb" => 1_000_000, + "gb" => 1_000_000_000, + "tb" => 1_000_000_000_000, + "kib" => 1 << 10, + "mib" => 1 << 20, + "gib" => 1 << 30, + "tib" => 1 << 40, + _ => return Err(bad()), + }; + let (whole, tenth) = match number.split_once('.') { + Some((whole, tenth)) if tenth.len() == 1 => (whole, tenth), + Some(_) => return Err(bad()), + None => (number, "0"), + }; + if whole.is_empty() || !whole.bytes().all(|b| b.is_ascii_digit()) || !tenth.bytes().all(|b| b.is_ascii_digit()) { + return Err(bad()); + } + let whole: u64 = whole.parse().map_err(|_| bad())?; + let tenth: u64 = tenth.parse().map_err(|_| bad())?; + // The decimal is a tenth of the unit, rounded down to a byte. + let bytes = whole + .checked_mul(multiplier) + .and_then(|b| b.checked_add(multiplier / 10 * tenth + (multiplier % 10) * tenth / 10)) + .ok_or_else(|| format!("{field} = {text:?} is too big (the most is about 16 EiB)"))?; + if bytes == 0 { + return Err(format!("{field} = {text:?} is empty: give a size like \"1 TiB\"")); + } + Ok(Self(bytes)) + } +} + +impl<'de> Deserialize<'de> for Capacity { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(SizeVisitor("storage.capacity")) + } +} + +/// Reads `storage.min_free` as a size, naming that setting in its messages. +fn min_free_size<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + deserializer.deserialize_any(SizeVisitor("storage.min_free")).map(Some) +} + +/// A size as text or a number of bytes, for the setting it names. +struct SizeVisitor(&'static str); + +impl serde::de::Visitor<'_> for SizeVisitor { + type Value = Capacity; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a size like \"1 TiB\" or a number of bytes") + } + + fn visit_str(self, text: &str) -> Result { + Capacity::parse_as(text, self.0).map_err(E::custom) + } + + fn visit_u64(self, bytes: u64) -> Result { + Capacity::parse_as(&bytes.to_string(), self.0).map_err(E::custom) + } + + fn visit_i64(self, bytes: i64) -> Result { + Capacity::parse_as(&bytes.to_string(), self.0).map_err(E::custom) + } +} + fn default_region() -> String { "us-east-1".to_owned() } @@ -164,7 +439,19 @@ impl std::fmt::Debug for Secret { impl Config { pub fn load(path: &Path) -> Result { let text = std::fs::read_to_string(path).map_err(|source| ConfigError::Read { path: path.to_owned(), source })?; - Self::parse(&text) + let mut config = Self::parse(&text)?; + // A relative path to the config file has a parent too, possibly empty: that is the working directory. + config.dir = Some(path.parent().map(Path::to_owned).unwrap_or_default()); + Ok(config) + } + + /// `path` of a `[two_factor]` key file as the server opens it: relative paths are relative to the config file's + /// directory (when the config was loaded from a file). + pub fn key_path(&self, path: &Path) -> PathBuf { + match (&self.dir, path.is_absolute()) { + (Some(dir), false) => dir.join(path), + _ => path.to_owned(), + } } pub fn parse(text: &str) -> Result { @@ -186,10 +473,43 @@ impl Config { { return Err(ConfigError::StoredBytesCapTooSmall(cap, MAX_CHUNK_BYTES)); } + if !(craft_cloud_meta::MIN_RETENTION_DAYS..=craft_cloud_meta::MAX_RETENTION_DAYS).contains(&config.audit.retention_days) { + return Err(ConfigError::AuditRetention( + config.audit.retention_days, + craft_cloud_meta::MIN_RETENTION_DAYS, + craft_cloud_meta::MAX_RETENTION_DAYS, + )); + } + let web = &config.web_admin; + for (name, value, low, high) in [("idle_minutes", web.idle_minutes, 5, 1440), ("absolute_hours", web.absolute_hours, 1, 720)] { + if !(low..=high).contains(&value) { + return Err(ConfigError::WebAdminRange(name, value, low, high)); + } + } + if !(MIN_TRASH_DAYS..=MAX_TRASH_DAYS).contains(&config.retention.trash_days) { + return Err(ConfigError::TrashDays(config.retention.trash_days)); + } + // The same rule as `ignore_forwarded_for`: a setting that would do nothing is refused, not ignored. + match &config.storage { + StorageConfig::Disk { refuse_when_full: false, min_free: Some(_), .. } => return Err(ConfigError::MinFreeWithoutRefusing), + StorageConfig::S3 { min_free: Some(_), .. } => return Err(ConfigError::MinFreeOnS3), + StorageConfig::S3 { refuse_when_full: true, capacity: None, .. } => return Err(ConfigError::RefuseOnS3WithoutCapacity), + _ => {} + } let mut config = config; if let Some(url) = &config.public_url { config.public_url = Some(public_url(url)?); } + if config.web_admin.enabled { + // The pages' address is an origin: https, a host and maybe a port, and no path to put in front of `/web`. + let origin_only = config.public_url.as_deref().and_then(|url| url.strip_prefix("https://")).is_some_and(|rest| !rest.contains('/')); + if !origin_only { + return Err(ConfigError::WebAdminPublicUrl); + } + if !config.behind_proxy { + return Err(ConfigError::WebAdminNeedsProxy); + } + } Ok(config) } } @@ -218,10 +538,30 @@ fn describe_toml_error(text: &str, e: &toml::de::Error) -> String { } impl StorageConfig { + /// The size the config says the store may hold, in bytes. + pub fn capacity(&self) -> Option { + match self { + StorageConfig::Disk { capacity, .. } | StorageConfig::S3 { capacity, .. } => capacity.map(|c| c.0), + } + } + + /// When the server refuses uploads because it is full (`refuse_when_full`), and where: `None` while the setting is off. + pub fn full_policy(&self) -> Option { + match self { + StorageConfig::Disk { refuse_when_full: true, min_free, capacity, .. } => { + Some(crate::full::Policy { capacity: capacity.map(|c| c.0), min_free: Some(min_free.map_or(crate::full::DEFAULT_MIN_FREE, |m| m.0)) }) + } + StorageConfig::S3 { refuse_when_full: true, capacity, .. } => { + Some(crate::full::Policy { capacity: capacity.map(|c| c.0), min_free: None }) + } + _ => None, + } + } + /// The S3 settings, with credentials from the config or else the standard `AWS_ACCESS_KEY_ID` / /// `AWS_SECRET_ACCESS_KEY` environment variables. `None` for disk storage. pub fn s3(&self) -> Result, ConfigError> { - let StorageConfig::S3 { bucket, prefix, region, endpoint, access_key_id, secret_access_key, allow_http } = self else { + let StorageConfig::S3 { bucket, prefix, region, endpoint, access_key_id, secret_access_key, allow_http, .. } = self else { return Ok(None); }; let access_key_id = match access_key_id { @@ -264,11 +604,162 @@ path = "/tmp/blobs" let c = Config::parse(DISK).unwrap(); assert_eq!(c.listen, "127.0.0.1:8080".parse().unwrap()); assert!(!c.behind_proxy && c.trusted_proxies.is_empty()); - assert_eq!(c.storage, StorageConfig::Disk { path: "/tmp/blobs".into() }); + assert_eq!(c.storage, StorageConfig::Disk { path: "/tmp/blobs".into(), capacity: None, refuse_when_full: false, min_free: None }); assert_eq!(c.storage.s3().unwrap(), None); assert_eq!(c.public_url, None); } + #[test] + fn the_capacity_is_optional_and_reads_sizes() { + assert_eq!(Config::parse(DISK).unwrap().storage.capacity(), None, "a config from before the option still loads"); + let with = |value: &str| { + Config::parse(&DISK.replace( + "path = \"/tmp/blobs\"", + &format!( + "path = \"/tmp/blobs\" +capacity = {value}" + ), + )) + }; + let bytes = |value: &str| with(value).unwrap().storage.capacity(); + assert_eq!(bytes("\"1 TiB\""), Some(1 << 40)); + assert_eq!(bytes("1000"), Some(1000), "an integer is bytes"); + assert_eq!(bytes("\"1000\""), Some(1000)); + assert_eq!(bytes("1_099_511_627_776"), Some(1 << 40)); + assert_eq!(bytes("\"1TiB\""), Some(1 << 40)); + assert_eq!(bytes("\"500 gb\""), Some(500_000_000_000)); + assert_eq!(bytes("\"2 TB\""), Some(2_000_000_000_000)); + assert_eq!(bytes("\"1.5 TB\""), Some(1_500_000_000_000)); + assert_eq!(bytes("\"0.5 GiB\""), Some(1 << 29)); + assert_eq!(bytes("\"1.5 KiB\""), Some(1536)); + assert_eq!(bytes("\"64 B\""), Some(64)); + assert_eq!(bytes("\"3 KB\""), Some(3000)); + assert_eq!(bytes("\"2 MiB\""), Some(2 << 20)); + assert_eq!(bytes("\"1 MB\""), Some(1_000_000)); + assert_eq!(bytes("\"1 KiB\""), Some(1024)); + assert_eq!(bytes("\"1 GB\""), Some(1_000_000_000)); + assert_eq!(bytes("\"1 GiB\""), Some(1 << 30)); + for bad in [ + "\"\"", + "\"1.5\"", + "0", + "-5", + "1.5", + "\"-5\"", + "\"TiB\"", + "\"1 XB\"", + "\"1.25 TB\"", + "\"1. TB\"", + "\".5 TB\"", + "\"-1 TB\"", + "\"1e3 GB\"", + "\"0 B\"", + "\"0.0 TB\"", + "\"99999999 TiB\"", + "true", + ] { + assert!(matches!(with(bad), Err(ConfigError::Parse(_))), "{bad}"); + } + let shown = with("\"1 XB\"").unwrap_err().to_string(); + assert!(shown.contains("storage.capacity") && shown.contains("1 XB") && shown.contains("line"), "{shown}"); + // S3 takes it too; without it S3 shows no size. + let s3 = "listen = \"127.0.0.1:8080\" +[database] +url = \"sqlite:/tmp/meta.db\" +[storage] +kind = \"s3\" +bucket = \"b\" +"; + assert_eq!(Config::parse(s3).unwrap().storage.capacity(), None); + assert_eq!( + Config::parse(&format!( + "{s3}capacity = \"10 TB\" +" + )) + .unwrap() + .storage + .capacity(), + Some(10_000_000_000_000) + ); + } + + #[test] + fn refusing_uploads_when_full_is_off_unless_asked_and_checked() { + let disk = |extra: &str| { + Config::parse(&DISK.replace( + "path = \"/tmp/blobs\"", + &format!( + "path = \"/tmp/blobs\" +{extra}" + ), + )) + }; + let s3 = |extra: &str| { + Config::parse(&format!( + "listen = \"127.0.0.1:8080\" +[database] +url = \"sqlite:/tmp/meta.db\" +[storage] +kind = \"s3\" +bucket = \"b\" +{extra} +" + )) + }; + use crate::full::{DEFAULT_MIN_FREE, Policy}; + assert_eq!(Config::parse(DISK).unwrap().storage.full_policy(), None, "off by default"); + assert_eq!(disk("refuse_when_full = false").unwrap().storage.full_policy(), None); + assert_eq!( + disk("refuse_when_full = true").unwrap().storage.full_policy(), + Some(Policy { capacity: None, min_free: Some(DEFAULT_MIN_FREE) }), + "a disk keeps 1 GiB by default" + ); + assert_eq!( + disk( + "refuse_when_full = true +min_free = \"5 GB\" +capacity = \"1 TB\"" + ) + .unwrap() + .storage + .full_policy(), + Some(Policy { capacity: Some(1_000_000_000_000), min_free: Some(5_000_000_000) }) + ); + assert_eq!( + s3("refuse_when_full = true +capacity = \"1 TB\"") + .unwrap() + .storage + .full_policy(), + Some(Policy { capacity: Some(1_000_000_000_000), min_free: None }) + ); + // Settings that would do nothing, or can't work, are refused. + assert!(matches!(disk("min_free = \"1 GiB\""), Err(ConfigError::MinFreeWithoutRefusing))); + assert!(matches!(s3("refuse_when_full = true"), Err(ConfigError::RefuseOnS3WithoutCapacity))); + assert!(matches!( + s3("refuse_when_full = true +capacity = \"1 TB\" +min_free = \"1 GB\""), + Err(ConfigError::MinFreeOnS3) + )); + // A bad reserve names its own setting. + let shown = disk( + "refuse_when_full = true +min_free = \"lots\"", + ) + .unwrap_err() + .to_string(); + assert!(shown.contains("storage.min_free") && !shown.contains("storage.capacity"), "{shown}"); + assert!(matches!( + disk( + "refuse_when_full = true +min_free = 0" + ), + Err(ConfigError::Parse(_)) + )); + assert!(matches!(disk("refuse_when_full = \"yes\""), Err(ConfigError::Parse(_)))); + } + #[test] fn the_public_url_is_normalised_or_refused() { let with = |url: &str| Config::parse(&format!("public_url = {url:?}\n{DISK}")); @@ -323,11 +814,96 @@ path = "/tmp/blobs" assert!(matches!(Config::parse(&no_proxy), Err(ConfigError::IgnoreForwardedForMisplaced))); } + #[test] + fn the_audit_section_is_optional_and_checked() { + assert_eq!(Config::parse(DISK).unwrap().audit.retention_days, 365, "a year by default"); + let with = |section: &str| Config::parse(&format!("{DISK}\n{section}\n")); + assert_eq!(with("[audit]").unwrap().audit.retention_days, 365); + assert_eq!(with("[audit]\nretention_days = 30").unwrap().audit.retention_days, 30); + for days in [1, 3650] { + assert_eq!(with(&format!("[audit]\nretention_days = {days}")).unwrap().audit.retention_days, days); + } + for days in ["0", "3651", "-1", "\"30\"", "1.5"] { + assert!(with(&format!("[audit]\nretention_days = {days}")).is_err(), "{days}"); + } + assert!(matches!(with("[audit]\nretention_days = 0"), Err(ConfigError::AuditRetention(0, 1, 3650)))); + assert!(matches!(with("[audit]\nkeep = 5"), Err(ConfigError::Parse(_))), "unknown keys are refused"); + } + + #[test] + fn the_web_admin_is_off_by_default_and_checked_when_on() { + let c = Config::parse(DISK).unwrap(); + assert_eq!(c.web_admin, WebAdminConfig { enabled: false, idle_minutes: 480, absolute_hours: 168 }, "off, 8 hours idle, 7 days at most"); + assert_eq!((c.web_admin.idle_ms(), c.web_admin.absolute_ms()), (8 * 3_600_000, 7 * 24 * 3_600_000)); + let proxied = DISK.replace("127.0.0.1:8080", "0.0.0.0:8080"); + let with = |top: &str, section: &str| Config::parse(&format!("{top}\n{proxied}\n[web_admin]\n{section}\n")); + let proxy = "behind_proxy = true\ntrusted_proxies = [\"10.0.0.1\"]"; + let good = format!("{proxy}\npublic_url = \"https://cloud.example.com\""); + let c = with(&good, "enabled = true\nidle_minutes = 30\nabsolute_hours = 12").unwrap(); + assert_eq!((c.web_admin.enabled, c.web_admin.idle_minutes, c.web_admin.absolute_hours), (true, 30, 12)); + // Each missing piece of the setup is refused with its own message. + assert!(matches!(with(proxy, "enabled = true"), Err(ConfigError::WebAdminPublicUrl)), "no public_url"); + for url in ["http://cloud.example.com", "https://cloud.example.com/cloud"] { + let bad = format!("{proxy}\npublic_url = \"{url}\""); + assert!(matches!(with(&bad, "enabled = true"), Err(ConfigError::WebAdminPublicUrl)), "{url}"); + } + let no_proxy = format!("public_url = \"https://cloud.example.com\"\n{DISK}\n[web_admin]\nenabled = true\n"); + assert!(matches!(Config::parse(&no_proxy), Err(ConfigError::WebAdminNeedsProxy))); + // Off, none of it matters; the lifetimes are always in range; unknown keys are errors. + assert!(!Config::parse(&format!("{DISK}\n[web_admin]\nenabled = false\n")).unwrap().web_admin.enabled); + for section in ["idle_minutes = 4", "idle_minutes = 1441", "absolute_hours = 0", "absolute_hours = 721"] { + assert!(matches!(Config::parse(&format!("{DISK}\n[web_admin]\n{section}\n")), Err(ConfigError::WebAdminRange(..))), "{section}"); + } + assert!(matches!(Config::parse(&format!("{DISK}\n[web_admin]\ncookie = 1\n")), Err(ConfigError::Parse(_)))); + } + + #[test] + fn the_trash_period_has_a_default_and_bounds() { + assert_eq!(Config::parse(DISK).unwrap().retention, RetentionConfig { trash_days: 30 }); + // A table has to come after the top-level keys, so the section goes at the end. + let at_end = |days: &str| Config::parse(&format!("{DISK}\n[retention]\ntrash_days = {days}\n")); + assert_eq!(at_end("1").unwrap().retention.trash_days, 1); + assert_eq!(at_end("3650").unwrap().retention.trash_days, 3650); + for bad in ["0", "3651", "-1", "\"30\"", "1.5"] { + assert!(at_end(bad).is_err(), "{bad}"); + } + assert!(matches!(at_end("0"), Err(ConfigError::TrashDays(0)))); + assert!(matches!(at_end("3651"), Err(ConfigError::TrashDays(3651)))); + // Nothing but `trash_days` is accepted in the section, and the section may be empty. + assert!(matches!(Config::parse(&format!("{DISK}\n[retention]\ngc_wait_hours = 24\n")), Err(ConfigError::Parse(_)))); + assert_eq!(Config::parse(&format!("{DISK}\n[retention]\n")).unwrap().retention.trash_days, 30); + } + #[test] fn unknown_keys_are_errors() { assert!(matches!(Config::parse(&format!("lisen = 1\n{DISK}")), Err(ConfigError::Parse(_)))); } + #[test] + fn the_two_factor_section_is_optional_and_strict() { + assert_eq!(Config::parse(DISK).unwrap().two_factor, None, "a config from before two-factor sign-in still loads"); + let with = |section: &str| Config::parse(&format!("{DISK}\n[two_factor]\n{section}\n")); + let c = with("key_file = \"two-factor.key\"").unwrap(); + assert_eq!(c.two_factor, Some(TwoFactorConfig { key_file: "two-factor.key".into(), key_file_old: None })); + let c = with("key_file = \"new.key\"\nkey_file_old = \"old.key\"").unwrap(); + assert_eq!(c.two_factor.unwrap().key_file_old, Some("old.key".into())); + assert!(matches!(with("key_file = \"k\"\nkey = \"x\""), Err(ConfigError::Parse(_))), "an unknown key in the section"); + assert!(matches!(with("key_file_old = \"old.key\""), Err(ConfigError::Parse(_))), "key_file is required"); + } + + #[test] + fn key_files_are_relative_to_the_config_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, format!("{DISK}\n[two_factor]\nkey_file = \"two-factor.key\"\n")).unwrap(); + let c = Config::load(&path).unwrap(); + assert_eq!(c.key_path(Path::new("two-factor.key")), dir.path().join("two-factor.key")); + let absolute = dir.path().join("elsewhere.key"); + assert_eq!(c.key_path(&absolute), absolute); + // A config that was parsed from text has no directory: the path stays as it is. + assert_eq!(Config::parse(DISK).unwrap().key_path(Path::new("k")), PathBuf::from("k")); + } + #[test] fn s3_config_hides_its_secret() { let text = r#" diff --git a/apps/craft-cloud-server/src/error.rs b/apps/craft-cloud-server/src/error.rs index 73d5a18f..e44dc22e 100644 --- a/apps/craft-cloud-server/src/error.rs +++ b/apps/craft-cloud-server/src/error.rs @@ -6,7 +6,7 @@ use axum::extract::rejection::{BytesRejection, JsonRejection, PathRejection, Que use axum::http::{HeaderValue, StatusCode, header}; use axum::response::{IntoResponse, Response}; use craft_cloud_meta::MetaError; -use craft_cloud_proto::api::{ErrorBody, QuotaUse, codes}; +use craft_cloud_proto::api::{ErrorBody, QuotaUse, ServerRoom, TwoFactorChallenge, codes}; use craft_cloud_proto::{ChunkId, Conflict}; use craft_cloud_store::StoreError; @@ -16,16 +16,21 @@ pub struct ApiError { pub code: &'static str, pub message: String, pub conflicts: Option>, - pub missing: Option>, + /// A boxed slice, to keep this type small: every handler's `Result` carries it. + pub missing: Option>, /// Boxed: rare, and every handler's `Result` carries this type. pub quota: Option>, + /// What the server knows of its room, for `server_full`. Boxed for the same reason as `quota`. + pub server: Option>, /// Seconds for the `Retry-After` header. pub retry_after: Option, + /// The challenge of `two_factor_required`. Boxed for the same reason as `quota`. + pub two_factor: Option>, } impl ApiError { pub fn new(status: StatusCode, code: &'static str, message: impl Into) -> Self { - Self { status, code, message: message.into(), conflicts: None, missing: None, quota: None, retry_after: None } + Self { status, code, message: message.into(), conflicts: None, missing: None, quota: None, server: None, retry_after: None, two_factor: None } } /// `507 quota_exceeded`: the account would store more than the server lets it. @@ -36,6 +41,22 @@ impl ApiError { e } + /// `507 server_full`: the server refuses new uploads because it is full (`refuse_when_full`). Not the account's quota. + pub fn server_full(room: ServerRoom) -> Self { + let mut e = Self::new( + StatusCode::INSUFFICIENT_STORAGE, + codes::SERVER_FULL, + "the server is full and is not taking new uploads: its admin needs to free space or raise its capacity", + ); + e.server = Some(Box::new(room)); + e + } + + /// `403 forbidden`: signed in, but not allowed to do this. + pub fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, codes::FORBIDDEN, message) + } + pub fn invalid(message: impl Into) -> Self { Self::new(StatusCode::BAD_REQUEST, codes::INVALID, message) } @@ -69,13 +90,56 @@ impl ApiError { Self::new(StatusCode::BAD_REQUEST, codes::PASSWORD_REQUIRED, "this invite is for a new account: choose a password for it") } + /// `401 two_factor_required`: the password (or code) was right and the account needs its second factor. No token. + /// `challenge` is the secret to send back to `POST /auth/login/second-factor` (empty for `/join`, which has none). + pub fn two_factor_required(challenge: String, expires_ms: i64) -> Self { + let mut e = Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, "this account needs a code from its authenticator app"); + e.two_factor = Some(Box::new(TwoFactorChallenge { challenge, expires_ms })); + e + } + + /// `403 two_factor_invalid`: a wrong code or recovery code on a signed-in route (never 401: on a signed-in route the + /// agent reads 401 as "this device was revoked"). [`ApiError::on_public_route`] makes it 401 for sign-in and join. + pub fn two_factor_invalid() -> Self { + Self::new(StatusCode::FORBIDDEN, codes::TWO_FACTOR_INVALID, "the code is wrong, or was used already") + } + + /// `429 two_factor_locked` with `Retry-After`. + pub fn two_factor_locked(retry_after_secs: u32) -> Self { + let mut e = Self::new( + StatusCode::TOO_MANY_REQUESTS, + codes::TWO_FACTOR_LOCKED, + "too many wrong codes: this account's second factor is locked for a while; try again later, or ask the server's admin", + ); + e.retry_after = Some(retry_after_secs); + e + } + + /// `503 two_factor_unavailable`: the server has no key for two-factor secrets. + pub fn two_factor_unavailable() -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + codes::TWO_FACTOR_UNAVAILABLE, + "this server has no key for two-factor secrets; the admin sets [two_factor] key_file in its config", + ) + } + + /// The answer on a route that is not signed in (sign-in's second step, join): a wrong second factor is `401` there, + /// where it is `403` on every signed-in route. + pub fn on_public_route(mut self) -> Self { + if self.code == codes::TWO_FACTOR_INVALID { + self.status = StatusCode::UNAUTHORIZED; + } + self + } + pub fn internal() -> Self { Self::new(StatusCode::INTERNAL_SERVER_ERROR, codes::INTERNAL, "the server hit an internal error; the admin can find details in its log") } pub fn missing_chunks(missing: Vec) -> Self { let mut e = Self::new(StatusCode::CONFLICT, codes::MISSING_CHUNKS, format!("{} chunk(s) must be uploaded first", missing.len())); - e.missing = Some(missing); + e.missing = Some(missing.into_boxed_slice()); e } } @@ -95,12 +159,29 @@ impl From for ApiError { MetaError::NotFound(what) => Self::not_found(what), MetaError::InvalidCode => Self::invalid_code(), MetaError::PasswordRequired => Self::password_required(), + MetaError::DeviceRevoked => Self::unauthorized(), + MetaError::Forbidden(message) => Self::forbidden(message), + MetaError::LastAdmin => Self::new(StatusCode::CONFLICT, codes::LAST_ADMIN, e.to_string()), + MetaError::AccountDisabled => Self::new(StatusCode::FORBIDDEN, codes::ACCOUNT_DISABLED, "this account is disabled; ask an administrator"), + // The status of a wrong second factor depends on the route: 403 here (every signed-in route), made 401 by + // `on_public_route` for sign-in and join. + MetaError::TwoFactorInvalid => Self::two_factor_invalid(), + MetaError::TwoFactorLocked { retry_after_ms } => Self::two_factor_locked(craft_cloud_meta::retry_after_secs(retry_after_ms)), + MetaError::TwoFactorPending => { + Self::new(StatusCode::CONFLICT, codes::TWO_FACTOR_PENDING, "a two-factor set-up is open on another device: finish it there, or wait") + } + // Only the join route can see this and it builds its own answer; a signed-in route never asks a second factor. + MetaError::TwoFactorRequired => Self::new(StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED, "this account needs its second factor"), + MetaError::ChallengeGone => { + Self::new(StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED, "this sign-in step doesn't work any more: sign in again") + } + MetaError::Conflict(message) => Self::new(StatusCode::CONFLICT, codes::CONFLICT, message), MetaError::UsernameTaken => Self::new(StatusCode::CONFLICT, codes::USERNAME_TAKEN, "the username is taken"), MetaError::QuotaExceeded { used, limit } => Self::quota_exceeded(used, limit), MetaError::DocFull { .. } | MetaError::LiveFull { .. } | MetaError::TooManyDocs { .. } | MetaError::TooManyDocRows { .. } => { Self::new(StatusCode::INSUFFICIENT_STORAGE, codes::QUOTA_EXCEEDED, e.to_string()) } - MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::UnsupportedUrl => { + MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::Missing(_) | MetaError::UnsupportedUrl => { tracing::error!(error = %e, "metadata error"); Self::internal() } @@ -158,8 +239,10 @@ impl IntoResponse for ApiError { error: self.code.to_owned(), message: self.message, conflicts: self.conflicts, - missing: self.missing, + missing: self.missing.map(Vec::from), quota: self.quota.map(|q| *q), + server: self.server.map(|r| *r), + two_factor: self.two_factor.map(|t| *t), }; let mut response = (self.status, Json(body)).into_response(); if let Some(secs) = self.retry_after { @@ -168,3 +251,42 @@ impl IntoResponse for ApiError { response } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_change_that_lost_a_race_is_a_conflict_and_a_signed_out_caller_is_unauthorized() { + let conflict = ApiError::from(MetaError::Conflict("changed meanwhile")); + assert_eq!((conflict.status, conflict.code), (StatusCode::CONFLICT, codes::CONFLICT)); + let revoked = ApiError::from(MetaError::DeviceRevoked); + assert_eq!((revoked.status, revoked.code), (StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED)); + let forbidden = ApiError::from(MetaError::Forbidden("no")); + assert_eq!((forbidden.status, forbidden.code), (StatusCode::FORBIDDEN, codes::FORBIDDEN)); + let last = ApiError::from(MetaError::LastAdmin); + assert_eq!((last.status, last.code), (StatusCode::CONFLICT, codes::LAST_ADMIN)); + let disabled = ApiError::from(MetaError::AccountDisabled); + assert_eq!((disabled.status, disabled.code), (StatusCode::FORBIDDEN, codes::ACCOUNT_DISABLED)); + } + + #[test] + fn second_factor_errors_are_403_on_signed_in_routes_and_401_on_public_ones() { + let invalid = ApiError::from(MetaError::TwoFactorInvalid); + assert_eq!((invalid.status, invalid.code), (StatusCode::FORBIDDEN, codes::TWO_FACTOR_INVALID)); + let public = invalid.on_public_route(); + assert_eq!((public.status, public.code), (StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_INVALID)); + let locked = ApiError::from(MetaError::TwoFactorLocked { retry_after_ms: 1500 }); + assert_eq!((locked.status, locked.code, locked.retry_after), (StatusCode::TOO_MANY_REQUESTS, codes::TWO_FACTOR_LOCKED, Some(2))); + let locked = locked.on_public_route(); + assert_eq!(locked.status, StatusCode::TOO_MANY_REQUESTS); + let pending = ApiError::from(MetaError::TwoFactorPending); + assert_eq!((pending.status, pending.code), (StatusCode::CONFLICT, codes::TWO_FACTOR_PENDING)); + assert_eq!(ApiError::two_factor_unavailable().status, StatusCode::SERVICE_UNAVAILABLE); + let gone = ApiError::from(MetaError::ChallengeGone); + assert_eq!((gone.status, gone.code), (StatusCode::UNAUTHORIZED, codes::UNAUTHORIZED)); + let required = ApiError::two_factor_required("challenge-secret".into(), 5); + assert_eq!((required.status, required.code), (StatusCode::UNAUTHORIZED, codes::TWO_FACTOR_REQUIRED)); + assert!(!format!("{required:?}").contains("challenge-secret"), "the challenge is hidden in Debug"); + } +} diff --git a/apps/craft-cloud-server/src/full.rs b/apps/craft-cloud-server/src/full.rs new file mode 100644 index 00000000..578e658d --- /dev/null +++ b/apps/craft-cloud-server/src/full.rs @@ -0,0 +1,316 @@ +//! A server that refuses new uploads when it is full (`[storage] refuse_when_full`, off by default). +//! +//! An upload (a chunk or a manifest, which is also how a live document's snapshot goes up) is refused with `507 +//! server_full` when either holds, after counting the bytes it would add: +//! +//! - the store's filesystem would be left with less free space than `min_free` (a disk store only; S3 has no free +//! space to measure), or +//! - the bytes the server stores would pass `capacity` (a disk or S3 store, when `capacity` is set). +//! +//! Reads, deletes, trash, garbage collection, live updates (they live in the database, not the blob store) and the +//! admin's calls are never refused, so an admin can free space while the server is full. +//! +//! The numbers are cached: they are read again at most every [`REFRESH`], and the bytes the server wrote itself since +//! are added in between, so the hot path costs a lock and a few additions. The state flips in the log at `warn` when +//! refusing starts and stops, with the reason and no names. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use craft_cloud_meta::Meta; +use craft_cloud_proto::api::ServerRoom; +use tokio::sync::Mutex; +use tokio::time::Instant; + +use crate::error::ApiError; +use crate::space::StorageInfo; + +/// The free space a disk store keeps when `min_free` is not set. +pub const DEFAULT_MIN_FREE: u64 = 1 << 30; + +/// How long the stored total and the free space count as current. +pub const REFRESH: Duration = Duration::from_secs(5); + +/// What the admin configured. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Policy { + /// The most the store may hold. + pub capacity: Option, + /// The free space to keep on the store's filesystem (disk only; `None`: not checked). + pub min_free: Option, +} + +/// Total and free bytes of the store's filesystem, `(None, None)` when unknown. Blocking. The tests' seam: they +/// replace the filesystem with numbers they choose. +pub type Probe = Arc (Option, Option) + Send + Sync>; + +/// Why the server is full. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Reason { + Capacity, + FreeSpace, +} + +#[derive(Default)] +struct Cache { + read_at: Option, + stored: u64, + total: Option, + free: Option, +} + +/// How long the free-space read may take before the filesystem counts as unknown (a dead network mount would +/// otherwise hold every upload). +pub const PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +/// Warnings that refusing started are at most this far apart, however close to the threshold the server sits. +const WARN_EVERY: Duration = Duration::from_secs(60); + +pub struct FullGuard { + policy: Policy, + probe: Probe, + /// How long the cached numbers count as current ([`REFRESH`]; tests use others). + refresh: Duration, + probe_timeout: Duration, + cache: Mutex, + /// Bytes this server stored since the numbers in the cache were read. + written: AtomicU64, + /// Whether a smallest upload would be refused now. Set from the real numbers only, never from what a request + /// declares, so no client can make the server say it is full. + refusing: AtomicBool, + /// A free-space read is running (it may be stuck on a dead mount); no second one is started meanwhile. + probing: Arc, + /// The free space was unknown at the last read, and that was said in the log. + unknown_logged: AtomicBool, + /// When the log last said refusing started, and whether it has not yet said it stopped. + warned: std::sync::Mutex<(Option, bool)>, +} + +impl FullGuard { + /// A guard for the store `info` describes: the filesystem of its directory is measured when there is one. + pub fn new(policy: Policy, info: &StorageInfo) -> Self { + let info = info.clone(); + Self::with_probe(policy, Arc::new(move || info.store_filesystem())) + } + + /// A guard that reads the filesystem through `probe` (tests). + pub fn with_probe(policy: Policy, probe: Probe) -> Self { + Self { + policy, + probe, + refresh: REFRESH, + probe_timeout: PROBE_TIMEOUT, + cache: Mutex::new(Cache::default()), + written: AtomicU64::new(0), + refusing: AtomicBool::new(false), + probing: Arc::new(AtomicBool::new(false)), + unknown_logged: AtomicBool::new(false), + warned: std::sync::Mutex::new((None, false)), + } + } + + /// The guard reading the numbers again after `refresh` instead of [`REFRESH`] (tests). + pub fn with_refresh(mut self, refresh: Duration) -> Self { + self.refresh = refresh; + self + } + + /// The guard waiting `timeout` for the free-space read instead of [`PROBE_TIMEOUT`] (tests). + pub fn with_probe_timeout(mut self, timeout: Duration) -> Self { + self.probe_timeout = timeout; + self + } + + /// Whether the server would refuse even the smallest upload, as of the last time the numbers were looked at. + pub fn refusing(&self) -> bool { + self.refusing.load(Ordering::Relaxed) + } + + /// Refuses an upload of `incoming` bytes when the server is full. + pub async fn check(&self, meta: &Meta, incoming: u64) -> Result<(), ApiError> { + match self.evaluate(meta, incoming).await? { + None => Ok(()), + Some(room) => Err(ApiError::server_full(room)), + } + } + + /// Whether the server is refusing uploads now (for the admin's overview). Reads the numbers if they are stale, so it + /// turns false soon after space was freed even when no upload has come since. + pub async fn status(&self, meta: &Meta) -> Result { + self.evaluate(meta, 0).await?; + Ok(self.refusing()) + } + + /// Counts bytes the server stored, until the numbers are read again. Called once the upload is recorded in the + /// database: a read of the stored total that came between the write and the record then still has them counted here, + /// and one that came after has them in the total (counted twice for up to one refresh, which errs toward refusing). + pub fn note_written(&self, bytes: u64) { + self.written.fetch_add(bytes, Ordering::Relaxed); + } + + /// `Some(room)` when an upload of `incoming` bytes would be refused. Also keeps `refusing` (for a one-byte upload). + async fn evaluate(&self, meta: &Meta, incoming: u64) -> Result, ApiError> { + let mut cache = self.cache.lock().await; + if cache.read_at.is_none_or(|at| at.elapsed() >= self.refresh) { + // Reset first: bytes stored while the numbers are read may then be counted twice, which errs toward refusing. + self.written.store(0, Ordering::Relaxed); + cache.stored = meta.admin_server_counts().await?.stored_bytes; + if self.policy.min_free.is_some() { + (cache.total, cache.free) = self.read_filesystem().await; + } + cache.read_at = Some(Instant::now()); + } + let written = self.written.load(Ordering::Relaxed); + let (stored, total, free) = (cache.stored, cache.total, cache.free); + drop(cache); + let reason_for = |incoming: u64| { + let added = written.saturating_add(incoming); + if self.policy.capacity.is_some_and(|capacity| stored.saturating_add(added) > capacity) { + Some(Reason::Capacity) + } else if let (Some(min_free), Some(free)) = (self.policy.min_free, free) + && free.saturating_sub(added) < min_free + { + Some(Reason::FreeSpace) + } else { + None + } + }; + self.note_state(reason_for(1)); + let room = ServerRoom { free_bytes: free, total_bytes: total, capacity_bytes: self.policy.capacity }; + Ok(reason_for(incoming).map(|_| room)) + } + + /// The store's filesystem: total and free bytes, or unknown when the read fails or takes longer than the timeout. + async fn read_filesystem(&self) -> (Option, Option) { + let known = if self.probing.swap(true, Ordering::SeqCst) { + // The last read is still stuck: don't pile up threads behind a dead mount. + None + } else { + let (probe, probing) = (self.probe.clone(), self.probing.clone()); + let read = tokio::task::spawn_blocking(move || { + let numbers = probe(); + probing.store(false, Ordering::SeqCst); + numbers + }); + match tokio::time::timeout(self.probe_timeout, read).await { + Ok(Ok(numbers)) => Some(numbers), + _ => None, + } + }; + let numbers = known.unwrap_or((None, None)); + // Said once while it lasts: a reserve that can't be checked is not checked. + if numbers.1.is_none() { + if !self.unknown_logged.swap(true, Ordering::Relaxed) { + tracing::warn!("can't read the free space of the store's filesystem: the `min_free` reserve is not checked until it can be"); + } + } else { + self.unknown_logged.store(false, Ordering::Relaxed); + } + numbers + } + + /// Logs when refusing starts or stops. A start is said at most once a minute, and a stop only after a start was said. + fn note_state(&self, reason: Option) { + let was = self.refusing.swap(reason.is_some(), Ordering::Relaxed); + if was == reason.is_some() { + return; + } + let mut warned = self.warned.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + match reason { + Some(reason) => { + if warned.0.is_none_or(|at| at.elapsed() >= WARN_EVERY) { + *warned = (Some(Instant::now()), true); + match reason { + Reason::Capacity => tracing::warn!("the server is full: it holds as much as `capacity` allows; refusing new uploads"), + Reason::FreeSpace => { + tracing::warn!("the server is full: free space is down to the `min_free` reserve; refusing new uploads") + } + } + } + } + None => { + if warned.1 { + warned.1 = false; + tracing::warn!("the server has room again; accepting uploads"); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn meta() -> Meta { + Meta::open("sqlite::memory:").await.unwrap() + } + + fn probe(free: Arc) -> Probe { + Arc::new(move || (Some(1000), Some(free.load(Ordering::Relaxed)))) + } + + #[tokio::test] + async fn the_reserve_refuses_the_write_that_would_cut_into_it_and_the_numbers_are_cached() { + let meta = meta().await; + let free = Arc::new(AtomicU64::new(300)); + let guard = + FullGuard::with_probe(Policy { capacity: None, min_free: Some(200) }, probe(free.clone())).with_refresh(Duration::from_millis(200)); + assert!(guard.check(&meta, 100).await.is_ok(), "300 - 100 = 200 stays at the reserve"); + guard.note_written(100); + // 200 free by what the server wrote itself, though the filesystem was not read again. + assert!(guard.check(&meta, 1).await.is_err()); + assert!(guard.refusing()); + assert!(guard.status(&meta).await.unwrap(), "the admin's overview says so too"); + // The disk is read again after the refresh time, not before. + free.store(900, Ordering::Relaxed); + assert!(guard.check(&meta, 1).await.is_err(), "still the cached numbers"); + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(!guard.status(&meta).await.unwrap(), "and stops saying so once space is back, with no upload needed"); + assert!(guard.check(&meta, 1).await.is_ok()); + assert!(!guard.refusing()); + } + + #[tokio::test] + async fn the_capacity_counts_what_is_stored_and_what_comes() { + let meta = meta().await; + let guard = FullGuard::with_probe(Policy { capacity: Some(10), min_free: None }, Arc::new(|| (None, None))); + assert!(guard.check(&meta, 10).await.is_ok(), "exactly the capacity fits"); + let err = guard.check(&meta, 11).await.unwrap_err(); + assert_eq!(err.code, "server_full"); + assert_eq!(err.server.unwrap().capacity_bytes, Some(10)); + } + + #[tokio::test] + async fn an_unknown_filesystem_never_refuses() { + let meta = meta().await; + let guard = FullGuard::with_probe(Policy { capacity: None, min_free: Some(u64::MAX) }, Arc::new(|| (None, None))); + assert!(guard.check(&meta, 1 << 40).await.is_ok()); + } + + #[tokio::test] + async fn what_a_request_declares_never_makes_the_server_say_it_is_full() { + let meta = meta().await; + let guard = FullGuard::with_probe(Policy { capacity: Some(100), min_free: None }, Arc::new(|| (None, None))); + assert!(guard.check(&meta, u64::MAX).await.is_err(), "that upload is refused"); + assert!(!guard.refusing(), "but a one-byte upload would fit, so the server is not full"); + assert!(!guard.status(&meta).await.unwrap()); + assert!(guard.check(&meta, 100).await.is_ok()); + } + + #[tokio::test] + async fn a_free_space_read_that_takes_too_long_counts_as_unknown() { + let meta = meta().await; + // A dead mount: the read outlasts the timeout, and the reserve is then not checked (not refused for). + let slow: Probe = Arc::new(|| { + std::thread::sleep(Duration::from_millis(400)); + (Some(1000), Some(0)) + }); + let guard = FullGuard::with_probe(Policy { capacity: None, min_free: Some(500) }, slow).with_probe_timeout(Duration::from_millis(50)); + let started = std::time::Instant::now(); + assert!(guard.check(&meta, 1).await.is_ok()); + assert!(started.elapsed() < Duration::from_millis(300), "{:?}", started.elapsed()); + assert!(!guard.refusing()); + } +} diff --git a/apps/craft-cloud-server/src/gc.rs b/apps/craft-cloud-server/src/gc.rs new file mode 100644 index 00000000..5e5b2c49 --- /dev/null +++ b/apps/craft-cloud-server/src/gc.rs @@ -0,0 +1,197 @@ +//! Read-only store inventory and garbage-collection preview. + +use craft_cloud_meta::{BlobKind, Meta, MetaError}; +use craft_cloud_store::{ListCursor, MAX_LIST_PAGE_SIZE, Store}; +use serde_json::{Value, json}; + +use crate::verify::{self, VerifyOptions}; + +const EXAMPLES: usize = 20; + +#[derive(Debug, thiserror::Error)] +pub enum GcError { + #[error(transparent)] + Meta(#[from] MetaError), +} + +#[derive(Default)] +struct Class { + count: u64, + bytes: u64, + examples: Vec, +} + +/// Objects the database records: how many and their stored bytes as `blobs.stored_size` has them (no object is read). +#[derive(Default)] +struct Recorded { + count: u64, + bytes: u64, +} + +pub struct Report { + chunks_recorded: Recorded, + manifests_recorded: Recorded, + chunks_orphans: Class, + manifests_orphans: Class, + foreign: Class, + inventory_error: Option, + db_check: verify::Report, + incomplete: bool, + chunk_count: u64, + chunk_candidates: u64, + chunk_candidate_bytes: u64, + chunk_protected: u64, + chunk_examples: Vec, +} + +fn add(class: &mut Class, id: String, bytes: u64) { + class.count = class.count.saturating_add(1); + class.bytes = class.bytes.saturating_add(bytes); + if class.examples.len() < EXAMPLES { + class.examples.push(id); + } +} + +pub async fn dry_run(meta: &Meta, store: &Store) -> Result { + let audit = meta.index_audit(EXAMPLES).await?; + let incomplete = audit.absent > 0 || audit.building > 0; + let mut chunks_recorded = Recorded::default(); + let mut manifests_recorded = Recorded::default(); + let mut chunks_orphans = Class::default(); + let mut manifests_orphans = Class::default(); + let mut foreign = Class::default(); + let mut inventory_error = None; + for (kind, label) in [(BlobKind::Chunk, "chunks"), (BlobKind::Manifest, "manifests")] { + let mut cursor: Option = None; + let mut page_no = 1u64; + loop { + let page = match store.list(kind, cursor.clone(), MAX_LIST_PAGE_SIZE).await { + Ok(p) => p, + Err(source) => { + inventory_error = Some(format!("store listing failed for {label} page {page_no}: {source}")); + break; + } + }; + let ids: Vec<[u8; 32]> = page.blobs.iter().map(|b| b.id).collect(); + let recorded = meta.recorded_blob_sizes(kind, &ids).await?; + let (recorded_total, orphans) = + if kind == BlobKind::Chunk { (&mut chunks_recorded, &mut chunks_orphans) } else { (&mut manifests_recorded, &mut manifests_orphans) }; + for blob in &page.blobs { + if let Some(size) = recorded.get(&blob.id) { + recorded_total.count = recorded_total.count.saturating_add(1); + recorded_total.bytes = recorded_total.bytes.saturating_add(*size); + continue; + } + match store.stored_size(*blob).await { + Ok(size) => add(orphans, hex(&blob.id), size), + Err(error) => { + add(orphans, hex(&blob.id), 0); + inventory_error = Some(format!("could not read stored size for {} object {}: {error}", label, hex(&blob.id))); + break; + } + } + } + if inventory_error.is_some() { + // A size that cannot be read (a backend without metadata-only sizes, or an I/O error) is reported once and + // ends the inventory; it is not retried for every remaining object. + break; + } + for key in &page.foreign { + let id = blake3::hash(key.as_bytes()).to_hex().to_string(); + add(&mut foreign, id, 0); + } + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + page_no = page_no.saturating_add(1); + } + if inventory_error.is_some() { + break; + } + } + let db_check = verify::verify(meta, store, VerifyOptions { read_chunks: true }).await?; + let chunk_count = audit.chunks; + let incomplete = incomplete || db_check.audit.absent > 0 || db_check.audit.building > 0; + let fail_closed = incomplete || db_check.problem_count() > 0; + // `index_audit` reads all reference roots in one read snapshot. The inventory is diagnostic; a later collector must + // recheck roots before claiming anything because store listing and metadata snapshots are not atomic together. + let chunk_candidates = if fail_closed { 0 } else { audit.unreferenced_chunks.count }; + let chunk_candidate_bytes = if fail_closed { 0 } else { audit.unreferenced_chunk_bytes }; + let chunk_protected = if fail_closed { chunk_count } else { chunk_count.saturating_sub(chunk_candidates) }; + let chunk_examples = if fail_closed { Vec::new() } else { audit.unreferenced_chunks.examples }; + Ok(Report { + chunks_recorded, + manifests_recorded, + chunks_orphans, + manifests_orphans, + foreign, + inventory_error, + db_check, + incomplete, + chunk_count, + chunk_candidates, + chunk_candidate_bytes, + chunk_protected, + chunk_examples, + }) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +impl Report { + pub fn has_problems(&self) -> bool { + self.inventory_error.is_some() || self.db_check.problem_count() > 0 + } + pub fn to_json(&self) -> Value { + json!({"schema_version":1,"store":{"chunk_recorded":{"count":self.chunks_recorded.count,"stored_bytes":self.chunks_recorded.bytes,"bytes_source":"database blobs.stored_size"},"manifest_recorded":{"count":self.manifests_recorded.count,"stored_bytes":self.manifests_recorded.bytes,"bytes_source":"database blobs.stored_size"},"chunk_orphans":{"count":self.chunks_orphans.count,"stored_bytes":self.chunks_orphans.bytes,"examples":self.chunks_orphans.examples},"manifest_orphans":{"count":self.manifests_orphans.count,"stored_bytes":self.manifests_orphans.bytes,"examples":self.manifests_orphans.examples},"foreign":{"count":self.foreign.count,"stored_bytes":null,"examples":self.foreign.examples},"listing_error":self.inventory_error},"database":{"problems":self.db_check.problem_count(),"verify":self.db_check.to_json()},"candidates":{"orphans":{"count":0,"potential_count":self.chunks_orphans.count.saturating_add(self.manifests_orphans.count),"reason":"eligible only after first-sight grace; would need two runs apart","protection":"first sight is not recorded"},"chunks":{"count":self.chunk_candidates,"stored_bytes":self.chunk_candidate_bytes,"examples":self.chunk_examples,"reason":"no complete-indexed manifest names chunk","protected_count":self.chunk_protected,"protection":if self.incomplete {"index incomplete: dependencies unknown"} else if self.db_check.problem_count()>0 {"protected: database verification reported problems"} else {"versions, saved versions, trash items, live snapshots and import snapshots in the database read snapshot; future collection must recheck"}},"manifests":{"count":0,"stored_bytes":0,"protected_count":self.db_check.audit.unrooted_manifests.count,"protection":"protected: may be a live document's asset"}},"index_incomplete":self.incomplete,"nothing_was_changed":true}) + } + pub fn to_text(&self) -> String { + let mut out = format!( + "store chunks: {} recorded, {} stored bytes (as the database records them)\nstore manifests: {} recorded, {} stored bytes (as the database records them)\nstore chunks: {} orphan(s), {} stored bytes\nstore manifests: {} orphan(s), {} stored bytes\nforeign objects: {}\n", + self.chunks_recorded.count, + self.chunks_recorded.bytes, + self.manifests_recorded.count, + self.manifests_recorded.bytes, + self.chunks_orphans.count, + self.chunks_orphans.bytes, + self.manifests_orphans.count, + self.manifests_orphans.bytes, + self.foreign.count + ); + out.push_str(&format!( + "orphan candidates: {}; potential: {}; protection: first sight is not recorded; would need two runs apart\nchunk candidates: {}; {} stored bytes; protected: {}\nforeign stored bytes: not reported\n", + 0, self.chunks_orphans.count.saturating_add(self.manifests_orphans.count), self.chunk_candidates, self.chunk_candidate_bytes, self.chunk_protected + )); + for (label, examples) in [ + ("chunk orphan IDs", &self.chunks_orphans.examples), + ("manifest orphan IDs", &self.manifests_orphans.examples), + ("foreign object pseudonyms", &self.foreign.examples), + ] { + if !examples.is_empty() { + out.push_str(&format!("{label}: {}\n", examples.join(", "))); + } + } + if self.incomplete { + out.push_str(&format!("chunk protection: index incomplete: dependencies unknown; protected {} recorded chunk(s)\n", self.chunk_count)); + } + out.push_str("chunk roots: current and saved file versions (including trash history), live snapshots and import snapshots\n"); + if !self.chunk_examples.is_empty() { + out.push_str(&format!("chunk candidate IDs from database snapshot: {}\n", self.chunk_examples.join(", "))); + } + out.push_str(&format!( + "manifest protection: protected: may be a live document's asset ({} unrooted recorded manifest(s))\n", + self.db_check.audit.unrooted_manifests.count + )); + if let Some(error) = &self.inventory_error { + out.push_str(&format!("listing error: {error}\n")); + } + if self.db_check.problem_count() > 0 { + out.push_str(&self.db_check.to_text()); + } + out.push_str(&format!("database problems: {}\nnothing was changed\n", self.db_check.problem_count())); + out + } +} diff --git a/apps/craft-cloud-server/src/join.rs b/apps/craft-cloud-server/src/join.rs index 41d57bdc..0bf3266b 100644 --- a/apps/craft-cloud-server/src/join.rs +++ b/apps/craft-cloud-server/src/join.rs @@ -4,11 +4,18 @@ //! - **One answer for every code that doesn't work**: wrong, expired, used, malformed, or a pairing whose maker has //! signed out are all `404 invalid_code` with the same message, so an answer never says which. //! - **Failed redeems are limited per client address** (behind trusted proxies, as sign-ins are): [`JOINS_PER_IP`] -//! in a minute, then `429 rate_limited`, even for a code that works. Only `invalid_code` counts. A successful -//! redeem doesn't, and neither does `password_required` or `weak_password`: they are only answered for a real, -//! unused invite, so they tell a guesser nothing a success wouldn't. Every try is counted before the code is -//! looked at, and given back once it turns out not to be `invalid_code`, so tries sent at the same moment are held -//! to the limit as well. +//! in a minute, then `429 rate_limited`, even for a code that works. `invalid_code` and `two_factor_invalid` count. A +//! successful redeem doesn't, and neither does `password_required`, `weak_password` or `two_factor_required`: they are +//! only answered for a real, unused invite, so they tell a guesser nothing a success wouldn't. Every try is counted +//! before the code is looked at, and given back once it turns out not to be one of those two, so tries sent at the same +//! moment are held to the limit as well. +//! - **An account with two-factor sign-in** needs its second factor (`totp_code`, or `recovery_code`) to redeem an invite or +//! a password reset code, unless the reset code was made to turn it off. Without it the answer is `401 +//! two_factor_required` and the code is **not** used up: the same body is sent again with the factor (there is no +//! challenge for `/join`, so the answer's `challenge` is empty). A wrong factor is `401 two_factor_invalid`, leaves the +//! code as it was and counts against the account (and its lock) and, with the client address, against the +//! second-factor limiter. A pairing code needs no second factor: only a signed-in device can make one, and turning +//! two-factor sign-in on signs the account's other devices out. //! - **An invite to a new account** must carry a password (the server's password rules), hashed with Argon2id under //! the same server-wide slots as sign-ins; it becomes the account's in the transaction that creates the device. //! - **Codes are never logged**, and neither are tokens or passwords; the database keeps only codes' hashes. @@ -21,16 +28,17 @@ use axum::extract::rejection::PathRejection; use axum::extract::{ConnectInfo, Extension, Path, Request, State}; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::IntoResponse; -use craft_cloud_meta::DeviceAuth; +use craft_cloud_meta::{CHALLENGE_TTL_MS, DeviceAuth, DeviceClient, MetaError}; use craft_cloud_proto::api::{JoinRequest, LoginResponse, PairingCreated, PairingStatus, codes}; use craft_cloud_proto::join::{Code, PAIRING_CHARS, PAIRING_TTL_MS}; use uuid::Uuid; -use crate::api::{PASSWORD_RETRY_SECS, SMALL_JSON_BYTES, check_device_name, read_json}; +use crate::api::{PASSWORD_RETRY_SECS, SMALL_JSON_BYTES, check_device_name, device_client, read_json}; use crate::auth::{check_password_rules, hash_password, new_token}; use crate::error::ApiError; use crate::ratelimit::{client_ip, ip_key}; use crate::state::AppState; +use crate::two_factor::{Prepared, ProofInput, SECOND_FACTORS_PER_IP, parse_proof}; /// Failed redeems one client address may make in a minute. pub const JOINS_PER_IP: u32 = 10; @@ -82,22 +90,50 @@ pub async fn join( ) -> Result, ApiError> { let req: JoinRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; check_device_name(&req.device_name)?; + // Checked before the try is counted, as the device name is: a body that is refused for it costs nothing. + let client = device_client(req.client_kind.as_deref(), req.platform.as_deref(), req.app_version.as_deref())?; + let proof = parse_proof(req.totp_code.clone(), req.recovery_code.clone())?; // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. let now = tokio::time::Instant::now().into_std(); let key = ip_key(client_ip(peer, &headers, &state.trusted_proxies)); // Counted before the redeem, so tries made at the same moment are held to the limit too; given back unless the // code didn't work. let Some(counted) = state.joins.count(&key, JOINS_PER_IP, now) else { - return Err(ApiError::rate_limited("too many tries with codes that don't work; wait a minute", None)); + return Err(limited("too many tries with codes that don't work; wait a minute", state.joins.retry_after(&key, now))); }; - let result = redeem(&state, &req).await; - if !result.as_ref().is_err_and(|e| e.code == codes::INVALID_CODE) { + // A second factor is also counted against the address's second-factor limiter: wrong ones stay counted. + let second = match proof { + Some(_) => match state.second_factors.count(&key, SECOND_FACTORS_PER_IP, now) { + Some(counted) => Some(counted), + None => { + state.joins.refund(&key, counted); + return Err(limited("too many wrong codes from this address; wait a few minutes", state.second_factors.retry_after(&key, now))); + } + }, + None => None, + }; + let result = redeem(&state, &req, &client, proof).await; + // Wrong codes (of the invite or of the second factor) stay counted; everything else is given back. + if !result.as_ref().is_err_and(|e| e.code == codes::INVALID_CODE || e.code == codes::TWO_FACTOR_INVALID) { state.joins.refund(&key, counted); } + if let Some(second) = second + && !result.as_ref().is_err_and(|e| e.code == codes::TWO_FACTOR_INVALID) + { + state.second_factors.refund(&key, second); + } result.map(Json) } -async fn redeem(state: &AppState, req: &JoinRequest) -> Result { +/// The answer for a join a limiter refuses: the same for every code (a real invite, a locked account's, a made-up one), with the +/// time until the limiter's window ends in `Retry-After`, so that the refusal tells a guesser nothing about codes or locks. +/// A lock is only told to a request the limiters let through (and counted), as at sign-in. +fn limited(message: &'static str, window: std::time::Duration) -> ApiError { + let secs = u32::try_from(window.as_secs().saturating_add(1)).unwrap_or(u32::MAX).max(1); + ApiError::rate_limited(message, Some(secs)) +} + +async fn redeem(state: &AppState, req: &JoinRequest, client: &DeviceClient, proof: Option) -> Result { let code = Code::parse_any(&req.code).ok_or_else(ApiError::invalid_code)?; let code_hash = code.hash(); let now_ms = state.now_ms(); @@ -110,8 +146,22 @@ async fn redeem(state: &AppState, req: &JoinRequest) -> Result None, }; let (token, token_hash) = new_token()?; - let redeemed = state.meta.redeem(&code_hash, &req.device_name, &token_hash, new_password_hash.as_deref(), now_ms).await?; - tracing::info!(device = %redeemed.device, "device joined with a code"); + let prepared = proof.map(|proof| proof.prepare(state, now_ms)); + let second_factor = prepared.as_ref().map(Prepared::proof); + let redeemed = state + .meta + .redeem_with_second_factor(&code_hash, &req.device_name, client, &token_hash, new_password_hash.as_deref(), second_factor.as_ref(), now_ms) + .await + .map_err(|e| match e { + // No challenge row for a join: the answer only says that the same code goes again with the second factor. + MetaError::TwoFactorRequired => ApiError::two_factor_required(String::new(), now_ms.saturating_add(CHALLENGE_TTL_MS)), + e => ApiError::from(e).on_public_route(), + })?; + // A reset code signed the account's other devices out: their live connections end now, not at their next request. + for device in &redeemed.revoked { + state.live.revoke(*device); + } + tracing::info!(device = %redeemed.device, revoked = redeemed.revoked.len(), "device joined with a code"); Ok(LoginResponse { device: redeemed.device, token }) } diff --git a/apps/craft-cloud-server/src/lib.rs b/apps/craft-cloud-server/src/lib.rs index 97ffd436..cf61affd 100644 --- a/apps/craft-cloud-server/src/lib.rs +++ b/apps/craft-cloud-server/src/lib.rs @@ -2,16 +2,27 @@ //! (`craft-cloud-store`), the live service on a WebSocket ([`live`]), plus the admin command line. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod account; +pub mod admin; pub mod api; +pub mod audit; pub mod auth; pub mod cli; pub mod config; pub mod error; +pub mod full; +pub mod gc; pub mod join; pub mod live; pub mod notify; pub mod ratelimit; +pub mod secretbox; +pub mod space; pub mod state; +pub mod totp; +pub mod two_factor; +pub mod verify; +pub mod web; use std::net::SocketAddr; use std::sync::Arc; diff --git a/apps/craft-cloud-server/src/live.rs b/apps/craft-cloud-server/src/live.rs index 859d5c88..e01e966f 100644 --- a/apps/craft-cloud-server/src/live.rs +++ b/apps/craft-cloud-server/src/live.rs @@ -768,13 +768,19 @@ pub async fn connect( return Err(ApiError::rate_limited(format!("a device may hold at most {MAX_CONNECTIONS_PER_DEVICE} live connections"), Some(5))); } let slot = ConnectionSlot { hub: state.clone(), device: auth.device }; + // Listening for revocations before the device is looked up once more, and before the upgrade: a revoke between + // the sign-in check and here is seen by the lookup, one after it by the receiver, so none is missed. + let revocations = state.live.revocations(); + if !state.meta.device_active(auth.device).await? { + return Err(ApiError::unauthorized()); + } // No frame is over an update and its envelope (snapshots travel through the chunk API). Ok(upgrade.max_message_size(MAX_FRAME_BYTES).max_frame_size(MAX_FRAME_BYTES).on_upgrade(move |socket| async move { let _slot = slot; // Listening before the session reads anything, so no update falls between a read and the notification. let notes = state.live.listen(auth.space); let presence = state.live.listen_presence(auth.space); - serve(socket, Session::new(state, &auth), notes, presence).await; + serve(socket, Session::new(state, &auth), notes, presence, revocations).await; })) } @@ -801,9 +807,14 @@ async fn send_all(socket: &mut WebSocket, frames: &[ServerFrame]) -> bool { /// The connection loop: frames in, frames out, the log pumped while a document has news, pings, and an end on /// shutdown, silence or a broken connection. -async fn serve(mut socket: WebSocket, mut session: Session, mut notes: broadcast::Receiver, mut presence: broadcast::Receiver) { +async fn serve( + mut socket: WebSocket, + mut session: Session, + mut notes: broadcast::Receiver, + mut presence: broadcast::Receiver, + mut revocations: broadcast::Receiver, +) { let mut stopping = session.state.notifier.shutdown_signal(); - let mut revocations = session.state.live.revocations(); let mut last_heard = Instant::now(); let mut ping = tokio::time::interval_at(Instant::now() + PING_EVERY, PING_EVERY); let hello_by = Instant::now() + HELLO_TIMEOUT; diff --git a/apps/craft-cloud-server/src/main.rs b/apps/craft-cloud-server/src/main.rs index 99384030..e53224be 100644 --- a/apps/craft-cloud-server/src/main.rs +++ b/apps/craft-cloud-server/src/main.rs @@ -5,6 +5,7 @@ use std::process::ExitCode; use clap::Parser; use craft_cloud_server::cli::{Cli, Command, open}; use craft_cloud_server::config::Config; +use craft_cloud_server::web::WebSettings; use craft_cloud_server::{AppState, serve}; #[tokio::main] @@ -35,9 +36,25 @@ async fn run_server(config_path: &std::path::Path) -> Result<(), String> { tracing::warn!("ignore_forwarded_for is set: every client shares one address for sign-in rate limits"); } let (meta, store) = open(&config).await.map_err(|e| e.to_string())?; - let state = AppState::new(meta, store, config.trusted_proxies.clone()).map_err(|e| e.message)?; + // Fail closed: a database that holds two-factor secrets the key can't open is not served. The server never makes the key. + let keys = craft_cloud_server::two_factor::load_keys(&config).map_err(|e| e.to_string())?; + let secrets = craft_cloud_server::two_factor::check_startup(&meta, keys, &config, craft_cloud_server::state::now_ms()).await?; + // `Config::load` has checked the setup the pages need (an https public_url, a proxy in front). + let web = match (config.web_admin.enabled, &config.public_url) { + (true, Some(public_url)) => Some(WebSettings::new(public_url, config.web_admin.idle_ms(), config.web_admin.absolute_ms())?), + _ => None, + }; + let storage_info = craft_cloud_server::space::StorageInfo::from_config(&config); + let state = AppState::new(meta, store, config.trusted_proxies.clone()) + .map_err(|e| e.message)? + .with_public_url(config.public_url.clone()) + .with_web_admin(web) + .with_two_factor(secrets) + .with_trash_days(config.retention.trash_days) + .with_storage_info(storage_info.clone()) + .with_full_guard(config.storage.full_policy().map(|policy| craft_cloud_server::full::FullGuard::new(policy, &storage_info))); let listener = tokio::net::TcpListener::bind(config.listen).await.map_err(|e| format!("can't listen on {}: {e}", config.listen))?; - tracing::info!(listen = %config.listen, behind_proxy = config.behind_proxy, "Craft Cloud server running"); + tracing::info!(listen = %config.listen, behind_proxy = config.behind_proxy, web_admin = config.web_admin.enabled, "Craft Cloud server running"); let shutdown = async { wait_for_stop_signal().await; tracing::info!("shutting down"); diff --git a/apps/craft-cloud-server/src/ratelimit.rs b/apps/craft-cloud-server/src/ratelimit.rs index 27e5b4ad..1a9f33c9 100644 --- a/apps/craft-cloud-server/src/ratelimit.rs +++ b/apps/craft-cloud-server/src/ratelimit.rs @@ -32,14 +32,16 @@ impl Limiter { Self { windows: Mutex::new(Windows { entries: HashMap::new(), last_sweep: None }), period, capacity } } - /// Counts one attempt for `key`; false once `max` attempts were made in the current window, or when + /// Counts one attempt for `key`; false once `max` attempts were counted in the current window, or when /// the key is new and the limiter is full. pub fn allow(&self, key: &str, max: u32, now: Instant) -> bool { self.count(key, max, now).is_some() } /// [`Limiter::allow`] for a caller that may give the attempt back: the window it was counted in, for - /// [`Limiter::refund`], or `None` when it is refused. + /// [`Limiter::refund`], or `None` when it is refused. A refused attempt is not counted: the count never goes past + /// `max`, so refused tries made at the same moment as a counted one cannot make that one's refund (or the window's + /// end) leave the key with more counted attempts than were accepted. pub fn count(&self, key: &str, max: u32, now: Instant) -> Option { let mut windows = self.windows.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let sweep_due = windows.last_sweep.is_none_or(|last| now.saturating_duration_since(last) >= self.period); @@ -55,8 +57,18 @@ impl Limiter { if now.saturating_duration_since(entry.0) >= self.period { *entry = (now, 0); } - entry.1 = entry.1.saturating_add(1); - (entry.1 <= max).then_some(Counted(entry.0)) + if entry.1 >= max { + return None; + } + entry.1 += 1; + Some(Counted(entry.0)) + } + + /// How long until `key`'s current window ends (when its count starts again from nothing): what a refused caller is + /// told in `Retry-After`. A key that has no window running is `Duration::ZERO`. + pub fn retry_after(&self, key: &str, now: Instant) -> Duration { + let windows = self.windows.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + windows.entries.get(key).map_or(Duration::ZERO, |(start, _)| (*start + self.period).saturating_duration_since(now)) } /// Gives back one attempt [`Limiter::count`] counted for `key`, if the window it was counted in is still the @@ -145,6 +157,21 @@ mod tests { /// An attempt counted near the end of a window whose redeem finishes in the next one: its refund would give the /// new window one try more than the limit. + #[test] + fn a_refused_attempt_is_not_counted_so_a_refund_frees_exactly_one_place() { + let limiter = Limiter::new(Duration::from_secs(60)); + let t0 = Instant::now(); + let first = limiter.count("user:a", 2, t0).unwrap(); + let _second = limiter.count("user:a", 2, t0).unwrap(); + // Tries made while the key is full are refused, however many there are. + for _ in 0..10 { + assert!(limiter.count("user:a", 2, t0).is_none()); + } + limiter.refund("user:a", first); + assert!(limiter.count("user:a", 2, t0).is_some(), "the refund freed one place"); + assert!(limiter.count("user:a", 2, t0).is_none(), "and only one"); + } + #[test] fn a_refund_never_crosses_into_a_later_window() { let limiter = Limiter::new(Duration::from_secs(60)); diff --git a/apps/craft-cloud-server/src/secretbox.rs b/apps/craft-cloud-server/src/secretbox.rs new file mode 100644 index 00000000..e2685f1b --- /dev/null +++ b/apps/craft-cloud-server/src/secretbox.rs @@ -0,0 +1,279 @@ +//! Sealing the TOTP secrets that the database keeps. +//! +//! - **The key** is 32 random bytes in a file the server's config names (`[two_factor] key_file`, see +//! [`crate::two_factor`]). It is never used as it is: the AES-256 key is [`blake3::derive_key`] of it under a fixed +//! context. A key shows in no log, no `Debug` output and no error message, and its bytes are wiped when it is dropped. +//! - **A sealed secret** is one byte naming the key (see below), a 12-byte random nonce and the AES-256-GCM ciphertext +//! with its tag. The associated data is `craft-cloud:totp:v1:` and the 16 bytes of the account's ID, so a ciphertext +//! copied to another account does not open. +//! - **The key's name** is the first byte of the BLAKE3 hash of the key file's bytes: one of 256 values, only there to tell +//! the current key from the older one, so two keys that share one are refused when the server starts. It does not +//! identify a key, and the authentication tag refuses a wrong key whatever the name says. With a second, older key (`key_file_old`) configured, the +//! name picks the key to open a secret with; sealing always uses the current key. The `two-factor rekey` command +//! rewrites the secrets under the current key, after which the old key can go. +//! - **Errors say nothing about their input**: not the key, not the plaintext, not the ciphertext. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use aes_gcm::aead::{Aead, KeyInit, Payload}; +use aes_gcm::{Aes256Gcm, Nonce}; +use craft_cloud_meta::UserId; +use zeroize::Zeroizing; + +/// The bytes a key file holds. +pub const KEY_BYTES: usize = 32; +const NONCE_BYTES: usize = 12; +const TAG_BYTES: usize = 16; +const KDF_CONTEXT: &str = "craft-cloud 2026 totp secret v1"; +const AAD_PREFIX: &[u8] = b"craft-cloud:totp:v1:"; + +/// A secret could not be sealed or opened. Carries nothing of what was sealed or opened. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("a two-factor secret could not be sealed or opened")] +pub struct SecretBoxError; + +/// A key file that cannot be used. The message names the file and what is wrong with it, never its content. +#[derive(Debug, thiserror::Error)] +pub enum KeyError { + #[error("the two-factor key file {path} does not exist")] + Missing { path: PathBuf }, + #[error("can't read the two-factor key file {path}: {source}")] + Read { path: PathBuf, source: std::io::Error }, + #[error("the two-factor key file {path} must hold exactly 32 bytes")] + Length { path: PathBuf }, + #[error( + "key_file and key_file_old are the same key, or two keys that share a name: make a new key for key_file (the first byte of the keys' BLAKE3 \ + hashes must differ)" + )] + SameName, +} + +/// A key as read from a key file. `Debug` shows nothing of it. +pub struct TwoFactorKey { + bytes: Zeroizing<[u8; KEY_BYTES]>, + id: u8, +} + +impl std::fmt::Debug for TwoFactorKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("TwoFactorKey()") + } +} + +impl TwoFactorKey { + /// A key from its 32 bytes. + pub fn from_bytes(bytes: [u8; KEY_BYTES]) -> Self { + let id = blake3::hash(&bytes).as_bytes().first().copied().unwrap_or(0); + Self { bytes: Zeroizing::new(bytes), id } + } + + /// A new random key (for `init`). + pub fn generate() -> Result { + let mut bytes = [0u8; KEY_BYTES]; + getrandom::fill(&mut bytes)?; + Ok(Self::from_bytes(bytes)) + } + + /// The key file's bytes: only for writing a new key file. + pub fn to_file_bytes(&self) -> Zeroizing<[u8; KEY_BYTES]> { + self.bytes.clone() + } + + /// Reads a key file: exactly [`KEY_BYTES`] bytes (a longer file is not read further). + pub fn from_file(path: &Path) -> Result { + let mut file = std::fs::File::open(path).map_err(|source| match source.kind() { + std::io::ErrorKind::NotFound => KeyError::Missing { path: path.to_owned() }, + _ => KeyError::Read { path: path.to_owned(), source }, + })?; + let mut bytes = Zeroizing::new(Vec::with_capacity(KEY_BYTES + 1)); + (&mut file).take(KEY_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|source| KeyError::Read { path: path.to_owned(), source })?; + let bytes: [u8; KEY_BYTES] = bytes.as_slice().try_into().map_err(|_| KeyError::Length { path: path.to_owned() })?; + Ok(Self::from_bytes(bytes)) + } + + /// The name sealed secrets carry for this key. + pub fn id(&self) -> u8 { + self.id + } +} + +struct KeyEntry { + id: u8, + cipher: Aes256Gcm, +} + +impl KeyEntry { + fn new(key: &TwoFactorKey) -> Result { + let derived = Zeroizing::new(blake3::derive_key(KDF_CONTEXT, key.bytes.as_slice())); + let cipher = Aes256Gcm::new_from_slice(derived.as_slice()).map_err(|_| SecretBoxError)?; + Ok(Self { id: key.id, cipher }) + } +} + +/// Seals and opens TOTP secrets with the current key and, for reading, an older one. +pub struct SecretBox { + current: KeyEntry, + old: Option, +} + +impl std::fmt::Debug for SecretBox { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SecretBox()") + } +} + +fn aad(user: UserId) -> Vec { + let mut aad = Vec::with_capacity(AAD_PREFIX.len() + 16); + aad.extend_from_slice(AAD_PREFIX); + aad.extend_from_slice(user.0.as_bytes()); + aad +} + +impl SecretBox { + /// A box for `current` and, optionally, the `old` key secrets are still sealed with. Two keys with one name are + /// refused. + pub fn new(current: &TwoFactorKey, old: Option<&TwoFactorKey>) -> Result { + if old.is_some_and(|old| old.id == current.id) { + return Err(KeyError::SameName); + } + let entry = |key: &TwoFactorKey| KeyEntry::new(key).map_err(|_| KeyError::SameName); + Ok(Self { current: entry(current)?, old: old.map(entry).transpose()? }) + } + + /// The name of the current key. + pub fn current_id(&self) -> u8 { + self.current.id + } + + /// The name of the older key, if one is configured. + pub fn old_id(&self) -> Option { + self.old.as_ref().map(|old| old.id) + } + + /// The name of the key a sealed secret was sealed with (`None` for something too short to be one). + pub fn key_id(sealed: &[u8]) -> Option { + (sealed.len() >= 1 + NONCE_BYTES + TAG_BYTES).then(|| sealed.first().copied()).flatten() + } + + /// Seals `plaintext` for `user` with the current key and a fresh random nonce. + pub fn seal(&self, user: UserId, plaintext: &[u8]) -> Result, SecretBoxError> { + let mut nonce = [0u8; NONCE_BYTES]; + getrandom::fill(&mut nonce).map_err(|_| SecretBoxError)?; + let ciphertext = + self.current.cipher.encrypt(Nonce::from_slice(&nonce), Payload { msg: plaintext, aad: &aad(user) }).map_err(|_| SecretBoxError)?; + let mut sealed = Vec::with_capacity(1 + NONCE_BYTES + ciphertext.len()); + sealed.push(self.current.id); + sealed.extend_from_slice(&nonce); + sealed.extend_from_slice(&ciphertext); + Ok(sealed) + } + + /// Opens a secret sealed for `user`. A secret of another account, a damaged or truncated one, one sealed with a key this + /// box doesn't hold: all the same error. + pub fn open(&self, user: UserId, sealed: &[u8]) -> Result>, SecretBoxError> { + let id = Self::key_id(sealed).ok_or(SecretBoxError)?; + let entry = if id == self.current.id { &self.current } else { self.old.as_ref().filter(|old| old.id == id).ok_or(SecretBoxError)? }; + let (nonce, ciphertext) = sealed.get(1..).and_then(|rest| rest.split_at_checked(NONCE_BYTES)).ok_or(SecretBoxError)?; + let plaintext = entry.cipher.decrypt(Nonce::from_slice(nonce), Payload { msg: ciphertext, aad: &aad(user) }).map_err(|_| SecretBoxError)?; + Ok(Zeroizing::new(plaintext)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn user() -> UserId { + UserId(uuid::Uuid::now_v7()) + } + + fn key(byte: u8) -> TwoFactorKey { + TwoFactorKey::from_bytes([byte; KEY_BYTES]) + } + + #[test] + fn a_sealed_secret_opens_again() { + let sb = SecretBox::new(&key(1), None).unwrap(); + let ana = user(); + let sealed = sb.seal(ana, b"12345678901234567890").unwrap(); + assert_eq!(sb.open(ana, &sealed).unwrap().as_slice(), b"12345678901234567890"); + assert_eq!(sealed.first().copied(), Some(key(1).id())); + assert_eq!(sealed.len(), 1 + NONCE_BYTES + 20 + TAG_BYTES); + assert_ne!(sealed, sb.seal(ana, b"12345678901234567890").unwrap(), "a fresh nonce each time"); + assert!(!sealed.windows(20).any(|w| w == b"12345678901234567890"), "no plaintext in the ciphertext"); + } + + #[test] + fn a_wrong_key_another_account_or_a_damaged_ciphertext_fails() { + let sb = SecretBox::new(&key(1), None).unwrap(); + let (ana, bob) = (user(), user()); + let sealed = sb.seal(ana, b"secret").unwrap(); + assert_eq!(sb.open(bob, &sealed), Err(SecretBoxError), "the account is bound in"); + let other = SecretBox::new(&key(2), None).unwrap(); + assert_eq!(other.open(ana, &sealed), Err(SecretBoxError), "wrong key"); + // A key with the same name but other bytes fails on the tag. + let mut bytes = [1u8; KEY_BYTES]; + let mut same_name = None; + for i in 0..=255u8 { + bytes[0] = i; + if TwoFactorKey::from_bytes(bytes).id() == key(1).id() && bytes != [1u8; KEY_BYTES] { + same_name = Some(TwoFactorKey::from_bytes(bytes)); + break; + } + } + if let Some(same_name) = same_name { + assert_eq!(SecretBox::new(&same_name, None).unwrap().open(ana, &sealed), Err(SecretBoxError)); + } + for cut in [0, 1, 12, 13, sealed.len() - 1] { + assert_eq!(sb.open(ana, &sealed[..cut]), Err(SecretBoxError), "cut at {cut}"); + } + let mut flipped = sealed.clone(); + if let Some(last) = flipped.last_mut() { + *last ^= 1; + } + assert_eq!(sb.open(ana, &flipped), Err(SecretBoxError)); + } + + #[test] + fn the_older_key_opens_what_it_sealed_and_the_new_one_seals() { + let ana = user(); + let old_only = SecretBox::new(&key(1), None).unwrap(); + let sealed_old = old_only.seal(ana, b"secret").unwrap(); + let rotated = SecretBox::new(&key(2), Some(&key(1))).unwrap(); + assert_eq!(rotated.open(ana, &sealed_old).unwrap().as_slice(), b"secret"); + assert_eq!(rotated.old_id(), Some(key(1).id())); + let sealed_new = rotated.seal(ana, b"secret").unwrap(); + assert_eq!(SecretBox::key_id(&sealed_new), Some(key(2).id())); + assert_eq!(old_only.open(ana, &sealed_new), Err(SecretBoxError), "the old key alone does not open the new"); + // Without the old key configured, what it sealed is a stranger's. + assert_eq!(SecretBox::new(&key(2), None).unwrap().open(ana, &sealed_old), Err(SecretBoxError)); + } + + #[test] + fn two_keys_with_one_name_are_refused() { + assert!(matches!(SecretBox::new(&key(1), Some(&key(1))), Err(KeyError::SameName))); + } + + #[test] + fn a_key_is_read_from_a_file_of_exactly_32_bytes() { + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.key"); + std::fs::write(&good, [7u8; KEY_BYTES]).unwrap(); + assert_eq!(TwoFactorKey::from_file(&good).unwrap().id(), key(7).id()); + for (name, len) in [("short.key", 31), ("long.key", 33), ("empty.key", 0), ("huge.key", 100_000)] { + let path = dir.path().join(name); + std::fs::write(&path, vec![7u8; len]).unwrap(); + assert!(matches!(TwoFactorKey::from_file(&path), Err(KeyError::Length { .. })), "{name}"); + } + assert!(matches!(TwoFactorKey::from_file(&dir.path().join("nope.key")), Err(KeyError::Missing { .. }))); + } + + #[test] + fn keys_and_boxes_show_nothing_in_debug_and_errors_say_nothing() { + let k = key(9); + assert_eq!(format!("{k:?}"), "TwoFactorKey()"); + assert_eq!(format!("{:?}", SecretBox::new(&k, None).unwrap()), "SecretBox()"); + assert_eq!(SecretBoxError.to_string(), "a two-factor secret could not be sealed or opened"); + } +} diff --git a/apps/craft-cloud-server/src/space.rs b/apps/craft-cloud-server/src/space.rs new file mode 100644 index 00000000..b32e51ea --- /dev/null +++ b/apps/craft-cloud-server/src/space.rs @@ -0,0 +1,179 @@ +//! How much room the server has, for the admin's storage overview (`GET /admin/storage`). +//! +//! The numbers are cheap: a filesystem query for the store's directory and for the directory of the SQLite file, and a +//! `stat` of the database files. Nothing scans the stored data, so the overview costs the same on an empty server and +//! a full one. A number the server can't get (a directory that vanished, a filesystem that doesn't say) is left out +//! rather than failing the overview. + +use std::path::{Path, PathBuf}; + +use crate::config::{Config, StorageConfig}; + +/// Where to measure, and what the admin said the store may hold. Built from the config at start; `Default` knows +/// nothing (an in-memory test server), so every number is `None`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StorageInfo { + capacity: Option, + /// The disk store's directory. `None` for S3 and memory. + store_dir: Option, + /// The SQLite file. `None` for Postgres and `sqlite::memory:`. + database_file: Option, +} + +/// What [`StorageInfo::measure`] found. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Measured { + pub store_fs_total: Option, + pub store_fs_free: Option, + pub server_fs_total: Option, + pub server_fs_free: Option, + pub database_bytes: Option, +} + +impl StorageInfo { + pub fn new(capacity: Option, store_dir: Option, database_file: Option) -> Self { + Self { capacity, store_dir, database_file } + } + + pub fn from_config(config: &Config) -> Self { + let store_dir = match &config.storage { + StorageConfig::Disk { path, .. } => Some(path.clone()), + StorageConfig::S3 { .. } => None, + }; + Self::new(config.storage.capacity(), store_dir, sqlite_file(&config.database.url)) + } + + /// What the admin set as `capacity`, in bytes. + pub fn capacity(&self) -> Option { + self.capacity + } + + /// Total and free bytes of the filesystem that holds the disk store; `(None, None)` for S3 and memory, or when it + /// can't be read. Blocking. + pub fn store_filesystem(&self) -> (Option, Option) { + self.store_dir.as_deref().map_or((None, None), filesystem) + } + + /// Reads the filesystem numbers and the database's size. Blocking: call it off the async threads. + pub fn measure(&self) -> Measured { + let mut measured = Measured::default(); + if let Some(dir) = &self.store_dir { + (measured.store_fs_total, measured.store_fs_free) = filesystem(dir); + } + if let Some(file) = &self.database_file { + let dir = match file.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + (measured.server_fs_total, measured.server_fs_free) = filesystem(dir); + measured.database_bytes = sqlite_bytes(file); + } + measured + } +} + +/// The SQLite file a database URL names (`sqlite:path`, `sqlite://path`, an optional `?query`); `None` for Postgres +/// and for the in-memory database. +pub fn sqlite_file(url: &str) -> Option { + let rest = url.strip_prefix("sqlite:")?; + let rest = rest.strip_prefix("//").unwrap_or(rest); + let (path, query) = rest.split_once('?').unwrap_or((rest, "")); + // `mode=memory` makes any name an in-memory database. + if query.split('&').any(|pair| pair.split_once('=').is_some_and(|(key, value)| key == "mode" && value == "memory")) { + return None; + } + if path.is_empty() || path.starts_with(":memory:") { + return None; + } + // As sqlx reads it: the path is percent-decoded, so a `?` or `#` can be in a file name. + let decoded = percent_encoding::percent_decode_str(path).decode_utf8().ok()?; + Some(PathBuf::from(decoded.as_ref())) +} + +/// Total and available bytes of the filesystem holding `dir`. +fn filesystem(dir: &Path) -> (Option, Option) { + // Windows answers for a directory that is not there with the numbers of the disk it would be on; a missing store + // is not a measurement. + if !dir.is_dir() { + tracing::debug!(dir = %dir.display(), "the directory to measure is not there"); + return (None, None); + } + match fs4::statvfs(dir) { + Ok(stats) => (Some(stats.total_space()), Some(stats.available_space())), + Err(e) => { + tracing::debug!(dir = %dir.display(), error = %e, "can't read the size of a filesystem"); + (None, None) + } + } +} + +/// The database file's size plus its write-ahead log and shared-memory file when they exist. `None` when the main +/// file can't be read. +fn sqlite_bytes(file: &Path) -> Option { + let mut total = std::fs::metadata(file).ok()?.len(); + for suffix in ["-wal", "-shm"] { + let mut name = file.as_os_str().to_owned(); + name.push(suffix); + if let Ok(meta) = std::fs::metadata(PathBuf::from(name)) { + total = total.saturating_add(meta.len()); + } + } + Some(total) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_sqlite_file_comes_out_of_the_url() { + assert_eq!(sqlite_file("sqlite:/var/lib/cc/meta.db"), Some(PathBuf::from("/var/lib/cc/meta.db"))); + assert_eq!(sqlite_file("sqlite:meta.db"), Some(PathBuf::from("meta.db"))); + assert_eq!(sqlite_file("sqlite:///var/lib/meta.db?mode=rwc"), Some(PathBuf::from("/var/lib/meta.db"))); + assert_eq!(sqlite_file("sqlite:C:/cc/meta.db"), Some(PathBuf::from("C:/cc/meta.db"))); + assert_eq!(sqlite_file("sqlite:/srv/a%20b/meta.db"), Some(PathBuf::from("/srv/a b/meta.db"))); + assert_eq!(sqlite_file("sqlite:/srv/a%3Fb.db?mode=rwc"), Some(PathBuf::from("/srv/a?b.db"))); + for none in [ + "sqlite:meta.db?mode=memory", + "sqlite:meta.db?cache=shared&mode=memory", + "sqlite::memory:", + "sqlite:", + "postgres://u:p@host/db", + "mysql://x", + ] { + assert_eq!(sqlite_file(none), None, "{none}"); + } + } + + #[test] + fn the_database_size_counts_the_wal_and_shm_files() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("meta.db"); + assert_eq!(sqlite_bytes(&db), None, "no file"); + std::fs::write(&db, [0u8; 100]).unwrap(); + assert_eq!(sqlite_bytes(&db), Some(100)); + std::fs::write(dir.path().join("meta.db-wal"), [0u8; 20]).unwrap(); + std::fs::write(dir.path().join("meta.db-shm"), [0u8; 3]).unwrap(); + assert_eq!(sqlite_bytes(&db), Some(123)); + } + + #[test] + fn a_disk_store_and_a_sqlite_file_have_filesystem_numbers() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("meta.db"); + std::fs::write(&db, [0u8; 10]).unwrap(); + let measured = StorageInfo::new(Some(7), Some(dir.path().to_owned()), Some(db)).measure(); + let (total, free) = (measured.store_fs_total.unwrap(), measured.store_fs_free.unwrap()); + assert!(total > 0 && free <= total, "{total} {free}"); + assert_eq!((measured.server_fs_total, measured.database_bytes), (Some(total), Some(10)), "same directory, same filesystem"); + assert!(measured.server_fs_free.is_some()); + } + + #[test] + fn what_is_not_there_is_left_out() { + assert_eq!(StorageInfo::default().measure(), Measured::default()); + let gone = tempfile::tempdir().unwrap().path().join("not-here"); + let measured = StorageInfo::new(None, Some(gone.clone()), Some(gone.join("meta.db"))).measure(); + assert_eq!(measured, Measured::default()); + } +} diff --git a/apps/craft-cloud-server/src/state.rs b/apps/craft-cloud-server/src/state.rs index 10d7edc8..8616a54c 100644 --- a/apps/craft-cloud-server/src/state.rs +++ b/apps/craft-cloud-server/src/state.rs @@ -10,11 +10,14 @@ use craft_cloud_proto::DeviceId; use craft_cloud_store::Store; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use crate::audit::AuditExportLimits; use crate::auth::hash_password; use crate::error::ApiError; use crate::live::LiveHub; use crate::notify::Notifier; use crate::ratelimit::Limiter; +use crate::secretbox::SecretBox; +use crate::space::StorageInfo; /// How much one user, or one request, may hold on the server at once. Each user's limits are their own, so a /// stalled or greedy client only ever slows its own account. @@ -109,10 +112,24 @@ pub struct AppState { pub logins: Limiter, /// Failed join-code redeems per client address, in one-minute windows ([`crate::join`]). pub joins: Limiter, + /// Password change tries per account, in 15-minute windows ([`crate::account`]). + pub password_changes: Limiter, + /// Wrong second factors per account and per client address, in five-minute windows ([`crate::two_factor`]). + pub second_factors: Limiter, + /// Sign-in attempts to the admin pages per client address and per account, in five-minute windows ([`crate::web`]). + pub web_logins: Limiter, + /// The admin pages' settings; `None`: they are off and `/web` does not exist. + pub web: Option, + /// Seals and opens the secrets of two-factor sign-in; `None`: no key is configured, so it can't be set up. + pub two_factor: Option>, pub notifier: Notifier, /// The live service's connections and subscriptions ([`crate::live`]). pub live: LiveHub, pub limits: Limits, + /// How the audit export is cut into pages and bounded ([`crate::audit`]). + pub audit_export: AuditExportLimits, + /// The address computers reach the server at (`public_url` in the config), for the join links of the admin API. + pub public_url: Option, /// Chunk and manifest bodies being received, per user ([`Limits::bodies_per_user`]). pub bodies: PerUser, /// Manifests being decoded and checked, per user ([`Limits::manifests_per_user`]). @@ -122,6 +139,12 @@ pub struct AppState { pub(crate) touches: Mutex>, /// Verified against when the username doesn't exist, so both cases cost the same. pub dummy_hash: String, + /// Days a deleted node stays in the trash before the server may remove it (`retention.trash_days`). + pub trash_days: u32, + /// Where to measure the room the server has, and the `capacity` the admin set (the admin's storage overview). + pub storage_info: StorageInfo, + /// Refuses new uploads while the server is full (`[storage] refuse_when_full`); `None`: off, the default. + pub full: Option, /// The time of day for everything a request records (a device's last use, a stored blob, a commit). clock: Clock, } @@ -135,14 +158,24 @@ impl AppState { trusted_proxies, logins: Limiter::new(Duration::from_secs(300)), joins: Limiter::new(Duration::from_secs(60)), + password_changes: Limiter::new(Duration::from_secs(900)), + second_factors: Limiter::new(Duration::from_secs(300)), + web_logins: Limiter::new(Duration::from_secs(300)), + web: None, + two_factor: None, notifier: Notifier::default(), live: LiveHub::default(), bodies: PerUser::new(limits.bodies_per_user), manifests: PerUser::new(limits.manifests_per_user), password_checks: Arc::new(Semaphore::new(limits.password_checks)), limits, + audit_export: AuditExportLimits::default(), + public_url: None, touches: Mutex::new(HashMap::new()), dummy_hash: hash_password("craft-cloud dummy password")?, + trash_days: 30, + storage_info: StorageInfo::default(), + full: None, clock: Arc::new(now_ms), }) } @@ -156,10 +189,69 @@ impl AppState { self } + /// The state with other bounds for the audit export (tests use small ones). + pub fn with_audit_export(mut self, limits: AuditExportLimits) -> Self { + self.audit_export = limits; + self + } + + /// The state with the key for two-factor secrets (`None`: none is configured and two-factor sign-in can't be set up). + pub fn with_two_factor(mut self, secrets: Option) -> Self { + self.two_factor = secrets.map(Arc::new); + self + } + + /// The state with the admin pages on (`None`: off, the default): their address, session lifetimes and the like. + pub fn with_web_admin(mut self, web: Option) -> Self { + self.web = web; + self + } + + /// The state with the server's public address (`public_url`), which the admin API puts in the join links it makes. + pub fn with_public_url(mut self, public_url: Option) -> Self { + self.public_url = public_url; + self + } + + /// The state with another trash period in days (`retention.trash_days`; the default is 30). + pub fn with_trash_days(mut self, days: u32) -> Self { + self.trash_days = days; + self + } + + /// The state with the places to measure and the configured capacity (`[storage] capacity`). The default measures nothing. + pub fn with_storage_info(mut self, info: StorageInfo) -> Self { + self.storage_info = info; + self + } + + /// The state that refuses uploads while it is full (`None`: it never does, the default). + pub fn with_full_guard(mut self, guard: Option) -> Self { + self.full = guard; + self + } + + /// Refuses an upload of `incoming` bytes with `507 server_full` when the server is full and was told to refuse. + pub async fn check_server_room(&self, incoming: u64) -> Result<(), ApiError> { + match &self.full { + Some(guard) => guard.check(&self.meta, incoming).await, + None => Ok(()), + } + } + + /// Counts bytes just stored toward the room the server believes it has left. + pub fn note_stored(&self, bytes: u64) { + if let Some(guard) = &self.full { + guard.note_written(bytes); + } + } + /// The state with another clock: a simulation's, so that what requests record and when a device's last use is /// written again follow simulated time, never the real one. A test seam (the `test-support` feature). #[cfg(feature = "test-support")] pub fn with_clock(mut self, clock: Clock) -> Self { + // The store judges a browser session's expiry inside an admin change by the same simulated time. + self.meta = self.meta.clone().with_clock(clock.clone()); self.clock = clock; self } diff --git a/apps/craft-cloud-server/src/totp.rs b/apps/craft-cloud-server/src/totp.rs new file mode 100644 index 00000000..aa7dd107 --- /dev/null +++ b/apps/craft-cloud-server/src/totp.rs @@ -0,0 +1,221 @@ +//! TOTP (RFC 6238) as authenticator apps speak it: HMAC-SHA-1, six digits, 30-second steps. +//! +//! - A code is accepted for the step the clock is in and the one before and after it (a clock a little off); the step +//! it matched is handed back, and the database lets an account accept a step only once, and only a later one than the +//! last (see `craft_cloud_meta`'s two-factor module), so a code can't be used twice. +//! - A code is exactly six ASCII digits. Anything else is refused before it is compared. +//! - Codes are compared in constant time. The secret is 20 random bytes, shown to the person as Base32 without padding. + +use hmac::{Hmac, KeyInit, Mac}; +use sha1::Sha1; +use subtle::ConstantTimeEq; +use zeroize::Zeroizing; + +/// Bytes of a new secret: 160 bits. +pub const SECRET_BYTES: usize = 20; +/// Digits of a code. +pub const DIGITS: usize = 6; +/// Seconds a code is good for. +pub const STEP_SECS: i64 = 30; +/// What the authenticator app shows as the service's name. +pub const ISSUER: &str = "Craft Cloud"; + +/// A new random secret. +pub fn generate_secret() -> Result, getrandom::Error> { + let mut secret = Zeroizing::new([0u8; SECRET_BYTES]); + getrandom::fill(secret.as_mut_slice())?; + Ok(secret) +} + +/// The time step of `now_ms` (milliseconds since the Unix epoch). +pub fn step_at(now_ms: i64) -> i64 { + now_ms.div_euclid(1000).div_euclid(STEP_SECS) +} + +/// Whether `code` has the shape of a code: exactly six ASCII digits. +pub fn is_code(code: &str) -> bool { + code.len() == DIGITS && code.bytes().all(|b| b.is_ascii_digit()) +} + +/// The code for time step `step` (HOTP's counter), with `secret`. `None` for a step before the epoch. +pub fn code_for(secret: &[u8], step: i64) -> Option { + let counter = u64::try_from(step).ok()?; + let mut mac = as KeyInit>::new_from_slice(secret).ok()?; + mac.update(&counter.to_be_bytes()); + let digest = mac.finalize().into_bytes(); + let offset = usize::from(digest.last().copied()? & 0x0f); + let word = digest.get(offset..offset.checked_add(4)?)?; + let value = u32::from_be_bytes([*word.first()?, *word.get(1)?, *word.get(2)?, *word.get(3)?]) & 0x7fff_ffff; + Some(format!("{:0width$}", value % 10u32.pow(DIGITS as u32), width = DIGITS)) +} + +/// The time step `code` is right for at `now_ms`, if it is right for the step of `now_ms` or the one before or after it +/// (the latest if several match, which is all but impossible); `None` for a code that is wrong or is not six digits. All +/// three steps are always computed and compared in constant time. +pub fn verify(secret: &[u8], code: &str, now_ms: i64) -> Option { + if !is_code(code) { + return None; + } + let now = step_at(now_ms); + let mut matched = None; + for step in [now.saturating_sub(1), now, now.saturating_add(1)] { + let expected = code_for(secret, step).unwrap_or_default(); + // Equal lengths (six) for a real code; `ct_eq` on slices of different lengths is false. + if bool::from(expected.as_bytes().ct_eq(code.as_bytes())) && !expected.is_empty() { + matched = Some(step); + } + } + matched +} + +const BASE32: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +/// RFC 4648 Base32 without padding. +pub fn base32_encode(bytes: &[u8]) -> String { + let mut out = String::with_capacity((bytes.len() * 8).div_ceil(5)); + let (mut buffer, mut bits) = (0u32, 0u32); + for &byte in bytes { + buffer = (buffer << 8) | u32::from(byte); + bits += 8; + while bits >= 5 { + bits -= 5; + if let Some(&c) = BASE32.get(((buffer >> bits) & 31) as usize) { + out.push(char::from(c)); + } + } + buffer &= (1 << bits) - 1; + } + if bits > 0 + && let Some(&c) = BASE32.get(((buffer << (5 - bits)) & 31) as usize) + { + out.push(char::from(c)); + } + out +} + +/// The inverse of [`base32_encode`] (upper or lower case, no padding, no spaces); `None` for anything else. +pub fn base32_decode(text: &str) -> Option> { + let mut out = Vec::with_capacity(text.len() * 5 / 8); + let (mut buffer, mut bits) = (0u32, 0u32); + for c in text.bytes() { + let value = BASE32.iter().position(|&b| b == c.to_ascii_uppercase())?; + buffer = (buffer << 5) | u32::try_from(value).ok()?; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push(u8::try_from((buffer >> bits) & 0xff).ok()?); + buffer &= (1 << bits) - 1; + } + } + // Leftover bits must be the zeros the encoder pads with. + (buffer == 0).then_some(out) +} + +/// `text` with everything but `A-Z a-z 0-9 - _ . ~` percent-encoded (UTF-8 bytes), for a URI's path and query. +pub fn percent_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// The `otpauth://` URI an authenticator app reads (from a QR code or typed in): the issuer and the account's username +/// as the label, both percent-encoded. +pub fn otpauth_uri(username: &str, secret_base32: &str) -> String { + let issuer = percent_encode(ISSUER); + format!( + "otpauth://totp/{issuer}:{}?secret={secret_base32}&issuer={issuer}&algorithm=SHA1&digits={DIGITS}&period={STEP_SECS}", + percent_encode(username) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// RFC 6238's SHA-1 key; its eight-digit test codes, cut to the last six digits. + const RFC_KEY: &[u8] = b"12345678901234567890"; + + #[test] + fn the_rfc_6238_vectors_hold_for_six_digits() { + for (time, code) in [ + (59i64, "287082"), + (1_111_111_109, "081804"), + (1_111_111_111, "050471"), + (1_234_567_890, "005924"), + (2_000_000_000, "279037"), + (20_000_000_000, "353130"), + ] { + assert_eq!(code_for(RFC_KEY, step_at(time * 1000)).as_deref(), Some(code), "t = {time}"); + } + // RFC 4226's own HOTP vectors, counter 0 and 1. + assert_eq!(code_for(RFC_KEY, 0).as_deref(), Some("755224")); + assert_eq!(code_for(RFC_KEY, 1).as_deref(), Some("287082")); + assert_eq!(code_for(RFC_KEY, -1), None); + } + + #[test] + fn the_window_is_one_step_either_way_and_reports_the_matched_step() { + let now_ms = 1_111_111_109_000; + let step = step_at(now_ms); + for (offset, accepted) in [(-2, false), (-1, true), (0, true), (1, true), (2, false)] { + let code = code_for(RFC_KEY, step + offset).unwrap(); + assert_eq!(verify(RFC_KEY, &code, now_ms), accepted.then_some(step + offset), "offset {offset}"); + } + // Just inside and just outside the step's edge. + let edge = step_at(1_111_111_109_000) * STEP_SECS * 1000; + assert_eq!(step_at(edge), step); + assert_eq!(step_at(edge - 1), step - 1); + } + + #[test] + fn only_six_ascii_digits_are_codes() { + for bad in ["", "12345", "1234567", "12 456", "١٢٣٤٥٦", "12345a", " 123456", "123456 ", "+12345"] { + assert!(!is_code(bad), "{bad:?}"); + assert_eq!(verify(RFC_KEY, bad, 59_000), None, "{bad:?}"); + } + assert!(is_code("000000")); + assert_eq!(verify(RFC_KEY, "287082", 59_000), Some(1)); + assert_eq!(verify(RFC_KEY, "287083", 59_000), None); + } + + #[test] + fn base32_round_trips_and_matches_the_rfc() { + assert_eq!(base32_encode(b""), ""); + assert_eq!(base32_encode(b"f"), "MY"); + assert_eq!(base32_encode(b"fo"), "MZXQ"); + assert_eq!(base32_encode(b"foo"), "MZXW6"); + assert_eq!(base32_encode(b"foob"), "MZXW6YQ"); + assert_eq!(base32_encode(b"fooba"), "MZXW6YTB"); + assert_eq!(base32_encode(b"foobar"), "MZXW6YTBOI"); + assert_eq!(base32_encode(RFC_KEY), "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ"); + for text in ["", "MY", "MZXQ", "MZXW6", "MZXW6YQ", "MZXW6YTB", "MZXW6YTBOI"] { + assert_eq!(base32_encode(&base32_decode(text).unwrap()), text); + } + assert_eq!(base32_decode("mzxw6ytboi").as_deref(), Some(b"foobar".as_slice())); + assert_eq!(base32_decode("MZXW6YTB1"), None, "1 is not in the alphabet"); + assert_eq!(base32_decode("MZXW6Y=="), None, "no padding"); + assert_eq!(base32_decode("MZ"), None, "the leftover bits must be zero"); + let secret = generate_secret().unwrap(); + assert_eq!(base32_decode(&base32_encode(secret.as_slice())).unwrap(), secret.to_vec()); + assert_eq!(base32_encode(secret.as_slice()).len(), 32); + } + + #[test] + fn the_otpauth_uri_encodes_the_label() { + let uri = otpauth_uri("ana", "GEZDGNBVGY3TQOJQ"); + assert_eq!(uri, "otpauth://totp/Craft%20Cloud:ana?secret=GEZDGNBVGY3TQOJQ&issuer=Craft%20Cloud&algorithm=SHA1&digits=6&period=30"); + let odd = otpauth_uri("a.b_c-d~e", "AAAA"); + assert!(odd.contains(":a.b_c-d~e?"), "{odd}"); + // Not a real username today, but the encoder must hold for whatever one comes to be. + let wild = otpauth_uri("a b&c=d/é?#:", "AAAA"); + assert!(wild.contains(":a%20b%26c%3Dd%2F%C3%A9%3F%23%3A?"), "{wild}"); + assert_eq!(wild.matches('?').count(), 1); + assert_eq!(wild.matches('#').count(), 0); + } +} diff --git a/apps/craft-cloud-server/src/two_factor.rs b/apps/craft-cloud-server/src/two_factor.rs new file mode 100644 index 00000000..98149aad --- /dev/null +++ b/apps/craft-cloud-server/src/two_factor.rs @@ -0,0 +1,617 @@ +//! Two-factor sign-in over HTTP: `GET /api/v1/account/two-factor`, `POST …/setup`, `…/confirm`, `…/disable`, +//! `…/recovery-codes`, and the second step of a sign-in, `POST /api/v1/auth/login/second-factor`. The admin API's +//! confirmation uses [`confirm_admin_second_factor`], `POST /join` the same parsing and proofs. The key for the secrets, +//! its checks at start and `two-factor rekey` are here too. +//! +//! - **TOTP and recovery codes** (see [`crate::totp`]): the secret is 160 random bits, shown once as Base32 and as an +//! `otpauth://` URI, and kept only sealed with the key from `[two_factor] key_file` ([`crate::secretbox`]). Ten recovery +//! codes of 16 characters (80 bits, the alphabet of the join codes) are shown once; the database keeps their BLAKE3 +//! hashes, so they work whatever happens to the key, and 80 bits can't be searched even from a leaked database. +//! - **Statuses**: a wrong password is `403 wrong_password`; a wrong code `403 two_factor_invalid` on a signed-in route and +//! `401` at sign-in and join; a body that is not what it should be, a code that is not six digits and a recovery code +//! that is not 16 letters and digits `400 invalid` (nothing is counted); a lock `429 two_factor_locked` with +//! `Retry-After`. 401 is otherwise the answer for a token that is revoked, and for a challenge that doesn't work. +//! - **Every change takes the account's lock in `craft_cloud_meta`** and checks again there what the handler checked +//! before (the device still signed in, the password still the verified one, the account's current state). Handlers end +//! the live connections of the devices a change signed out only after it committed. +//! - **Counting**: a wrong password counts against `password_changes` (as a password change does), a wrong second factor +//! against the account (the lock) and, at sign-in's second step, join and the admin's confirmation, against +//! [`AppState::second_factors`] ([`SECOND_FACTORS_PER_USER`] per account and [`SECOND_FACTORS_PER_IP`] per client +//! address in five minutes; a right proof is given back). Wrong codes at the second step also count against the +//! sign-in limiter. +//! - **Nothing secret is logged**: not a secret, a code, a recovery code, a challenge, the key or a password. +//! - **The key**: [`load_keys`] reads the key files, [`check_startup`] refuses to start a server that has secrets it can't +//! open (all of them are opened, not a sample), and [`rekey`] rewrites the secrets under the current key. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::{ConnectInfo, Extension, FromRequest, Request, State}; +use axum::http::{HeaderMap, StatusCode}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use craft_cloud_meta::{DeviceActor, DeviceAuth, Meta, MetaError, Proof, RECOVERY_CODES, TotpVerifier, UnreadableSecret, UserId}; +use craft_cloud_proto::api::{ + LoginResponse, RecoveryCodes, SecondFactorRequest, TwoFactorConfirmRequest, TwoFactorProofRequest, TwoFactorSetupRequest, TwoFactorSetupResponse, + TwoFactorStatus, +}; +use craft_cloud_proto::join::{Code, INVITE_CHARS}; + +use crate::account::confirm_password; +use crate::api::{LOGINS_PER_IP, LOGINS_PER_USER, TINY_JSON_BYTES, body_too_slow}; +use crate::auth::{new_token, token_hash}; +use crate::config::Config; +use crate::error::ApiError; +use crate::ratelimit::{client_ip, ip_key}; +use crate::secretbox::{KeyError, SecretBox, TwoFactorKey}; +use crate::state::AppState; +use crate::totp; + +/// Wrong second factors one account may give in [`AppState::second_factors`]'s window (five minutes). +pub const SECOND_FACTORS_PER_USER: u32 = 10; +/// Wrong second factors one client address may give in the same window. +pub const SECOND_FACTORS_PER_IP: u32 = 30; + +type Shared = Arc; + +// ---- reading and checking what a client sends ---- + +/// Reads a tiny JSON body (at most [`TINY_JSON_BYTES`]) under the usual deadline. Whatever is wrong with its shape (not +/// JSON, a missing or mistyped field) is `400 invalid`, one too long `413 too_large`, one too slow `408`. +pub(crate) async fn read_tiny(state: &AppState, request: Request) -> Result { + let bytes = match tokio::time::timeout(state.limits.body_time(TINY_JSON_BYTES), Bytes::from_request(request, &())).await { + Ok(bytes) => bytes?, + Err(_) => return Err(body_too_slow()), + }; + serde_json::from_slice(&bytes).map_err(|_| ApiError::invalid("the body must be a JSON object with the fields of this call")) +} + +/// A second factor as a client sent it, checked for its shape only. +pub(crate) enum ProofInput { + /// Six digits. + Totp(String), + /// The BLAKE3 hash of the normalised recovery code. + Recovery([u8; 32]), +} + +/// The second factor of a body: `totp_code` (six ASCII digits) or `recovery_code` (16 letters and digits, case, dashes and +/// spaces forgiven, `O` for `0` and `I`, `L` for `1`), never both. `None` when the body has neither. Anything else is `400 +/// invalid`, before anything is counted. +pub(crate) fn parse_proof(totp_code: Option, recovery_code: Option) -> Result, ApiError> { + match (totp_code, recovery_code) { + (None, None) => Ok(None), + (Some(_), Some(_)) => Err(ApiError::invalid("send either totp_code or recovery_code, not both")), + (Some(code), None) => { + if !totp::is_code(&code) { + return Err(ApiError::invalid("totp_code is the six digits from the authenticator app")); + } + Ok(Some(ProofInput::Totp(code))) + } + (None, Some(code)) => match Code::parse(&code, INVITE_CHARS) { + Some(code) => Ok(Some(ProofInput::Recovery(code.hash()))), + None => Err(ApiError::invalid("recovery_code is 16 letters and digits, like ABCD-EFGH-JKMN-PQRS")), + }, + } +} + +/// A second factor ready to hand to `craft_cloud_meta`: a recovery code as its hash, a TOTP code as the check that opens +/// the account's stored secret (under the account's lock, in the transaction) and compares. +pub(crate) enum Prepared { + Totp(Box>), + Recovery([u8; 32]), +} + +impl Prepared { + pub(crate) fn proof(&self) -> Proof<'_> { + match self { + Prepared::Totp(verify) => Proof::Totp(&**verify), + Prepared::Recovery(hash) => Proof::Recovery(*hash), + } + } +} + +impl ProofInput { + pub(crate) fn prepare(self, state: &AppState, now_ms: i64) -> Prepared { + match self { + ProofInput::Recovery(hash) => Prepared::Recovery(hash), + ProofInput::Totp(code) => Prepared::Totp(totp_check(state.two_factor.clone(), code, now_ms)), + } + } +} + +/// The TOTP check for `code` at `now_ms`: opens the stored secret (sealed for the account in question), compares, and +/// hands back the time step. No key, or a secret the key doesn't open, is [`UnreadableSecret`]. +fn totp_check(secrets: Option>, code: String, now_ms: i64) -> Box> { + Box::new(move |user: UserId, sealed: &[u8]| { + let secrets = secrets.as_ref().ok_or(UnreadableSecret)?; + let secret = secrets.open(user, sealed).map_err(|_| UnreadableSecret)?; + Ok(totp::verify(&secret, &code, now_ms)) + }) +} + +fn random_failed(e: getrandom::Error) -> ApiError { + tracing::error!(error = %e, "the system random source failed"); + ApiError::internal() +} + +fn sealing_failed(_: crate::secretbox::SecretBoxError) -> ApiError { + tracing::error!("a two-factor secret could not be sealed"); + ApiError::internal() +} + +/// Ten new, distinct recovery codes. +fn new_recovery_codes() -> Result, ApiError> { + let mut codes: Vec = Vec::with_capacity(RECOVERY_CODES); + while codes.len() < RECOVERY_CODES { + let code = Code::generate(INVITE_CHARS).map_err(random_failed)?; + // A repeated code is all but impossible (80 bits), and is drawn again if it comes. + if !codes.iter().any(|known| known.hash() == code.hash()) { + codes.push(code); + } + } + Ok(codes) +} + +/// The codes as shown once, and their hashes as stored. +fn shown(codes: &[Code]) -> (RecoveryCodes, Vec<[u8; 32]>) { + (RecoveryCodes { recovery_codes: codes.iter().map(Code::display).collect() }, codes.iter().map(Code::hash).collect()) +} + +fn no_second_factor() -> ApiError { + ApiError::invalid("send a code in totp_code or a recovery code in recovery_code") +} + +// ---- the account's own routes ---- + +/// `GET /api/v1/account/two-factor`. +pub async fn get_status(State(state): State, Extension(auth): Extension) -> Result, ApiError> { + let info = state.meta.two_factor_info(auth.user, state.now_ms()).await?; + Ok(Json(TwoFactorStatus { + enabled: info.enabled_ms.is_some(), + enabled_ms: info.enabled_ms, + recovery_codes_left: info.recovery_codes_left, + pending: info.pending, + })) +} + +/// `POST /api/v1/account/two-factor/setup`: the password starts a set-up and gets the new secret, shown once. Nothing is on +/// until `confirm`. +pub async fn setup( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: TwoFactorSetupRequest = read_tiny(&state, request).await?; + let secrets = state.two_factor.clone().ok_or_else(ApiError::two_factor_unavailable)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let secret = totp::generate_secret().map_err(random_failed)?; + let sealed = secrets.seal(auth.user, secret.as_slice()).map_err(sealing_failed)?; + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + let expires_ms = state.meta.two_factor_begin_setup(&actor, &sealed, state.now_ms()).await?; + let encoded = totp::base32_encode(secret.as_slice()); + tracing::info!(device = %auth.device, "two-factor set-up started"); + Ok(Json(TwoFactorSetupResponse { otpauth_uri: totp::otpauth_uri(&auth.username, &encoded), secret: encoded, expires_ms })) +} + +/// `POST /api/v1/account/two-factor/confirm`: the first code turns two-factor sign-in on and shows the recovery codes, +/// once. Every other device is signed out. +pub async fn confirm(State(state): State, Extension(auth): Extension, request: Request) -> Result, ApiError> { + let req: TwoFactorConfirmRequest = read_tiny(&state, request).await?; + let Some(ProofInput::Totp(code)) = parse_proof(Some(req.totp_code), None)? else { return Err(no_second_factor()) }; + let now_ms = state.now_ms(); + let verify = totp_check(state.two_factor.clone(), code, now_ms); + let (answer, hashes) = shown(&new_recovery_codes()?); + let revoked = state.meta.two_factor_confirm(auth.user, auth.device, &*verify, &hashes, now_ms).await?; + // Only after the commit: their live connections end now, not at their next request. + end_live_connections(&state, &revoked); + tracing::info!(device = %auth.device, revoked = revoked.len(), "two-factor sign-in turned on"); + Ok(Json(answer)) +} + +/// `POST /api/v1/account/two-factor/disable`: the password and a code or recovery code turn it off. Every other device is +/// signed out. +pub async fn disable(State(state): State, Extension(auth): Extension, request: Request) -> Result { + let req: TwoFactorProofRequest = read_tiny(&state, request).await?; + let proof = parse_proof(req.totp_code, req.recovery_code)?.ok_or_else(no_second_factor)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let now_ms = state.now_ms(); + let prepared = proof.prepare(&state, now_ms); + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + let revoked = state.meta.two_factor_disable(&actor, &prepared.proof(), now_ms).await?; + end_live_connections(&state, &revoked); + tracing::info!(device = %auth.device, revoked = revoked.len(), "two-factor sign-in turned off"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /api/v1/account/two-factor/recovery-codes`: the password and a code make ten new recovery codes, shown once; the +/// old ones stop working. +pub async fn recovery_codes( + State(state): State, + Extension(auth): Extension, + request: Request, +) -> Result, ApiError> { + let req: TwoFactorProofRequest = read_tiny(&state, request).await?; + if req.recovery_code.is_some() { + return Err(ApiError::invalid("new recovery codes are made with a code from the authenticator app (totp_code)")); + } + let proof = parse_proof(req.totp_code, None)?.ok_or_else(no_second_factor)?; + let password_hash = confirm_password(&state, auth.user, &auth.username, req.password).await?; + let now_ms = state.now_ms(); + let prepared = proof.prepare(&state, now_ms); + let (answer, hashes) = shown(&new_recovery_codes()?); + let actor = DeviceActor { user: auth.user, device: auth.device, password_hash }; + state.meta.two_factor_new_recovery_codes(&actor, &prepared.proof(), &hashes, now_ms).await?; + tracing::info!(device = %auth.device, "new recovery codes made"); + Ok(Json(answer)) +} + +fn end_live_connections(state: &AppState, devices: &[craft_cloud_proto::DeviceId]) { + for device in devices { + state.live.revoke(*device); + } +} + +// ---- the second step of a sign-in ---- + +/// The hash of a challenge token as the client sent it; `None` for text that is no token (it is answered like an unknown +/// challenge). +pub(crate) fn challenge_hash(challenge: &str) -> Option<[u8; 32]> { + let is_token = challenge.len() == 43 && URL_SAFE_NO_PAD.decode(challenge).is_ok(); + is_token.then(|| token_hash(challenge)) +} + +/// `POST /api/v1/auth/login/second-factor` (no sign-in): the challenge `POST /auth/login` gave and a code or recovery +/// code make the device, answered as a sign-in is. A challenge that is unknown, expired, used, from before a password +/// change or a reset, or fit no more is `401 unauthorized`, the same for each. A wrong code is `401 two_factor_invalid` +/// (five of them delete the challenge), a lock `429 two_factor_locked`. +pub async fn second_factor( + State(state): State, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + request: Request, +) -> Result, ApiError> { + let req: SecondFactorRequest = read_tiny(&state, request).await?; + let proof = parse_proof(req.totp_code, req.recovery_code)?.ok_or_else(no_second_factor)?; + // tokio's clock: the system's monotonic clock in the server, the simulated one in a simulation. + let now = tokio::time::Instant::now().into_std(); + let now_ms = state.now_ms(); + let ip = ip_key(client_ip(peer, &headers, &state.trusted_proxies)); + // Counted before anything is looked at, so tries at the same moment are held to the limits too, and given back for a + // right proof or a lock (which checked nothing). + let Some(ip_counted) = state.second_factors.count(&ip, SECOND_FACTORS_PER_IP, now) else { + return Err(ApiError::rate_limited("too many wrong codes from this address; wait a few minutes", None)); + }; + let gone = || ApiError::from(MetaError::ChallengeGone); + let Some(hash) = challenge_hash(&req.challenge) else { + crate::audit::challenge_failed(&state, None).await; + return Err(gone()); + }; + let Some((user, username)) = state.meta.challenge_owner(&hash, now_ms).await? else { + crate::audit::challenge_failed(&state, None).await; + return Err(gone()); + }; + let user_key = format!("user:{username}"); + let Some(user_counted) = state.second_factors.count(&user_key, SECOND_FACTORS_PER_USER, now) else { + state.second_factors.refund(&ip, ip_counted); + // Ten wrong codes both fill this window and lock the account: say the lock, which tells how long. + if let Some(locked_ms) = state.meta.two_factor_locked_for(user, now_ms).await? { + return Err(ApiError::two_factor_locked(craft_cloud_meta::retry_after_secs(locked_ms))); + } + return Err(ApiError::rate_limited("too many wrong codes for this account; wait a few minutes", None)); + }; + let (token, new_token_hash) = new_token()?; + let prepared = proof.prepare(&state, now_ms); + let result = state.meta.complete_second_factor(&hash, &prepared.proof(), &new_token_hash, now_ms).await; + match result { + Ok(done) => { + state.second_factors.refund(&ip, ip_counted); + state.second_factors.refund(&user_key, user_counted); + // The username is the only personal identifier the server could log here: kept to debug. + tracing::debug!(%username, "sign-in with a second factor"); + tracing::info!(device = %done.device, "device signed in with a second factor"); + Ok(Json(LoginResponse { device: done.device, token })) + } + Err(MetaError::TwoFactorInvalid) => { + // A wrong second factor counts against sign-in too. + let _ = state.logins.allow(&ip, LOGINS_PER_IP, now); + let _ = state.logins.allow(&user_key, LOGINS_PER_USER, now); + Err(ApiError::from(MetaError::TwoFactorInvalid).on_public_route()) + } + Err(MetaError::TwoFactorLocked { retry_after_ms }) => { + state.second_factors.refund(&ip, ip_counted); + state.second_factors.refund(&user_key, user_counted); + Err(ApiError::from(MetaError::TwoFactorLocked { retry_after_ms })) + } + Err(MetaError::ChallengeGone) => { + // The challenge was found a moment ago and has gone since (used, out of tries, expired, or the account changed). + crate::audit::challenge_failed(&state, Some(user)).await; + Err(gone()) + } + Err(e) => Err(e.into()), + } +} + +// ---- the admin's confirmation ---- + +/// The second factor of an admin's confirmation (see [`crate::admin`], the account is a device's or a browser session's): when the admin's account has two-factor sign-in, +/// `proof` is checked and used up (`403 two_factor_invalid` when it is missing, wrong or a time step that was used, `429 +/// two_factor_locked` while locked), counted against [`AppState::second_factors`] for the account. Whether the account +/// has it is read again under its lock when the proof is checked. The answer says whether it had: the change compares +/// that with the account's state in its own transaction. +pub(crate) async fn confirm_admin_second_factor(state: &AppState, user: UserId, username: &str, proof: Option) -> Result { + let enabled = state.meta.two_factor_enabled(user).await?; + if !enabled { + // A code given to an account that has none is not looked at: the change compares the account's state again anyway. + return Ok(false); + } + let Some(proof) = proof else { + let mut e = ApiError::two_factor_invalid(); + e.message = "this account has two-factor sign-in: add a code in totp_code (or a recovery code in recovery_code)".to_owned(); + return Err(e); + }; + let now = tokio::time::Instant::now().into_std(); + let key = format!("user:{username}"); + let now_ms = state.now_ms(); + let Some(counted) = state.second_factors.count(&key, SECOND_FACTORS_PER_USER, now) else { + if let Some(locked_ms) = state.meta.two_factor_locked_for(user, now_ms).await? { + return Err(ApiError::two_factor_locked(craft_cloud_meta::retry_after_secs(locked_ms))); + } + return Err(ApiError::rate_limited("too many wrong codes; wait a few minutes", None)); + }; + let prepared = proof.prepare(state, now_ms); + match state.meta.confirm_second_factor(user, Some(&prepared.proof()), now_ms).await { + Ok(had) => { + state.second_factors.refund(&key, counted); + Ok(had) + } + Err(MetaError::TwoFactorInvalid) => Err(MetaError::TwoFactorInvalid.into()), + Err(e) => { + state.second_factors.refund(&key, counted); + Err(e.into()) + } + } +} + +// ---- the key ---- + +/// Where the server's key for two-factor secrets stands, from its config. +#[derive(Debug)] +pub enum KeyState { + /// The config has no `[two_factor]` section. + NotConfigured, + /// The config names a key file that is not there. + Missing(PathBuf), + /// The key (and the older one, if named) is read. + Loaded(Box), +} + +/// Reads the key files the config names. A key file that is not there is [`KeyState::Missing`] (whether that is fatal +/// depends on whether the database holds secrets, see [`check_startup`]); one that can't be read, has the wrong length, or +/// shares its name with the older key is an error, naming the file. +pub fn load_keys(config: &Config) -> Result { + let Some(section) = &config.two_factor else { return Ok(KeyState::NotConfigured) }; + let path = config.key_path(§ion.key_file); + let current = match TwoFactorKey::from_file(&path) { + Ok(key) => key, + Err(KeyError::Missing { path }) => return Ok(KeyState::Missing(path)), + Err(e) => return Err(e), + }; + warn_if_readable_by_others(&path); + let old = match §ion.key_file_old { + Some(old) => { + let old = config.key_path(old); + warn_if_readable_by_others(&old); + Some(TwoFactorKey::from_file(&old)?) + } + None => None, + }; + Ok(KeyState::Loaded(Box::new(SecretBox::new(¤t, old.as_ref())?))) +} + +#[cfg(unix)] +fn warn_if_readable_by_others(path: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(path) + && meta.permissions().mode() & 0o077 != 0 + { + tracing::warn!(path = %path.display(), "the two-factor key file is readable by other users: make it owner-only (chmod 600)"); + } +} + +#[cfg(not(unix))] +fn warn_if_readable_by_others(_path: &std::path::Path) {} + +/// Refuses to start a server that holds two-factor secrets it can't open: with secrets in the database and no key (not +/// configured, or the file is missing), or a key that doesn't open **every** one of them. The message names the key file +/// the config expects. Without secrets there is nothing to open, and a server without a key runs, with set-up answering `503 +/// two_factor_unavailable`. Returns the box to give [`AppState::with_two_factor`]. +pub async fn check_startup(meta: &Meta, keys: KeyState, config: &Config, now_ms: i64) -> Result, String> { + let secrets = meta.two_factor_secrets(now_ms).await.map_err(|e| e.to_string())?; + let active: Vec<_> = secrets.iter().filter_map(|row| row.active.as_ref().map(|sealed| (row.user, sealed))).collect(); + let expected = config.two_factor.as_ref().map(|section| config.key_path(§ion.key_file)); + match keys { + KeyState::Loaded(secret_box) => { + let unreadable = active.iter().filter(|(user, sealed)| secret_box.open(*user, sealed).is_err()).count(); + if unreadable > 0 { + let path = expected.map(|p| p.display().to_string()).unwrap_or_default(); + return Err(format!( + "{unreadable} of {} two-factor secrets in the database can't be opened with the key in {path}{}: this is not the key they were sealed \ + with. Put the right key there (and a previous one in key_file_old while rotating); the server won't start without it", + active.len(), + if config.two_factor.as_ref().is_some_and(|section| section.key_file_old.is_some()) { " or key_file_old" } else { "" }, + )); + } + Ok(Some(*secret_box)) + } + KeyState::Missing(path) => { + if active.is_empty() { + tracing::warn!(path = %path.display(), "the two-factor key file is missing: two-factor sign-in can't be set up"); + Ok(None) + } else { + Err(format!( + "{} accounts use two-factor sign-in but the key file {} does not exist: restore it (it is not part of the database), or the server won't start", + active.len(), + path.display() + )) + } + } + KeyState::NotConfigured => { + if active.is_empty() { + Ok(None) + } else { + Err(format!( + "{} accounts use two-factor sign-in but the config has no [two_factor] key_file: set it to the key file the server had (it is not part of the \ + database), or the server won't start", + active.len() + )) + } + } + } +} + +/// What [`rekey`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Rekeyed { + /// Secrets rewritten under the current key. + pub rewritten: usize, + /// Secrets that were under it already. + pub already_current: usize, +} + +/// Rewrites every secret (the active ones and the set-ups', expired ones too) that is sealed with the older key under the +/// current one, one account at a time, each in a transaction that reads the account's ciphertexts fresh under its lock and +/// rewrites both, so a confirmation that races it can't leave one behind. It refuses, before changing anything, when a secret +/// is sealed with a key that is neither (a key that was lost) or can't be opened. It then scans again and goes round again +/// (at most five times) until no ciphertext is under any key but the current one, and fails if some still is: only a +/// successful return means `key_file_old` can be removed **and** that no server still running holds only the old key, which +/// is the operator's to see to (every instance must run with both keys before this, see `docs/server.md`). Neither the +/// secrets nor the keys are printed. It can be run again. +pub async fn rekey(meta: &Meta, secret_box: &SecretBox) -> Result { + rekey_hooked(meta, secret_box, &|| {}).await +} + +/// [`rekey`] with a test seam: `after_scan` is called once, after the first scan of what is stored and before anything is +/// rewritten, so a test can let other changes (a confirmation moving a set-up's secret) land exactly there. It blocks the +/// thread it runs on, so a test runs the rest on others. +pub async fn rekey_hooked(meta: &Meta, secret_box: &SecretBox, after_scan: &(dyn Fn() + Sync)) -> Result { + let neither = || { + "some secrets are sealed with a key that is neither key_file nor key_file_old: nothing was changed; put that key in key_file_old first" + .to_owned() + }; + let mut already_current = 0usize; + for row in meta.two_factor_all_secrets().await.map_err(|e| e.to_string())? { + for sealed in [row.active, row.pending].into_iter().flatten() { + let id = SecretBox::key_id(&sealed); + if id == Some(secret_box.current_id()) { + // Under the current key: it must open, or the rest of the work would hide a lost key. + secret_box.open(row.user, &sealed).map_err(|_| "a secret that carries the current key's name does not open with it".to_owned())?; + already_current += 1; + } else if id.is_some() && id == secret_box.old_id() { + secret_box.open(row.user, &sealed).map_err(|_| "a secret does not open with key_file_old: nothing was changed".to_owned())?; + } else { + return Err(neither()); + } + } + } + after_scan(); + let current = secret_box.current_id(); + let rewrite = |user: UserId, sealed: &[u8]| -> Result>, UnreadableSecret> { + if SecretBox::key_id(sealed) == Some(current) { + return Ok(None); + } + let secret = secret_box.open(user, sealed).map_err(|_| UnreadableSecret)?; + secret_box.seal(user, &secret).map(Some).map_err(|_| UnreadableSecret) + }; + let mut rewritten = 0usize; + for _round in 0..5 { + let behind: Vec = meta + .two_factor_all_secrets() + .await + .map_err(|e| e.to_string())? + .into_iter() + .filter(|row| [&row.active, &row.pending].into_iter().flatten().any(|sealed| SecretBox::key_id(sealed) != Some(current))) + .map(|row| row.user) + .collect(); + if behind.is_empty() { + return Ok(Rekeyed { rewritten, already_current }); + } + for user in behind { + let (done, _) = meta.two_factor_rewrite(user, &rewrite).await.map_err(|e| e.to_string())?; + rewritten += done; + } + } + // One last look: a secret still under another key means the migration is not complete. + let left = meta + .two_factor_all_secrets() + .await + .map_err(|e| e.to_string())? + .iter() + .flat_map(|row| [&row.active, &row.pending]) + .flatten() + .filter(|sealed| SecretBox::key_id(sealed) != Some(current)) + .count(); + if left == 0 { + Ok(Rekeyed { rewritten, already_current }) + } else { + Err(format!( + "{left} secret(s) are still sealed with another key after rewriting: a server may still be sealing new ones with the old key. \ + Restart every instance with key_file and key_file_old, and run this again; do not remove key_file_old yet" + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_proof_has_the_shape_of_one_code_only() { + assert!(matches!(parse_proof(None, None), Ok(None))); + assert!(matches!(parse_proof(Some("123456".into()), None), Ok(Some(ProofInput::Totp(c))) if c == "123456")); + for bad in ["12345", "1234567", "abcdef", " 12345", "١٢٣٤٥٦", ""] { + assert!(parse_proof(Some(bad.into()), None).is_err(), "{bad:?}"); + } + assert!(parse_proof(Some("123456".into()), Some("ABCD-EFGH-JKMN-PQRS".into())).is_err(), "both"); + let a = parse_proof(None, Some("abcd-efgh-jkmn-pqrs".into())).unwrap(); + let b = parse_proof(None, Some("ABCDEFGHJKMNPQRS".into())).unwrap(); + let (Some(ProofInput::Recovery(a)), Some(ProofInput::Recovery(b))) = (a, b) else { panic!("recovery codes") }; + assert_eq!(a, b, "case and dashes don't matter"); + // O is 0, and I and L are 1, as in the join codes. + let zero = parse_proof(None, Some("0000-0000-0000-0000".into())).unwrap(); + let letter = parse_proof(None, Some("OOOO-OOOO-OOOO-OOOO".into())).unwrap(); + let (Some(ProofInput::Recovery(zero)), Some(ProofInput::Recovery(letter))) = (zero, letter) else { panic!("recovery codes") }; + assert_eq!(zero, letter); + for bad in ["ABCD-EFGH-JKMN-PQR", "ABCD-EFGH-JKMN-PQRSA", "ABCD-EFGH-JKMN-PQRU", ""] { + assert!(parse_proof(None, Some(bad.into())).is_err(), "{bad:?}"); + } + } + + #[test] + fn ten_distinct_recovery_codes_of_16_characters_are_shown_in_groups() { + let codes = new_recovery_codes().unwrap(); + assert_eq!(codes.len(), 10); + let (answer, hashes) = shown(&codes); + assert_eq!(answer.recovery_codes.len(), 10); + let unique: std::collections::BTreeSet<_> = hashes.iter().collect(); + assert_eq!(unique.len(), 10); + for (shown, code) in answer.recovery_codes.iter().zip(&codes) { + assert_eq!(shown.len(), 19, "16 characters in four groups"); + assert_eq!(shown.replace('-', ""), code.as_str()); + assert_eq!(Code::parse(shown, INVITE_CHARS).map(|c| c.hash()), Some(code.hash())); + } + // The hash is BLAKE3 of the normalised code, as the join codes' are, and reproducible. + assert_eq!(hashes.first().copied(), codes.first().map(|c| *blake3::hash(c.as_str().as_bytes()).as_bytes())); + } + + #[test] + fn a_challenge_is_a_43_character_token_or_nothing() { + let (token, hash) = new_token().unwrap(); + assert_eq!(challenge_hash(&token), Some(hash)); + for bad in ["", "short", &"x".repeat(44), &format!("{}=", &token[..42]), &format!("{}!", &token[..42])] { + assert_eq!(challenge_hash(bad), None, "{bad:?}"); + } + } +} diff --git a/apps/craft-cloud-server/src/verify.rs b/apps/craft-cloud-server/src/verify.rs new file mode 100644 index 00000000..a40edcb7 --- /dev/null +++ b/apps/craft-cloud-server/src/verify.rs @@ -0,0 +1,370 @@ +//! The invariant checker behind `craft-cloud-server verify`, and the index builder behind `craft-cloud-server index +//! build`. +//! +//! `verify` reads and reports; it changes nothing: the database and the storage must exist already (a wrong path is an +//! error, not a new empty server with no problems), and a database whose schema is older is refused, not upgraded. It +//! checks what a later garbage collector would have to trust: that every manifest's list of +//! chunks in the database is complete and equal to the manifest object in the store, that every chunk a manifest names +//! is recorded and its object is there and intact, and that nothing in the database names a manifest or a chunk that does +//! not exist. Anything it cannot establish is reported, not guessed: a manifest it cannot read or whose index is not +//! complete has unknown dependencies, and unknown means kept. +//! +//! Two kinds of finding. *Problems* are inconsistencies and make `verify` fail. *Notices* are facts a reader needs but +//! that are not wrong by themselves: manifests whose index is not complete yet (`index build` completes it), manifests +//! no version or snapshot names (a live document's assets are named only inside the document, where the server cannot +//! read them, so these are not garbage and must not be treated as garbage), and chunks no complete manifest names. + +use std::collections::BTreeMap; + +use craft_cloud_meta::{EdgeCheck, Finding, IndexAudit, IndexState, Meta, MetaError}; +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use craft_cloud_store::{BlobKey, Store, StoreError}; +use serde_json::{Value, json}; + +/// Problems listed in full in a report; further ones are only counted. +const MAX_LISTED_PROBLEMS: usize = 1000; +/// Examples of each audit finding. +const EXAMPLES: usize = 20; +/// IDs one metadata query covers. +const PAGE: usize = 200; + +/// What `verify` should look at. +#[derive(Debug, Clone, Copy)] +pub struct VerifyOptions { + /// Read every chunk object from the store, which checks its hash. Without it only manifests are read and chunks are + /// checked against the database. + pub read_chunks: bool, +} + +impl Default for VerifyOptions { + fn default() -> Self { + Self { read_chunks: true } + } +} + +/// One inconsistency. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Problem { + pub kind: &'static str, + /// The IDs involved in hex (`manifest` or `manifest/chunk`). + pub subject: String, + pub detail: Option, +} + +/// The result of a `verify` run. +#[derive(Debug, Clone)] +pub struct Report { + pub audit: IndexAudit, + /// The first [`MAX_LISTED_PROBLEMS`] problems. + pub problems: Vec, + /// Every problem, by kind. + pub counts: BTreeMap<&'static str, u64>, + pub manifests_checked: u64, + pub chunks_checked: u64, + pub chunk_objects_read: bool, +} + +impl Report { + /// How many problems there are in all. + pub fn problem_count(&self) -> u64 { + self.counts.values().sum() + } + + /// Whether the index covers every recorded manifest, completely. + pub fn index_complete(&self) -> bool { + self.audit.manifests == self.audit.complete + } + + fn add(&mut self, kind: &'static str, subject: String, detail: Option) { + *self.counts.entry(kind).or_insert(0) += 1; + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject, detail }); + } + } + + /// An audit finding that is a problem: counted in full, its examples listed. + fn add_finding(&mut self, kind: &'static str, finding: &Finding) { + if finding.count == 0 { + return; + } + *self.counts.entry(kind).or_insert(0) += finding.count; + for example in &finding.examples { + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject: example.clone(), detail: None }); + } + } + } + + /// The report as JSON. + pub fn to_json(&self) -> Value { + let notice = |f: &Finding| json!({ "count": f.count, "examples": f.examples }); + json!({ + "ok": self.problem_count() == 0, + "index": { + "manifests": self.audit.manifests, + "complete": self.audit.complete, + "building": self.audit.building, + "absent": self.audit.absent, + "all_complete": self.index_complete(), + }, + "problems": { + "total": self.problem_count(), + "by_kind": self.counts, + "listed": self.problems.iter().map(|p| json!({ "kind": p.kind, "id": p.subject, "detail": p.detail })).collect::>(), + "truncated": self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX), + }, + "notices": { + "manifests_without_complete_index": self.audit.absent.saturating_add(self.audit.building), + "roots_not_indexed": notice(&self.audit.roots_not_indexed), + "unrooted_manifests": notice(&self.audit.unrooted_manifests), + "unreferenced_chunks": notice(&self.audit.unreferenced_chunks), + }, + "checked": { + "manifests": self.manifests_checked, + "chunks": self.chunks_checked, + "chunk_objects_read": self.chunk_objects_read, + }, + }) + } + + /// The report for people. + pub fn to_text(&self) -> String { + let a = &self.audit; + let mut out = format!( + "manifests: {} recorded, index complete for {}, being built for {}, not indexed for {}\nchecked {} manifest(s) and {} chunk(s){}\n", + a.manifests, + a.complete, + a.building, + a.absent, + self.manifests_checked, + self.chunks_checked, + if self.chunk_objects_read { " (chunk objects read and hashed)" } else { " (chunk objects not read)" }, + ); + if self.problem_count() == 0 { + out.push_str("no problems found\n"); + } else { + out.push_str(&format!("{} problem(s) found:\n", self.problem_count())); + for (kind, count) in &self.counts { + out.push_str(&format!(" {kind}: {count}\n")); + } + for p in &self.problems { + match &p.detail { + Some(detail) => out.push_str(&format!(" {} {} ({detail})\n", p.kind, p.subject)), + None => out.push_str(&format!(" {} {}\n", p.kind, p.subject)), + } + } + if self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX) { + out.push_str(" (more problems than are listed; use --json for counts by kind)\n"); + } + } + if !self.index_complete() { + out.push_str("notice: the index is not complete; `craft-cloud-server index build` completes it. Manifests without a complete index have unknown dependencies.\n"); + } + for (what, finding) in [ + ( + "manifests named by no version or snapshot (live-document assets are named only inside their documents, so these are not garbage)", + &a.unrooted_manifests, + ), + ("recorded chunks no complete manifest names", &a.unreferenced_chunks), + ] { + if finding.count > 0 { + out.push_str(&format!("notice: {} {what}\n", finding.count)); + } + } + out + } +} + +/// Why a manifest or chunk object could not be read. +fn store_problem(error: &StoreError) -> (&'static str, Option) { + match error { + StoreError::NotFound(_) => ("object_missing", None), + StoreError::HashMismatch(_) | StoreError::Corrupt { .. } | StoreError::TooLarge { .. } => ("object_corrupt", Some(error.to_string())), + StoreError::Io(_) | StoreError::Backend(_) => ("store_error", Some(error.to_string())), + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Reads a manifest object and decodes it; `Err` says what is wrong with it. +pub async fn read_manifest(store: &Store, id: &ManifestId) -> Result)> { + read_manifest_sized(store, id).await.map(|(manifest, _)| manifest) +} + +/// Like [`read_manifest`], with the bytes the object is stored as (the format byte included). +async fn read_manifest_sized(store: &Store, id: &ManifestId) -> Result<(Manifest, u64), (&'static str, Option)> { + let (bytes, stored) = store.get_sized(BlobKey::manifest(id)).await.map_err(|e| { + let (kind, detail) = store_problem(&e); + ( + if kind == "object_missing" { + "manifest_object_missing" + } else if kind == "object_corrupt" { + "manifest_object_corrupt" + } else { + "store_error" + }, + detail, + ) + })?; + let manifest = Manifest::decode(&bytes).map_err(|e| ("manifest_not_decodable", Some(e.to_string())))?; + if manifest.id() != *id { + return Err(("manifest_id_mismatch", None)); + } + Ok((manifest, stored)) +} + +/// Checks the database and the store against each other and reports. Changes nothing. +pub async fn verify(meta: &Meta, store: &Store, options: VerifyOptions) -> Result { + let audit = meta.index_audit(EXAMPLES).await?; + let mut report = Report { + audit: audit.clone(), + problems: Vec::new(), + counts: BTreeMap::new(), + manifests_checked: 0, + chunks_checked: 0, + chunk_objects_read: options.read_chunks, + }; + report.add_finding("index_without_manifest", &audit.index_without_manifest); + report.add_finding("orphan_edges", &audit.edges_without_index); + report.add_finding("index_wrong_count", &audit.complete_with_wrong_count); + report.add_finding("unknown_dependency", &audit.unknown_dependencies); + report.add_finding("root_manifest_unknown", &audit.roots_without_manifest); + report.add_finding("space_blob_without_blob", &audit.space_blobs_without_blob); + + // Every recorded manifest: its object, its index, the chunks it names. + let mut after: Option<[u8; 32]> = None; + loop { + let records = meta.blob_records_after(BlobKind::Manifest, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; + after = Some(last); + for record in records { + let raw = record.id; + report.manifests_checked += 1; + let id = ManifestId::from_bytes(raw); + let (manifest, stored) = match read_manifest_sized(store, &id).await { + Ok(read) => read, + Err((kind, detail)) => { + report.add(kind, hex(&raw), detail); + continue; + } + }; + // What the database records of the manifest is what the object is: the bytes it is stored as, and the size + // of the file it describes. + if record.stored_size != stored { + report.add( + "manifest_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + if record.file_size != Some(manifest.size()) { + let recorded = record.file_size.map_or_else(|| "none".to_owned(), |n| n.to_string()); + report.add( + "manifest_file_size_mismatch", + hex(&raw), + Some(format!("the database records a file of {recorded} bytes, the manifest says {}", manifest.size())), + ); + } + if let IndexState::Complete { .. } = meta.index_state(&id).await? + && let EdgeCheck::Mismatch(problem) = meta.index_check(&manifest).await? + { + report.add("index_mismatch", hex(&raw), Some(format!("ordinal {}: {}", problem.ordinal, problem.kind.as_str()))); + } + for refs in manifest.chunks().chunks(500) { + let part: Vec = refs.iter().map(|c| c.id).collect(); + let recorded = meta.recorded_chunk_sizes(&part).await?; + for chunk in &part { + if !recorded.contains_key(chunk) { + report.add("chunk_not_recorded", format!("{}/{}", hex(&raw), hex(chunk.as_bytes())), None); + } + } + // The manifest says how long each chunk is; the database records the bytes it is stored as, one more + // than that (the format byte). With the chunk objects read below, this ties the length the manifest + // names to the bytes in the store. + for chunk in refs { + if let Some(stored) = recorded.get(&chunk.id) + && *stored != u64::from(chunk.len).saturating_add(1) + { + report.add( + "chunk_length_mismatch", + format!("{}/{}", hex(&raw), hex(chunk.id.as_bytes())), + Some(format!("the manifest says {} bytes, the database records {stored} stored", chunk.len)), + ); + } + } + } + } + } + + // Every recorded chunk's object (it also finds chunks no manifest names, which a manifest walk would not). + let mut after: Option<[u8; 32]> = None; + loop { + let records = meta.blob_records_after(BlobKind::Chunk, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; + after = Some(last); + for record in records { + let raw = record.id; + report.chunks_checked += 1; + if !options.read_chunks { + continue; + } + match store.get_sized(BlobKey::chunk(&ChunkId::from_bytes(raw))).await { + // Read, and its hash is right: its length is what the database records. + Ok((_, stored)) => { + if stored != record.stored_size { + report.add( + "chunk_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + } + Err(e) => { + let (kind, detail) = store_problem(&e); + let kind = match kind { + "object_missing" => "chunk_object_missing", + "object_corrupt" => "chunk_object_corrupt", + other => other, + }; + report.add(kind, hex(&raw), detail); + } + } + } + } + Ok(report) +} + +/// What an `index build` run did. +#[derive(Debug, Clone, Default)] +pub struct BuildReport { + /// Manifests whose index is complete after the run that were not before. + pub indexed: u64, + /// Manifests that could not be indexed: `(manifest, why)`. Their index stays incomplete. + pub unreadable: Vec<(String, &'static str)>, +} + +/// Completes the index of every recorded manifest whose index is not complete, in batches of at most the index budget +/// per transaction, so it can run beside the server and be stopped and run again. A manifest is read from the store and +/// checked against its ID first; one that cannot be read is reported and left unindexed (its chunks stay unknown). Only +/// the derived index is written. +pub async fn build_index(meta: &Meta, store: &Store) -> Result { + let mut report = BuildReport::default(); + let mut after: Option = None; + loop { + let ids = meta.manifests_to_index(after, PAGE).await?; + let Some(last) = ids.last().copied() else { break }; + after = Some(last); + for id in ids { + match read_manifest(store, &id).await { + Ok(manifest) => { + meta.index_complete(&manifest).await?; + report.indexed += 1; + } + Err((kind, _)) => report.unreadable.push((hex(id.as_bytes()), kind)), + } + } + } + Ok(report) +} diff --git a/apps/craft-cloud-server/src/web/html.rs b/apps/craft-cloud-server/src/web/html.rs new file mode 100644 index 00000000..eb05fcff --- /dev/null +++ b/apps/craft-cloud-server/src/web/html.rs @@ -0,0 +1,255 @@ +//! The pages' building blocks: escaping, the layout, and the form fields. +//! +//! Pages are plain `format!`ed HTML. The rule that keeps them safe is small: **every value that is not a literal in this +//! code goes through [`esc`]** before it is put into markup, and the helpers here escape their own arguments, so a page +//! built from them has no place to forget it. There is no script, no inline style and no external file anywhere (the +//! CSP, see [`super::secure_headers`], forbids all three), so a value that did slip through could still not run code. + +use std::fmt::Write; + +/// `text` escaped for HTML text and for attribute values in double or single quotes. +pub fn esc(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + // Control characters other than the usual whitespace have no place in a page. + c if c.is_control() && !matches!(c, '\n' | '\t') => out.push('\u{fffd}'), + c => out.push(c), + } + } + out +} + +/// A time in milliseconds since the Unix epoch as `2026-11-04 13:05:09 UTC`, or `-` for one that is out of range. +pub fn fmt_time(ms: i64) -> String { + let secs = ms.div_euclid(1000); + let (days, rest) = (secs.div_euclid(86_400), secs.rem_euclid(86_400)); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm), in checked arithmetic. + let date = || -> Option<(i64, i64, i64)> { + let z = days.checked_add(719_468)?; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = yoe.checked_add(era.checked_mul(400)?)?.checked_add(i64::from(month <= 2))?; + Some((year, month, day)) + }; + match date() { + Some((y, m, d)) if (0..=9999).contains(&y) => format!("{y:04}-{m:02}-{d:02} {:02}:{:02}:{:02} UTC", rest / 3600, rest % 3600 / 60, rest % 60), + _ => "-".to_owned(), + } +} + +/// A size in bytes as `1.5 GiB`, exact below one KiB. +pub fn fmt_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["KiB", "MiB", "GiB", "TiB", "PiB"]; + if bytes < 1024 { + return format!("{bytes} B"); + } + // Tenths of the unit, in 128 bits so that nothing overflows. + let mut tenths = u128::from(bytes) * 10 / 1024; + let mut unit = 0usize; + while tenths >= 10_240 && unit + 1 < UNITS.len() { + tenths /= 1024; + unit += 1; + } + format!("{}.{} {}", tenths / 10, tenths % 10, UNITS.get(unit).copied().unwrap_or("PiB")) +} + +/// What a page shows about the signed-in admin. +pub struct Ctx { + pub username: String, + /// The CSRF token the forms of this session carry. + pub csrf: String, +} + +/// The whole page: `body` is markup built with this module's helpers, `title` plain text. +pub fn page(ctx: Option<&Ctx>, title: &str, body: &str) -> String { + let nav = match ctx { + Some(ctx) => format!( + "", + esc(&ctx.csrf), + esc(&ctx.username) + ), + None => String::new(), + }; + format!( + "\n\ + {} - Craft Cloud admin\ + Skip to the content\ +

Craft Cloud admin

{nav}
{body}
\n", + esc(title) + ) +} + +/// A page heading and its first paragraph. +pub fn heading(text: &str) -> String { + format!("

{}

", esc(text)) +} + +/// A notice that something worked (`role=status`) or failed (`role=alert`). +pub fn notice(text: &str, error: bool) -> String { + let (class, role) = if error { ("notice error", "alert") } else { ("notice", "status") }; + format!("

{}

", esc(text)) +} + +/// A link to a path made by this code (never to a value from the outside as it is). +pub fn link(href: &str, text: &str) -> String { + format!("{}", esc(href), esc(text)) +} + +/// Percent-encoding for a value that goes into a query string. +pub fn url_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + out.push(char::from(byte)); + } else { + let _ = write!(out, "%{byte:02X}"); + } + } + out +} + +/// A labelled text field. `kind` is an input type from this code (`text`, `password`, `search`, `number`). +pub fn field(form: &str, name: &str, label: &str, kind: &str, value: &str, extra: &str) -> String { + let id = esc(&format!("{form}-{name}")); + format!( + "

", + esc(label), + esc(name), + esc(value) + ) +} + +/// A labelled select; `options` are `(value, text)` pairs and `selected` the value that is chosen. +pub fn select(form: &str, name: &str, label: &str, options: &[(&str, &str)], selected: &str) -> String { + let id = esc(&format!("{form}-{name}")); + let mut out = format!("

"); + out +} + +/// A labelled checkbox. +pub fn checkbox(form: &str, name: &str, label: &str) -> String { + let id = esc(&format!("{form}-{name}")); + format!( + "

", + esc(name), + esc(label) + ) +} + +/// The confirmation every change asks for: the admin's password and, when their account has two-factor sign-in, a code or a +/// recovery code. The server asks again whatever the page showed. +pub fn confirm_fields(form: &str, two_factor: bool) -> String { + let mut out = + field(form, "password", "Your password, to confirm", "password", "", "required autocomplete=\"current-password\" maxlength=\"1024\""); + if two_factor { + out.push_str(&field( + form, + "totp_code", + "Code from your authenticator app", + "text", + "", + "inputmode=\"numeric\" autocomplete=\"one-time-code\" pattern=\"[0-9]{6}\" maxlength=\"6\"", + )); + out.push_str(&field(form, "recovery_code", "Or a recovery code", "text", "", "autocomplete=\"off\" maxlength=\"32\" spellcheck=\"false\"")); + } + out +} + +/// A form that posts to `action` with the session's CSRF token, `inner` being markup built with the helpers above. +pub fn post_form(ctx: &Ctx, action: &str, class: &str, inner: &str) -> String { + format!( + "
{inner}
", + esc(action), + esc(class), + esc(&ctx.csrf) + ) +} + +/// A table with a caption and column headers. Every cell is markup already built with [`esc`]. +pub fn table(caption: &str, headers: &[&str], rows: &[Vec]) -> String { + let mut out = format!("", esc(caption)); + for header in headers { + let _ = write!(out, "", esc(header)); + } + out.push_str(""); + for row in rows { + out.push_str(""); + for (n, cell) in row.iter().enumerate() { + if n == 0 { + let _ = write!(out, ""); + } else { + let _ = write!(out, ""); + } + } + out.push_str(""); + } + out.push_str("
{}
{}
{cell}{cell}
"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn markup_characters_are_escaped_everywhere_a_value_goes() { + let nasty = "'&"; + assert_eq!(esc(nasty), "<script>alert("x")</script>'&"); + assert_eq!(esc("a\u{0}b\u{7}c\nd"), "a\u{fffd}b\u{fffd}c\nd"); + for html in [ + heading(nasty), + notice(nasty, true), + link(nasty, nasty), + field("f", nasty, nasty, "text", nasty, ""), + select("f", nasty, nasty, &[(nasty, nasty)], nasty), + checkbox("f", nasty, nasty), + page(Some(&Ctx { username: nasty.to_owned(), csrf: nasty.to_owned() }), nasty, ""), + post_form(&Ctx { username: String::new(), csrf: nasty.to_owned() }, nasty, nasty, ""), + table(nasty, &[nasty], &[]), + ] { + assert!(!html.contains("".into(), + stored_bytes: 12, + quota_bytes: Some(100), + }], + manifest_index: craft_cloud_meta::StorageIndexCounts { complete: 1, building: 0, not_indexed: 0 }, + }; + let html = storage_section(&storage); + assert!(html.contains("<script>alert('x')</script>"), "{html}"); + assert!(!html.contains(""; + let platform = "&\""; + let res = s + .api( + None, + Method::POST, + "/auth/login", + Some(json!({ "username": "ana", "password": PASSWORD, "device_name": name, "platform": platform, "app_version": "" })), + ) + .await; + assert_eq!(res.status(), StatusCode::OK); + // A username can't hold markup, but a row in a damaged or hand-edited database could: the pages must hold anyway. + use sqlx::Connection; + sqlx::any::install_default_drivers(); + let mut conn = sqlx::AnyConnection::connect(&s.url).await.unwrap(); + sqlx::query("UPDATE users SET username = $1 WHERE username = 'ana'").bind("").execute(&mut conn).await.unwrap(); + drop(conn); + let pages = [ + s.get("/web", Some(&sess.token)).await, + s.get("/web/users", Some(&sess.token)).await, + s.get(&format!("/web/users/{}", ana.id.0), Some(&sess.token)).await, + s.get("/web/audit", Some(&sess.token)).await, + s.get("/web/users?query=%3Cscript%3Ealert(2)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/audit?action=%22%3E%3Cscript%3Ealert(3)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/audit?user=%3Cscript%3Ealert(4)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get("/web/%3Cscript%3Ealert(5)%3C%2Fscript%3E", Some(&sess.token)).await, + s.get(&format!("/web/users/{}?notice=%3Cscript%3E", ana.id.0), Some(&sess.token)).await, + ]; + let mut shown = String::new(); + for res in pages { + let body = res.text().await.unwrap(); + assert!(!body.contains(""), "{body}"); + shown.push_str(&body); + } + assert!(shown.contains("<script>alert(1)</script>"), "the user name is shown, escaped"); + assert!(shown.contains("<script>alert('d')</script>"), "the device name is shown, escaped"); + assert!(shown.contains("<b onmouseover=x>&""), "the platform is shown, escaped"); + assert!(shown.contains("<script>alert(2)</script>"), "the search is shown back, escaped"); + assert!(shown.contains("Storage") && shown.contains("Top accounts by stored bytes"), "the storage overview is shown"); + assert!(shown.contains("<script>alert(1)</script>"), "the top-account name is escaped"); + assert!( + shown.contains("Effective quota") && shown.contains("Trash estimate (overlapping)") && shown.contains("History estimate (overlapping)"), + "the account usage is shown" + ); +} + +async fn old_sessions_are_cleaned_up_at_sign_in(s: &Web) { + let root = s.add_admin("root").await; + s.clock.store(craft_cloud_server::state::now_ms(), Ordering::SeqCst); + for _ in 0..5 { + s.sign_in("root").await; + } + let out = s.sign_in("root").await; + s.post("/web/logout", Some(&out.token), &[("csrf", &out.csrf)]).await; + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 1, "the logged-out one"); + s.advance(IDLE_MS + 1); + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 6, "five idle ones and the logged-out one"); + // The next sign-in deletes them on its way and is itself unharmed. + let fresh = s.sign_in("root").await; + assert_eq!(s.meta.web_sessions_count_over(s.at(), IDLE_MS).await.unwrap(), 0); + assert!(signed_in(s, &fresh).await); + assert_eq!(s.meta.web_sessions_open(root.id, s.at(), IDLE_MS).await.unwrap(), 1); +} + +async fn form_bodies_are_small_and_odd_ones_refused(s: &Web) { + s.add_admin("root").await; + let sess = s.sign_in("root").await; + // More than 4 KiB is 413, and no form at all is not a sign-in. + let big = "x".repeat(5000); + let res = s.post("/web/login", None, &[("username", "root"), ("password", &big)]).await; + assert_eq!(res.status(), StatusCode::PAYLOAD_TOO_LARGE); + let res = s + .client + .post(s.url("/web/login")) + .header("origin", ORIGIN) + .header("content-type", "application/json") + .body("{\"username\":\"root\",\"password\":\"x\"}") + .send() + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST, "a body that is not a form is a malformed request"); + let res = s + .client + .post(s.url("/web/logout")) + .header("origin", ORIGIN) + .header("cookie", format!("{COOKIE_NAME}={}", sess.token)) + .header("content-type", "application/x-www-form-urlencoded") + .body(vec![0xff, 0xfe, 0x25, 0x00]) + .send() + .await + .unwrap(); + assert!(res.status().is_client_error()); + assert!(signed_in(s, &sess).await); + // Parameters of the wrong kind in a query are a 400 page. + assert_eq!(s.get("/web/users?query=a&query=b&cursor=%FF", Some(&sess.token)).await.status().as_u16() / 100, 4); +} + +// ---- off by default, and its own tests ---- + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_pages_do_not_exist_unless_the_server_was_set_up_for_them() { + let s = sqlite_with(vec![], false).await; + for path in ["/web", "/web/", "/web/login", "/web/style.css", "/web/users"] { + let res = s.get(path, None).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND, "{path}"); + assert_eq!(header(&res, "content-type"), "application/json", "{path}: the API's own not-found answer"); + } + let res = s.post("/web/login", None, &[("username", "root"), ("password", PASSWORD)]).await; + assert_eq!(res.status(), StatusCode::NOT_FOUND); + // The API is unharmed. + s.add_admin("root").await; + assert!(!s.device("root").await.is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_client_address_behind_a_trusted_proxy_is_what_the_limits_count() { + let s = sqlite_with(vec!["127.0.0.1".parse().unwrap()], true).await; + s.add_admin("root").await; + // Thirty wrong tries from one forwarded address use it up, and another forwarded address still signs in. + for n in 0..30 { + let name = format!("guess{n}"); + let res = s + .client + .post(s.url("/web/login")) + .header("origin", ORIGIN) + .header("x-forwarded-for", "203.0.113.7") + .header("content-type", "application/x-www-form-urlencoded") + .body(encode(&[("username", &name), ("password", "x")])) + .send() + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); + } + let post = |from: &'static str| { + let (client, url) = (s.client.clone(), s.url("/web/login")); + async move { + client + .post(url) + .header("origin", ORIGIN) + .header("x-forwarded-for", from) + .header("content-type", "application/x-www-form-urlencoded") + .body(encode(&[("username", "root"), ("password", PASSWORD)])) + .send() + .await + .unwrap() + } + }; + assert_eq!(post("203.0.113.7").await.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(post("203.0.113.8").await.status(), StatusCode::SEE_OTHER); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn wrong_confirmation_passwords_are_limited_per_account_on_sqlite() { + wrong_confirmation_passwords_are_limited_per_account(&sqlite_web().await).await; +} + +// ---- review fixes ---- + +async fn an_account_without_a_password_never_signs_in(s: &Web) { + // What an invite makes: an account whose password hash is empty. It is answered like a name nobody has, whatever is typed, + // the server's own dummy password included (which is what an unknown name is checked against). + s.meta.create_user("pending", "", Role::Admin, 0).await.unwrap(); + let unknown = refused(s.post("/web/login", None, &[("username", "nobody"), ("password", PASSWORD)]).await).await; + for password in [PASSWORD, "", "craft-cloud dummy password", "$argon2id$"] { + let answer = refused(s.post("/web/login", None, &[("username", "pending"), ("password", password)]).await).await; + assert_eq!(answer, unknown, "{password:?}"); + } + for password in [PASSWORD, "craft-cloud dummy password"] { + let res = s.api(None, Method::POST, "/auth/login", Some(json!({ "username": "pending", "password": password, "device_name": "pc" }))).await; + assert_eq!(res.status(), StatusCode::UNAUTHORIZED, "{password:?}: the JSON sign-in too"); + } +} + +async fn the_slash_path_has_the_same_headers_and_policy(s: &Web) { + let get = s.get("/web/", None).await; + assert_eq!((get.status(), location(&get).as_str()), (StatusCode::SEE_OTHER, "/web")); + let head = s.client.head(s.url("/web/")).send().await.unwrap(); + assert_eq!((head.status(), location(&head).as_str()), (StatusCode::SEE_OTHER, "/web")); + let wrong_method = s.post("/web/", None, &[]).await; + assert_eq!(wrong_method.status(), StatusCode::METHOD_NOT_ALLOWED); + let no_origin = s.post_from("/web/", None, None, None, &[]).await; + assert_eq!(no_origin.status(), StatusCode::FORBIDDEN, "a post from nowhere is refused here too"); + for res in [get, head, wrong_method, no_origin] { + let csp = header(&res, "content-security-policy"); + assert!(csp.contains("script-src 'none'") && csp.contains("frame-ancestors 'none'"), "{}: {csp}", res.status()); + assert_eq!(header(&res, "x-frame-options"), "DENY"); + assert_eq!(header(&res, "x-content-type-options"), "nosniff"); + assert_eq!(header(&res, "referrer-policy"), "no-referrer"); + assert_eq!(header(&res, "cache-control"), "no-store"); + } +} + +/// A change whose body is still arriving when its session ends is refused, though the session was valid when the request began. +async fn a_change_that_outlives_its_session_is_refused(s: &Web) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + s.add_admin("root").await; + let vic = s.add_user("vic", Role::Member).await; + s.clock.store(craft_cloud_server::state::now_ms(), Ordering::SeqCst); + let sess = s.sign_in("root").await; + let body = encode(&[("csrf", &sess.csrf), ("password", PASSWORD)]); + let (first, rest) = body.split_at(8); + let host = s.base.trim_start_matches("http://").to_owned(); + let mut stream = tokio::net::TcpStream::connect(&host).await.unwrap(); + let head = format!( + "POST /web/users/{}/disable HTTP/1.1\r\nHost: {host}\r\nOrigin: {ORIGIN}\r\nCookie: {COOKIE_NAME}={}\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{first}", + vic.id.0, + sess.token, + body.len() + ); + stream.write_all(head.as_bytes()).await.unwrap(); + // The server has looked at the session by now (it does so before it reads the form); the session ends; the body goes on. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + s.advance(IDLE_MS + 1); + stream.write_all(rest.as_bytes()).await.unwrap(); + let mut answer = String::new(); + stream.read_to_string(&mut answer).await.unwrap(); + assert!(answer.starts_with("HTTP/1.1 401"), "{answer}"); + assert!(s.meta.admin_user(vic.id, s.at()).await.unwrap().unwrap().disabled_ms.is_none(), "nothing changed"); +} diff --git a/apps/craft-cloud-server/tests/web_log.rs b/apps/craft-cloud-server/tests/web_log.rs new file mode 100644 index 00000000..216a80b4 --- /dev/null +++ b/apps/craft-cloud-server/tests/web_log.rs @@ -0,0 +1,248 @@ +//! What the admin pages write to the log: events with the kind and the target's ID, and no password, session cookie, CSRF +//! token, challenge, code, recovery code, invite or reset code, or join link. The subscriber is global, so this is the only +//! test in its binary. + +use std::sync::{Arc, Mutex}; + +use craft_cloud_meta::{Meta, Role}; +use craft_cloud_proto::api::{LoginResponse, RecoveryCodes, TwoFactorSetupResponse}; +use craft_cloud_server::AppState; +use craft_cloud_server::auth::hash_password; +use craft_cloud_server::secretbox::{SecretBox, TwoFactorKey}; +use craft_cloud_server::totp; +use craft_cloud_server::web::{COOKIE_NAME, WebSettings}; +use craft_cloud_store::Store; +use craft_cloud_store::disk::DiskBackend; +use reqwest::{Client, StatusCode}; +use serde_json::json; + +#[derive(Clone, Default)] +struct Capture(Arc>>); + +impl std::io::Write for Capture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Capture { + type Writer = Capture; + + fn make_writer(&'a self) -> Capture { + self.clone() + } +} + +const PASSWORD: &str = "correct horse battery"; +const ORIGIN: &str = "https://cloud.example.org"; + +fn form(fields: &[(&str, &str)]) -> String { + fields + .iter() + .map(|(k, v)| { + let esc: String = v + .bytes() + .map(|b| { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~') { char::from(b).to_string() } else { format!("%{b:02X}") } + }) + .collect(); + format!("{k}={esc}") + }) + .collect::>() + .join("&") +} + +async fn post_form(client: &Client, base: &str, path: &str, token: Option<&str>, fields: Vec<(&'static str, String)>) -> reqwest::Response { + let borrowed: Vec<(&str, &str)> = fields.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let mut req = client + .post(format!("{base}{path}")) + .header("origin", ORIGIN) + .header("content-type", "application/x-www-form-urlencoded") + .body(form(&borrowed)); + if let Some(token) = token { + req = req.header("cookie", format!("{COOKIE_NAME}={token}")); + } + req.send().await.unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_admin_pages_log_no_secret() { + let log = Capture::default(); + tracing::subscriber::set_global_default( + tracing_subscriber::fmt().with_writer(log.clone()).with_max_level(tracing::Level::DEBUG).with_ansi(false).finish(), + ) + .unwrap(); + + let db = tempfile::tempdir().unwrap(); + let blobs = tempfile::tempdir().unwrap(); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]) + .unwrap() + .with_public_url(Some(ORIGIN.to_owned())) + .with_web_admin(Some(WebSettings::new(ORIGIN, 30 * 60_000, 4 * 3_600_000).unwrap())) + .with_two_factor(Some(SecretBox::new(&TwoFactorKey::from_bytes([42; 32]), None).unwrap())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(craft_cloud_server::serve(state, listener, std::future::pending::<()>())); + let client = Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap(); + meta.create_user("root", &hash_password(PASSWORD).unwrap(), Role::Admin, 0).await.unwrap(); + meta.create_user("ana", &hash_password(PASSWORD).unwrap(), Role::Member, 0).await.unwrap(); + let ana_id = meta.user_by_name("ana").await.unwrap().unwrap().id.0; + + // Two-factor sign-in for root, from a device; its token and codes are secrets too. + let device = client + .post(format!("{base}/api/v1/auth/login")) + .json(&json!({ "username": "root", "password": PASSWORD, "device_name": "pc" })) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .token; + let setup: TwoFactorSetupResponse = client + .post(format!("{base}/api/v1/account/two-factor/setup")) + .bearer_auth(&device) + .json(&json!({ "password": PASSWORD })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let secret = totp::base32_decode(&setup.secret).unwrap(); + let first_code = totp::code_for(&secret, totp::step_at(craft_cloud_server::state::now_ms())).unwrap(); + let recovery: RecoveryCodes = client + .post(format!("{base}/api/v1/account/two-factor/confirm")) + .bearer_auth(&device) + .json(&json!({ "totp_code": first_code })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let recovery_code = recovery.recovery_codes.first().unwrap().clone(); + + // A wrong password, the password step with a challenge, a wrong code, a recovery code that signs in. + let wrong = post_form(&client, &base, "/web/login", None, vec![("username", "root".into()), ("password", "a wrong password".into())]).await; + assert_eq!(wrong.status(), StatusCode::UNAUTHORIZED); + let step = + post_form(&client, &base, "/web/login", None, vec![("username", "root".into()), ("password", PASSWORD.into())]).await.text().await.unwrap(); + let marker = "name=\"challenge\" value=\""; + let challenge = step[step.find(marker).unwrap() + marker.len()..].split('"').next().unwrap().to_owned(); + let bad = + post_form(&client, &base, "/web/login/second-factor", None, vec![("challenge", challenge.clone()), ("totp_code", "000000".into())]).await; + assert_eq!(bad.status(), StatusCode::FORBIDDEN); + let signed = + post_form(&client, &base, "/web/login/second-factor", None, vec![("challenge", challenge.clone()), ("recovery_code", recovery_code.clone())]) + .await; + assert_eq!(signed.status(), StatusCode::SEE_OTHER); + let line = signed.headers().get("set-cookie").unwrap().to_str().unwrap().to_owned(); + let token = line.strip_prefix(&format!("{COOKIE_NAME}=")).unwrap().split(';').next().unwrap().to_owned(); + let activity: serde_json::Value = + client.get(format!("{base}/api/v1/account/activity?limit=100")).bearer_auth(&device).send().await.unwrap().json().await.unwrap(); + assert!(activity["events"].as_array().unwrap().iter().any(|event| event["action"] == "auth.login" && event["details"]["client_kind"] == "web")); + let audit: serde_json::Value = + client.get(format!("{base}/api/v1/admin/audit?limit=200")).bearer_auth(&device).send().await.unwrap().json().await.unwrap(); + let events = audit["events"].as_array().unwrap(); + assert!(events.iter().any(|event| event["action"] == "auth.login" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.login_failed" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.second_factor_failed" && event["details"]["client_kind"] == "web")); + assert!(events.iter().any(|event| event["action"] == "auth.recovery_code_used" && event["details"]["client_kind"] == "web")); + let export = client.get(format!("{base}/api/v1/admin/audit/export")).bearer_auth(&device).send().await.unwrap().text().await.unwrap(); + assert!(export.lines().any(|line| line.contains("\"action\":\"auth.login\"") && line.contains("\"client_kind\":\"web\"")), "{export}"); + assert!(export.lines().any(|line| line.contains("\"action\":\"auth.login_failed\"") && line.contains("\"client_kind\":\"web\"")), "{export}"); + assert!( + export.lines().any(|line| line.contains("\"action\":\"auth.second_factor_failed\"") && line.contains("\"client_kind\":\"web\"")), + "{export}" + ); + assert!( + export.lines().any(|line| line.contains("\"action\":\"auth.recovery_code_used\"") && line.contains("\"client_kind\":\"web\"")), + "{export}" + ); + let audit_page = + client.get(format!("{base}/web/audit")).header("cookie", format!("{COOKIE_NAME}={token}")).send().await.unwrap().text().await.unwrap(); + assert!(audit_page.contains(""client_kind":"web""), "{audit_page}"); + let page = client.get(format!("{base}/web")).header("cookie", format!("{COOKIE_NAME}={token}")).send().await.unwrap().text().await.unwrap(); + let marker = "name=\"csrf\" value=\""; + let csrf = page[page.find(marker).unwrap() + marker.len()..].split('"').next().unwrap().to_owned(); + + // Changes: a new account with its invite, a reset code, a wrong confirmation, a disable, and the sign-out. + let code_of = |html: &str| html.split("").nth(1).unwrap().split("").next().unwrap().to_owned(); + // Every change is confirmed with a recovery code of its own (each works once). + let mut spare = recovery.recovery_codes.iter().skip(1).cloned(); + let mut fields = |extra: Vec<(&'static str, String)>| { + let mut all = extra; + all.push(("csrf", csrf.clone())); + all.push(("password", PASSWORD.to_owned())); + all.push(("recovery_code", spare.next().unwrap())); + all + }; + let created = post_form(&client, &base, "/web/users", Some(&token), fields(vec![("username", "newbie".into()), ("role", "member".into())])).await; + assert_eq!(created.status(), StatusCode::OK); + let invite = code_of(&created.text().await.unwrap()); + let reset = post_form(&client, &base, &format!("/web/users/{ana_id}/password-reset"), Some(&token), fields(vec![])).await; + assert_eq!(reset.status(), StatusCode::OK); + let reset_code = code_of(&reset.text().await.unwrap()); + let refused = post_form( + &client, + &base, + &format!("/web/users/{ana_id}/disable"), + Some(&token), + vec![("csrf", csrf.clone()), ("password", "another wrong password".into()), ("totp_code", "000000".into())], + ) + .await; + assert_eq!(refused.status(), StatusCode::FORBIDDEN); + let disabled = post_form(&client, &base, &format!("/web/users/{ana_id}/disable"), Some(&token), fields(vec![])).await; + assert_eq!(disabled.status(), StatusCode::SEE_OTHER); + let out = post_form(&client, &base, "/web/logout", Some(&token), vec![("csrf", csrf.clone())]).await; + assert_eq!(out.status(), StatusCode::SEE_OTHER); + + let text = String::from_utf8(log.0.lock().unwrap().clone()).unwrap(); + // The events are there, with the target's ID. + assert!( + text.contains("web admin signed in") && text.contains("admin (web): account created") && text.contains("admin (web): account disabled"), + "{text}" + ); + assert!(text.contains(&ana_id.to_string()), "the target's ID is logged"); + // Nothing secret is. + let join_url = format!("{ORIGIN}/join/{invite}"); + let all_codes = recovery.recovery_codes.clone(); + let secrets = [ + PASSWORD, + "a wrong password", + "another wrong password", + token.as_str(), + csrf.as_str(), + challenge.as_str(), + invite.as_str(), + reset_code.as_str(), + recovery_code.as_str(), + device.as_str(), + setup.secret.as_str(), + first_code.as_str(), + ]; + for secret in secrets { + assert!(!text.contains(secret), "{secret} in the log"); + } + for code in &all_codes { + assert!( + !text.contains(code.as_str()) && !text.to_ascii_lowercase().contains(&code.replace('-', "").to_ascii_lowercase()), + "a recovery code in the log" + ); + } + let lower = text.to_ascii_lowercase(); + for code in [&invite, &reset_code, &recovery_code.replace('-', "")] { + assert!(!lower.contains(&code.to_ascii_lowercase()), "a code in the log"); + } + assert!(!text.contains(&join_url) && !text.contains("/join/")); + // The usernames are not logged at info or above. + assert!(!text.lines().any(|l| l.contains(" INFO ") && (l.contains("newbie") || l.contains("root") || l.contains("ana"))), "{text}"); +} diff --git a/crates/api/src/api.rs b/crates/api/src/api.rs index 699dfae0..ecb8b2a3 100644 --- a/crates/api/src/api.rs +++ b/crates/api/src/api.rs @@ -1,9 +1,11 @@ //! The parts of the HTTP API's bodies (`/api/v1/…`) that apps see through the agent's local API: the error codes, the -//! error answer's code and message, the account's devices and add-a-computer codes. The rest of the HTTP API (commit +//! error answer's code and message, the account's devices and add-a-computer codes, and the admin API's bodies (which the +//! agent passes through for the Craft Cloud app's admin pages). The rest of the HTTP API (commit //! batches, manifests, the full error body) is the server's and the agent's, in `craft_cloud_proto::api`, which //! re-exports these. use serde::{Deserialize, Serialize}; +use uuid::Uuid; use crate::ids::DeviceId; @@ -26,21 +28,121 @@ pub mod codes { /// A chunk or manifest would take the account over the bytes the server lets it store (HTTP 507); `quota` says /// how much it stores and may store. pub const QUOTA_EXCEEDED: &str = "quota_exceeded"; + /// The server itself is full and refuses new uploads (HTTP 507; the admin set `refuse_when_full`): its disk is + /// down to the reserve, or its store holds as much as the admin said it may. Not the account's quota; `server` + /// says what the server knows of its room. Older clients don't know the code and see a server error (HTTP 507). + pub const SERVER_FULL: &str = "server_full"; /// `join`: the code is wrong, expired or used (never which). pub const INVALID_CODE: &str = "invalid_code"; /// `join`: an invite to an account without a password needs one; send it again with `password`. pub const PASSWORD_REQUIRED: &str = "password_required"; /// `join`: the new password breaks the server's rules (the message says which). pub const WEAK_PASSWORD: &str = "weak_password"; + /// A signed-in route that asks for the account's password (changing it) got a wrong one (HTTP 403, never 401: a + /// 401 on a signed-in route means the device was revoked, and the agent signs out on it). The local API's + /// `login` uses the same code when the server refuses the username or password. + pub const WRONG_PASSWORD: &str = "wrong_password"; + /// The device is signed in but may not do this (HTTP 403): a member calling an `/admin` route, or an admin doing + /// what their account may not. The local API refuses with it a request that didn't come from a local program. + pub const FORBIDDEN: &str = "forbidden"; + /// A sign-in or a join with the right password or code, for an account an admin disabled or marked for removal + /// (HTTP 403). Said only after the password or code was right, never to a guesser. + pub const ACCOUNT_DISABLED: &str = "account_disabled"; + /// An admin change would leave the server without an enabled administrator (HTTP 409). + pub const LAST_ADMIN: &str = "last_admin"; + /// A password sign-in (or the redeeming of an invite or reset code for an account with two-factor sign-in) was + /// right, but needs a second factor (HTTP 401, no token). The answer's `two_factor` carries the challenge. + pub const TWO_FACTOR_REQUIRED: &str = "two_factor_required"; + /// A code or recovery code was wrong, or a code was used before (HTTP 401 at sign-in and join, which are not + /// signed in, and 403 on a signed-in route, where a 401 would read as "this device was revoked"). + pub const TWO_FACTOR_INVALID: &str = "two_factor_invalid"; + /// The server has no key for two-factor secrets (`[two_factor] key_file`), so it can't set it up (HTTP 503). + pub const TWO_FACTOR_UNAVAILABLE: &str = "two_factor_unavailable"; + /// A set-up is running on another device (HTTP 409). + pub const TWO_FACTOR_PENDING: &str = "two_factor_pending"; + /// Too many wrong codes: every check of this account's second factor is refused until `Retry-After` is over + /// (HTTP 429). A lock never lasts for good. + pub const TWO_FACTOR_LOCKED: &str = "two_factor_locked"; pub const INTERNAL: &str = "internal"; } /// What a client reads of every error answer: `error`, a stable code (one of [`codes`], or the local API's), and -/// `message`, for people. Every other field is ignored (the full body is `craft_cloud_proto::api::ErrorBody`). +/// `message`, for people. The local trash restore call also keeps `name_taken` when a server `conflict` includes that +/// cause; any other server-only fields are ignored (the full body is `craft_cloud_proto::api::ErrorBody`). #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub struct ErrorAnswer { pub error: String, pub message: String, + /// Whether the server's conflict details say that no available restored name remains. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name_taken: Option, +} + +/// The second step of a sign-in: what `two_factor_required` carries. `challenge` is a secret, good once and until +/// `expires_ms` (milliseconds since the Unix epoch, by the server's clock); the code or recovery code goes with it to +/// `POST /auth/login/second-factor`. Its `Debug` output hides the challenge. For `POST /join` there is no challenge +/// (the code is simply sent again with the second factor): `challenge` is empty there. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorChallenge { + pub challenge: String, + pub expires_ms: i64, +} + +impl std::fmt::Debug for TwoFactorChallenge { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorChallenge").field("challenge", &"").field("expires_ms", &self.expires_ms).finish() + } +} + +/// The answer to a set-up: the secret as Base32 without padding and as an `otpauth://` URI (for a QR code), good until +/// `expires_ms`. Shown once. Its `Debug` output hides both. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorSetupResponse { + pub secret: String, + pub otpauth_uri: String, + pub expires_ms: i64, +} + +impl std::fmt::Debug for TwoFactorSetupResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorSetupResponse") + .field("secret", &"") + .field("otpauth_uri", &"") + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// Ten recovery codes, shown once (in groups of four, like `ABCD-EFGH-JKMN-PQRS`; any case, with or without dashes is +/// accepted back). Each works once. Its `Debug` output hides them. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecoveryCodes { + pub recovery_codes: Vec, +} + +impl std::fmt::Debug for RecoveryCodes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RecoveryCodes").field("recovery_codes", &format_args!("<{} redacted>", self.recovery_codes.len())).finish() + } +} + +/// `GET /account/two-factor`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorStatus { + pub enabled: bool, + /// When it was turned on. + #[serde(default)] + pub enabled_ms: Option, + /// Recovery codes not used yet (0 while it is off). + pub recovery_codes_left: u32, + /// A set-up is running (it ends after ten minutes or five wrong codes). + pub pending: bool, +} + +/// `POST /api/v1/devices/revoke-others` (device auth): how many other devices were signed out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct RevokedOthers { + pub revoked: u32, } /// Where an add-a-computer code stands, in [`PairingStatus`]. @@ -64,7 +166,8 @@ pub struct PairingStatus { pub device_name: Option, } -/// One of the signed-in user's devices. +/// One of the signed-in user's devices. The fields after `current` are newer than the first release of the API: an +/// older server leaves them out, and a client that doesn't know them ignores them. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DeviceInfo { pub id: DeviceId, @@ -74,16 +177,568 @@ pub struct DeviceInfo { pub revoked: bool, /// The device making this request. pub current: bool, + /// What the device said about its client when it signed in: `agent`, `desktop` or `cli`, or empty. Display only. + #[serde(default)] + pub client_kind: String, + /// The platform the device said it runs on, free text, empty when it didn't say. + #[serde(default)] + pub platform: String, + /// The version of the client the device said it runs, free text, empty when it didn't say. + #[serde(default)] + pub app_version: String, + /// When the device was revoked, milliseconds since the Unix epoch, by the server's clock. + #[serde(default)] + pub revoked_ms: Option, + /// Why it was revoked: `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`. A client + /// treats a reason it doesn't know as "revoked". + #[serde(default)] + pub revoke_reason: Option, +} + +/// `GET /api/v1/account` (device auth): the signed-in account. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountInfo { + pub user_id: Uuid, + /// The username, in lowercase. + pub username: String, + /// `admin` or `member`. + pub role: String, + pub created_ms: i64, + /// When the password last changed through the server; `None` while it is still the first one. + #[serde(default)] + pub password_changed_ms: Option, + /// Whether signing in asks for a second factor. + #[serde(default)] + pub two_factor_enabled: bool, +} + +// ---- the admin API (`/api/v1/admin/…`, an admin's device token) ---- +// +// Every change asks for a fresh confirmation in its body: the acting admin's `password`. A device token never expires, +// so a stolen admin token alone must not be enough. A missing or mistyped field is `400 invalid`, a wrong password `403 +// wrong_password` (never 401). An admin with two-factor sign-in adds `totp_code` (or `recovery_code`) to the same +// bodies; a body without them keeps working for an admin without it, and fields a server does not know are ignored. + +/// The body of an admin change that needs nothing but the confirmation: `POST …/disable`, `POST …/enable`, +/// `DELETE /admin/users/{id}`, `POST` and `DELETE …/invite`, `POST …/reset-two-factor` and `DELETE …/devices/{id}`. Its +/// `Debug` output hides the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminConfirm { + /// The acting admin's current password. Missing: `400 invalid`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// The acting admin's code, when their account has two-factor sign-in. Missing or wrong then: `403 + /// two_factor_invalid`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + /// A recovery code instead of `totp_code`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for AdminConfirm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminConfirm") + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /admin/users/{id}/password-reset`: the confirmation as in [`AdminConfirm`], and whether redeeming the code also +/// turns the account's two-factor sign-in off (`reset_two_factor`, default `false`: an account that has it then needs +/// its second factor to redeem the code). Its `Debug` output hides the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminPasswordReset { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, + #[serde(default)] + pub reset_two_factor: bool, +} + +impl std::fmt::Debug for AdminPasswordReset { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminPasswordReset") + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .field("reset_two_factor", &self.reset_two_factor) + .finish() + } +} + +/// `GET /admin/server`: counters and kinds, nothing about paths, addresses or settings. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminServerInfo { + pub version: String, + /// Accounts that are not marked for removal. + pub users: u64, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + pub live_bytes: u64, + /// `disk`, `s3` or `memory`. + pub storage_kind: String, + /// `sqlite` or `postgres`. + pub database_kind: String, +} + +/// `GET /admin/storage`: server-wide storage counts and the ten largest accounts, and how much room the server has. +/// +/// The `Option` fields at the end are the room: each is left out of the JSON when the server has no number for it, and +/// reads as `None` when an older server did not send it, so a client and a server of different versions keep working. +/// The numbers are cheap to get (one filesystem query each, no scan of the stored data) and are read when asked. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminStorageOverview { + pub accounts: u64, + pub stored_bytes: u64, + pub default_quota_bytes: Option, + pub accounts_with_override: u64, + /// Accounts at or above 90% of their effective quota. + pub accounts_near_quota: u64, + /// Accounts at or above their effective quota. + pub accounts_over_quota: u64, + pub top_accounts: Vec, + pub manifest_index: AdminManifestIndex, + /// Where the blobs are: `disk`, `s3` or `memory` (as `storage_kind` of [`AdminServerInfo`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_kind: Option, + /// What the admin says the store may hold (`capacity` in the server's `[storage]` section). `None`: not set. It is + /// the only size an S3 store has; for a disk store the filesystem's own numbers are below. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capacity_bytes: Option, + /// Total size of the filesystem that holds a disk store. `None` for S3 and memory, or when it can't be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_fs_total_bytes: Option, + /// Bytes still available to the server on that filesystem (what the server's own user may use, which can be less + /// than the unused bytes on a disk that reserves room for another user). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_fs_free_bytes: Option, + /// Total size of the filesystem that holds the server's own data: the SQLite file. `None` for Postgres (the + /// database server's disk is not this server's to measure), or when it can't be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server_fs_total_bytes: Option, + /// Bytes still available to the server on that filesystem. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server_fs_free_bytes: Option, + /// Size of the SQLite database on disk: the main file plus its `-wal` and `-shm` files when they exist. `None` for + /// Postgres. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_bytes: Option, +} + +/// One account in [`AdminStorageOverview`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminStorageAccount { + pub id: Uuid, + pub username: String, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +/// Global manifest-index counts from the metadata database. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminManifestIndex { + pub complete: u64, + pub building: u64, + pub not_indexed: u64, +} + +/// `GET /account/usage` and `GET /admin/users/{id}/usage`: stored bytes and labeled estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountUsage { + pub stored_bytes: u64, + pub quota_bytes: Option, + pub trash_bytes: Option, + pub history_bytes: Option, + pub estimates: UsageEstimates, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UsageEstimates { + pub index_complete: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UsageEstimateReason { + IndexIncomplete, +} + +/// An account in the admin API. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUserInfo { + pub id: Uuid, + pub username: String, + /// `admin` or `member`. + pub role: String, + pub created_ms: i64, + /// When an admin disabled it (`null`: enabled). + pub disabled_ms: Option, + /// Whether the account has two-factor sign-in on. + pub two_factor_enabled: bool, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + /// The quota an admin set for the account (`null`: the server-wide cap, if any, applies). + pub quota_bytes: Option, + /// An unexpired invite is open. + pub invite_pending: bool, +} + +/// `GET /admin/users?cursor&limit&query`: a page of accounts by username. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUserList { + pub users: Vec, + /// Opaque; give it back as `cursor` for the next page. Missing on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /admin/users`: a new account without a password, for an invite. Its `Debug` output hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCreateUser { + pub username: String, + /// `admin` or `member`. + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for AdminCreateUser { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminCreateUser") + .field("username", &self.username) + .field("role", &self.role) + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// A join code the admin made, shown once. `join_url` is the whole join string when the server knows its public +/// address, and missing otherwise. Its `Debug` output hides the code and the join string. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCode { + pub code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub join_url: Option, + pub expires_ms: i64, +} + +impl std::fmt::Debug for AdminCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminCode") + .field("code", &"") + .field("join_url", &self.join_url.as_ref().map(|_| "")) + .field("expires_ms", &self.expires_ms) + .finish() + } +} + +/// The answer to `POST /admin/users`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminCreatedUser { + pub user: AdminUserInfo, + pub invite: AdminCode, +} + +/// `PATCH /admin/users/{id}`: a new role and/or quota. At least one of them; a missing `quota_bytes` leaves the quota, +/// `null` removes it. Its `Debug` output hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminUpdateUser { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, deserialize_with = "present", skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +/// A field that is there, `null` or not: `Some(None)` for `null`, `Some(Some(x))` for a value (a missing field is +/// `None`, from `#[serde(default)]`). +fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +impl std::fmt::Debug for AdminUpdateUser { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminUpdateUser") + .field("role", &self.role) + .field("quota_bytes", &self.quota_bytes) + .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `GET /admin/users/{id}/devices`: every device of the account, revoked ones with when and why. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdminDevices { + pub devices: Vec, +} + +/// The typed values of a security event (`details`). Every field is a closed list of words or a number: nothing a client +/// typed is ever in it. Fields an event does not have are missing, and fields a later release adds are ignored. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditDetails { + /// `agent`, `desktop` or `cli`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + /// `windows`, `macos`, `linux`, `android`, `ios` or `other`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + /// Why a device was signed out: `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// How far a failed sign-in got: `password`, `totp`, `recovery` or `challenge_expired`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step: Option, + /// `admin` or `member`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_from: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_to: Option, + /// The quota an admin set: `null` when they removed it, missing when the event is not about a quota. + #[serde(default, deserialize_with = "present", skip_serializing_if = "Option::is_none")] + pub quota_bytes: Option>, + /// How many devices an event signed out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revoked: Option, + /// The device an admin or the command line signed out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device: Option, + /// Set (to `true`) when redeeming a password reset code also turned the account's two-factor sign-in off. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor_removed: Option, + /// Set when the values were too long for the log and whole fields were left out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncated: Option, +} + +/// One security event, as `GET /account/activity`, `GET /admin/audit` and the export show it. No address, name of a file or +/// device, or secret is ever in it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditEvent { + pub id: Uuid, + pub at_ms: i64, + /// The account that acted; `null` for the command line, and for a sign-in that failed (nobody proved who they were). + pub actor_user: Option, + pub actor_device: Option, + /// A name from the closed list in `docs/design/sync.md`, like `auth.login`. + pub action: String, + /// `user` or `device`, or empty. + pub target_kind: String, + pub target_id: Option, + /// `success` or `failure`. + pub outcome: String, + pub details: AuditDetails, + /// How many tries the event stands for: failed sign-ins are one event per account and minute. 1 for every other event. + pub count: u64, +} + +/// A page of `GET /account/activity` or `GET /admin/audit`, newest first. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditPage { + pub events: Vec, + /// Opaque; give it back as `cursor` for the next (older) page. Missing on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// The last line of `GET /admin/audit/export` (the lines before it are [`AuditEvent`]s). `complete` says the export holds +/// every event up to the moment it started; `truncated` says there were more than the export holds. A stream that ends +/// without either line was cut off and is not a complete export. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuditExportEnd { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub complete: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub truncated: Option, + /// The events before this line. + pub exported: u64, +} + +/// The names of the security events, the closed list the server writes and `GET /admin/audit` filters by (`docs/design/sync.md`, +/// "Security audit log"). A filter is one of these, or a prefix of some of them that ends in a dot, like `auth.`. +pub const AUDIT_ACTIONS: [&str; 30] = [ + "auth.login", + "auth.login_failed", + "auth.second_factor_failed", + "auth.recovery_code_used", + "auth.two_factor_locked", + "device.revoked", + "device.revoked_others", + "account.password_changed", + "account.two_factor_enabled", + "account.two_factor_disabled", + "account.recovery_codes_regenerated", + "admin.user_created", + "admin.user_disabled", + "admin.user_enabled", + "admin.user_removed", + "admin.role_changed", + "admin.quota_changed", + "admin.invite_issued", + "admin.invite_revoked", + "admin.password_reset_issued", + "admin.two_factor_reset", + "admin.device_revoked", + "join.invite_redeemed", + "join.reset_redeemed", + "join.pairing_redeemed", + "cli.password_reset", + "cli.two_factor_reset", + "cli.device_revoked", + "cli.devices_revoked_all", + "cli.user_removed", +]; + +/// Whether `text` is an audit filter the server takes for `action`: an event name, or a prefix ending in a dot that at least +/// one name starts with. Anything else is refused by the server as `400 invalid`; checking here keeps it from being sent. +pub fn valid_audit_action(text: &str) -> bool { + AUDIT_ACTIONS.contains(&text) + || (text.ends_with('.') + && text.bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.') + && AUDIT_ACTIONS.iter().any(|a| a.starts_with(text))) } #[cfg(test)] mod tests { use super::*; + #[test] + fn a_device_from_an_older_server_reads_with_empty_new_fields() { + let id = DeviceId::generate(); + let json = serde_json::json!({ "id": id, "name": "pc", "created_ms": 1, "last_used_ms": 2, "revoked": false, "current": true }); + let device: DeviceInfo = serde_json::from_value(json).unwrap(); + assert_eq!((device.client_kind.as_str(), device.platform.as_str(), device.app_version.as_str()), ("", "", "")); + assert_eq!((device.revoked_ms, device.revoke_reason), (None, None)); + } + + #[test] + fn the_new_error_codes_are_stable() { + assert_eq!(codes::WRONG_PASSWORD, "wrong_password"); + assert_eq!(codes::FORBIDDEN, "forbidden"); + assert_eq!(codes::ACCOUNT_DISABLED, "account_disabled"); + assert_eq!(codes::LAST_ADMIN, "last_admin"); + } + + #[test] + fn the_two_factor_codes_are_stable_and_a_challenge_hides_itself() { + assert_eq!(codes::TWO_FACTOR_REQUIRED, "two_factor_required"); + assert_eq!(codes::TWO_FACTOR_INVALID, "two_factor_invalid"); + assert_eq!(codes::TWO_FACTOR_UNAVAILABLE, "two_factor_unavailable"); + assert_eq!(codes::TWO_FACTOR_PENDING, "two_factor_pending"); + assert_eq!(codes::TWO_FACTOR_LOCKED, "two_factor_locked"); + let shown = format!("{:?}", TwoFactorChallenge { challenge: "challenge-secret".into(), expires_ms: 5 }); + assert!(!shown.contains("challenge-secret") && shown.contains(""), "{shown}"); + } + #[test] fn an_error_answer_reads_any_error_body() { let json = serde_json::json!({ "error": "conflict", "message": "m", "conflicts": [{ "op": 0, "kind": "root" }], "quota": { "used": 1, "limit": 2 } }); let answer: ErrorAnswer = serde_json::from_value(json).unwrap(); - assert_eq!(answer, ErrorAnswer { error: "conflict".into(), message: "m".into() }); + assert_eq!(answer, ErrorAnswer { error: "conflict".into(), message: "m".into(), name_taken: None }); + } + + #[test] + fn an_error_answer_keeps_the_optional_trash_name_conflict() { + let answer: ErrorAnswer = serde_json::from_value(serde_json::json!({ + "error": "conflict", "message": "no free conflict name", "name_taken": true + })) + .unwrap(); + assert_eq!(answer.name_taken, Some(true)); + let old: ErrorAnswer = serde_json::from_value(serde_json::json!({ "error": "conflict", "message": "m" })).unwrap(); + assert_eq!(old.name_taken, None, "older agents omit the optional field"); + } + + #[test] + fn admin_bodies_carry_the_second_factor_and_hide_every_secret() { + let confirm = + AdminConfirm { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), recovery_code: Some("ABCD-EFGH".into()) }; + let shown = format!("{confirm:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && !shown.contains("ABCD-EFGH"), "{shown}"); + assert_eq!(shown.matches("").count(), 3); + let reset = AdminPasswordReset { + password: Some("admin-secret-pw".into()), + totp_code: Some("918273".into()), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + let shown = format!("{reset:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && shown.contains("reset_two_factor: true"), "{shown}"); + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), totp_code: Some("918273".into()), ..AdminCreateUser::default() }; + assert!(!format!("{create:?}").contains("918273")); + let update = AdminUpdateUser { recovery_code: Some("ABCD-EFGH".into()), ..AdminUpdateUser::default() }; + assert!(!format!("{update:?}").contains("ABCD-EFGH")); + // Every factor field is optional and absent from a body that doesn't have it; the reset flag defaults to false. + assert_eq!( + serde_json::to_value(AdminConfirm { password: Some("p".into()), ..AdminConfirm::default() }).unwrap(), + serde_json::json!({ "password": "p" }) + ); + let reset: AdminPasswordReset = serde_json::from_value(serde_json::json!({ "password": "p" })).unwrap(); + assert!(!reset.reset_two_factor && reset.totp_code.is_none()); + let user: AdminUserInfo = serde_json::from_value(serde_json::json!({ + "id": Uuid::nil(), "username": "ana", "role": "member", "created_ms": 1, "disabled_ms": null, + "two_factor_enabled": true, "devices_active": 0, "stored_bytes": 0, "quota_bytes": null, "invite_pending": false + })) + .unwrap(); + assert!(user.two_factor_enabled); + } + + #[test] + fn an_update_tells_a_missing_quota_from_a_null_one() { + let missing: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "role": "admin", "password": "p" })).unwrap(); + assert_eq!(missing.quota_bytes, None); + let null: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": null })).unwrap(); + assert_eq!(null.quota_bytes, Some(None)); + let set: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": 5_000_000 })).unwrap(); + assert_eq!(set.quota_bytes, Some(Some(5_000_000))); + assert!(serde_json::from_value::(serde_json::json!({ "quota_bytes": -1 })).is_err()); + } + + #[test] + fn audit_bodies_read_what_the_server_writes_and_ignore_what_is_newer() { + let details: AuditDetails = serde_json::from_str(r#"{"revoked":2,"a_later_field":"x","quota_bytes":null}"#).unwrap(); + assert_eq!((details.revoked, details.quota_bytes), (Some(2), Some(None))); + let end: AuditExportEnd = serde_json::from_str(r#"{"complete":true,"exported":3}"#).unwrap(); + assert_eq!((end.complete, end.truncated, end.exported), (Some(true), None, 3)); + let page: AuditPage = serde_json::from_str(r#"{"events":[]}"#).unwrap(); + assert_eq!(page.next, None); + } + + #[test] + fn audit_filters_are_the_closed_list_or_a_prefix_of_it() { + assert_eq!(AUDIT_ACTIONS.len(), 30); + assert!(AUDIT_ACTIONS.iter().all(|a| valid_audit_action(a))); + for prefix in ["auth.", "admin.", "cli.", "join.", "device.", "account."] { + assert!(valid_audit_action(prefix), "{prefix}"); + } + for bad in ["", "auth", "auth.log", "nope.", "AUTH.", "auth.login ", "admin.user_created.", "x' OR 1=1 --", "a.b."] { + assert!(!valid_audit_action(bad), "{bad:?}"); + } } } diff --git a/crates/api/src/local.rs b/crates/api/src/local.rs index c2b98742..ff44ff85 100644 --- a/crates/api/src/local.rs +++ b/crates/api/src/local.rs @@ -13,10 +13,13 @@ //! `POST /v1/pairings` → [`PairingInfo`], `GET /v1/pairings/{id}` → [`PairingStatus`](crate::api::PairingStatus), //! `DELETE /v1/pairings/{id}` → 204; `GET /v1/devices` → a list of [`DeviceInfo`](crate::api::DeviceInfo), //! `DELETE /v1/devices/{id}` → the list after; `POST /v1/pause`, `/v1/resume` and `/v1/quit` → [`Status`]; +//! `GET /v1/usage` → [`AccountUsage`](crate::api::AccountUsage) from the signed-in account's server; //! `POST /v1/autostart` ([`AutostartRequest`]) → [`Status`]; `GET /v1/recents`, upgraded to a WebSocket of JSON //! text messages: [`RecentsRequest`] in, [`RecentsEvent`] out; `GET /v1/docs`, upgraded to a WebSocket of live //! documents ([`docs`]: [`DocsRequest`] and binary frames in, [`DocsEvent`] and binary frames out); `PUT /v1/blobs` -//! and `GET /v1/blobs/{hash}`: binary content kept out of live documents ([`blobs`]). Errors are the HTTP API's error +//! and `GET /v1/blobs/{hash}`: binary content kept out of live documents ([`blobs`]); the admin calls under +//! `/v1/admin` ([`paths::ADMIN_SERVER`], [`paths::ADMIN_USERS`]), which the agent passes on to the server's admin API +//! with the same bodies and answers (see `docs/agent.md`). Errors are the HTTP API's error //! body ([`ErrorAnswer`](crate::api::ErrorAnswer) reads its code and message) with its codes and [`codes`]. //! - **Versions**: [`Status::api`] is the local API version the agent speaks; a client refuses one outside //! [`LOCAL_API_MIN`]..=[`LOCAL_API_MAX`] when it connects. @@ -26,7 +29,7 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use crate::ids::NodeId; +use crate::ids::{DeviceId, Epoch, NodeId, VersionId}; use crate::limits::MAX_TREE_DEPTH; use crate::name::Name; @@ -52,14 +55,16 @@ pub const MAX_TITLE_BYTES: usize = 256; pub const MAX_MACHINE_BYTES: usize = 128; /// Largest JSON body or WebSocket message of the local API, either way. pub const MAX_LOCAL_JSON_BYTES: usize = 1024 * 1024; +/// Largest body of an admin call (the server's own limit is tiny too: a confirmation and a few fields). +pub const MAX_ADMIN_JSON_BYTES: usize = 8 * 1024; /// Error codes of the local API (the `error` field of [`ErrorAnswer`](crate::api::ErrorAnswer)), beside the HTTP API's /// `invalid`, `unauthorized` (the token), `not_found`, `too_large` and `internal`. pub mod codes { /// A `Host` that isn't `127.0.0.1:`, or an `Origin` header: the request didn't come from a local program. - pub const FORBIDDEN: &str = "forbidden"; + pub const FORBIDDEN: &str = crate::api::codes::FORBIDDEN; /// `login`: the server refused the username or password. - pub const WRONG_PASSWORD: &str = "wrong_password"; + pub const WRONG_PASSWORD: &str = crate::api::codes::WRONG_PASSWORD; /// `login` while signed in: sign out first. pub const ALREADY_SIGNED_IN: &str = "already_signed_in"; /// `logout`, or recents, while signed out. @@ -86,6 +91,23 @@ pub mod codes { pub const FOLDER_HAS_FILES: &str = "folder_has_files"; /// `GET /v1/blobs/{hash}`: neither this machine nor the account's server has that blob. pub const BLOB_NOT_FOUND: &str = "blob_not_found"; + /// `login` or `join`: the account has two-factor sign-in. The answer's `two_factor` (see [`ErrorExtras`](super::ErrorExtras)) + /// holds the challenge for `POST /v1/login/second-factor` (empty for `join`: send the same join again with the code). + pub const TWO_FACTOR_REQUIRED: &str = crate::api::codes::TWO_FACTOR_REQUIRED; + /// A code or recovery code was wrong (HTTP 401 for `login`, `login/second-factor` and `join`, 403 on the account calls). + /// It never means the device was revoked. + pub const TWO_FACTOR_INVALID: &str = crate::api::codes::TWO_FACTOR_INVALID; + /// Too many wrong codes: try again after the answer's `retry_after_secs` (HTTP 429, also the `Retry-After` header). + pub const TWO_FACTOR_LOCKED: &str = crate::api::codes::TWO_FACTOR_LOCKED; + /// The server has no key for two-factor secrets, so it can't be set up (HTTP 503). + pub const TWO_FACTOR_UNAVAILABLE: &str = crate::api::codes::TWO_FACTOR_UNAVAILABLE; + /// A set-up is open on another computer (HTTP 409). + pub const TWO_FACTOR_PENDING: &str = crate::api::codes::TWO_FACTOR_PENDING; + /// `login/second-factor`: the challenge doesn't work any more (it lasts five minutes and takes five wrong codes, and a + /// password change ends it): sign in again from the first step (HTTP 409; a 401 would read as a bad local token). + pub const TWO_FACTOR_EXPIRED: &str = "two_factor_expired"; + /// A state clash on the server (two-factor already on, no set-up to confirm, a password that changed meanwhile), HTTP 409. + pub const CONFLICT: &str = crate::api::codes::CONFLICT; /// Never sent by an agent: what a client reports when the agent answers a call it doesn't know (`not_found` for /// the call itself), as one from before that call answers it. Update Craft Cloud. pub const AGENT_TOO_OLD: &str = "agent_too_old"; @@ -101,11 +123,34 @@ pub mod paths { pub const DOCS: &str = "/v1/docs"; /// `POST`: join with a join string ([`super::JoinRequest`]) → [`super::Status`]. pub const JOIN: &str = "/v1/join"; + /// `POST` ([`super::SecondFactorLoginRequest`]): the second step of a sign-in that answered `two_factor_required` → + /// [`super::Status`]. + pub const LOGIN_SECOND_FACTOR: &str = "/v1/login/second-factor"; + /// `GET` → [`crate::api::TwoFactorStatus`]; with `/setup`, `/confirm`, `/disable` and `/recovery-codes` below it: the + /// signed-in account's two-factor sign-in, passed on to the server ([`super::PasswordRequest`], [`super::TotpRequest`], + /// [`super::TwoFactorProofRequest`]). Nothing is stored or logged on the way. + pub const TWO_FACTOR: &str = "/v1/account/two-factor"; + pub const TWO_FACTOR_SETUP: &str = "/v1/account/two-factor/setup"; + pub const TWO_FACTOR_CONFIRM: &str = "/v1/account/two-factor/confirm"; + pub const TWO_FACTOR_DISABLE: &str = "/v1/account/two-factor/disable"; + pub const TWO_FACTOR_RECOVERY_CODES: &str = "/v1/account/two-factor/recovery-codes"; + /// `GET` → [`crate::api::AccountInfo`]: the signed-in account (its role, and whether it has two-factor sign-in), as the + /// server says it. + pub const ACCOUNT: &str = "/v1/account"; + /// `GET /v1/account/activity?cursor&before_ms&limit` → [`crate::api::AuditPage`]: the signed-in account's own security + /// events, for every account. + pub const ACCOUNT_ACTIVITY: &str = "/v1/account/activity"; + /// `POST` ([`super::ChangePasswordRequest`]) → 204. + pub const ACCOUNT_PASSWORD: &str = "/v1/account/password"; + /// `POST` → [`crate::api::RevokedOthers`]: sign every other computer of the account out. + pub const DEVICES_REVOKE_OTHERS: &str = "/v1/devices/revoke-others"; /// `POST`: a new add-a-computer code → [`super::PairingInfo`]; `GET`/`DELETE` `/v1/pairings/{id}`: its state, or /// cancel it. pub const PAIRINGS: &str = "/v1/pairings"; /// `GET`: the account's computers; `DELETE /v1/devices/{id}`: remove one (never this one). pub const DEVICES: &str = "/v1/devices"; + /// `GET`: the signed-in account's stored bytes and labeled estimates, fetched from its server. + pub const USAGE: &str = "/v1/usage"; /// `POST`: hold syncing (kept across restarts) → [`super::Status`]. pub const PAUSE: &str = "/v1/pause"; /// `POST`: sync again after a pause → [`super::Status`]. @@ -116,6 +161,129 @@ pub mod paths { pub const AUTOSTART: &str = "/v1/autostart"; /// `PUT`: a blob → [`super::blobs::Stored`]; `GET /v1/blobs/{hash}`: its bytes ([`super::blobs`]). pub const BLOBS: &str = "/v1/blobs"; + /// `GET`: the server's counters and kinds (an admin's; any other account is refused with `forbidden`) → + /// [`AdminServerInfo`](crate::api::AdminServerInfo). + pub const ADMIN_SERVER: &str = "/v1/admin/server"; + /// `GET /v1/admin/storage` → [`AdminStorageOverview`](crate::api::AdminStorageOverview). + pub const ADMIN_STORAGE: &str = "/v1/admin/storage"; + /// `GET /v1/admin/users?cursor&limit&query` → [`AdminUserList`](crate::api::AdminUserList); `POST` (an + /// [`AdminCreateUser`](crate::api::AdminCreateUser)) → [`AdminCreatedUser`](crate::api::AdminCreatedUser). + /// Under `/v1/admin/users/{id}`: `PATCH` ([`AdminUpdateUser`](crate::api::AdminUpdateUser)) → the account's + /// [`AdminUserInfo`](crate::api::AdminUserInfo); `DELETE`, and `POST` to `…/disable`, `…/enable`, `…/invite` and + /// `…/password-reset`, and `DELETE …/invite` and `…/devices/{device_id}`, each with an + /// [`AdminConfirm`](crate::api::AdminConfirm) (the invite and the reset code answer an + /// [`AdminCode`](crate::api::AdminCode), the others have no body); `GET …/devices` → + /// [`AdminDevices`](crate::api::AdminDevices). The server's status and error code are the agent's. + pub const ADMIN_USERS: &str = "/v1/admin/users"; + /// `GET /v1/admin/users/{id}/usage` → [`AccountUsage`](crate::api::AccountUsage). + pub const ADMIN_USER_USAGE: &str = "/v1/admin/users"; + /// `GET /v1/admin/audit?cursor&before_ms&limit&user&action` → [`AuditPage`](crate::api::AuditPage): the whole security log, + /// newest first, filtered (an admin's). + pub const ADMIN_AUDIT: &str = "/v1/admin/audit"; + /// `GET /v1/admin/audit/export?before_ms&user&action`: the log as NDJSON, streamed through (chunked, never held whole by the + /// agent): [`AuditEvent`](crate::api::AuditEvent) lines and a last [`AuditExportEnd`](crate::api::AuditExportEnd) line. + /// A stream that ends without that last line was cut off. + pub const ADMIN_AUDIT_EXPORT: &str = "/v1/admin/audit/export"; + /// `GET`: version history for a synced file, addressed by its path relative to the sync root. + pub const FILE_HISTORY: &str = "/v1/files/history"; + /// `POST`: restore a prior version of a synced file, addressed by its path relative to the sync root. + pub const FILE_RESTORE: &str = "/v1/files/restore"; + /// `GET`: the server's deleted nodes; `POST` restore and remove calls address node IDs. + pub const TRASH: &str = "/v1/trash"; + pub const TRASH_RESTORE: &str = "/v1/trash/restore"; + pub const TRASH_REMOVE: &str = "/v1/trash/remove"; +} + +/// One version in a file's newest-first history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryVersion { + pub id: VersionId, + pub parent_version: Option, + pub epoch: Epoch, + pub seq: u64, + pub size: u64, + pub mtime_ms: i64, + pub device: DeviceId, + pub created_ms: i64, + pub save: bool, + pub pruned: bool, +} + +/// A page of a file's newest-first version history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryResponse { + pub versions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// The path and page cursor for `GET /v1/files/history`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileHistoryRequest { + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +/// `POST /v1/files/restore`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreRequest { + pub path: String, + pub version: VersionId, + pub expected_version: VersionId, +} + +/// The new version made by a successful restore. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreResponse { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// One deleted node from the server's trash. `path` is a local display hint and can be absent. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashItem { + pub id: NodeId, + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + pub deleted_ms: i64, + pub purge_after_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// A page of the server's trash, newest deletion first. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashResponse { + pub items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `{id}` in a local trash restore or remove request. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashNodeRequest { + pub id: String, +} + +/// The server's answer after restoring a node from trash. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashRestoreResponse { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub seq: u64, + pub epoch: Epoch, } /// Binary content an app keeps out of a live document (images, audio, a font): `PUT /v1/blobs` with the bytes as the @@ -228,8 +396,14 @@ pub enum RunState { Syncing, /// The last pass failed for a reason that may pass by itself; it is tried again. Retrying { message: String }, - /// Stopped until someone puts something right. - Paused { reason: String }, + /// Stopped until someone puts something right. `reason_code` names the cause where it is one an app may want to + /// treat on its own (`server_full`, `quota_exceeded`: the [`crate::api::codes`] of the same name), so it need not + /// match the words of `reason`; absent for other causes, and from an older agent. + Paused { + reason: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + reason_code: Option, + }, /// Paused by the user (`POST /v1/pause`), until `POST /v1/resume`, also after a restart: no sync passes and no /// live connection; the local API and apps' recents on this machine keep working. Not a problem to put right, /// unlike [`RunState::Paused`]. @@ -324,6 +498,12 @@ pub struct JoinRequest { /// The new account's password, for an invite that needs one (answered `password_required` without it). #[serde(default)] pub password: Option, + /// The second factor of an account that has two-factor sign-in (answered `two_factor_required` without it, the code not + /// used up): a code from the authenticator app, or a recovery code instead. Send the same join again with one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, } impl std::fmt::Debug for JoinRequest { @@ -334,10 +514,128 @@ impl std::fmt::Debug for JoinRequest { .field("device_name", &self.device_name) .field("adopt", &self.adopt) .field("password", &self.password.as_ref().map(|_| "")) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) .finish() } } +/// What an error answer carries beyond `error` and `message` ([`ErrorAnswer`](crate::api::ErrorAnswer) reads those two): the +/// challenge of `two_factor_required` and the wait of `two_factor_locked` (also in the `Retry-After` header). A client reads +/// it from the same body; every field is missing from answers that have no use for it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ErrorExtras { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_after_secs: Option, +} + +/// `POST /v1/login/second-factor`: the second step of a sign-in the agent answered `two_factor_required`. The agent keeps +/// nothing between the two steps, so the first step's `server`, `folder`, `device_name` and `adopt` come again (`user` is +/// the first step's and is only checked to be sane), with the `challenge` from the answer and a code (`totp_code`) or a recovery +/// code. Answered with [`Status`]. Its `Debug` output hides the challenge and the codes. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SecondFactorLoginRequest { + pub server: String, + pub user: String, + #[serde(default)] + pub folder: Option, + #[serde(default)] + pub device_name: Option, + #[serde(default)] + pub adopt: bool, + pub challenge: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for SecondFactorLoginRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorLoginRequest") + .field("server", &self.server) + .field("user", &self.user) + .field("folder", &self.folder) + .field("device_name", &self.device_name) + .field("adopt", &self.adopt) + .field("challenge", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /v1/account/two-factor/setup`: the account's password, to start setting two-factor sign-in up. Answered with +/// [`TwoFactorSetupResponse`](crate::api::TwoFactorSetupResponse). Its `Debug` output hides the password. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PasswordRequest { + pub password: String, +} + +impl std::fmt::Debug for PasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PasswordRequest").field("password", &"").finish() + } +} + +/// `POST /v1/account/two-factor/confirm`: the first code of the set-up; answered with +/// [`RecoveryCodes`](crate::api::RecoveryCodes), shown once. Its `Debug` output hides the code. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TotpRequest { + pub totp_code: String, +} + +impl std::fmt::Debug for TotpRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TotpRequest").field("totp_code", &"").finish() + } +} + +/// `POST /v1/account/two-factor/disable` (answered 204) and `/recovery-codes` (answered with +/// [`RecoveryCodes`](crate::api::RecoveryCodes), with `totp_code` only): the password and one second factor. Its `Debug` output +/// hides all three. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorProofRequest { + pub password: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for TwoFactorProofRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorProofRequest") + .field("password", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /v1/account/password`: the account's password change; answered 204, and every other computer is signed out. Its +/// `Debug` output hides both passwords. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangePasswordRequest { + pub current_password: String, + pub new_password: String, +} + +impl std::fmt::Debug for ChangePasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChangePasswordRequest").field("current_password", &"").field("new_password", &"").finish() + } +} + +/// `PATCH /v1/devices/{id}`: a new name for one of the account's computers; answered with its +/// [`DeviceInfo`](crate::api::DeviceInfo). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenameDeviceRequest { + pub name: String, +} + /// `POST /v1/pairings`: a new add-a-computer code of this account, to enter on the computer being added. It /// replaces this computer's earlier one. Its `Debug` output hides the code. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -736,9 +1034,29 @@ mod tests { device_name: None, adopt: false, password: Some("hunter2-secret".into()), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERY-SECRET".into()), }; let shown = format!("{join:?}"); - assert!(!shown.contains("K7M4") && !shown.contains("hunter2-secret"), "{shown}"); + assert!(!shown.contains("K7M4") && !shown.contains("hunter2-secret") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let second = SecondFactorLoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + folder: None, + device_name: None, + adopt: false, + challenge: "challenge-secret".into(), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERY-SECRET".into()), + }; + let shown = format!("{second:?}"); + assert!(!shown.contains("challenge-secret") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let proof = + TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: Some("918273".into()), recovery_code: Some("RECOVERY-SECRET".into()) }; + let shown = format!("{proof:?} {:?} {:?}", PasswordRequest { password: "pw-secret-1".into() }, TotpRequest { totp_code: "918273".into() }); + assert!(!shown.contains("pw-secret-1") && !shown.contains("918273") && !shown.contains("RECOVERY"), "{shown}"); + let change = format!("{:?}", ChangePasswordRequest { current_password: "old-secret-pw".into(), new_password: "new-secret-pw".into() }); + assert!(!change.contains("secret-pw"), "{change}"); let pairing = PairingInfo { id: uuid::Uuid::nil(), join: "cloud.example.com/K7M4-Q9TX".into(), @@ -747,7 +1065,20 @@ mod tests { }; assert!(!format!("{pairing:?}").contains("K7M4")); let minimal: JoinRequest = serde_json::from_value(serde_json::json!({ "join": "x/y" })).unwrap(); - assert!(!minimal.adopt && minimal.password.is_none()); + assert!(!minimal.adopt && minimal.password.is_none() && minimal.totp_code.is_none() && minimal.recovery_code.is_none()); + // An older app's body reads, and a new body without a second factor is the old one. + assert_eq!( + serde_json::to_value(&minimal).unwrap(), + serde_json::json!({ "join": "x/y", "folder": null, "device_name": null, "adopt": false, "password": null }) + ); + // The extras of an error answer are optional and ignored by a reader that doesn't know them. + let extras: ErrorExtras = serde_json::from_value(serde_json::json!({ "error": "x", "message": "m" })).unwrap(); + assert_eq!(extras, ErrorExtras::default()); + let extras: ErrorExtras = serde_json::from_value( + serde_json::json!({ "error": "two_factor_required", "two_factor": { "challenge": "c", "expires_ms": 5 }, "retry_after_secs": 7 }), + ) + .unwrap(); + assert_eq!((extras.two_factor.map(|t| t.expires_ms), extras.retry_after_secs), (Some(5), Some(7))); } #[test] @@ -776,6 +1107,14 @@ mod tests { assert_eq!(serde_json::from_value::(json).unwrap(), status); let paused = serde_json::to_value(RunState::PausedByUser).unwrap(); assert_eq!(paused, serde_json::json!({ "kind": "paused_by_user" })); + // A pause names its cause by code where there is one; an older agent sends none, and none is not written. + let coded = RunState::Paused { reason: "full".into(), reason_code: Some("server_full".into()) }; + let json = serde_json::to_value(&coded).unwrap(); + assert_eq!(json, serde_json::json!({ "kind": "paused", "reason": "full", "reason_code": "server_full" })); + assert_eq!(serde_json::from_value::(json).unwrap(), coded); + let plain = RunState::Paused { reason: "x".into(), reason_code: None }; + assert_eq!(serde_json::to_value(&plain).unwrap(), serde_json::json!({ "kind": "paused", "reason": "x" })); + assert_eq!(serde_json::from_value::(serde_json::json!({ "kind": "paused", "reason": "x" })).unwrap(), plain); let older: Status = serde_json::from_value(serde_json::json!({ "api": 1, "signed_in": false, "online": false, "run_state": { "kind": "signed_out" } })) diff --git a/crates/app/src/admin.rs b/crates/app/src/admin.rs new file mode 100644 index 00000000..a7c0c050 --- /dev/null +++ b/crates/app/src/admin.rs @@ -0,0 +1,166 @@ +//! The admin calls of the agent's local API (`/v1/admin/…`), which the agent passes on to the server's admin API for the +//! Craft Cloud app's admin pages. Every one asks the server (an Argon2 check of the admin's password among it), so call +//! them **off the UI thread**. +//! +//! The server decides everything: a member's call is `Refused { code: "forbidden" }`, a changing call without the +//! admin's current password, or with a wrong one, is `invalid` or `wrong_password` (HTTP 403, never 401), and a change +//! that would leave the server without an enabled admin is `last_admin`. A `rate_limited` or `two_factor_locked` refusal +//! carries the seconds to wait ([`Error::retry_after_secs`]). The password is sent in the body of each change +//! ([`AdminConfirm::password`] and the same field of the other bodies), goes to the server and is kept by neither the +//! agent nor this crate; the types' `Debug` output hides it. + +use craft_cloud_api::local::paths; + +use crate::{ + AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminPasswordReset, AdminServerInfo, + AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, Client, DeviceId, Error, Uuid, http, +}; + +/// Which accounts [`Client::admin_users`] lists: a page by username, optionally those whose name contains `query`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UserQuery { + /// The `next` of the page before (opaque); none for the first page. + pub cursor: Option, + /// Accounts on the page, 1 to 200; none: the server's default (50). + pub limit: Option, + /// Only usernames containing this text (case doesn't matter, at most 64 bytes); none or empty: all. + pub query: Option, +} + +impl UserQuery { + /// The path with its query string. + fn path(&self) -> String { + let mut pairs = Vec::new(); + if let Some(cursor) = self.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(format!("cursor={}", encode(cursor))); + } + if let Some(limit) = self.limit { + pairs.push(format!("limit={limit}")); + } + if let Some(query) = self.query.as_deref().filter(|q| !q.is_empty()) { + pairs.push(format!("query={}", encode(query))); + } + if pairs.is_empty() { paths::ADMIN_USERS.to_owned() } else { format!("{}?{}", paths::ADMIN_USERS, pairs.join("&")) } + } +} + +/// `text` percent-encoded for a query string (everything but unreserved characters). +pub(crate) fn encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn body(value: &T) -> Result, Error> { + serde_json::to_vec(value).map_err(|e| Error::Protocol(e.to_string())) +} + +fn user_path(id: Uuid) -> String { + format!("{}/{id}", paths::ADMIN_USERS) +} + +impl Client { + /// The server's counters and kinds. Only for an admin: any other account is `Refused { code: "forbidden" }`, which + /// is how the app tells whether to show its admin pages. + pub fn admin_server(&self) -> Result { + self.call("GET", paths::ADMIN_SERVER, None, http::SLOW_TIMEOUT) + } + + /// The server-wide storage overview; admins only. + pub fn admin_storage(&self) -> Result { + self.call("GET", paths::ADMIN_STORAGE, None, http::SLOW_TIMEOUT) + } + + /// One account's stored bytes and labeled estimates; admins only. + pub fn admin_user_usage(&self, id: Uuid) -> Result { + self.call("GET", &format!("{}/{id}/usage", paths::ADMIN_USER_USAGE), None, http::SLOW_TIMEOUT) + } + + /// A page of accounts by username ([`UserQuery`]); `next` of the answer is the cursor of the following page. + pub fn admin_users(&self, query: &UserQuery) -> Result { + self.call("GET", &query.path(), None, http::SLOW_TIMEOUT) + } + + /// A new account without a password, with an invite (its code is in the answer, once). `request.password` is the + /// acting admin's current password. + pub fn admin_create_user(&self, request: &AdminCreateUser) -> Result { + self.call("POST", paths::ADMIN_USERS, Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// A new role and/or quota for an account (`request.password` is the acting admin's current password). + pub fn admin_update_user(&self, id: Uuid, request: &AdminUpdateUser) -> Result { + self.call("PATCH", &user_path(id), Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// Disables an account: its devices are signed out, and it can't sign in. + pub fn admin_disable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/disable", user_path(id)), confirm) + } + + /// Enables a disabled account again (its signed-out devices stay signed out). + pub fn admin_enable_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/enable", user_path(id)), confirm) + } + + /// Marks an account for removal; nothing it stored is deleted. + pub fn admin_remove_user(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &user_path(id), confirm) + } + + /// A new invite for an account that has no password yet; the code is in the answer, once. + pub fn admin_issue_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result { + self.call("POST", &format!("{}/invite", user_path(id)), Some(body(confirm)?), http::SLOW_TIMEOUT) + } + + /// Ends the account's open invite or reset code. + pub fn admin_revoke_invite(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &format!("{}/invite", user_path(id)), confirm) + } + + /// A password reset code for an account (good for an hour); it is in the answer, once. `request.reset_two_factor` says + /// whether redeeming it also turns the account's two-factor sign-in off. + pub fn admin_issue_reset(&self, id: Uuid, request: &AdminPasswordReset) -> Result { + self.call("POST", &format!("{}/password-reset", user_path(id)), Some(body(request)?), http::SLOW_TIMEOUT) + } + + /// Takes an account's two-factor sign-in away and signs every device of it out (the server refuses it for the + /// admin's own account). + pub fn admin_reset_two_factor(&self, id: Uuid, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("POST", &format!("{}/reset-two-factor", user_path(id)), confirm) + } + + /// Every device of an account, revoked ones with when and why. + pub fn admin_devices(&self, id: Uuid) -> Result { + self.call("GET", &format!("{}/devices", user_path(id)), None, http::SLOW_TIMEOUT) + } + + /// Signs one device of an account out. + pub fn admin_revoke_device(&self, id: Uuid, device: DeviceId, confirm: &AdminConfirm) -> Result<(), Error> { + self.confirmed("DELETE", &format!("{}/devices/{device}", user_path(id)), confirm) + } + + /// A change whose only body is the confirmation, answered without one. + fn confirmed(&self, method: &str, path: &str, confirm: &AdminConfirm) -> Result<(), Error> { + self.answer(method, path, Some(body(confirm)?), http::SLOW_TIMEOUT).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_list_query_is_written_as_a_query_string() { + assert_eq!(UserQuery::default().path(), "/v1/admin/users"); + let all = UserQuery { cursor: Some("YW5h".into()), limit: Some(25), query: Some("a b&c=é".into()) }; + assert_eq!(all.path(), "/v1/admin/users?cursor=YW5h&limit=25&query=a%20b%26c%3D%C3%A9"); + let empty = UserQuery { cursor: Some(String::new()), limit: None, query: Some(String::new()) }; + assert_eq!(empty.path(), "/v1/admin/users", "empty means none"); + } +} diff --git a/crates/app/src/audit.rs b/crates/app/src/audit.rs new file mode 100644 index 00000000..1b95d9cf --- /dev/null +++ b/crates/app/src/audit.rs @@ -0,0 +1,368 @@ +//! The security audit log through the agent's local API: the signed-in account's own activity +//! ([`Client::account_activity`], for every account), and for an admin the whole log ([`Client::admin_audit`]) and its export +//! ([`Client::admin_audit_export`]). The server decides who may read what; this crate only asks. Every call asks the server: +//! call them **off the UI thread**. +//! +//! **The export is streamed**: the agent passes the server's NDJSON through as it comes (chunked), and this client writes each +//! line to the writer it is given as it reads it, with nothing held but one line (at most [`MAX_LINE_BYTES`]) and a small +//! buffer. It counts the export as whole only with the server's last line, `{"complete":true,"exported":n}`, when `n` is the +//! number of event lines it saw; a stream that ends without that line, breaks in the middle, or writes a line this crate can't +//! read is an error ([`Error::Protocol`] or [`Error::Io`]), and what was written to the writer is the caller's to throw away. +//! `{"truncated":true,…}` is a good export of the newest events that is not the whole log ([`ExportOutcome::truncated`]). + +use std::io::{BufRead, Read, Write}; + +use craft_cloud_api::api::{AuditEvent, AuditExportEnd, AuditPage, valid_audit_action}; +use craft_cloud_api::local::paths; + +use crate::{Client, Error, Uuid, http}; + +/// The longest line of an export this client reads, in bytes (an event is far smaller: its details are capped at 2 KiB). +pub const MAX_LINE_BYTES: usize = 64 * 1024; +/// The most events one export may hold before this client gives up (the server's own limit is 100,000). +pub const MAX_EXPORT_EVENTS: u64 = 1_000_000; + +/// Which events a list or an export asks for. A `None` leaves the filter out. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AuditQuery { + /// The `next` of the page before (opaque, given back as it came); none for the first page. Lists only. + pub cursor: Option, + /// Only events before this time (milliseconds since the Unix epoch); it limits the first page, a cursor wins over it. + pub before_ms: Option, + /// Events on the page, 1 to 200 (an account's own activity: 100); none: the server's default (50). Lists only. + pub limit: Option, + /// Only events about this account (an admin's filter). + pub user: Option, + /// Only this event name, or a prefix ending in a dot like `auth.` ([`craft_cloud_api::api::AUDIT_ACTIONS`]; an admin's filter). + pub action: Option, +} + +impl AuditQuery { + /// What is wrong with the query, before anything is sent: an action that is not in the closed list, a time before 1970, a + /// limit outside 1 to 200. + pub fn check(&self) -> Result<(), Error> { + let bad = + |message: &str| Error::Refused { code: craft_cloud_api::api::codes::INVALID.to_owned(), message: message.to_owned(), name_taken: None }; + if self.action.as_deref().is_some_and(|a| !valid_audit_action(a)) { + return Err(bad("action is an event name like auth.login, or a prefix like auth.")); + } + if self.before_ms.is_some_and(|t| t < 0) { + return Err(bad("before_ms is a time in milliseconds")); + } + if self.limit.is_some_and(|l| !(1..=200).contains(&l)) { + return Err(bad("limit is 1 to 200")); + } + Ok(()) + } + + /// `path` with the query string. + fn path(&self, path: &str) -> String { + let mut pairs = Vec::new(); + if let Some(cursor) = self.cursor.as_deref().filter(|c| !c.is_empty()) { + pairs.push(format!("cursor={}", crate::admin::encode(cursor))); + } + if let Some(before) = self.before_ms { + pairs.push(format!("before_ms={before}")); + } + if let Some(limit) = self.limit { + pairs.push(format!("limit={limit}")); + } + if let Some(user) = self.user { + pairs.push(format!("user={user}")); + } + if let Some(action) = self.action.as_deref().filter(|a| !a.is_empty()) { + pairs.push(format!("action={}", crate::admin::encode(action))); + } + if pairs.is_empty() { path.to_owned() } else { format!("{path}?{}", pairs.join("&")) } + } +} + +/// How an export ended, when it ended with the server's last line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExportOutcome { + /// Event lines written. + pub events: u64, + /// The log holds more events than an export does (the server's `truncated` line): these are the newest ones. + pub truncated: bool, +} + +impl Client { + /// The signed-in account, as the server says it: its role and whether it has two-factor sign-in. + pub fn account(&self) -> Result { + self.call("GET", paths::ACCOUNT, None, http::SLOW_TIMEOUT) + } + + /// A page of the signed-in account's own security events, newest first. Any account may; `user` and `action` of `query` + /// are not sent (they are the admin's filters). + pub fn account_activity(&self, query: &AuditQuery) -> Result { + let own = AuditQuery { user: None, action: None, ..query.clone() }; + own.check()?; + self.call("GET", &own.path(paths::ACCOUNT_ACTIVITY), None, http::SLOW_TIMEOUT) + } + + /// A page of the whole security log, newest first, filtered (an admin's: a member is `Refused { code: "forbidden" }`). + pub fn admin_audit(&self, query: &AuditQuery) -> Result { + query.check()?; + self.call("GET", &query.path(paths::ADMIN_AUDIT), None, http::SLOW_TIMEOUT) + } + + /// Writes the log, filtered by `query.user`, `query.action` and `query.before_ms`, to `out` as NDJSON (one event a line, + /// the server's own text), reading the stream as it comes (see the module's docs). Whole only when it returns + /// `Ok`; on any `Err`, `out` holds an unfinished export. A refusal before the stream starts is the usual `Refused`. + pub fn admin_audit_export(&self, query: &AuditQuery, out: &mut dyn Write) -> Result { + let export = AuditQuery { cursor: None, limit: None, ..query.clone() }; + export.check()?; + let streamed = http::request_stream(self.addr(), self.token(), "GET", &export.path(paths::ADMIN_AUDIT_EXPORT), http::SLOW_TIMEOUT)?; + match streamed { + http::Streamed::Refused(answer) => Err(crate::refusal(answer).err().unwrap_or_else(|| Error::Protocol("the export was refused".into()))), + http::Streamed::Body(body) => copy_export(body, out), + } + } +} + +/// Reads `body` (the NDJSON of an export) line by line into `out`. +fn copy_export(body: impl Read, out: &mut dyn Write) -> Result { + let mut lines = std::io::BufReader::with_capacity(8 * 1024, body); + let mut events = 0u64; + let mut end: Option = None; + let mut line = Vec::new(); + loop { + line.clear(); + // One more byte than the longest line, to tell a line that is too long from one that just fits. + let read = + (&mut lines).take(MAX_LINE_BYTES as u64 + 1).read_until(b'\n', &mut line).map_err(|e| Error::Io(format!("the export broke off: {e}")))?; + if read == 0 { + break; + } + if line.last() != Some(&b'\n') { + return if read > MAX_LINE_BYTES { + Err(Error::Protocol("the export has a line that is too long".into())) + } else { + Err(Error::Protocol("the export was cut off in the middle of a line".into())) + }; + } + let text = line.strip_suffix(b"\n").unwrap_or(&line); + let text = text.strip_suffix(b"\r").unwrap_or(text); + if text.is_empty() { + continue; + } + if end.is_some() { + return Err(Error::Protocol("the export goes on after its last line".into())); + } + let value: serde_json::Value = + serde_json::from_slice(text).map_err(|e| Error::Protocol(format!("the export has a line that doesn't read: {e}")))?; + if value.get("exported").is_some() { + end = Some(serde_json::from_value(value).map_err(|e| Error::Protocol(format!("the export's last line doesn't read: {e}")))?); + continue; + } + // An event: read as one, so a stream of anything else is no export; what is written is the server's own line. + serde_json::from_value::(value).map_err(|e| Error::Protocol(format!("the export has an event that doesn't read: {e}")))?; + events = events.saturating_add(1); + if events > MAX_EXPORT_EVENTS { + return Err(Error::Protocol("the export has more events than this client takes".into())); + } + out.write_all(text).and_then(|()| out.write_all(b"\n")).map_err(|e| Error::Io(format!("can't write the export: {e}")))?; + } + out.flush().map_err(|e| Error::Io(format!("can't write the export: {e}")))?; + let Some(end) = end else { + return Err(Error::Protocol(format!("the export was cut off after {events} events: it has no last line"))); + }; + if end.exported != events { + return Err(Error::Protocol(format!("the export says {} events and has {events}", end.exported))); + } + match (end.complete, end.truncated) { + (Some(true), _) => Ok(ExportOutcome { events, truncated: false }), + (_, Some(true)) => Ok(ExportOutcome { events, truncated: true }), + _ => Err(Error::Protocol("the export's last line says neither complete nor truncated".into())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(n: u128) -> String { + format!( + r#"{{"id":"{}","at_ms":{n},"actor_user":null,"actor_device":null,"action":"auth.login","target_kind":"","target_id":null,"outcome":"success","details":{{}},"count":1}}"#, + Uuid::from_u128(n) + ) + } + + fn copy(text: &str) -> (Result, String) { + let mut out = Vec::new(); + let result = copy_export(text.as_bytes(), &mut out); + (result, String::from_utf8(out).unwrap()) + } + + #[test] + fn a_query_is_written_as_a_query_string_and_checked_first() { + let q = + AuditQuery { cursor: Some("YWJj".into()), before_ms: Some(5), limit: Some(20), user: Some(Uuid::nil()), action: Some("auth.".into()) }; + assert_eq!( + q.path("/v1/admin/audit"), + "/v1/admin/audit?cursor=YWJj&before_ms=5&limit=20&user=00000000-0000-0000-0000-000000000000&action=auth." + ); + assert_eq!(AuditQuery::default().path("/v1/admin/audit"), "/v1/admin/audit"); + assert!(q.check().is_ok()); + for bad in [ + AuditQuery { action: Some("auth.x".into()), ..AuditQuery::default() }, + AuditQuery { before_ms: Some(-1), ..AuditQuery::default() }, + AuditQuery { limit: Some(0), ..AuditQuery::default() }, + AuditQuery { limit: Some(201), ..AuditQuery::default() }, + ] { + assert!(matches!(bad.check(), Err(Error::Refused { ref code, .. }) if code == "invalid"), "{bad:?}"); + } + } + + #[test] + fn a_whole_export_is_copied_line_by_line() { + let text = format!("{}\n{}\n{{\"complete\":true,\"exported\":2}}\n", event(2), event(1)); + let (result, out) = copy(&text); + assert_eq!(result.unwrap(), ExportOutcome { events: 2, truncated: false }); + assert_eq!(out, format!("{}\n{}\n", event(2), event(1)), "the end line is not part of the export"); + let (result, _) = copy("{\"complete\":true,\"exported\":0}\n"); + assert_eq!(result.unwrap(), ExportOutcome { events: 0, truncated: false }); + } + + #[test] + fn a_truncated_export_is_good_but_not_whole() { + let text = format!("{}\n{{\"truncated\":true,\"exported\":1}}\n", event(1)); + assert_eq!(copy(&text).0.unwrap(), ExportOutcome { events: 1, truncated: true }); + } + + #[test] + fn a_stream_that_is_cut_off_or_wrong_is_no_export() { + // No last line (the server broke off, or the connection did). + let (result, _) = copy(&format!("{}\n{}\n", event(2), event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("cut off after 2 events")), "{result:?}"); + // Cut in the middle of a line. + let (result, _) = copy(&format!("{}\n{}", event(2), &event(1)[..20])); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("middle of a line")), "{result:?}"); + // A last line that disagrees with what came. + let (result, _) = copy(&format!("{}\n{{\"complete\":true,\"exported\":2}}\n", event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("says 2 events and has 1")), "{result:?}"); + // Neither complete nor truncated; lines after the end; lines that are not events; a line that is too long. + let (result, _) = copy("{\"exported\":0}\n"); + assert!(result.is_err()); + let (result, _) = copy(&format!("{{\"complete\":true,\"exported\":0}}\n{}\n", event(1))); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("goes on")), "{result:?}"); + let (result, _) = copy("{\"hello\":1}\n{\"complete\":true,\"exported\":0}\n"); + assert!(matches!(result, Err(Error::Protocol(_))), "{result:?}"); + let (result, _) = copy("not json\n"); + assert!(matches!(result, Err(Error::Protocol(_))), "{result:?}"); + let long = format!("{}\n", "x".repeat(MAX_LINE_BYTES + 10)); + let (result, _) = copy(&long); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("too long")), "{result:?}"); + } + + /// A reader that breaks after what it has (a connection that dropped): the export fails with an I/O error; a writer that + /// fails (a full disk) fails it too. + #[test] + fn a_broken_connection_or_a_full_disk_fails_the_export() { + struct Breaks(std::io::Cursor>); + impl Read for Breaks { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self.0.read(buf)? { + 0 => Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset")), + n => Ok(n), + } + } + } + let mut out = Vec::new(); + let result = copy_export(Breaks(std::io::Cursor::new(format!("{}\n", event(1)).into_bytes())), &mut out); + assert!(matches!(result, Err(Error::Io(ref m)) if m.contains("broke off")), "{result:?}"); + struct Full; + impl Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let result = copy_export(format!("{}\n", event(1)).as_bytes(), &mut Full); + assert!(matches!(result, Err(Error::Io(ref m)) if m.contains("can't write")), "{result:?}"); + } + + /// One answer from a listener of this test: the bytes it writes (the request is read first), then it closes. + fn serve_once(answer: Vec) -> std::net::SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut seen = Vec::new(); + let mut chunk = [0u8; 1024]; + while !seen.windows(4).any(|w| w == b"\r\n\r\n") { + let n = stream.read(&mut chunk).unwrap(); + seen.extend_from_slice(&chunk[..n]); + } + stream.write_all(&answer).unwrap(); + }); + addr + } + + fn chunk(text: &str) -> Vec { + format!("{:x}\r\n{text}\r\n", text.len()).into_bytes() + } + + fn exported(answer: Vec) -> (Result, String) { + let addr = serve_once(answer); + let mut out = Vec::new(); + let streamed = http::request_stream(addr, "token", "GET", "/v1/admin/audit/export", std::time::Duration::from_secs(5)).unwrap(); + let result = match streamed { + http::Streamed::Body(body) => copy_export(body, &mut out), + http::Streamed::Refused(answer) => Err(crate::refusal(answer).unwrap_err()), + }; + (result, String::from_utf8(out).unwrap()) + } + + fn head_again() -> Vec { + b"HTTP/1.1 200 OK\r\ncontent-type: application/x-ndjson\r\ntransfer-encoding: chunked\r\n\r\n".to_vec() + } + + /// Over a real connection: a chunked export that ends properly, one whose connection drops in the middle (no + /// terminating chunk: a failed export, whatever came before), and a refusal before it starts. + #[test] + fn an_export_over_a_connection_is_whole_only_when_it_ends_properly() { + let head = b"HTTP/1.1 200 OK\r\ncontent-type: application/x-ndjson\r\ntransfer-encoding: chunked\r\n\r\n".to_vec(); + let mut whole = head.clone(); + whole.extend(chunk(&format!("{}\n", event(2)))); + whole.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":2}}\n", event(1)))); + whole.extend(b"0\r\n\r\n"); + let (result, out) = exported(whole); + assert_eq!(result.unwrap(), ExportOutcome { events: 2, truncated: false }); + assert_eq!(out, format!("{}\n{}\n", event(2), event(1))); + + // The last line came, but the stream was cut before its terminating chunk: not whole (the end is not the server's). + let mut cut = head.clone(); + cut.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":1}}\n", event(1)))); + let (result, _) = exported(cut); + assert!(matches!(result, Err(Error::Io(_))), "{result:?}"); + // Cut off in the middle of a chunk. + let mut broken = head.clone(); + broken.extend(b"ff\r\n"); + broken.extend(event(1).as_bytes()); + let (result, out) = exported(broken); + assert!(result.is_err(), "{result:?}"); + assert!(out.is_empty(), "no half line is written: {out:?}"); + // Whole lines but no last line, and a clean end: still no export. + let mut no_end = head; + no_end.extend(chunk(&format!("{}\n", event(1)))); + no_end.extend(b"0\r\n\r\n"); + let (result, _) = exported(no_end); + assert!(matches!(result, Err(Error::Protocol(ref m)) if m.contains("no last line")), "{result:?}"); + // The last chunk, then the connection ends before the body's last line break, or inside a trailer: not complete. + for tail in [&b"0\r\n"[..], &b"0\r\nX-Trailer: y"[..], &b"0\r\nX-Trailer: y\r\n"[..]] { + let mut cut_at_the_end = head_again(); + cut_at_the_end.extend(chunk(&format!("{}\n{{\"complete\":true,\"exported\":1}}\n", event(1)))); + cut_at_the_end.extend(tail); + let (result, _) = exported(cut_at_the_end); + assert!(matches!(result, Err(Error::Io(_))), "{tail:?}: {result:?}"); + } + // A refusal: the agent's JSON, with the server's wait. + let body = r#"{"error":"rate_limited","message":"too many tries"}"#; + let refused = format!("HTTP/1.1 429 Too Many Requests\r\nretry-after: 30\r\ncontent-length: {}\r\n\r\n{body}", body.len()); + let (result, _) = exported(refused.into_bytes()); + assert!(matches!(&result, Err(e) if e.code() == Some("rate_limited") && e.retry_after_secs() == Some(30)), "{result:?}"); + } +} diff --git a/crates/app/src/blobs.rs b/crates/app/src/blobs.rs index 27698701..f3dd75be 100644 --- a/crates/app/src/blobs.rs +++ b/crates/app/src/blobs.rs @@ -28,7 +28,7 @@ pub struct Blobs { /// What an agent from before blobs answers. fn not_found() -> Error { - Error::Refused { code: craft_cloud_api::api::codes::NOT_FOUND.into(), message: "no such call".into() } + Error::Refused { code: craft_cloud_api::api::codes::NOT_FOUND.into(), message: "no such call".into(), name_taken: None } } /// An older agent's "no such call" as [`codes::AGENT_TOO_OLD`]. @@ -37,6 +37,7 @@ fn too_old(e: Error) -> Error { Error::Refused { code, .. } if code == craft_cloud_api::api::codes::NOT_FOUND => Error::Refused { code: codes::AGENT_TOO_OLD.into(), message: "the Craft Cloud agent on this computer is too old for binary content in live documents: update Craft Cloud".into(), + name_taken: None, }, other => other, } @@ -69,7 +70,11 @@ impl Blobs { /// agent too old for blobs is found out before a byte of them is sent ([`codes::AGENT_TOO_OLD`]). pub fn put(&self, bytes: &[u8]) -> Result { if bytes.is_empty() || bytes.len() > MAX_BLOB_BYTES { - return Err(Error::Refused { code: "too_large".into(), message: format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes") }); + return Err(Error::Refused { + code: "too_large".into(), + message: format!("a blob is 1 byte to {MAX_BLOB_BYTES} bytes"), + name_taken: None, + }); } self.check_agent()?; let answer = diff --git a/crates/app/src/fake.rs b/crates/app/src/fake.rs index ee5b8de2..930b64a3 100644 --- a/crates/app/src/fake.rs +++ b/crates/app/src/fake.rs @@ -876,7 +876,7 @@ mod tests { let stored = a.client().blobs().put(b"blob bytes").unwrap(); assert_eq!( b.client().blobs().get(&stored.hash, stored.len).unwrap_err(), - crate::Error::Refused { code: "offline".into(), message: "the server can't be reached to fetch the blob".into() } + crate::Error::Refused { code: "offline".into(), message: "the server can't be reached to fetch the blob".into(), name_taken: None } ); b.set_online(true); assert_eq!(b.client().blobs().get(&stored.hash, stored.len).unwrap(), b"blob bytes"); diff --git a/crates/app/src/http.rs b/crates/app/src/http.rs index 48aad8d5..fdd4cd0c 100644 --- a/crates/app/src/http.rs +++ b/crates/app/src/http.rs @@ -1,7 +1,7 @@ //! One HTTP/1.1 request to the agent's local API and its answer, over a plain loopback TCP connection: written out by //! hand (the agent is the only peer, and it answers with a `Content-Length`), read with `httparse`, every size capped. -use std::io::{Read, Write}; +use std::io::{BufRead, Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::time::Duration; @@ -20,11 +20,12 @@ pub(crate) const SLOW_TIMEOUT: Duration = Duration::from_secs(90); /// Most bytes of an answer's head. const MAX_HEAD_BYTES: usize = 16 * 1024; -/// An answer: status and body. +/// An answer: status and body, and the wait in seconds its `Retry-After` header names (a number of seconds; a date isn't read). #[derive(Debug)] pub(crate) struct Answer { pub status: u16, pub body: Vec, + pub retry_after_secs: Option, } /// Sends `method path` with the token and an optional JSON body to the agent at `addr`; the answer must come within @@ -74,7 +75,7 @@ pub(crate) fn read_answer(stream: &mut impl Read) -> Result { pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result { let mut buf = Vec::with_capacity(4096); let mut chunk = [0u8; 4096]; - let (status, length, head_len) = loop { + let (status, length, head_len, retry_after_secs) = loop { let n = stream.read(&mut chunk).map_err(io)?; if n == 0 { return Err(Error::Protocol("the agent closed the connection before answering".into())); @@ -86,15 +87,18 @@ pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result { let status = response.code.ok_or_else(|| Error::Protocol("an answer without a status".into()))?; let mut length = None; + let mut retry_after_secs = None; for header in response.headers.iter() { if header.name.eq_ignore_ascii_case("content-length") { let text = std::str::from_utf8(header.value).map_err(|_| Error::Protocol("a bad Content-Length".into()))?; length = Some(text.trim().parse::().map_err(|_| Error::Protocol("a bad Content-Length".into()))?); + } else if header.name.eq_ignore_ascii_case("retry-after") { + retry_after_secs = std::str::from_utf8(header.value).ok().and_then(|text| text.trim().parse::().ok()); } else if header.name.eq_ignore_ascii_case("transfer-encoding") { return Err(Error::Protocol("the agent's answer is chunked, which this client doesn't read".into())); } } - break (status, length, head_len); + break (status, length, head_len, retry_after_secs); } Ok(httparse::Status::Partial) if buf.len() <= MAX_HEAD_BYTES => continue, Ok(httparse::Status::Partial) => return Err(Error::Protocol("the answer's head is too long".into())), @@ -124,7 +128,192 @@ pub(crate) fn read_answer_max(stream: &mut impl Read, max: usize) -> Result>), +} + +/// Most bytes of a refusal's body read from a stream request. +const MAX_REFUSAL_BYTES: usize = 64 * 1024; +/// Longest chunk-size line (hex digits and extensions) read. +const MAX_CHUNK_LINE: usize = 64; + +/// `method path` with the token to the agent at `addr`, and its answer read as it comes: for a long body (an audit export) +/// that must not be held whole. A good answer (2xx) is its [`BodyReader`] (chunked, with a `Content-Length`, or to the end of +/// the connection); any other is read whole (at most [`MAX_REFUSAL_BYTES`]) as an [`Answer`]. `timeout` is how long a +/// single read or write may wait, not the whole transfer. +pub(crate) fn request_stream(addr: SocketAddr, token: &str, method: &str, path: &str, timeout: Duration) -> Result { + let mut stream = TcpStream::connect_timeout(&addr, CONNECT_TIMEOUT).map_err(|e| Error::NotRunning(format!("can't reach it: {e}")))?; + stream.set_read_timeout(Some(timeout)).map_err(io)?; + stream.set_write_timeout(Some(timeout)).map_err(io)?; + let head = format!("{method} {path} HTTP/1.1\r\nHost: {addr}\r\nAuthorization: Bearer {token}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + stream.write_all(head.as_bytes()).map_err(io)?; + stream.flush().map_err(io)?; + let mut reader = std::io::BufReader::with_capacity(8 * 1024, stream); + // The head: lines up to the empty one, at most MAX_HEAD_BYTES. + let mut raw = Vec::new(); + loop { + let mut line = Vec::new(); + let n = (&mut reader).take(MAX_HEAD_BYTES as u64).read_until(b'\n', &mut line).map_err(io)?; + if n == 0 { + return Err(Error::Protocol("the agent closed the connection before answering".into())); + } + raw.extend_from_slice(&line); + if raw.len() > MAX_HEAD_BYTES { + return Err(Error::Protocol("the answer's head is too long".into())); + } + if line == b"\r\n" || line == b"\n" { + break; + } + } + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut response = httparse::Response::new(&mut headers); + match response.parse(&raw) { + Ok(httparse::Status::Complete(_)) => {} + Ok(httparse::Status::Partial) => return Err(Error::Protocol("the answer's head is cut short".into())), + Err(e) => return Err(Error::Protocol(format!("the answer doesn't parse: {e}"))), + } + let status = response.code.ok_or_else(|| Error::Protocol("an answer without a status".into()))?; + let (mut length, mut chunked, mut retry_after_secs) = (None, false, None); + for header in response.headers.iter() { + let text = std::str::from_utf8(header.value).unwrap_or_default().trim(); + if header.name.eq_ignore_ascii_case("content-length") { + length = Some(text.parse::().map_err(|_| Error::Protocol("a bad Content-Length".into()))?); + } else if header.name.eq_ignore_ascii_case("transfer-encoding") { + chunked = text.eq_ignore_ascii_case("chunked"); + } else if header.name.eq_ignore_ascii_case("retry-after") { + retry_after_secs = text.parse::().ok(); + } + } + let mut body = if chunked { + BodyReader::chunked(reader) + } else if let Some(n) = length { + BodyReader::with_length(reader, n) + } else { + BodyReader::to_end(reader) + }; + if (200..300).contains(&status) { + return Ok(Streamed::Body(body)); + } + let mut bytes = Vec::new(); + // A refusal's body is small; a longer one is cut (the code and message are at its start). + let _ = (&mut body).take(MAX_REFUSAL_BYTES as u64).read_to_end(&mut bytes); + Ok(Streamed::Refused(Answer { status, body: bytes, retry_after_secs })) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Framing { + Length, + Chunked, + ToEnd, +} + +/// The body of an answer as it arrives, with its framing (chunks, or a length) taken off. A body that ends before its framing +/// says it should is an error, not an end. +pub(crate) struct BodyReader { + inner: R, + framing: Framing, + /// Bytes left of the length, or of the chunk being read. + left: u64, + done: bool, +} + +impl BodyReader { + pub(crate) fn chunked(inner: R) -> Self { + Self { inner, framing: Framing::Chunked, left: 0, done: false } + } + + pub(crate) fn with_length(inner: R, length: u64) -> Self { + Self { inner, framing: Framing::Length, left: length, done: length == 0 } + } + + pub(crate) fn to_end(inner: R) -> Self { + Self { inner, framing: Framing::ToEnd, left: 0, done: false } + } + + /// The next chunk's size (a line of hex digits, extensions ignored); `0` ends the body, after its trailers. + fn next_chunk(&mut self) -> std::io::Result { + let bad = |why: &str| std::io::Error::new(std::io::ErrorKind::InvalidData, why.to_owned()); + let mut line = Vec::new(); + let n = (&mut self.inner).take(MAX_CHUNK_LINE as u64 + 1).read_until(b'\n', &mut line)?; + if n == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body ended inside its chunks")); + } + if line.last() != Some(&b'\n') { + return Err(bad("a chunk size line is too long or cut off")); + } + let text = std::str::from_utf8(&line).map_err(|_| bad("a chunk size isn't text"))?; + let size = text.trim().split(';').next().unwrap_or_default().trim(); + let size = u64::from_str_radix(size, 16).map_err(|_| bad("a chunk size isn't a number"))?; + if size == 0 { + // The trailers, up to the empty line that ends the body: a stream that ends before it, or inside a trailer, is cut + // off (an error, not an end), and the trailers are bounded as the head is. + let mut seen = 0usize; + loop { + let mut trailer = Vec::new(); + let n = (&mut self.inner).take(MAX_HEAD_BYTES as u64 + 1).read_until(b'\n', &mut trailer)?; + if n == 0 || trailer.last() != Some(&b'\n') { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body ended before its last line break")); + } + if trailer == b"\r\n" || trailer == b"\n" { + break; + } + seen = seen.saturating_add(n); + if seen > MAX_HEAD_BYTES { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "the body's trailers are too long")); + } + } + } + Ok(size) + } + + /// The line break after a chunk's data. + fn end_of_chunk(&mut self) -> std::io::Result<()> { + let mut line = Vec::new(); + (&mut self.inner).take(3).read_until(b'\n', &mut line)?; + if line == b"\r\n" || line == b"\n" { + Ok(()) + } else { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "a chunk doesn't end where it should")) + } + } +} + +impl Read for BodyReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if buf.is_empty() || self.done { + return Ok(0); + } + if self.framing == Framing::ToEnd { + return self.inner.read(buf); + } + if self.framing == Framing::Chunked && self.left == 0 { + let size = self.next_chunk()?; + if size == 0 { + self.done = true; + return Ok(0); + } + self.left = size; + } + let max = usize::try_from(self.left).unwrap_or(usize::MAX).min(buf.len()); + let n = self.inner.read(buf.get_mut(..max).unwrap_or_default())?; + if n == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "the body is shorter than its framing says")); + } + self.left = self.left.saturating_sub(n as u64); + if self.left == 0 { + match self.framing { + Framing::Chunked => self.end_of_chunk()?, + _ => self.done = true, + } + } + Ok(n) + } } #[cfg(test)] @@ -146,5 +335,51 @@ mod tests { assert!(read_answer(&mut &b""[..]).is_err()); let no_length = b"HTTP/1.1 401 Unauthorized\r\n\r\n{\"error\":\"unauthorized\"}"; assert_eq!(read_answer(&mut &no_length[..]).unwrap().status, 401); + let slow = b"HTTP/1.1 429 Too Many Requests +Retry-After: 30 +content-length: 2 + +{}"; + assert_eq!(read_answer(&mut &slow[..]).unwrap().retry_after_secs, Some(30)); + assert_eq!(read_answer(&mut &no_length[..]).unwrap().retry_after_secs, None); + } + + #[test] + fn a_retry_after_header_is_kept_when_it_is_a_number_of_seconds() { + let text = b"HTTP/1.1 429 Too Many Requests\r\nRetry-After: 840\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &text[..]).unwrap().retry_after_secs, Some(840)); + let date = b"HTTP/1.1 429 Too Many Requests\r\nretry-after: Wed, 21 Oct 2026 07:28:00 GMT\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &date[..]).unwrap().retry_after_secs, None); + let none = b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}"; + assert_eq!(read_answer(&mut &none[..]).unwrap().retry_after_secs, None); + } + + #[test] + fn bodies_are_read_by_their_framing() { + let read = |mut r: BodyReader<&[u8]>| { + let mut out = Vec::new(); + r.read_to_end(&mut out).map(|_| out) + }; + let chunked = b"5\r\nhello\r\n6;ext=1\r\n world\r\n0\r\nTrailer: x\r\n\r\nrest"; + assert_eq!(read(BodyReader::chunked(&chunked[..])).unwrap(), b"hello world"); + assert_eq!(read(BodyReader::with_length(&b"abcdef"[..], 4)).unwrap(), b"abcd"); + assert_eq!(read(BodyReader::to_end(&b"abc"[..])).unwrap(), b"abc"); + // Cut off inside a chunk, between chunks, or before the length: errors, never an end. + assert!(read(BodyReader::chunked(&b"5\r\nhel"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello"[..])).is_err()); + // The last chunk without the line break that ends the body, or cut inside a trailer: cut off, not complete. + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\n"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\nTrailer: x"[..])).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\nTrailer: x\r\n"[..])).is_err()); + assert_eq!(read(BodyReader::chunked(&b"5\r\nhello\r\n0\r\n\r\n"[..])).unwrap(), b"hello"); + let long = format!("5\r\nhello\r\n0\r\n{}\r\n\r\n", "X: y\r\n".repeat(5_000)); + assert!(read(BodyReader::chunked(long.as_bytes())).is_err(), "trailers are bounded"); + assert!(read(BodyReader::with_length(&b"abc"[..], 4)).is_err()); + // Garbage and over-long size lines. + assert!(read(BodyReader::chunked(&b"zz\r\n"[..])).is_err()); + let long = format!("{}\r\n", "1".repeat(200)); + assert!(read(BodyReader::chunked(long.as_bytes())).is_err()); + assert!(read(BodyReader::chunked(&b"5\r\nhelloXX0\r\n\r\n"[..])).is_err()); } } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 3f5d9af6..6e069909 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -13,7 +13,7 @@ //! [`Client::set_autostart`] (a few seconds at most). **Off the UI thread**, since they wait for the server (up to //! 90 s): [`Client::login`], [`Client::join`], [`Client::logout`], and the add-a-computer and computer-list calls //! ([`Client::pairing`], [`Client::pairing_state`], [`Client::cancel_pairing`], [`Client::devices`], -//! [`Client::remove_device`]), and opening a live document ([`live::Documents::open`], [`live::Importer::import`]: up +//! [`Client::remove_device`]), the admin calls ([`admin`]), and opening a live document ([`live::Documents::open`], [`live::Importer::import`]: up //! to 30 s). A live document's own calls ([`live::Document::send`], [`live::Document::try_recv`]) don't wait for an //! answer. //! @@ -49,6 +49,8 @@ //! TLS: the agent is on this machine. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod admin; +mod audit; pub mod blobs; #[cfg(feature = "fake")] pub mod fake; @@ -61,11 +63,25 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use craft_cloud_api::api::ErrorAnswer; -use craft_cloud_api::local::{AutostartRequest, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, paths}; +use craft_cloud_api::local::{ + AutostartRequest, ChangePasswordRequest, ErrorExtras, LOCAL_API_FILE, LOCAL_API_MAX, LOCAL_API_MIN, LocalApiFile, LogoutRequest, PasswordRequest, + RenameDeviceRequest, TotpRequest, paths, +}; +pub use admin::UserQuery; +pub use audit::{AuditQuery, ExportOutcome}; pub use craft_cloud_api::DeviceId; -pub use craft_cloud_api::api::{DeviceInfo, PairingState, PairingStatus}; -pub use craft_cloud_api::local::{JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RunState, Status, codes}; +pub use craft_cloud_api::api::{ + AUDIT_ACTIONS, AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminManifestIndex, + AdminPasswordReset, AdminServerInfo, AdminStorageAccount, AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditEvent, + AuditPage, DeviceInfo, PairingState, PairingStatus, RecoveryCodes, RevokedOthers, TwoFactorChallenge, TwoFactorSetupResponse, TwoFactorStatus, + UsageEstimateReason, UsageEstimates, valid_audit_action, +}; +pub use craft_cloud_api::local::{ + HistoryResponse, HistoryVersion, JoinRequest, LoginRequest, NewRecent, PairingInfo, RecentEntry, RestoreResponse, RunState, + SecondFactorLoginRequest, Status, TrashItem, TrashNodeRequest, TrashResponse, TrashRestoreResponse, TwoFactorProofRequest, codes, +}; +pub use craft_cloud_api::{NodeId, VersionId}; pub use recents::{RecentsEvent, RecentsWatch}; pub use uuid::Uuid; @@ -78,9 +94,15 @@ pub enum Error { /// The agent speaks another local API version: one of the two needs an update. #[error("the Craft Cloud agent speaks local API version {0}, this app {LOCAL_API_MIN}-{LOCAL_API_MAX}: update the older one")] Version(u32), - /// The agent refused the request: `code` is one of [`codes`] (or the HTTP API's), `message` is for people. + /// The agent refused the request: `code` is one of [`codes`] (or the HTTP API's), `message` is for people, and + /// `name_taken` identifies a trash-restore conflict when the server provides it. #[error("{message}")] - Refused { code: String, message: String }, + Refused { code: String, message: String, name_taken: Option }, + /// A refusal that carries more: the challenge of `two_factor_required` and/or the wait of `two_factor_locked` + /// (`retry_after_secs`). Only answers with one of those are this; every other refusal is [`Error::Refused`], so code that + /// matches `Refused { code, .. }` goes on working. [`Error::code`] reads the code of both. + #[error("{message}")] + RefusedWith { code: String, message: String, two_factor: Option, retry_after_secs: Option }, /// The connection broke, or the agent didn't answer in time. #[error("the connection to the Craft Cloud agent broke: {0}")] Io(String), @@ -89,6 +111,32 @@ pub enum Error { Protocol(String), } +impl Error { + /// The code of a refusal ([`Error::Refused`] or [`Error::RefusedWith`]); `None` for any other error. + pub fn code(&self) -> Option<&str> { + match self { + Error::Refused { code, .. } | Error::RefusedWith { code, .. } => Some(code), + _ => None, + } + } + + /// The challenge of a `two_factor_required` refusal. + pub fn challenge(&self) -> Option<&TwoFactorChallenge> { + match self { + Error::RefusedWith { two_factor, .. } => two_factor.as_ref(), + _ => None, + } + } + + /// How many seconds a `two_factor_locked` (or other) refusal says to wait. + pub fn retry_after_secs(&self) -> Option { + match self { + Error::RefusedWith { retry_after_secs, .. } => *retry_after_secs, + _ => None, + } + } +} + /// The running agent, as `local-api.json` names it. #[derive(Debug, Clone)] pub struct Client { @@ -147,6 +195,11 @@ impl Client { serde_json::from_slice(&answer.body).map_err(|e| Error::Protocol(e.to_string())) } + /// A call with a body that is answered with no body (`204`). + fn call_unit(&self, method: &str, path: &str, body: Option>, timeout: Duration) -> Result<(), Error> { + self.answer(method, path, body, timeout).map(|_| ()) + } + /// A call answered without a body (`204`). fn call_empty(&self, method: &str, path: &str, timeout: Duration) -> Result<(), Error> { self.answer(method, path, None, timeout).map(|_| ()) @@ -183,6 +236,67 @@ impl Client { self.call("POST", paths::LOGIN, Some(body), http::SLOW_TIMEOUT) } + /// The second step of a sign-in that answered `Refused`/`RefusedWith { code: "two_factor_required" }`: the first step's + /// server, folder, computer name and `adopt` again, with the answer's challenge ([`Error::challenge`]) and a code from + /// the authenticator app or a recovery code. A wrong code is `two_factor_invalid` (the challenge takes five), too many + /// is `two_factor_locked` with [`Error::retry_after_secs`]; neither signs anything out. Waits for the server: call it off + /// the UI thread. + pub fn login_second_factor(&self, request: &SecondFactorLoginRequest) -> Result { + let body = serde_json::to_vec(request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::LOGIN_SECOND_FACTOR, Some(body), http::SLOW_TIMEOUT) + } + + /// Whether the signed-in account has two-factor sign-in on, how many recovery codes are left and whether a set-up is + /// open. Asks the server: call it off the UI thread. A server from before two-factor sign-in answers `not_found`. + pub fn two_factor_status(&self) -> Result { + self.call("GET", paths::TWO_FACTOR, None, http::SLOW_TIMEOUT) + } + + /// Starts setting two-factor sign-in up with the account's password: the secret (Base32 and as an `otpauth://` URI, for + /// a QR code), good for 10 minutes. `wrong_password` (403) is no sign-out. The secret is shown once and never stored. + pub fn two_factor_setup(&self, password: &str) -> Result { + let body = serde_json::to_vec(&PasswordRequest { password: password.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_SETUP, Some(body), http::SLOW_TIMEOUT) + } + + /// Turns two-factor sign-in on with the first code; the ten recovery codes, shown once (the account's other computers + /// are signed out). + pub fn two_factor_confirm(&self, totp_code: &str) -> Result { + let body = serde_json::to_vec(&TotpRequest { totp_code: totp_code.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_CONFIRM, Some(body), http::SLOW_TIMEOUT) + } + + /// Turns two-factor sign-in off: the password and a code or a recovery code. + pub fn two_factor_disable(&self, request: &TwoFactorProofRequest) -> Result<(), Error> { + let body = serde_json::to_vec(request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call_unit("POST", paths::TWO_FACTOR_DISABLE, Some(body), http::SLOW_TIMEOUT) + } + + /// Ten new recovery codes (the old ones stop working): the password and a code from the authenticator app. + pub fn two_factor_recovery_codes(&self, password: &str, totp_code: &str) -> Result { + let request = TwoFactorProofRequest { password: password.to_owned(), totp_code: Some(totp_code.to_owned()), recovery_code: None }; + let body = serde_json::to_vec(&request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TWO_FACTOR_RECOVERY_CODES, Some(body), http::SLOW_TIMEOUT) + } + + /// Changes the account's password; the account's other computers are signed out. + pub fn change_password(&self, current_password: &str, new_password: &str) -> Result<(), Error> { + let request = ChangePasswordRequest { current_password: current_password.to_owned(), new_password: new_password.to_owned() }; + let body = serde_json::to_vec(&request).map_err(|e| Error::Protocol(e.to_string()))?; + self.call_unit("POST", paths::ACCOUNT_PASSWORD, Some(body), http::SLOW_TIMEOUT) + } + + /// Signs every other computer of the account out; how many were. + pub fn revoke_other_devices(&self) -> Result { + self.call("POST", paths::DEVICES_REVOKE_OTHERS, None, http::SLOW_TIMEOUT) + } + + /// Renames one of the account's computers that isn't signed out. + pub fn rename_device(&self, id: DeviceId, name: &str) -> Result { + let body = serde_json::to_vec(&RenameDeviceRequest { name: name.to_owned() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("PATCH", &format!("{}/{id}", paths::DEVICES), Some(body), http::SLOW_TIMEOUT) + } + /// Signs the agent out (the files stay). `local`: without reaching the server (the device stays valid there). /// Waits for the server (up to 90 s): call it off the UI thread. pub fn logout(&self, local: bool) -> Result { @@ -221,6 +335,11 @@ impl Client { self.call("GET", paths::DEVICES, None, http::SLOW_TIMEOUT) } + /// The signed-in account's storage usage and labeled estimates. Asks the server each time. + pub fn usage(&self) -> Result { + self.call("GET", paths::USAGE, None, http::SLOW_TIMEOUT) + } + /// Removes another computer from the account (its next request is refused; its files stay there), and gives the /// list after. This computer is `Refused { code: "cannot_remove_self" }`: that is [`Client::logout`]. Asks the /// server: call it off the UI thread. @@ -228,6 +347,52 @@ impl Client { self.call("DELETE", &format!("{}/{id}", paths::DEVICES), None, http::SLOW_TIMEOUT) } + /// The newest-first server history for a file in the synced folder. `cursor` is the opaque `next` value from the + /// previous page. Asks the server: call it off the UI thread. + pub fn file_history(&self, path: &str, cursor: Option<&str>, limit: Option) -> Result { + let mut query = format!("path={}", query_component(path)); + if let Some(cursor) = cursor { + query.push_str(&format!("&cursor={}", query_component(cursor))); + } + if let Some(limit) = limit { + query.push_str(&format!("&limit={limit}")); + } + self.call("GET", &format!("{}?{query}", paths::FILE_HISTORY), None, http::SLOW_TIMEOUT) + } + + /// Restores `version` of a file in the synced folder, if `expected_version` is still current. Asks the server: + /// call it off the UI thread. + pub fn restore_file(&self, path: &str, version: VersionId, expected_version: VersionId) -> Result { + let body = serde_json::to_vec(&craft_cloud_api::local::RestoreRequest { path: path.to_owned(), version, expected_version }) + .map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::FILE_RESTORE, Some(body), http::SLOW_TIMEOUT) + } + + /// The server's newest-first trash page. `cursor` is the opaque `next` value from the preceding page. + pub fn trash(&self, cursor: Option<&str>, limit: Option) -> Result { + let mut query = Vec::new(); + if let Some(cursor) = cursor { + query.push(format!("cursor={}", query_component(cursor))); + } + if let Some(limit) = limit { + query.push(format!("limit={limit}")); + } + let path = if query.is_empty() { paths::TRASH.to_owned() } else { format!("{}?{}", paths::TRASH, query.join("&")) }; + self.call("GET", &path, None, http::SLOW_TIMEOUT) + } + + /// Restores a node from the server's trash. Call off the UI thread. + pub fn restore_from_trash(&self, id: NodeId) -> Result { + let body = serde_json::to_vec(&TrashNodeRequest { id: id.to_string() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.call("POST", paths::TRASH_RESTORE, Some(body), http::SLOW_TIMEOUT) + } + + /// Marks a node as removed from the server's trash. This does not free its stored data yet. + pub fn remove_from_trash(&self, id: NodeId) -> Result<(), Error> { + let body = serde_json::to_vec(&TrashNodeRequest { id: id.to_string() }).map_err(|e| Error::Protocol(e.to_string()))?; + self.answer("POST", paths::TRASH_REMOVE, Some(body), http::SLOW_TIMEOUT).map(|_| ()) + } + /// Pauses syncing until [`Client::resume`], also across restarts; recents keep working. Answered by the agent /// itself (a few seconds at most): fine from a UI thread. pub fn pause(&self) -> Result { @@ -283,13 +448,34 @@ impl Client { } } +fn query_component(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b'/') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + /// `answer` if it succeeded, else the agent's refusal (its code and message). fn refusal(answer: http::Answer) -> Result { if (200..300).contains(&answer.status) { Ok(answer) } else { match serde_json::from_slice::(&answer.body) { - Ok(body) => Err(Error::Refused { code: body.error, message: body.message }), + Ok(body) => { + let extras = serde_json::from_slice::(&answer.body).unwrap_or_default(); + // The wait is in the answer's JSON; the `Retry-After` header is the same wait, used when the JSON lacks it. + let retry_after_secs = extras.retry_after_secs.or(answer.retry_after_secs); + if extras.two_factor.is_none() && retry_after_secs.is_none() { + Err(Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }) + } else { + Err(Error::RefusedWith { code: body.error, message: body.message, two_factor: extras.two_factor, retry_after_secs }) + } + } Err(_) => Err(Error::Protocol(format!("HTTP {}", answer.status))), } } @@ -300,6 +486,23 @@ mod tests { use super::*; use std::io::{Read, Write}; + #[test] + fn a_refusal_takes_its_wait_from_the_retry_after_header_when_the_json_has_none() { + let body = |text: &str| text.as_bytes().to_vec(); + let only_header = http::Answer { status: 429, body: body(r#"{"error":"two_factor_locked","message":"wait"}"#), retry_after_secs: Some(900) }; + assert_eq!(refusal(only_header).unwrap_err().retry_after_secs(), Some(900)); + // The JSON wins when both are there. + let both = http::Answer { + status: 429, + body: body(r#"{"error":"two_factor_locked","message":"wait","retry_after_secs":840}"#), + retry_after_secs: Some(900), + }; + assert_eq!(refusal(both).unwrap_err().retry_after_secs(), Some(840)); + // Neither: a plain refusal. + let neither = http::Answer { status: 403, body: body(r#"{"error":"two_factor_invalid","message":"no"}"#), retry_after_secs: None }; + assert!(matches!(refusal(neither).unwrap_err(), Error::Refused { .. })); + } + /// What the fake agent was asked: method, path and body. type Asked = (String, String, String); @@ -379,8 +582,20 @@ mod tests { let device = craft_cloud_api::DeviceId::generate(); let info = PairingInfo { id, join: "c.example/K7M4-Q9TX".into(), link: "https://c.example/join/K7M4-Q9TX".into(), expires_ms: 9 }; let used = PairingStatus { state: PairingState::Used, device_name: Some("lptp1".into()) }; - let list = vec![DeviceInfo { id: device, name: "lptp1".into(), created_ms: 1, last_used_ms: 2, revoked: false, current: false }]; - let refused = json(&ErrorAnswer { error: "password_required".into(), message: "choose a password".into() }); + let list = vec![DeviceInfo { + id: device, + name: "lptp1".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }]; + let refused = json(&ErrorAnswer { error: "password_required".into(), message: "choose a password".into(), name_taken: None }); let script = vec![ (400, refused), (200, json(&status(true))), @@ -397,8 +612,15 @@ mod tests { let (_dir, agent, asked) = fake_agent(script); let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); - let request = - JoinRequest { join: "c.example/K7M4-Q9TX".into(), folder: None, device_name: Some("lptp1".into()), adopt: true, password: None }; + let request = JoinRequest { + join: "c.example/K7M4-Q9TX".into(), + folder: None, + device_name: Some("lptp1".into()), + adopt: true, + password: None, + totp_code: None, + recovery_code: None, + }; let refused = agent.join(&request); assert!(matches!(refused, Err(Error::Refused { ref code, .. }) if code == "password_required"), "{refused:?}"); let (method, path, body) = next(); @@ -430,6 +652,357 @@ mod tests { assert_eq!(serde_json::from_str::(&body).unwrap(), AutostartRequest { on: true }); } + /// The two-factor calls send what the local API wants and read what it answers; a refusal keeps its challenge or its wait. + #[test] + fn the_two_factor_calls_send_their_requests_and_keep_what_a_refusal_carries() { + let recovery = RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into(); 10] }; + let setup = TwoFactorSetupResponse { secret: "JBSWY3DPEHPK3PXP".into(), otpauth_uri: "otpauth://totp/x".into(), expires_ms: 5 }; + let state = TwoFactorStatus { enabled: true, enabled_ms: Some(3), recovery_codes_left: 9, pending: false }; + let device = craft_cloud_api::DeviceId::generate(); + let renamed = DeviceInfo { + id: device, + name: "new name".into(), + created_ms: 1, + last_used_ms: 2, + revoked: false, + current: false, + client_kind: String::new(), + platform: String::new(), + app_version: String::new(), + revoked_ms: None, + revoke_reason: None, + }; + let required = + json(&serde_json::json!({ "error": "two_factor_required", "message": "m", "two_factor": { "challenge": "chal", "expires_ms": 9 } })); + let locked = json(&serde_json::json!({ "error": "two_factor_locked", "message": "wait", "retry_after_secs": 840 })); + let plain = json(&ErrorAnswer { error: "two_factor_invalid".into(), message: "wrong".into(), name_taken: None }); + let script = vec![ + (401, required), + (200, json(&status(true))), + (429, locked), + (403, plain), + (200, json(&state)), + (200, json(&setup)), + (200, json(&recovery)), + (204, String::new()), + (200, json(&recovery)), + (204, String::new()), + (200, json(&RevokedOthers { revoked: 2 })), + (200, json(&renamed)), + ]; + let (_dir, agent, asked) = fake_agent(script); + let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); + + let login = LoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + password: "pw-secret-1".into(), + folder: None, + device_name: None, + adopt: false, + }; + let refused = agent.login(&login).unwrap_err(); + assert_eq!(refused.code(), Some("two_factor_required")); + assert_eq!(refused.challenge().map(|c| (c.challenge.as_str(), c.expires_ms)), Some(("chal", 9))); + assert!(!format!("{refused:?}").contains("\"chal\""), "the challenge is hidden in Debug"); + let _ = next(); + let second = SecondFactorLoginRequest { + server: "https://c.example".into(), + user: "ana".into(), + folder: None, + device_name: None, + adopt: false, + challenge: "chal".into(), + totp_code: Some("123456".into()), + recovery_code: None, + }; + assert!(agent.login_second_factor(&second).unwrap().signed_in); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::LOGIN_SECOND_FACTOR)); + assert_eq!(serde_json::from_str::(&body).unwrap(), second); + + // A lock keeps its wait; a plain refusal stays a plain `Refused` (code and message only). + let err = agent.login_second_factor(&second).unwrap_err(); + assert_eq!((err.code(), err.retry_after_secs(), err.challenge()), (Some("two_factor_locked"), Some(840), None)); + let _ = next(); + let err = agent.login_second_factor(&second).unwrap_err(); + assert!(matches!(&err, Error::Refused { code, .. } if code == "two_factor_invalid"), "{err:?}"); + let _ = next(); + + assert_eq!(agent.two_factor_status().unwrap(), state); + assert_eq!(next(), ("GET".into(), paths::TWO_FACTOR.to_owned(), String::new())); + assert_eq!(agent.two_factor_setup("pw-secret-1").unwrap(), setup); + let (_, path, body) = next(); + assert_eq!( + (path.as_str(), serde_json::from_str::(&body).unwrap().password.as_str()), + (paths::TWO_FACTOR_SETUP, "pw-secret-1") + ); + assert_eq!(agent.two_factor_confirm("123456").unwrap(), recovery); + let (_, path, body) = next(); + assert_eq!((path.as_str(), serde_json::from_str::(&body).unwrap().totp_code.as_str()), (paths::TWO_FACTOR_CONFIRM, "123456")); + let off = TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: None, recovery_code: Some("ABCD-EFGH-JKMN-PQRS".into()) }; + agent.two_factor_disable(&off).unwrap(); + let (_, path, body) = next(); + assert_eq!((path.as_str(), serde_json::from_str::(&body).unwrap()), (paths::TWO_FACTOR_DISABLE, off)); + assert_eq!(agent.two_factor_recovery_codes("pw-secret-1", "654321").unwrap(), recovery); + let (_, path, body) = next(); + let sent = serde_json::from_str::(&body).unwrap(); + assert_eq!((path.as_str(), sent.totp_code.as_deref(), sent.recovery_code), (paths::TWO_FACTOR_RECOVERY_CODES, Some("654321"), None)); + agent.change_password("old-secret-pw", "new-secret-pw").unwrap(); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::ACCOUNT_PASSWORD)); + assert_eq!(serde_json::from_str::(&body).unwrap().new_password, "new-secret-pw"); + assert_eq!(agent.revoke_other_devices().unwrap(), RevokedOthers { revoked: 2 }); + assert_eq!(next(), ("POST".into(), paths::DEVICES_REVOKE_OTHERS.to_owned(), String::new())); + assert_eq!(agent.rename_device(device, "new name").unwrap(), renamed); + let (method, path, body) = next(); + assert_eq!((method.as_str(), path), ("PATCH", format!("{}/{device}", paths::DEVICES))); + assert_eq!(serde_json::from_str::(&body).unwrap().name, "new name"); + } + + /// The admin calls: each sends its request to its path with the confirmation in the body, and reads its answer; a + /// refusal comes back with the server's code, and the wait of a `Retry-After` is kept. + #[test] + fn each_admin_call_sends_its_request_and_reads_its_answer() { + let id = uuid::Uuid::now_v7(); + let device = craft_cloud_api::DeviceId::generate(); + let user = AdminUserInfo { + id, + username: "ana".into(), + role: "member".into(), + created_ms: 1, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 1, + stored_bytes: 5, + quota_bytes: None, + invite_pending: false, + }; + let code = AdminCode { code: "ABCDEFGHJKMNPQRS".into(), join_url: None, expires_ms: 9 }; + let list = AdminUserList { users: vec![user.clone()], next: Some("YW5h".into()) }; + let forbidden = json(&ErrorAnswer { error: "wrong_password".into(), message: "that is not your password".into(), name_taken: None }); + let script = vec![ + (200, json(&list)), + (200, json(&list)), + (201, json(&AdminCreatedUser { user: user.clone(), invite: code.clone() })), + (200, json(&user)), + (204, String::new()), + (204, String::new()), + (202, String::new()), + (200, json(&code)), + (204, String::new()), + (200, json(&code)), + (204, String::new()), + (200, json(&AdminDevices { devices: Vec::new() })), + (204, String::new()), + (403, forbidden), + ]; + let (_dir, agent, asked) = fake_agent(script); + let next = || asked.recv_timeout(Duration::from_secs(5)).unwrap(); + let confirm = AdminConfirm { password: Some("admin-secret-pw".into()), ..AdminConfirm::default() }; + let password = |body: &str| serde_json::from_str::(body).unwrap()["password"] == "admin-secret-pw"; + + assert_eq!(agent.admin_users(&UserQuery::default()).unwrap(), list); + assert_eq!(next(), ("GET".into(), "/v1/admin/users".into(), String::new())); + let page = UserQuery { cursor: Some("YW5h".into()), limit: Some(10), query: Some("an".into()) }; + agent.admin_users(&page).unwrap(); + assert_eq!(next().1, "/v1/admin/users?cursor=YW5h&limit=10&query=an"); + let create = + AdminCreateUser { username: "bo".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + assert_eq!(agent.admin_create_user(&create).unwrap().invite, code); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("POST", "/v1/admin/users")); + assert!(password(&sent) && sent.contains("\"bo\""), "{sent}"); + let update = AdminUpdateUser { quota_bytes: Some(None), password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + assert_eq!(agent.admin_update_user(id, &update).unwrap(), user); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("PATCH", format!("/v1/admin/users/{id}").as_str())); + assert!(password(&sent) && sent.contains("\"quota_bytes\":null"), "{sent}"); + agent.admin_disable_user(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/disable")); + agent.admin_enable_user(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/enable")); + agent.admin_remove_user(id, &confirm).unwrap(); + let (method, path, sent) = next(); + assert_eq!((method.as_str(), path.as_str()), ("DELETE", format!("/v1/admin/users/{id}").as_str())); + assert!(password(&sent)); + assert_eq!(agent.admin_issue_invite(id, &confirm).unwrap(), code); + assert_eq!(next().1, format!("/v1/admin/users/{id}/invite")); + agent.admin_revoke_invite(id, &confirm).unwrap(); + assert_eq!(next(), ("DELETE".into(), format!("/v1/admin/users/{id}/invite"), json(&confirm))); + let reset = AdminPasswordReset { + password: Some("admin-secret-pw".into()), + totp_code: Some("123456".into()), + reset_two_factor: true, + ..AdminPasswordReset::default() + }; + assert_eq!(agent.admin_issue_reset(id, &reset).unwrap(), code); + let (_, path, sent) = next(); + assert_eq!(path, format!("/v1/admin/users/{id}/password-reset")); + assert_eq!(serde_json::from_str::(&sent).unwrap(), reset, "the factor and the reset flag go with it"); + agent.admin_reset_two_factor(id, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/reset-two-factor")); + assert!(agent.admin_devices(id).unwrap().devices.is_empty()); + assert_eq!(next(), ("GET".into(), format!("/v1/admin/users/{id}/devices"), String::new())); + agent.admin_revoke_device(id, device, &confirm).unwrap(); + assert_eq!(next().1, format!("/v1/admin/users/{id}/devices/{device}")); + let wrong = agent.admin_disable_user(id, &confirm); + assert!(matches!(wrong, Err(Error::Refused { ref code, .. }) if code == "wrong_password"), "{wrong:?}"); + } + + #[test] + fn storage_admin_calls_use_the_local_admin_routes() { + let id = uuid::Uuid::now_v7(); + let overview = AdminStorageOverview { + accounts: 1, + stored_bytes: 17, + default_quota_bytes: Some(99), + accounts_with_override: 0, + accounts_near_quota: 0, + accounts_over_quota: 0, + top_accounts: vec![AdminStorageAccount { id, username: "ana".into(), stored_bytes: 17, quota_bytes: Some(99) }], + manifest_index: AdminManifestIndex { complete: 3, building: 1, not_indexed: 2 }, + store_kind: Some("disk".into()), + capacity_bytes: Some(1 << 40), + store_fs_total_bytes: Some(2 << 40), + store_fs_free_bytes: Some(1 << 39), + server_fs_total_bytes: Some(1 << 38), + server_fs_free_bytes: Some(1 << 37), + database_bytes: Some(123_456), + }; + let usage = AccountUsage { + stored_bytes: 17, + quota_bytes: Some(99), + trash_bytes: Some(4), + history_bytes: Some(8), + estimates: UsageEstimates { index_complete: true, reason: None }, + }; + let (_dir, agent, asked) = fake_agent(vec![(200, json(&overview)), (200, json(&usage))]); + assert_eq!(agent.admin_storage().unwrap(), overview); + assert_eq!(asked.recv_timeout(Duration::from_secs(5)).unwrap(), ("GET".into(), "/v1/admin/storage".into(), String::new())); + assert_eq!(agent.admin_user_usage(id).unwrap(), usage); + assert_eq!(asked.recv_timeout(Duration::from_secs(5)).unwrap(), ("GET".into(), format!("/v1/admin/users/{id}/usage"), String::new())); + } + + #[test] + fn history_and_restore_use_relative_paths_and_typed_bodies() { + let restored = RestoreResponse { + version: VersionId::generate(), + parent_version: VersionId::generate(), + seq: 3, + epoch: craft_cloud_api::Epoch::generate(), + }; + let (_dir, agent, asked) = fake_agent(vec![ + (200, json(&HistoryResponse { versions: vec![], next: Some("7.0192f0c4-0000-7000-8000-000000000000".into()) })), + (200, json(&restored)), + ]); + let history = agent.file_history("Art Folder/paint #1.craft", None, Some(25)).unwrap(); + assert_eq!(history.next.as_deref(), Some("7.0192f0c4-0000-7000-8000-000000000000")); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(method, "GET"); + assert_eq!(path, "/v1/files/history?path=Art%20Folder/paint%20%231.craft&limit=25"); + assert!(body.is_empty()); + + let expected = VersionId::generate(); + assert_eq!(agent.restore_file("Art Folder/paint #1.craft", restored.version, expected).unwrap(), restored); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::FILE_RESTORE)); + assert_eq!( + serde_json::from_str::(&body).unwrap(), + craft_cloud_api::local::RestoreRequest { + path: "Art Folder/paint #1.craft".into(), + version: restored.version, + expected_version: expected, + } + ); + } + + #[test] + fn trash_calls_use_paging_and_typed_node_ids() { + let id = NodeId::generate(); + let restored = TrashRestoreResponse { + node: id, + parent: NodeId::generate(), + name: "restored.txt".into(), + renamed: true, + version: Some(VersionId::generate()), + seq: 4, + epoch: craft_cloud_api::Epoch::generate(), + }; + let item = craft_cloud_api::local::TrashItem { + id, + parent: restored.parent, + name: "gone.txt".into(), + is_dir: false, + size: Some(3), + deleted_ms: 10, + purge_after_ms: 20, + path: Some("folder/gone.txt".into()), + }; + let (_dir, agent, asked) = fake_agent(vec![ + (200, json(&TrashResponse { items: vec![item], next: Some("cursor value".into()) })), + (200, json(&restored)), + (204, String::new()), + ]); + assert_eq!(agent.trash(None, Some(25)).unwrap().next.as_deref(), Some("cursor value")); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("GET", "/v1/trash?limit=25")); + assert!(body.is_empty()); + + assert_eq!(agent.restore_from_trash(id).unwrap(), restored); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::TRASH_RESTORE)); + assert_eq!(serde_json::from_str::(&body).unwrap(), TrashNodeRequest { id: id.to_string() }); + + agent.remove_from_trash(id).unwrap(); + let (method, path, body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("POST", paths::TRASH_REMOVE)); + assert_eq!(serde_json::from_str::(&body).unwrap(), TrashNodeRequest { id: id.to_string() }); + } + + #[test] + fn trash_cursor_with_reserved_characters_is_percent_encoded() { + let (_dir, agent, asked) = fake_agent(vec![(200, json(&TrashResponse { items: vec![], next: None }))]); + agent.trash(Some("a+b/c=d&e f%g"), Some(25)).unwrap(); + let (method, path, _body) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(method, "GET"); + assert_eq!(path, "/v1/trash?cursor=a%2Bb/c%3Dd%26e%20f%25g&limit=25"); + // The agent reads the query as URL-encoded pairs: decoding the value must give the cursor back. + let value = path.split_once("cursor=").and_then(|(_, rest)| rest.split_once('&')).map(|(value, _)| value).unwrap(); + let mut decoded = Vec::new(); + let bytes = value.as_bytes(); + let mut at = 0; + while at < bytes.len() { + if bytes[at] == b'%' { + decoded.push(u8::from_str_radix(&value[at + 1..at + 3], 16).unwrap()); + at += 3; + } else { + decoded.push(bytes[at]); + at += 1; + } + } + assert_eq!(String::from_utf8(decoded).unwrap(), "a+b/c=d&e f%g"); + } + + #[test] + fn trash_restore_keeps_the_server_name_taken_detail() { + let id = NodeId::generate(); + let refused = json(&ErrorAnswer { error: "conflict".into(), message: "no free conflict name".into(), name_taken: Some(true) }); + let (_dir, agent, _asked) = fake_agent(vec![(409, refused)]); + let result = agent.restore_from_trash(id); + assert!(matches!(result, Err(Error::Refused { ref code, name_taken: Some(true), .. }) if code == "conflict"), "{result:?}"); + } + + #[test] + fn usage_maps_unauthorized_and_uses_the_local_route() { + let answer = json(&ErrorAnswer { error: "unauthorized".into(), message: "not signed in".into(), name_taken: None }); + let (_dir, agent, asked) = fake_agent(vec![(401, answer)]); + let result = agent.usage(); + assert!(matches!(result, Err(Error::Refused { ref code, .. }) if code == "unauthorized"), "{result:?}"); + let (method, path, _) = asked.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!((method.as_str(), path.as_str()), ("GET", paths::USAGE)); + } + /// An agent that takes the connection but never answers (hung, or something else on its port) costs a /// UI thread at most a couple of seconds; it used to wait 90 s, the sign-in's time. #[test] diff --git a/crates/app/src/live.rs b/crates/app/src/live.rs index 1e3eba68..18f371c6 100644 --- a/crates/app/src/live.rs +++ b/crates/app/src/live.rs @@ -214,7 +214,9 @@ impl Shared { } } Ok(DocsEvent::Error { id: Some(id), seq: Some(seq), code, message }) => self.route(id, Event::Refused { seq, code, message }), - Ok(DocsEvent::Error { id: Some(id), seq: None, code, message }) => self.answer(id, Answer::Refused(Error::Refused { code, message })), + Ok(DocsEvent::Error { id: Some(id), seq: None, code, message }) => { + self.answer(id, Answer::Refused(Error::Refused { code, message, name_taken: None })) + } Ok(DocsEvent::Error { id: None, .. }) | Err(_) => {} }, Message::Binary(bytes) => { @@ -317,7 +319,7 @@ impl Documents { let status = response.status().as_u16(); let body = response.into_body().unwrap_or_default(); return Err(match serde_json::from_slice::(&body) { - Ok(body) => Error::Refused { code: body.error, message: body.message }, + Ok(body) => Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }, Err(_) => Error::Protocol(format!("HTTP {status}")), }); } @@ -451,7 +453,7 @@ impl Importer { Ok(Document { id: self.id, head, snapshot, updates, events: self.events, shared: self.shared, seq: AtomicU64::new(0) }) } Answer::NotLive(reason) => { - Err(Error::Refused { code: "not_live".into(), message: format!("the file opens as a plain file ({reason:?})") }) + Err(Error::Refused { code: "not_live".into(), message: format!("the file opens as a plain file ({reason:?})"), name_taken: None }) } Answer::NeedsImport => Err(Error::Protocol("asked for the import again".into())), Answer::Refused(e) => Err(e), diff --git a/crates/app/src/recents.rs b/crates/app/src/recents.rs index 30552fef..e769abd1 100644 --- a/crates/app/src/recents.rs +++ b/crates/app/src/recents.rs @@ -75,7 +75,7 @@ type Socket = WebSocket; fn refused(event: Wire) -> Error { match event { - Wire::Error { code, message, .. } => Error::Refused { code, message }, + Wire::Error { code, message, .. } => Error::Refused { code, message, name_taken: None }, other => Error::Protocol(format!("unexpected {other:?}")), } } @@ -110,7 +110,7 @@ fn open(client: &Client, app: &str, follow: bool) -> Result<(Socket, (Vec(&body) { - Ok(body) => Error::Refused { code: body.error, message: body.message }, + Ok(body) => Error::Refused { code: body.error, message: body.message, name_taken: body.name_taken }, Err(_) => Error::Protocol(format!("HTTP {status}")), }); } @@ -142,7 +142,7 @@ fn open(client: &Client, app: &str, follow: bool) -> Result<(Socket, (Vec bool { match e { Error::NotRunning(_) | Error::Io(_) => true, - Error::Refused { code, .. } => code == codes::NOT_SIGNED_IN, + Error::Refused { code, .. } | Error::RefusedWith { code, .. } => code == codes::NOT_SIGNED_IN, Error::Version(_) | Error::Protocol(_) => false, } } @@ -332,7 +332,7 @@ impl Link { } Ok(Some(Wire::Error { id: None, code, message })) => { if code == codes::NOT_SIGNED_IN { - break Error::Refused { code, message }; + break Error::Refused { code, message, name_taken: None }; } } Ok(None) => {} diff --git a/crates/kit/src/session.rs b/crates/kit/src/session.rs index 3a55e985..459791be 100644 --- a/crates/kit/src/session.rs +++ b/crates/kit/src/session.rs @@ -506,7 +506,7 @@ impl Session { let file_hash = file_bytes.map(|b| *blake3::hash(b).as_bytes()); let opened = match docs.open_with(path, D::APP, &OpenOptions { file_hash }) { Ok(o) => o, - Err(craft_cloud_app::Error::Refused { code, message }) => return Err(KitError::Agent(format!("{code}: {message}"))), + Err(craft_cloud_app::Error::Refused { code, message, .. }) => return Err(KitError::Agent(format!("{code}: {message}"))), Err(e) => return not_running(e), }; let (document, snapshot) = match opened { diff --git a/crates/kit/src/value.rs b/crates/kit/src/value.rs index faa47539..6d7a0281 100644 --- a/crates/kit/src/value.rs +++ b/crates/kit/src/value.rs @@ -95,7 +95,7 @@ impl BlobStore for AgentBlobs { /// An agent's refusal, its code kept in the message (`agent_too_old: …`). fn agent_error(e: craft_cloud_app::Error) -> KitError { match e { - craft_cloud_app::Error::Refused { code, message } => KitError::Agent(format!("{code}: {message}")), + craft_cloud_app::Error::Refused { code, message, .. } => KitError::Agent(format!("{code}: {message}")), other => KitError::from(other), } } diff --git a/crates/meta/Cargo.toml b/crates/meta/Cargo.toml index 318a1996..72c435c3 100644 --- a/crates/meta/Cargo.toml +++ b/crates/meta/Cargo.toml @@ -14,6 +14,7 @@ serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true tokio.workspace = true +tracing.workspace = true uuid.workspace = true [features] diff --git a/crates/meta/migrations/postgres/0007_devices_v2.sql b/crates/meta/migrations/postgres/0007_devices_v2.sql new file mode 100644 index 00000000..21abd244 --- /dev/null +++ b/crates/meta/migrations/postgres/0007_devices_v2.sql @@ -0,0 +1,14 @@ +-- Device sessions v2. A device now says what kind of client it is (`agent`, `desktop` or `cli`, empty when the +-- client didn't say), on what platform and in what version: shown in the device list and never a basis for any +-- permission. A revoked device keeps when and why it was revoked (`user`, `others`, `password`, `admin`, +-- `disabled`, `two_factor`, `idle` or `reset`), set together with `revoked` and only by the first revoke. The +-- `revoked` flag stays the one the server checks. A user keeps when the password last changed through the +-- server (`NULL` when never) and, for a later release, when the account was disabled (`NULL` while it is not). +-- Existing rows stay valid: the new text columns read as empty and the new times as `NULL`. +ALTER TABLE devices ADD COLUMN client_kind TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN platform TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN app_version TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN revoked_ms BIGINT; +ALTER TABLE devices ADD COLUMN revoke_reason TEXT; +ALTER TABLE users ADD COLUMN password_changed_ms BIGINT; +ALTER TABLE users ADD COLUMN disabled_ms BIGINT; diff --git a/crates/meta/migrations/postgres/0008_admin_resets.sql b/crates/meta/migrations/postgres/0008_admin_resets.sql new file mode 100644 index 00000000..fec1d19f --- /dev/null +++ b/crates/meta/migrations/postgres/0008_admin_resets.sql @@ -0,0 +1,12 @@ +-- Admin API. A row of `invites` now says what it is for: `invite` (a new computer for an account, as before) or +-- `reset` (a password reset code the admin made, good for an hour and redeemed through the same join route). A +-- space may carry a quota of its own that replaces the server-wide cap for that account (`NULL`: the cap applies), +-- and a user may be marked for removal (`removed_ms`, `NULL` while it is not): the account is disabled and left +-- out of the lists, and its data stays until a later release frees it. Existing rows stay valid: every old +-- invite reads as `invite`, and no space has a quota or user a removal mark. `users.auth_gen` counts the changes +-- that end what a sign-in checked before them (a disable, an enable, a removal, a password change or reset): a +-- sign-in that read it before verifying the password creates its device only if it is still the same afterwards. +ALTER TABLE invites ADD COLUMN purpose TEXT NOT NULL DEFAULT 'invite'; +ALTER TABLE spaces ADD COLUMN quota_bytes BIGINT; +ALTER TABLE users ADD COLUMN removed_ms BIGINT; +ALTER TABLE users ADD COLUMN auth_gen BIGINT NOT NULL DEFAULT 0; diff --git a/crates/meta/migrations/postgres/0009_audit_events.sql b/crates/meta/migrations/postgres/0009_audit_events.sql new file mode 100644 index 00000000..62e3a055 --- /dev/null +++ b/crates/meta/migrations/postgres/0009_audit_events.sql @@ -0,0 +1,26 @@ +-- The security audit log. One row per event: when, who did it (`actor_user`, `actor_device`, `NULL` for the command +-- line or a guess at a name nobody has), what (`action`, a name from a closed list in the code), what it was done to +-- (`target_kind`, `target_id`), whether it worked, and a small JSON text of typed values (`details`) that never holds +-- an address, a name of a file or device, a secret or anything a client typed. There are no foreign keys, so an +-- event outlives the account it is about. A failed sign-in is one row per account and minute: `agg_key` names +-- that row and `agg_count` counts the tries in it (rows of other events have a `NULL` key, which a unique index +-- allows many times). Nothing updates or deletes a row except the pruning of old events and that counter. +CREATE TABLE audit_events ( + id BYTEA NOT NULL PRIMARY KEY, + at_ms BIGINT NOT NULL, + actor_user BYTEA, + actor_device BYTEA, + action TEXT NOT NULL, + target_kind TEXT NOT NULL DEFAULT '', + target_id BYTEA, + outcome TEXT NOT NULL, + details TEXT NOT NULL DEFAULT '{}', + agg_key TEXT, + agg_count BIGINT NOT NULL DEFAULT 1 +); + +CREATE INDEX audit_events_at ON audit_events (at_ms); + +CREATE INDEX audit_events_actor ON audit_events (actor_user, at_ms); + +CREATE UNIQUE INDEX audit_events_agg ON audit_events (agg_key); diff --git a/crates/meta/migrations/postgres/0010_two_factor.sql b/crates/meta/migrations/postgres/0010_two_factor.sql new file mode 100644 index 00000000..961a0d28 --- /dev/null +++ b/crates/meta/migrations/postgres/0010_two_factor.sql @@ -0,0 +1,32 @@ +CREATE TABLE user_two_factor ( + user_id BYTEA NOT NULL PRIMARY KEY REFERENCES users(id), + secret_ciphertext BYTEA, + enabled_ms BIGINT, + pending_ciphertext BYTEA, + pending_device BYTEA, + pending_expires_ms BIGINT, + pending_attempts BIGINT NOT NULL DEFAULT 0, + last_accepted_step BIGINT NOT NULL DEFAULT 0, + failed_attempts BIGINT NOT NULL DEFAULT 0, + locked_until_ms BIGINT +); +CREATE TABLE user_recovery_codes ( + user_id BYTEA NOT NULL REFERENCES users(id), + code_hash BYTEA NOT NULL, + used_ms BIGINT, + PRIMARY KEY (user_id, code_hash) +); +CREATE TABLE login_challenges ( + id_hash BYTEA NOT NULL PRIMARY KEY, + user_id BYTEA NOT NULL REFERENCES users(id), + kind TEXT NOT NULL, + client_key TEXT NOT NULL, + device_name TEXT NOT NULL, + client_kind TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + app_version TEXT NOT NULL DEFAULT '', + attempts BIGINT NOT NULL DEFAULT 0, + expires_ms BIGINT NOT NULL +); +CREATE INDEX login_challenges_expiry ON login_challenges (expires_ms); +ALTER TABLE invites ADD COLUMN reset_two_factor BIGINT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/crates/meta/migrations/postgres/0011_web_sessions.sql b/crates/meta/migrations/postgres/0011_web_sessions.sql new file mode 100644 index 00000000..999659b1 --- /dev/null +++ b/crates/meta/migrations/postgres/0011_web_sessions.sql @@ -0,0 +1,18 @@ +-- Browser sessions of the web admin. A session is a random token in a cookie, of which only the BLAKE3 hash is kept +-- here (`id_hash`). `auth_gen` is the account's generation when the session was made, so every change that raises +-- `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off) +-- ends the session with it. `last_seen_ms` is moved at most once a minute and carries the idle expiry, +-- `absolute_expires_ms` the hard one, `revoked_ms` is a logout. `mfa_verified_ms` is when a second factor was given +-- at sign-in (`NULL` for an account without one). There is no password, no token and no address in a row. +CREATE TABLE web_sessions ( + id_hash BYTEA NOT NULL PRIMARY KEY, + user_id BYTEA NOT NULL REFERENCES users(id), + auth_gen BIGINT NOT NULL, + created_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL, + absolute_expires_ms BIGINT NOT NULL, + revoked_ms BIGINT, + mfa_verified_ms BIGINT +); +CREATE INDEX web_sessions_user ON web_sessions (user_id); +CREATE INDEX web_sessions_expiry ON web_sessions (absolute_expires_ms) diff --git a/crates/meta/migrations/postgres/0012_versions_node_seq.sql b/crates/meta/migrations/postgres/0012_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/postgres/0012_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/migrations/postgres/0013_trash.sql b/crates/meta/migrations/postgres/0013_trash.sql new file mode 100644 index 00000000..7fd42245 --- /dev/null +++ b/crates/meta/migrations/postgres/0013_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/migrations/postgres/0014_manifest_index.sql b/crates/meta/migrations/postgres/0014_manifest_index.sql new file mode 100644 index 00000000..340a97ac --- /dev/null +++ b/crates/meta/migrations/postgres/0014_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BYTEA NOT NULL, + ordinal BIGINT NOT NULL, + chunk BYTEA NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BYTEA NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/migrations/sqlite/0007_devices_v2.sql b/crates/meta/migrations/sqlite/0007_devices_v2.sql new file mode 100644 index 00000000..21abd244 --- /dev/null +++ b/crates/meta/migrations/sqlite/0007_devices_v2.sql @@ -0,0 +1,14 @@ +-- Device sessions v2. A device now says what kind of client it is (`agent`, `desktop` or `cli`, empty when the +-- client didn't say), on what platform and in what version: shown in the device list and never a basis for any +-- permission. A revoked device keeps when and why it was revoked (`user`, `others`, `password`, `admin`, +-- `disabled`, `two_factor`, `idle` or `reset`), set together with `revoked` and only by the first revoke. The +-- `revoked` flag stays the one the server checks. A user keeps when the password last changed through the +-- server (`NULL` when never) and, for a later release, when the account was disabled (`NULL` while it is not). +-- Existing rows stay valid: the new text columns read as empty and the new times as `NULL`. +ALTER TABLE devices ADD COLUMN client_kind TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN platform TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN app_version TEXT NOT NULL DEFAULT ''; +ALTER TABLE devices ADD COLUMN revoked_ms BIGINT; +ALTER TABLE devices ADD COLUMN revoke_reason TEXT; +ALTER TABLE users ADD COLUMN password_changed_ms BIGINT; +ALTER TABLE users ADD COLUMN disabled_ms BIGINT; diff --git a/crates/meta/migrations/sqlite/0008_admin_resets.sql b/crates/meta/migrations/sqlite/0008_admin_resets.sql new file mode 100644 index 00000000..fec1d19f --- /dev/null +++ b/crates/meta/migrations/sqlite/0008_admin_resets.sql @@ -0,0 +1,12 @@ +-- Admin API. A row of `invites` now says what it is for: `invite` (a new computer for an account, as before) or +-- `reset` (a password reset code the admin made, good for an hour and redeemed through the same join route). A +-- space may carry a quota of its own that replaces the server-wide cap for that account (`NULL`: the cap applies), +-- and a user may be marked for removal (`removed_ms`, `NULL` while it is not): the account is disabled and left +-- out of the lists, and its data stays until a later release frees it. Existing rows stay valid: every old +-- invite reads as `invite`, and no space has a quota or user a removal mark. `users.auth_gen` counts the changes +-- that end what a sign-in checked before them (a disable, an enable, a removal, a password change or reset): a +-- sign-in that read it before verifying the password creates its device only if it is still the same afterwards. +ALTER TABLE invites ADD COLUMN purpose TEXT NOT NULL DEFAULT 'invite'; +ALTER TABLE spaces ADD COLUMN quota_bytes BIGINT; +ALTER TABLE users ADD COLUMN removed_ms BIGINT; +ALTER TABLE users ADD COLUMN auth_gen BIGINT NOT NULL DEFAULT 0; diff --git a/crates/meta/migrations/sqlite/0009_audit_events.sql b/crates/meta/migrations/sqlite/0009_audit_events.sql new file mode 100644 index 00000000..7b0d86e0 --- /dev/null +++ b/crates/meta/migrations/sqlite/0009_audit_events.sql @@ -0,0 +1,26 @@ +-- The security audit log. One row per event: when, who did it (`actor_user`, `actor_device`, `NULL` for the command +-- line or a guess at a name nobody has), what (`action`, a name from a closed list in the code), what it was done to +-- (`target_kind`, `target_id`), whether it worked, and a small JSON text of typed values (`details`) that never holds +-- an address, a name of a file or device, a secret or anything a client typed. There are no foreign keys, so an +-- event outlives the account it is about. A failed sign-in is one row per account and minute: `agg_key` names +-- that row and `agg_count` counts the tries in it (rows of other events have a `NULL` key, which a unique index +-- allows many times). Nothing updates or deletes a row except the pruning of old events and that counter. +CREATE TABLE audit_events ( + id BLOB NOT NULL PRIMARY KEY, + at_ms BIGINT NOT NULL, + actor_user BLOB, + actor_device BLOB, + action TEXT NOT NULL, + target_kind TEXT NOT NULL DEFAULT '', + target_id BLOB, + outcome TEXT NOT NULL, + details TEXT NOT NULL DEFAULT '{}', + agg_key TEXT, + agg_count BIGINT NOT NULL DEFAULT 1 +); + +CREATE INDEX audit_events_at ON audit_events (at_ms); + +CREATE INDEX audit_events_actor ON audit_events (actor_user, at_ms); + +CREATE UNIQUE INDEX audit_events_agg ON audit_events (agg_key); diff --git a/crates/meta/migrations/sqlite/0010_two_factor.sql b/crates/meta/migrations/sqlite/0010_two_factor.sql new file mode 100644 index 00000000..308e19d9 --- /dev/null +++ b/crates/meta/migrations/sqlite/0010_two_factor.sql @@ -0,0 +1,32 @@ +CREATE TABLE user_two_factor ( + user_id BLOB NOT NULL PRIMARY KEY REFERENCES users(id), + secret_ciphertext BLOB, + enabled_ms BIGINT, + pending_ciphertext BLOB, + pending_device BLOB, + pending_expires_ms BIGINT, + pending_attempts INTEGER NOT NULL DEFAULT 0, + last_accepted_step BIGINT NOT NULL DEFAULT 0, + failed_attempts INTEGER NOT NULL DEFAULT 0, + locked_until_ms BIGINT +); +CREATE TABLE user_recovery_codes ( + user_id BLOB NOT NULL REFERENCES users(id), + code_hash BLOB NOT NULL, + used_ms BIGINT, + PRIMARY KEY (user_id, code_hash) +); +CREATE TABLE login_challenges ( + id_hash BLOB NOT NULL PRIMARY KEY, + user_id BLOB NOT NULL REFERENCES users(id), + kind TEXT NOT NULL, + client_key TEXT NOT NULL, + device_name TEXT NOT NULL, + client_kind TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT '', + app_version TEXT NOT NULL DEFAULT '', + attempts INTEGER NOT NULL DEFAULT 0, + expires_ms BIGINT NOT NULL +); +CREATE INDEX login_challenges_expiry ON login_challenges (expires_ms); +ALTER TABLE invites ADD COLUMN reset_two_factor INTEGER NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/crates/meta/migrations/sqlite/0011_web_sessions.sql b/crates/meta/migrations/sqlite/0011_web_sessions.sql new file mode 100644 index 00000000..a2ee36a0 --- /dev/null +++ b/crates/meta/migrations/sqlite/0011_web_sessions.sql @@ -0,0 +1,18 @@ +-- Browser sessions of the web admin. A session is a random token in a cookie, of which only the BLAKE3 hash is kept +-- here (`id_hash`). `auth_gen` is the account's generation when the session was made, so every change that raises +-- `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off) +-- ends the session with it. `last_seen_ms` is moved at most once a minute and carries the idle expiry, +-- `absolute_expires_ms` the hard one, `revoked_ms` is a logout. `mfa_verified_ms` is when a second factor was given +-- at sign-in (`NULL` for an account without one). There is no password, no token and no address in a row. +CREATE TABLE web_sessions ( + id_hash BLOB NOT NULL PRIMARY KEY, + user_id BLOB NOT NULL REFERENCES users(id), + auth_gen BIGINT NOT NULL, + created_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL, + absolute_expires_ms BIGINT NOT NULL, + revoked_ms BIGINT, + mfa_verified_ms BIGINT +); +CREATE INDEX web_sessions_user ON web_sessions (user_id); +CREATE INDEX web_sessions_expiry ON web_sessions (absolute_expires_ms) diff --git a/crates/meta/migrations/sqlite/0012_versions_node_seq.sql b/crates/meta/migrations/sqlite/0012_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/sqlite/0012_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/migrations/sqlite/0013_trash.sql b/crates/meta/migrations/sqlite/0013_trash.sql new file mode 100644 index 00000000..6da29e88 --- /dev/null +++ b/crates/meta/migrations/sqlite/0013_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = CAST(strftime('%s', 'now') AS BIGINT) * 1000 WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/migrations/sqlite/0014_manifest_index.sql b/crates/meta/migrations/sqlite/0014_manifest_index.sql new file mode 100644 index 00000000..cd491700 --- /dev/null +++ b/crates/meta/migrations/sqlite/0014_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BLOB NOT NULL, + ordinal BIGINT NOT NULL, + chunk BLOB NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BLOB NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/src/admin.rs b/crates/meta/src/admin.rs new file mode 100644 index 00000000..b08fe343 --- /dev/null +++ b/crates/meta/src/admin.rs @@ -0,0 +1,623 @@ +//! What an administrator does through the admin API: server counters, listing and creating accounts, changing a role or +//! a quota, disabling, enabling and removing an account, invites and password reset codes, and a user's devices. +//! +//! - **One `Meta` method per change, with a typed result**, and each writes its audit event in its own transaction, before +//! the commit, so a change whose event cannot be written is not made. A change that changes nothing (disabling an account +//! that is disabled, signing out a device that is out already, giving the role an account has) writes no event. +//! - **The acting admin is checked again inside every change** ([`AdminActor`], `begin_admin`): the role the middleware +//! read and the password the handler verified (a slow Argon2id check) can be stale by the time the transaction runs. +//! The device (or the browser session) must still be signed in, the account neither disabled nor marked for removal, the role still `admin` +//! and the password hash still the one that was verified. +//! - **Lock order** in every change: (1) the `last_admin` transaction lock, if the change can take away an +//! administrator; (2) the rows of the acting admin and of the target, in ascending ID; (3) rows derived from them +//! (devices, invites, pairings). Sign-ins, redeems and pairings take the user's row first as well (see +//! `users::lock_user`), so none of these can deadlock with another, and a disable can never be followed by a device +//! that an earlier check let in. +//! - **The last administrator** cannot be demoted, disabled or removed. A count cannot be locked with `FOR UPDATE`, and +//! locking the rows of two different targets does not serialise two changes that each look at the other's row. So +//! every change that can take an administrator away takes one transaction-wide lock first and counts after it: on +//! Postgres `pg_advisory_xact_lock`, on SQLite the write transaction (`BEGIN IMMEDIATE`, held by one writer at a +//! time) already is that lock. + +use craft_cloud_proto::limits::MAX_CHUNK_BYTES; +use craft_cloud_proto::{DeviceId, NodeId, SpaceId}; +use sqlx::Row; +use uuid::Uuid; + +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; +use crate::codec::{to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::join::{Purpose, put_code}; +use crate::two_factor; +use crate::users::{delete_join_codes, insert_user, lock_user, revoke_devices}; +use crate::web::session_is_live; +use crate::{DeviceRecord, Dialect, Meta, MetaError, RevokeReason, Role, UserId, WriteTx, valid_username}; + +/// The key of the Postgres advisory lock that serialises changes which can take an administrator away. Arbitrary and +/// fixed, and different from the migration lock's. +const LAST_ADMIN_LOCK_KEY: i64 = 7240318150662; + +/// The smallest quota an admin may give an account: one maximal chunk, like the server-wide cap. +pub const MIN_QUOTA_BYTES: u64 = MAX_CHUNK_BYTES as u64; + +/// The longest username search an admin may make, in bytes. +pub const MAX_SEARCH_BYTES: usize = 64; + +/// What an admin's request came with: the device token of the JSON API, or the browser session of the web admin. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum AdminVia { + /// The device the request came with; it must still be signed in. + Device(DeviceId), + /// The hash of the session token the request came with and the idle lifetime of sessions; the session must still be valid + /// when the change runs (not logged out, under the account's current `auth_gen`, inside its absolute lifetime and not idle + /// for `idle_ms`, all judged at the time the change is made, so a request that began before the session ended cannot + /// change anything after). No device acts, so the events it writes name none. + Web { id_hash: [u8; 32], idle_ms: i64 }, +} + +impl std::fmt::Debug for AdminVia { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AdminVia::Device(device) => f.debug_tuple("Device").field(device).finish(), + AdminVia::Web { idle_ms, .. } => f.debug_struct("Web").field("id_hash", &"").field("idle_ms", idle_ms).finish(), + } + } +} + +/// The admin who asks, as the handler knew them when it verified their password. +#[derive(Clone, PartialEq, Eq)] +pub struct AdminActor { + pub user: UserId, + /// How the request signed in; it must still be so. + pub via: AdminVia, + /// The stored password hash the confirmation was verified against; it must still be the account's. + pub password_hash: String, + /// Whether the acting admin's account had two-factor sign-in on when their confirmation was checked + /// ([`Meta::confirm_second_factor`]); it must still be so (turning it on or off meanwhile is `Conflict`). + pub two_factor: bool, +} + +impl std::fmt::Debug for AdminActor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminActor") + .field("user", &self.user) + .field("via", &self.via) + .field("password_hash", &"") + .field("two_factor", &self.two_factor) + .finish() + } +} + +/// An account as the admin lists it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdminUser { + pub id: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + pub disabled_ms: Option, + /// When the account was marked for removal (it is disabled too); the lists leave such accounts out, a single lookup does not. + pub removed_ms: Option, + /// Two-factor sign-in is on. + pub two_factor_enabled: bool, + /// Devices that are not revoked. + pub devices_active: u64, + pub stored_bytes: u64, + /// The quota an admin set for the account (`None`: the server-wide cap, if any, applies). + pub quota_bytes: Option, + /// An unexpired invite is open (a reset code is not an invite). + pub invite_pending: bool, +} + +/// The counters `GET /admin/server` shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServerCounts { + pub users: u64, + pub devices_active: u64, + pub stored_bytes: u64, + pub live_bytes: u64, +} + +/// One page of [`Meta::admin_list_users`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserPage { + pub users: Vec, + /// The username to continue after, when there are more. + pub next_after: Option, +} + +/// A code an admin made: when it stops working. The code itself is never stored, only its hash. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Issued { + pub expires_ms: i64, +} + +/// How long a password reset code works: one hour. +pub const RESET_TTL_MS: i64 = 60 * 60 * 1000; + +struct Target { + role: Role, + disabled: bool, + removed: bool, + has_password: bool, +} + +const USER_SELECT: &str = "SELECT u.id, u.username, u.role, u.created_ms, u.disabled_ms, u.removed_ms, s.stored_bytes, s.quota_bytes, \ + (SELECT COUNT(*) FROM user_two_factor t WHERE t.user_id = u.id AND t.enabled_ms IS NOT NULL) AS two_factor, \ + (SELECT COUNT(*) FROM devices d WHERE d.user_id = u.id AND d.revoked = 0) AS devices_active, \ + (SELECT COUNT(*) FROM invites i WHERE i.user_id = u.id AND i.purpose = 'invite' AND i.expires_ms > $1) AS invites \ + FROM users u JOIN spaces s ON s.owner = u.id"; + +fn admin_user_from(row: &sqlx::any::AnyRow) -> Result { + let quota: Option = row.try_get("quota_bytes")?; + Ok(AdminUser { + id: UserId(uuid_from(&row.try_get::, _>("id")?)?), + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + disabled_ms: row.try_get("disabled_ms")?, + removed_ms: row.try_get("removed_ms")?, + two_factor_enabled: row.try_get::("two_factor")? > 0, + devices_active: to_u64(row.try_get("devices_active")?)?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + quota_bytes: quota.map(to_u64).transpose()?, + invite_pending: row.try_get::("invites")? > 0, + }) +} + +/// `text` as a `LIKE` pattern that matches it literally (`\` is the escape character, see the queries). +fn escape_like(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + if matches!(c, '\\' | '%' | '_') { + out.push('\\'); + } + out.push(c); + } + out +} + +/// An event the acting admin makes to `target`. +fn by_admin(actor: &AdminActor, action: AuditAction, target: UserId) -> NewEvent { + let device = match actor.via { + AdminVia::Device(device) => Some(device), + AdminVia::Web { .. } => None, + }; + NewEvent::new(action).by(actor.user, device).on_user(target) +} + +fn same_user(a: UserId, b: UserId) -> bool { + a.0 == b.0 +} + +impl Meta { + /// Starts the write transaction of a change the admin `actor` makes (to `target`, if it has one): takes the + /// locks in the order the module describes and checks the actor again. `reduces_admins`: the change can take an + /// administrator away, so it takes the `last_admin` lock first. + async fn begin_admin(&self, actor: &AdminActor, target: Option, reduces_admins: bool) -> Result { + let mut w = self.begin_write().await?; + // SQLite: the write transaction already serialises every change, so nothing more is needed there. + if reduces_admins && self.dialect == Dialect::Postgres { + sqlx::query("SELECT pg_advisory_xact_lock($1)").bind(LAST_ADMIN_LOCK_KEY).execute(&mut *w.tx).await?; + } + let mut ids = vec![actor.user]; + if let Some(target) = target + && !same_user(target, actor.user) + { + ids.push(target); + } + ids.sort_by_key(|id| *id.0.as_bytes()); + for id in ids { + // A target that isn't there is the caller's `NotFound`; the actor's own row is checked just below. + lock_user(&mut w.tx, id).await?; + } + let row = sqlx::query("SELECT role, password_hash, disabled_ms, removed_ms FROM users WHERE id = $1") + .bind(uuid_bytes(&actor.user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(row) = row else { return Err(MetaError::DeviceRevoked) }; + let still_signed_in = match actor.via { + AdminVia::Device(device) => sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&actor.user.0)) + .fetch_optional(&mut *w.tx) + .await? + .is_some(), + // The time is read here, after every lock wait above and in the order the module describes, so a session that ended + // while this change waited does not authorise it. + AdminVia::Web { id_hash, idle_ms } => session_is_live(&mut w, actor.user, &id_hash, self.clock_ms(), idle_ms).await?, + }; + if !still_signed_in || row.try_get::, _>("disabled_ms")?.is_some() || row.try_get::, _>("removed_ms")?.is_some() { + return Err(MetaError::DeviceRevoked); + } + if Role::parse(&row.try_get::("role")?)? != Role::Admin { + return Err(MetaError::Forbidden("you are no longer an administrator")); + } + if row.try_get::("password_hash")? != actor.password_hash { + return Err(MetaError::Conflict("your password changed while this request was running; try again")); + } + // The confirmation asked for a second factor, or didn't, by what the account had then. + if two_factor::is_enabled(&mut w.tx, actor.user).await? != actor.two_factor { + return Err(MetaError::Conflict("your two-factor sign-in changed while this request was running; try again")); + } + Ok(w) + } + + async fn target(w: &mut WriteTx, user: UserId) -> Result { + let row = sqlx::query("SELECT role, password_hash, disabled_ms, removed_ms FROM users WHERE id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("user"))?; + Ok(Target { + role: Role::parse(&row.try_get::("role")?)?, + disabled: row.try_get::, _>("disabled_ms")?.is_some(), + removed: row.try_get::, _>("removed_ms")?.is_some(), + has_password: !row.try_get::("password_hash")?.is_empty(), + }) + } + + /// `LastAdmin` unless an administrator who is not disabled and not marked for removal is left, counted in `w` + /// after the `last_admin` lock was taken and the change was made. + async fn keep_an_admin(w: &mut WriteTx) -> Result<(), MetaError> { + let row = sqlx::query("SELECT COUNT(*) AS n FROM users WHERE role = 'admin' AND disabled_ms IS NULL AND removed_ms IS NULL") + .fetch_one(&mut *w.tx) + .await?; + if row.try_get::("n")? < 1 { Err(MetaError::LastAdmin) } else { Ok(()) } + } + + /// The counters of `GET /admin/server`: accounts that are not marked for removal, devices that are not revoked, and + /// the bytes accounts store and their live documents hold. + pub async fn admin_server_counts(&self) -> Result { + let count = |row: sqlx::any::AnyRow, column: &str| -> Result { to_u64(row.try_get(column)?) }; + // Postgres sums BIGINT as NUMERIC, which the `Any` driver can't read: cast back. + let users = sqlx::query("SELECT COUNT(*) AS n FROM users WHERE removed_ms IS NULL").fetch_one(&self.pool).await?; + let devices = sqlx::query("SELECT COUNT(*) AS n FROM devices WHERE revoked = 0").fetch_one(&self.pool).await?; + let bytes = sqlx::query( + "SELECT CAST(COALESCE(SUM(stored_bytes), 0) AS BIGINT) AS stored, CAST(COALESCE(SUM(live_bytes), 0) AS BIGINT) AS live FROM spaces", + ) + .fetch_one(&self.pool) + .await?; + Ok(ServerCounts { + users: count(users, "n")?, + devices_active: count(devices, "n")?, + stored_bytes: to_u64(bytes.try_get("stored")?)?, + live_bytes: to_u64(bytes.try_get("live")?)?, + }) + } + + /// One page of accounts that are not marked for removal, by username ascending: those after `after` (exclusive) + /// whose username contains `search` (case does not matter, `%` and `_` are ordinary characters), at most `limit`. + /// `next_after` is set when there are more. A `search` over [`MAX_SEARCH_BYTES`] is `Invalid`. + pub async fn admin_list_users(&self, after: Option<&str>, search: Option<&str>, limit: u32, now_ms: i64) -> Result { + let search = search.unwrap_or_default(); + if search.len() > MAX_SEARCH_BYTES { + return Err(MetaError::Invalid("the search is at most 64 bytes")); + } + let pattern = format!("%{}%", escape_like(&search.to_ascii_lowercase())); + let sql = + format!("{USER_SELECT} WHERE u.removed_ms IS NULL AND u.username > $2 AND u.username LIKE $3 ESCAPE '\\' ORDER BY u.username LIMIT $4"); + let fetch = i64::from(limit).saturating_add(1); + let rows = sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(now_ms) + .bind(after.unwrap_or_default()) + .bind(pattern) + .bind(fetch) + .fetch_all(&self.pool) + .await?; + let mut users = rows.iter().map(admin_user_from).collect::, _>>()?; + let more = users.len() > limit as usize; + users.truncate(limit as usize); + let next_after = if more { users.last().map(|u| u.username.clone()) } else { None }; + Ok(UserPage { users, next_after }) + } + + /// One account, marked for removal or not; `None` if there is no such user. + pub async fn admin_user(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("{USER_SELECT} WHERE u.id = $2"))) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(admin_user_from).transpose() + } + + /// Makes an account for `username` with `role`, no password and its own space, and an invite under `code_hash` + /// (good for `ttl_ms`) the person redeems to choose a password. A name that is taken, by any account (one marked for + /// removal too), is `UsernameTaken`. + pub async fn admin_create_user( + &self, + actor: &AdminActor, + username: &str, + role: Role, + code_hash: &[u8; 32], + now_ms: i64, + ttl_ms: i64, + ) -> Result<(AdminUser, Issued), MetaError> { + if !valid_username(username) { + return Err(MetaError::Invalid("usernames are 1-64 letters, digits, '.', '_' or '-'")); + } + let username = username.to_ascii_lowercase(); + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the invite's lifetime is too long"))?; + let mut w = self.begin_admin(actor, None, false).await?; + let taken = sqlx::query("SELECT 1 FROM users WHERE username = $1").bind(username.as_str()).fetch_optional(&mut *w.tx).await?; + if taken.is_some() { + return Err(MetaError::UsernameTaken); + } + let ids = (UserId(Uuid::now_v7()), SpaceId::generate(), NodeId::generate()); + let user = insert_user(&mut w.tx, &username, "", role, now_ms, ids).await?.id; + put_code(&mut w.tx, user, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; + let created = by_admin(actor, AuditAction::AdminUserCreated, user).with(AuditDetails { role_to: Some(role), ..AuditDetails::default() }); + record_in(&mut w.tx, now_ms, created).await?; + self.commit_audited(w, now_ms).await?; + let made = self.admin_user(user, now_ms).await?.ok_or(MetaError::Corrupt("a new account is missing"))?; + Ok((made, Issued { expires_ms })) + } + + /// Changes an account's role and/or its quota: `role` `None` leaves the role, `quota` `None` leaves the quota, + /// `Some(None)` removes it, `Some(Some(n))` sets it (at least [`MIN_QUOTA_BYTES`], at most `i64::MAX`). Demoting the + /// last administrator is `LastAdmin`; an account marked for removal is `Conflict`. + pub async fn admin_update_user( + &self, + actor: &AdminActor, + target: UserId, + role: Option, + quota: Option>, + now_ms: i64, + ) -> Result { + if role.is_none() && quota.is_none() { + return Err(MetaError::Invalid("give a role or a quota_bytes to change")); + } + if let Some(Some(bytes)) = quota + && !(MIN_QUOTA_BYTES..=i64::MAX as u64).contains(&bytes) + { + return Err(MetaError::Invalid("quota_bytes is at least one chunk (4194304) and at most 9223372036854775807")); + } + let mut w = self.begin_admin(actor, Some(target), role == Some(Role::Member)).await?; + let found = Self::target(&mut w, target).await?; + if found.removed { + return Err(MetaError::Conflict("this account is marked for removal")); + } + if let Some(role) = role { + sqlx::query("UPDATE users SET role = $1 WHERE id = $2").bind(role.as_str()).bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if found.role == Role::Admin && role == Role::Member { + Self::keep_an_admin(&mut w).await?; + } + // One event for the role and one for the quota when both change, in this transaction. A role the account has + // already is no change and has none. + if role != found.role { + let details = AuditDetails { role_from: Some(found.role), role_to: Some(role), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminRoleChanged, target).with(details)).await?; + } + } + if let Some(quota) = quota { + let quota = quota.map(to_i64).transpose()?; + let before: Option = sqlx::query("SELECT quota_bytes FROM spaces WHERE owner = $1") + .bind(uuid_bytes(&target.0)) + .fetch_optional(&mut *w.tx) + .await? + .map(|row| row.try_get("quota_bytes")) + .transpose()? + .flatten(); + sqlx::query("UPDATE spaces SET quota_bytes = $1 WHERE owner = $2").bind(quota).bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if quota != before { + let details = AuditDetails { quota_bytes: Some(quota.map(to_u64).transpose()?), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminQuotaChanged, target).with(details)).await?; + } + } + self.commit_audited(w, now_ms).await?; + self.admin_user(target, now_ms).await?.ok_or(MetaError::NotFound("user")) + } + + /// Disables an account: sets `disabled_ms`, signs every device out ([`RevokeReason::Disabled`]) and deletes the + /// account's invite, reset code and pairings. Disabling one's own account is `Invalid`, the last enabled + /// administrator `LastAdmin`; an account that is disabled already is left as it is. Returns the devices signed out. + pub async fn admin_disable_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("you can't disable your own account")); + } + let mut w = self.begin_admin(actor, Some(target), true).await?; + let found = Self::target(&mut w, target).await?; + sqlx::query("UPDATE users SET disabled_ms = COALESCE(disabled_ms, $1), auth_gen = auth_gen + 1 WHERE id = $2") + .bind(now_ms) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + Self::keep_an_admin(&mut w).await?; + let revoked = revoke_devices(&mut w.tx, target, None, RevokeReason::Disabled, now_ms).await?; + delete_join_codes(&mut w.tx, target).await?; + // The devices signed out with it are counted in the event. An account that was disabled already is no change. + if !found.disabled { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserDisabled, target).with(AuditDetails::revoked(revoked.len()))).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Enables a disabled account. Devices that were signed out stay signed out. One marked for removal is `Conflict`. + /// An account that is not disabled is left as it is and writes no event. + pub async fn admin_enable_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed { + return Err(MetaError::Conflict("this account is marked for removal")); + } + sqlx::query("UPDATE users SET disabled_ms = NULL, auth_gen = auth_gen + 1 WHERE id = $1") + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + if found.disabled { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserEnabled, target)).await?; + } + self.commit_audited(w, now_ms).await + } + + /// Marks an account for removal: as [`Meta::admin_disable_user`], and also sets `removed_ms`, which leaves it out of + /// the lists and cannot be undone here. Nothing it stored is deleted; its username stays taken. Removing one's + /// own account is `Invalid`, the last enabled administrator `LastAdmin`. Returns the devices signed out. + pub async fn admin_remove_user(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("you can't remove your own account")); + } + let mut w = self.begin_admin(actor, Some(target), true).await?; + let found = Self::target(&mut w, target).await?; + sqlx::query( + "UPDATE users SET removed_ms = COALESCE(removed_ms, $1), disabled_ms = COALESCE(disabled_ms, $1), auth_gen = auth_gen + 1 WHERE id = $2", + ) + .bind(now_ms) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + Self::keep_an_admin(&mut w).await?; + let revoked = revoke_devices(&mut w.tx, target, None, RevokeReason::Disabled, now_ms).await?; + delete_join_codes(&mut w.tx, target).await?; + // The event outlives the account, which stays only as a mark. Marked already: no change, no event. + if !found.removed { + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminUserRemoved, target).with(AuditDetails::revoked(revoked.len()))).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// An invite under `code_hash` for an account that has no password yet (one nobody redeemed an invite for), replacing + /// its open invite or reset code. An account with a password is `Conflict`: an invite adds a computer without asking + /// for the password, which would give whoever holds it the account's files; a reset code is the way in there. A + /// disabled account, or one marked for removal, is `Conflict` too. + pub async fn admin_issue_invite( + &self, + actor: &AdminActor, + target: UserId, + code_hash: &[u8; 32], + now_ms: i64, + ttl_ms: i64, + ) -> Result { + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the invite's lifetime is too long"))?; + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed || found.disabled { + return Err(MetaError::Conflict("this account is disabled or marked for removal")); + } + if found.has_password { + return Err(MetaError::Conflict("this account has a password: make a password reset code for it instead")); + } + put_code(&mut w.tx, target, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminInviteIssued, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Issued { expires_ms }) + } + + /// Deletes the account's open invite or reset code; `NotFound` when there is none (so nothing changed and nothing is + /// written). + pub async fn admin_revoke_invite(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + Self::target(&mut w, target).await?; + let done = sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&target.0)).execute(&mut *w.tx).await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("invite")); + } + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminInviteRevoked, target)).await?; + self.commit_audited(w, now_ms).await + } + + /// A password reset code under `code_hash`, good for `ttl_ms`, replacing the account's open invite or reset code. + /// Redeeming it sets a new password and signs every device of the account out ([`Meta::redeem`]). Nothing changes + /// until then. With `reset_two_factor`, redeeming it also turns the account's two-factor sign-in off; without, an + /// account that has it needs its second factor to redeem the code. Resetting one's own password is `Invalid` (change + /// it), another administrator's `Forbidden` unless the acting admin has two-factor sign-in on themselves. A disabled + /// account, or one marked for removal, is `Conflict`. + pub async fn admin_issue_reset( + &self, + actor: &AdminActor, + target: UserId, + code_hash: &[u8; 32], + reset_two_factor: bool, + now_ms: i64, + ttl_ms: i64, + ) -> Result { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("change your own password in your account instead")); + } + let expires_ms = now_ms.checked_add(ttl_ms).ok_or(MetaError::Invalid("the code's lifetime is too long"))?; + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + if found.removed || found.disabled { + return Err(MetaError::Conflict("this account is disabled or marked for removal")); + } + Self::admin_target_needs_two_factor(&found, actor)?; + put_code(&mut w.tx, target, code_hash, Purpose::Reset, reset_two_factor, now_ms, expires_ms).await?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminPasswordResetIssued, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Issued { expires_ms }) + } + + /// Takes away another account's two-factor sign-in: the secret, the codes and the challenges go, every device is signed + /// out ([`RevokeReason::TwoFactor`]) and the invite and pairings are deleted, in one transaction (the target's lock + /// and the acting admin's checks are as for every admin change). Returns the devices signed out, for the caller to end + /// their live connections. Your own account is `Invalid` (turn it off in your account), an account without two-factor + /// sign-in `Conflict` (nothing is signed out), another administrator's `Forbidden` unless the acting admin has + /// two-factor sign-in on themselves. + pub async fn admin_reset_two_factor(&self, actor: &AdminActor, target: UserId, now_ms: i64) -> Result, MetaError> { + if same_user(actor.user, target) { + return Err(MetaError::Invalid("turn your own two-factor sign-in off in your account instead")); + } + let mut w = self.begin_admin(actor, Some(target), false).await?; + let found = Self::target(&mut w, target).await?; + Self::admin_target_needs_two_factor(&found, actor)?; + let revoked = two_factor::reset_in(&mut w.tx, target, now_ms).await?.ok_or(MetaError::Conflict("this account has no two-factor sign-in"))?; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminTwoFactorReset, target)).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Resetting another **administrator's** password or two-factor sign-in takes the acting admin's own two-factor + /// sign-in: a stolen admin token and password alone can't take over the other admins' accounts. (`begin_admin` has + /// compared `actor.two_factor` with the account's state under the lock, so it is the truth here.) + fn admin_target_needs_two_factor(target: &Target, actor: &AdminActor) -> Result<(), MetaError> { + if target.role == Role::Admin && !actor.two_factor { + return Err(MetaError::Forbidden( + "resetting another administrator needs two-factor sign-in on your own account: turn it on first, or use the command line on the server", + )); + } + Ok(()) + } + + /// All of an account's devices, revoked ones too; `NotFound` for a user that isn't there. + pub async fn admin_devices(&self, target: UserId) -> Result, MetaError> { + let found = sqlx::query("SELECT 1 FROM users WHERE id = $1").bind(uuid_bytes(&target.0)).fetch_optional(&self.pool).await?; + if found.is_none() { + return Err(MetaError::NotFound("user")); + } + self.list_devices(target).await + } + + /// Signs one of `target`'s devices out ([`RevokeReason::Admin`]) and deletes the pairing it made. A device that is + /// revoked already stays as it is. A device of another account, or an unknown user, is `NotFound`. + pub async fn admin_revoke_device(&self, actor: &AdminActor, target: UserId, device: DeviceId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_admin(actor, Some(target), false).await?; + Self::target(&mut w, target).await?; + let done = sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE id = $3 AND user_id = $4 AND revoked = 0") + .bind(now_ms) + .bind(RevokeReason::Admin.as_str()) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&target.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + let owned = sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&target.0)) + .fetch_optional(&mut *w.tx) + .await?; + if owned.is_none() { + return Err(MetaError::NotFound("device")); + } + } + sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; + // The event is about the account, whose own activity then shows it, and names the device. A device signed out + // already is no change. + if done.rows_affected() > 0 { + let details = AuditDetails { device: Some(device), reason: Some(RevokeReason::Admin), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, by_admin(actor, AuditAction::AdminDeviceRevoked, target).with(details)).await?; + } + self.commit_audited(w, now_ms).await + } +} diff --git a/crates/meta/src/audit.rs b/crates/meta/src/audit.rs new file mode 100644 index 00000000..cfd7452d --- /dev/null +++ b/crates/meta/src/audit.rs @@ -0,0 +1,990 @@ +//! The security audit log: events in the `audit_events` table, written in the transaction of the change they record, +//! and read back by the account's own activity, the admin's list and export, and the command line. +//! +//! - **Names are a closed list** ([`AuditAction`]); no event has a name a client typed. +//! - **`details` holds typed values only** ([`AuditDetails`]): every field is a closed enumeration or a number, each +//! event allows only its own fields ([`AuditAction::fields`]), and nothing a client sent as text ever reaches it. In +//! particular `platform` is reduced to [`Platform`] (anything unknown is `other`), because a device may call its +//! platform any printable text of 32 bytes, an address or a file name included. What is written is at most +//! [`MAX_DETAILS_BYTES`] of JSON: when it would be more, whole fields go (the least important first) and +//! `"truncated": true` says so; bytes are never cut off, which would leave invalid JSON. +//! - **No addresses, file or device names, user names of unknown accounts, secrets** are stored anywhere in a row. +//! An account is named by its ID, which outlives the account (there are no foreign keys). +//! - **Changes write their event in their own transaction, before the commit, and fail with it** (fail closed): an event +//! that cannot be written means the change is not made. A failed sign-in changes nothing, so its event is written +//! after the fact in a transaction of its own ([`Meta::audit_record_failure`]), and a failure to write it is for the +//! caller to log: it must never change the answer or undo the abuse protection around it. +//! - **Append-only, in the code.** Nothing here updates or deletes a row, with two named exceptions: the counter of an +//! aggregated failure (one row per account, action, client kind and minute, `agg_count + 1` on a collision) and +//! [`Meta::audit_prune`], which deletes events older than the retention in batches of their own, never inside the +//! transaction of a security change. A test reads this crate's sources and allows exactly those two statements. + +use craft_cloud_proto::DeviceId; +use serde_json::{Map, Value}; +use sqlx::Row; +use uuid::Uuid; + +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::users::{DeviceClient, RevokeReason, Role, UserId}; +use crate::{Meta, MetaError}; + +/// The most serialized UTF-8 bytes of an event's `details`. +pub const MAX_DETAILS_BYTES: usize = 2048; + +/// Rows one pruning statement deletes. +pub const PRUNE_BATCH: i64 = 1000; +/// Batches one opportunistic pruning (after a write) runs before it leaves the rest to the next one. +pub const OPPORTUNISTIC_PRUNE_BATCHES: usize = 20; +/// How often at most a write starts a pruning. +pub const PRUNE_EVERY_MS: i64 = 60 * 60 * 1000; +/// The shortest and longest retention `[audit] retention_days` may name. +pub const MIN_RETENTION_DAYS: u32 = 1; +pub const MAX_RETENTION_DAYS: u32 = 3650; + +const MINUTE_MS: i64 = 60_000; + +/// What an event is. The names are the ones in `docs/design/sync.md`; a name is never reused for something else. +/// +/// Not every name has a writer yet: the two-factor events (`auth.second_factor_failed`, `auth.recovery_code_used`, +/// `auth.two_factor_locked`, `account.two_factor_*`, `account.recovery_codes_regenerated`, `admin.two_factor_reset`, +/// `cli.two_factor_reset`) and `cli.user_removed` are written by the changes that add those operations. The names are +/// listed here so that the list a client may filter by does not change then. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AuditAction { + AuthLogin, + AuthLoginFailed, + AuthSecondFactorFailed, + AuthRecoveryCodeUsed, + AuthTwoFactorLocked, + DeviceRevoked, + DeviceRevokedOthers, + AccountPasswordChanged, + AccountTwoFactorEnabled, + AccountTwoFactorDisabled, + AccountRecoveryCodesRegenerated, + AdminUserCreated, + AdminUserDisabled, + AdminUserEnabled, + AdminUserRemoved, + AdminRoleChanged, + AdminQuotaChanged, + AdminInviteIssued, + AdminInviteRevoked, + AdminPasswordResetIssued, + AdminTwoFactorReset, + AdminDeviceRevoked, + JoinInviteRedeemed, + JoinResetRedeemed, + JoinPairingRedeemed, + CliPasswordReset, + CliTwoFactorReset, + CliDeviceRevoked, + CliDevicesRevokedAll, + CliUserRemoved, +} + +impl AuditAction { + /// Every action, in the order the documentation lists them. + pub const ALL: [AuditAction; 30] = [ + AuditAction::AuthLogin, + AuditAction::AuthLoginFailed, + AuditAction::AuthSecondFactorFailed, + AuditAction::AuthRecoveryCodeUsed, + AuditAction::AuthTwoFactorLocked, + AuditAction::DeviceRevoked, + AuditAction::DeviceRevokedOthers, + AuditAction::AccountPasswordChanged, + AuditAction::AccountTwoFactorEnabled, + AuditAction::AccountTwoFactorDisabled, + AuditAction::AccountRecoveryCodesRegenerated, + AuditAction::AdminUserCreated, + AuditAction::AdminUserDisabled, + AuditAction::AdminUserEnabled, + AuditAction::AdminUserRemoved, + AuditAction::AdminRoleChanged, + AuditAction::AdminQuotaChanged, + AuditAction::AdminInviteIssued, + AuditAction::AdminInviteRevoked, + AuditAction::AdminPasswordResetIssued, + AuditAction::AdminTwoFactorReset, + AuditAction::AdminDeviceRevoked, + AuditAction::JoinInviteRedeemed, + AuditAction::JoinResetRedeemed, + AuditAction::JoinPairingRedeemed, + AuditAction::CliPasswordReset, + AuditAction::CliTwoFactorReset, + AuditAction::CliDeviceRevoked, + AuditAction::CliDevicesRevokedAll, + AuditAction::CliUserRemoved, + ]; + + pub fn as_str(self) -> &'static str { + match self { + AuditAction::AuthLogin => "auth.login", + AuditAction::AuthLoginFailed => "auth.login_failed", + AuditAction::AuthSecondFactorFailed => "auth.second_factor_failed", + AuditAction::AuthRecoveryCodeUsed => "auth.recovery_code_used", + AuditAction::AuthTwoFactorLocked => "auth.two_factor_locked", + AuditAction::DeviceRevoked => "device.revoked", + AuditAction::DeviceRevokedOthers => "device.revoked_others", + AuditAction::AccountPasswordChanged => "account.password_changed", + AuditAction::AccountTwoFactorEnabled => "account.two_factor_enabled", + AuditAction::AccountTwoFactorDisabled => "account.two_factor_disabled", + AuditAction::AccountRecoveryCodesRegenerated => "account.recovery_codes_regenerated", + AuditAction::AdminUserCreated => "admin.user_created", + AuditAction::AdminUserDisabled => "admin.user_disabled", + AuditAction::AdminUserEnabled => "admin.user_enabled", + AuditAction::AdminUserRemoved => "admin.user_removed", + AuditAction::AdminRoleChanged => "admin.role_changed", + AuditAction::AdminQuotaChanged => "admin.quota_changed", + AuditAction::AdminInviteIssued => "admin.invite_issued", + AuditAction::AdminInviteRevoked => "admin.invite_revoked", + AuditAction::AdminPasswordResetIssued => "admin.password_reset_issued", + AuditAction::AdminTwoFactorReset => "admin.two_factor_reset", + AuditAction::AdminDeviceRevoked => "admin.device_revoked", + AuditAction::JoinInviteRedeemed => "join.invite_redeemed", + AuditAction::JoinResetRedeemed => "join.reset_redeemed", + AuditAction::JoinPairingRedeemed => "join.pairing_redeemed", + AuditAction::CliPasswordReset => "cli.password_reset", + AuditAction::CliTwoFactorReset => "cli.two_factor_reset", + AuditAction::CliDeviceRevoked => "cli.device_revoked", + AuditAction::CliDevicesRevokedAll => "cli.devices_revoked_all", + AuditAction::CliUserRemoved => "cli.user_removed", + } + } + + /// The action with this exact name. + pub fn parse(name: &str) -> Option { + Self::ALL.into_iter().find(|action| action.as_str() == name) + } + + /// The `details` fields this event may have; the writer drops any other. Listed in the order of importance, which is + /// the order they are written in and the reverse of the order they are dropped in when the text is too long. + pub fn fields(self) -> &'static [Field] { + use Field::*; + match self { + AuditAction::AuthLogin | AuditAction::JoinInviteRedeemed | AuditAction::JoinPairingRedeemed => &[ClientKind, Platform], + AuditAction::JoinResetRedeemed => &[Revoked, TwoFactorRemoved, ClientKind, Platform], + AuditAction::AuthLoginFailed | AuditAction::AuthSecondFactorFailed => &[ClientKind, Step], + AuditAction::AuthRecoveryCodeUsed => &[ClientKind], + AuditAction::DeviceRevoked => &[Reason], + AuditAction::DeviceRevokedOthers => &[Revoked, Reason], + AuditAction::AdminUserCreated => &[RoleTo], + AuditAction::AdminRoleChanged => &[RoleFrom, RoleTo], + AuditAction::AdminQuotaChanged => &[QuotaBytes], + AuditAction::AdminDeviceRevoked => &[Device, Reason], + AuditAction::CliDeviceRevoked => &[Device, Reason], + AuditAction::AccountPasswordChanged + | AuditAction::AdminUserDisabled + | AuditAction::AdminUserRemoved + | AuditAction::CliPasswordReset + | AuditAction::CliTwoFactorReset + | AuditAction::CliDevicesRevokedAll + | AuditAction::CliUserRemoved => &[Revoked], + AuditAction::AccountTwoFactorEnabled + | AuditAction::AccountTwoFactorDisabled + | AuditAction::AccountRecoveryCodesRegenerated + | AuditAction::AdminUserEnabled + | AuditAction::AdminInviteIssued + | AuditAction::AdminInviteRevoked + | AuditAction::AdminPasswordResetIssued + | AuditAction::AdminTwoFactorReset => &[], + AuditAction::AuthTwoFactorLocked => &[ClientKind], + } + } +} + +/// What a `details` field is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Field { + ClientKind, + Platform, + Reason, + Step, + RoleFrom, + RoleTo, + QuotaBytes, + Revoked, + Device, + /// A reset code redeemed for an account that had two-factor sign-in turned it off. + TwoFactorRemoved, +} + +/// The platforms an event names. A device may call its platform any printable text; only these reach the log. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + Windows, + Macos, + Linux, + Android, + Ios, + Other, +} + +impl Platform { + pub fn as_str(self) -> &'static str { + match self { + Platform::Windows => "windows", + Platform::Macos => "macos", + Platform::Linux => "linux", + Platform::Android => "android", + Platform::Ios => "ios", + Platform::Other => "other", + } + } + + /// The platform a client named: `None` for none at all, one of the list when it is that (case does not matter), + /// `Other` for anything else, whatever it looks like. + pub fn normalize(platform: &str) -> Option { + if platform.is_empty() { + return None; + } + Some( + [Platform::Windows, Platform::Macos, Platform::Linux, Platform::Android, Platform::Ios] + .into_iter() + .find(|known| known.as_str().eq_ignore_ascii_case(platform)) + .unwrap_or(Platform::Other), + ) + } +} + +/// The client kinds an event names. Device kinds are [`crate::CLIENT_KINDS`]; `Web` is assigned only by web sign-in code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientKind { + Agent, + Desktop, + Cli, + Web, +} + +impl ClientKind { + pub fn as_str(self) -> &'static str { + match self { + ClientKind::Agent => "agent", + ClientKind::Desktop => "desktop", + ClientKind::Cli => "cli", + ClientKind::Web => "web", + } + } + + /// Parses device-sent kinds only. Web is assigned by the server, never accepted from a device. + fn parse(kind: &str) -> Option { + [ClientKind::Agent, ClientKind::Desktop, ClientKind::Cli].into_iter().find(|known| known.as_str() == kind) + } +} + +/// How far a failed sign-in got. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Step { + Password, + Totp, + Recovery, + ChallengeExpired, +} + +impl Step { + pub fn as_str(self) -> &'static str { + match self { + Step::Password => "password", + Step::Totp => "totp", + Step::Recovery => "recovery", + Step::ChallengeExpired => "challenge_expired", + } + } +} + +/// The typed values of an event. Which of them an event keeps is decided by [`AuditAction::fields`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AuditDetails { + pub client_kind: Option, + pub platform: Option, + pub reason: Option, + pub step: Option, + pub role_from: Option, + pub role_to: Option, + /// `Some(None)`: the quota was removed. + pub quota_bytes: Option>, + pub revoked: Option, + pub device: Option, + /// Redeeming a reset code also took the account's two-factor sign-in away (only ever set to `true`). + pub two_factor_removed: bool, +} + +impl AuditDetails { + /// What a device says about its client, reduced to the closed lists. + pub fn client(client: &DeviceClient) -> Self { + Self { client_kind: ClientKind::parse(&client.kind), platform: Platform::normalize(&client.platform), ..Self::default() } + } + + pub fn revoked(count: usize) -> Self { + Self { revoked: Some(u32::try_from(count).unwrap_or(u32::MAX)), ..Self::default() } + } + + pub fn reason(reason: RevokeReason) -> Self { + Self { reason: Some(reason), ..Self::default() } + } + + pub fn step(step: Step) -> Self { + Self { step: Some(step), ..Self::default() } + } + + pub fn web() -> Self { + Self { client_kind: Some(ClientKind::Web), ..Self::default() } + } + + /// The value of `field`, if it is set. + fn value(&self, field: Field) -> Option { + match field { + Field::ClientKind => self.client_kind.map(|v| Value::from(v.as_str())), + Field::Platform => self.platform.map(|v| Value::from(v.as_str())), + Field::Reason => self.reason.map(|v| Value::from(v.as_str())), + Field::Step => self.step.map(|v| Value::from(v.as_str())), + Field::RoleFrom => self.role_from.map(|v| Value::from(v.as_str())), + Field::RoleTo => self.role_to.map(|v| Value::from(v.as_str())), + Field::QuotaBytes => self.quota_bytes.map(|v| v.map_or(Value::Null, Value::from)), + Field::Revoked => self.revoked.map(Value::from), + Field::Device => self.device.map(|v| Value::from(v.to_string())), + Field::TwoFactorRemoved => self.two_factor_removed.then_some(Value::Bool(true)), + } + } + + fn name(field: Field) -> &'static str { + match field { + Field::ClientKind => "client_kind", + Field::Platform => "platform", + Field::Reason => "reason", + Field::Step => "step", + Field::RoleFrom => "role_from", + Field::RoleTo => "role_to", + Field::QuotaBytes => "quota_bytes", + Field::Revoked => "revoked", + Field::Device => "device", + Field::TwoFactorRemoved => "two_factor_removed", + } + } + + /// The JSON text stored for `action`: the fields it allows that are set, within [`MAX_DETAILS_BYTES`]. + pub fn to_json(&self, action: AuditAction) -> String { + let fields = action.fields().iter().filter_map(|field| self.value(*field).map(|value| (Self::name(*field), value))).collect(); + serialize_fields(fields, MAX_DETAILS_BYTES) + } +} + +/// `fields` (most important first) as a JSON object of at most `max` bytes. When it would be longer, fields are dropped +/// from the end, one at a time, and `"truncated": true` is added; no text is ever cut short. +pub(crate) fn serialize_fields(mut fields: Vec<(&'static str, Value)>, max: usize) -> String { + let mut truncated = false; + loop { + let mut object = Map::new(); + for (name, value) in &fields { + object.insert((*name).to_owned(), value.clone()); + } + if truncated { + object.insert("truncated".to_owned(), Value::Bool(true)); + } + let text = Value::Object(object).to_string(); + if text.len() <= max { + return text; + } + if fields.pop().is_none() { + // Not reachable for any `max` this crate uses (the flag alone is 18 bytes), but never invalid JSON. + return "{\"truncated\":true}".to_owned(); + } + truncated = true; + } +} + +/// Whether the event is about something that worked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditOutcome { + Success, + Failure, +} + +impl AuditOutcome { + pub fn as_str(self) -> &'static str { + match self { + AuditOutcome::Success => "success", + AuditOutcome::Failure => "failure", + } + } + + fn parse(s: &str) -> Result { + match s { + "success" => Ok(AuditOutcome::Success), + "failure" => Ok(AuditOutcome::Failure), + _ => Err(MetaError::Corrupt("unknown audit outcome")), + } + } +} + +/// What an event was done to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetKind { + User, + Device, +} + +impl TargetKind { + pub fn as_str(self) -> &'static str { + match self { + TargetKind::User => "user", + TargetKind::Device => "device", + } + } +} + +/// Who made a change that a signed-in account or the command line can make, for its event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Origin { + /// The account's own act, from a device the caller does not say. + Account, + /// The account's own act, from this device. + Device(DeviceId), + /// The admin's command line on the server's host: no account acts. + Cli, +} + +/// An event on its way into the log. +#[derive(Debug, Clone, Copy)] +pub(crate) struct NewEvent { + pub action: AuditAction, + pub actor_user: Option, + pub actor_device: Option, + pub target: Option<(TargetKind, Uuid)>, + pub details: AuditDetails, +} + +impl NewEvent { + pub fn new(action: AuditAction) -> Self { + Self { action, actor_user: None, actor_device: None, target: None, details: AuditDetails::default() } + } + + pub fn by(mut self, user: UserId, device: Option) -> Self { + self.actor_user = Some(user); + self.actor_device = device; + self + } + + pub fn on_user(mut self, user: UserId) -> Self { + self.target = Some((TargetKind::User, user.0)); + self + } + + pub fn on_device(mut self, device: DeviceId) -> Self { + self.target = Some((TargetKind::Device, *device.as_uuid())); + self + } + + pub fn with(mut self, details: AuditDetails) -> Self { + self.details = details; + self + } +} + +const INSERT: &str = concat!( + "INSERT INTO audit_events (id, at_ms, actor_user, actor_device, action, target_kind, target_id, outcome, details, agg_key, agg_count) ", + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 1)" +); + +/// Inserts `event` on `conn` (a transaction's connection). `aggregate`: the key of an aggregated failure, whose row +/// only counts up when the key is there already; `None` for every other event, whose key is `NULL`. +async fn insert( + conn: &mut sqlx::AnyConnection, + event: &NewEvent, + outcome: AuditOutcome, + now_ms: i64, + aggregate: Option, +) -> Result<(), MetaError> { + // Qualified: on Postgres a bare `agg_count` is ambiguous between the table's column and `excluded`'s. Only the + // counter changes, so the row keeps the identity, time and details of its first try. + let sql = if aggregate.is_some() { + format!("{INSERT} ON CONFLICT (agg_key) DO UPDATE SET agg_count = audit_events.agg_count + 1") + } else { + INSERT.to_owned() + }; + sqlx::query(sqlx::AssertSqlSafe(sql)) + .bind(uuid_bytes(&Uuid::now_v7())) + .bind(now_ms) + .bind(event.actor_user.map(|u| uuid_bytes(&u.0))) + .bind(event.actor_device.map(|d| uuid_bytes(d.as_uuid()))) + .bind(event.action.as_str()) + .bind(event.target.map_or("", |(kind, _)| kind.as_str())) + .bind(event.target.map(|(_, id)| uuid_bytes(&id))) + .bind(outcome.as_str()) + .bind(event.details.to_json(event.action)) + .bind(aggregate) + .execute(conn) + .await?; + Ok(()) +} + +/// Writes `event` as a success in `tx`, the transaction of the change it records. Its error fails the change. +pub(crate) async fn record_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, now_ms: i64, event: NewEvent) -> Result<(), MetaError> { + insert(tx, &event, AuditOutcome::Success, now_ms, None).await +} + +/// A failed sign-in step, counted by action, client kind and minute. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditFailure { + /// `AuthLoginFailed` or `AuthSecondFactorFailed`; any other action is `Invalid`. + pub action: AuditAction, + /// The account the guess was about; `None` for a name no account has, which is all the log may say of it. + pub target: Option, + pub step: Step, +} + +/// Where a page of the log starts: strictly after (older than) this event, in the order `(at_ms, id)` descending. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AuditCursor { + pub at_ms: i64, + pub id: Uuid, +} + +impl AuditCursor { + pub const BYTES: usize = 24; + + pub fn to_bytes(self) -> [u8; Self::BYTES] { + let mut out = [0u8; Self::BYTES]; + out[..8].copy_from_slice(&self.at_ms.to_be_bytes()); + out[8..].copy_from_slice(self.id.as_bytes()); + out + } + + /// `None` unless `bytes` is exactly what [`AuditCursor::to_bytes`] makes (and a time that can be one). + pub fn from_bytes(bytes: &[u8]) -> Option { + let bytes: &[u8; Self::BYTES] = bytes.try_into().ok()?; + let at_ms = i64::from_be_bytes(bytes.get(..8)?.try_into().ok()?); + let id = Uuid::from_slice(bytes.get(8..)?).ok()?; + (at_ms >= 0).then_some(Self { at_ms, id }) + } +} + +/// Which actions a list is limited to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ActionFilter { + /// This action only. + Exact(AuditAction), + /// The actions whose name starts with this (it ends in a dot, like `auth.`). + Prefix(&'static str), +} + +impl ActionFilter { + /// An exact name, or a prefix ending in a dot that at least one name starts with. Anything else is `None`, so + /// nothing a client typed ever reaches a query. + pub fn parse(text: &str) -> Option { + if let Some(action) = AuditAction::parse(text) { + return Some(ActionFilter::Exact(action)); + } + if !text.ends_with('.') || !text.bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.') { + return None; + } + // The prefix is kept as the `'static` start of the first name it matches: the same characters, from the closed list. + AuditAction::ALL + .iter() + .find(|action| action.as_str().starts_with(text)) + .and_then(|action| action.as_str().get(..text.len())) + .map(ActionFilter::Prefix) + } + + /// The names the filter lets through. + pub fn actions(self) -> Vec { + match self { + ActionFilter::Exact(action) => vec![action], + ActionFilter::Prefix(prefix) => AuditAction::ALL.into_iter().filter(|action| action.as_str().starts_with(prefix)).collect(), + } + } +} + +/// One page of the log, asked for. +#[derive(Debug, Clone, Copy, Default)] +pub struct AuditQuery { + /// Events the account acted in or was the target of. + pub user: Option, + pub action: Option, + /// Only events before this time (the first call of a listing). + pub before_ms: Option, + /// Only events up to and including this time (an export's fixed end). + pub until_ms: Option, + /// Only events strictly after this one in the listing's order. + pub after: Option, + pub limit: u32, +} + +/// An event as read back. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuditEvent { + pub id: Uuid, + pub at_ms: i64, + pub actor_user: Option, + pub actor_device: Option, + pub action: AuditAction, + /// `user`, `device`, or empty. + pub target_kind: String, + pub target_id: Option, + pub outcome: AuditOutcome, + /// A JSON object text, as written. + pub details: String, + /// The tries one aggregated row counts (1 for every other event). + pub count: u64, +} + +impl AuditEvent { + /// Where a page continues after this event. + pub fn cursor(&self) -> AuditCursor { + AuditCursor { at_ms: self.at_ms, id: self.id } + } +} + +/// What a pruning did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Pruned { + pub deleted: u64, + /// There was more to delete than the batches allowed. + pub more: bool, +} + +const SELECT: &str = "SELECT id, at_ms, actor_user, actor_device, action, target_kind, target_id, outcome, details, agg_count FROM audit_events"; + +fn event_from(row: &sqlx::any::AnyRow) -> Result { + let actor_user: Option> = row.try_get("actor_user")?; + let actor_device: Option> = row.try_get("actor_device")?; + let target_id: Option> = row.try_get("target_id")?; + Ok(AuditEvent { + id: uuid_from(&row.try_get::, _>("id")?)?, + at_ms: row.try_get("at_ms")?, + actor_user: actor_user.map(|b| uuid_from(&b).map(UserId)).transpose()?, + actor_device: actor_device.map(|b| uuid_from(&b).map(DeviceId::from_uuid)).transpose()?, + action: AuditAction::parse(&row.try_get::("action")?).ok_or(MetaError::Corrupt("unknown audit action"))?, + target_kind: row.try_get("target_kind")?, + target_id: target_id.map(|b| uuid_from(&b)).transpose()?, + outcome: AuditOutcome::parse(&row.try_get::("outcome")?)?, + details: row.try_get("details")?, + count: to_u64(row.try_get("agg_count")?)?, + }) +} + +enum Arg { + Int(i64), + Bytes(Vec), +} + +/// How long a pruning that is running holds the right to run, and what the last complete one's time was. +#[derive(Debug)] +pub(crate) struct PruneState { + last_done_ms: std::sync::atomic::AtomicI64, + running: std::sync::atomic::AtomicBool, +} + +impl PruneState { + const NEVER: i64 = i64::MIN; + + pub fn new() -> Self { + Self { last_done_ms: std::sync::atomic::AtomicI64::new(Self::NEVER), running: std::sync::atomic::AtomicBool::new(false) } + } +} + +/// Gives the right to run back when dropped, whether the pruning finished, failed or was cancelled. +struct Running<'a>(&'a PruneState); + +impl Drop for Running<'_> { + fn drop(&mut self) { + self.0.running.store(false, std::sync::atomic::Ordering::Release); + } +} + +impl Meta { + /// The store that also prunes events older than `days` days after a write, at most once an hour and in bounded + /// batches ([`Meta::audit_prune_if_due`]). Without it nothing is pruned but by [`Meta::audit_prune`]. + pub fn with_audit_retention_days(mut self, days: u32) -> Self { + self.audit_retention_ms = Some(i64::from(days).saturating_mul(24 * 60 * 60 * 1000)); + self + } + + /// Writes a failed sign-in step in a transaction of its own, after the fact: one row per account (or per minute, for + /// names no account has), action, client kind and minute, whose `count` goes up by one with every further try. The caller logs a + /// failure and goes on: the answer to the client and the limits around the sign-in do not depend on this. + pub async fn audit_record_failure(&self, failure: AuditFailure, now_ms: i64) -> Result<(), MetaError> { + self.audit_record_failure_kind(failure, None, now_ms).await + } + + pub async fn audit_record_web_failure(&self, failure: AuditFailure, now_ms: i64) -> Result<(), MetaError> { + self.audit_record_failure_kind(failure, Some(ClientKind::Web), now_ms).await + } + + async fn audit_record_failure_kind(&self, failure: AuditFailure, kind: Option, now_ms: i64) -> Result<(), MetaError> { + if !matches!(failure.action, AuditAction::AuthLoginFailed | AuditAction::AuthSecondFactorFailed) { + return Err(MetaError::Invalid("only failed sign-in steps are aggregated")); + } + let mut details = AuditDetails::step(failure.step); + details.client_kind = kind; + let mut event = NewEvent::new(failure.action).with(details); + if let Some(user) = failure.target { + event = event.on_user(user); + } + let who = failure.target.map_or_else(|| "-".to_owned(), |u| u.0.simple().to_string()); + let kind = kind.map_or("device", ClientKind::as_str); + let key = format!("{}:{kind}:{who}:{}", failure.action.as_str(), now_ms.div_euclid(MINUTE_MS)); + let mut w = self.begin_write().await?; + insert(&mut w.tx, &event, AuditOutcome::Failure, now_ms, Some(key)).await?; + w.commit().await?; + self.prune_soon(now_ms); + Ok(()) + } + + /// One page of events, newest first, in the order `(at_ms, id)` descending: at most `query.limit` (and never more + /// than 1000). + pub async fn audit_list(&self, query: &AuditQuery) -> Result, MetaError> { + let mut sql = format!("{SELECT} WHERE 1 = 1"); + let mut args = Vec::new(); + let mut next = |arg: Arg| -> usize { + args.push(arg); + args.len() + }; + if let Some(user) = query.user { + let n = next(Arg::Bytes(uuid_bytes(&user.0))); + sql.push_str(&format!(" AND (actor_user = ${n} OR target_id = ${n})")); + } + if let Some(filter) = query.action { + // The names come from the closed list, never from a request, so they are safe to write into the text. + let names: Vec = filter.actions().into_iter().map(|a| format!("'{}'", a.as_str())).collect(); + sql.push_str(&format!(" AND action IN ({})", names.join(", "))); + } + if let Some(before) = query.before_ms { + let n = next(Arg::Int(before)); + sql.push_str(&format!(" AND at_ms < ${n}")); + } + if let Some(until) = query.until_ms { + let n = next(Arg::Int(until)); + sql.push_str(&format!(" AND at_ms <= ${n}")); + } + if let Some(after) = query.after { + let at = next(Arg::Int(after.at_ms)); + let id = next(Arg::Bytes(uuid_bytes(&after.id))); + sql.push_str(&format!(" AND (at_ms < ${at} OR (at_ms = ${at} AND id < ${id}))")); + } + let limit = next(Arg::Int(i64::from(query.limit.min(1000)))); + sql.push_str(&format!(" ORDER BY at_ms DESC, id DESC LIMIT ${limit}")); + let mut q = sqlx::query(sqlx::AssertSqlSafe(sql)); + for arg in args { + q = match arg { + Arg::Int(i) => q.bind(i), + Arg::Bytes(b) => q.bind(b), + }; + } + q.fetch_all(&self.pool).await?.iter().map(event_from).collect() + } + + /// How many events are older than `before_ms` (what [`Meta::audit_prune`] would delete). + pub async fn audit_count_before(&self, before_ms: i64) -> Result { + let row = sqlx::query("SELECT COUNT(*) AS n FROM audit_events WHERE at_ms < $1").bind(before_ms).fetch_one(&self.pool).await?; + to_u64(row.try_get("n")?) + } + + /// Deletes every event older than `before_ms`, in batches of [`PRUNE_BATCH`] rows, each in a transaction of its own. + /// Returns how many went. It is the one way a row is deleted (a test checks the sources). + pub async fn audit_prune(&self, before_ms: i64) -> Result { + let mut total = 0u64; + loop { + let done = self.audit_prune_batches(before_ms, 64).await?; + total = total.saturating_add(done.deleted); + if !done.more { + return Ok(total); + } + } + } + + /// [`Meta::audit_prune`] for at most `max_batches` batches: `more` says whether events older than `before_ms` are + /// left. A batch holds the write lock only for its own statement, so a sign-in never waits for a long pruning. + pub async fn audit_prune_batches(&self, before_ms: i64, max_batches: usize) -> Result { + let mut deleted = 0u64; + for _ in 0..max_batches { + let mut w = self.begin_write().await?; + let done = sqlx::query("DELETE FROM audit_events WHERE id IN (SELECT id FROM audit_events WHERE at_ms < $1 ORDER BY at_ms LIMIT $2)") + .bind(before_ms) + .bind(PRUNE_BATCH) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + deleted = deleted.saturating_add(done.rows_affected()); + if done.rows_affected() < PRUNE_BATCH as u64 { + return Ok(Pruned { deleted, more: false }); + } + } + Ok(Pruned { deleted, more: true }) + } + + /// Prunes events past the retention ([`Meta::with_audit_retention_days`]) if the last complete pruning was an hour or + /// more before `now_ms` (or there was none), in at most [`OPPORTUNISTIC_PRUNE_BATCHES`] batches. The time of the last + /// one is set only when it deleted everything that was due, so a backlog or an error lets the next write go on. + /// Returns whether a pruning ran. Called after the commit of a write, never inside a transaction. + pub async fn audit_prune_if_due(&self, now_ms: i64) -> Result { + self.audit_prune_if_due_in(now_ms, OPPORTUNISTIC_PRUNE_BATCHES).await + } + + /// [`Meta::audit_prune_if_due`] with its own limit of batches, so that a test can make a backlog without thousands of rows. + pub async fn audit_prune_if_due_in(&self, now_ms: i64, max_batches: usize) -> Result { + use std::sync::atomic::Ordering; + let Some(keep_ms) = self.audit_retention_ms else { return Ok(false) }; + let state = &self.prune_state; + let last = state.last_done_ms.load(Ordering::Acquire); + if last != PruneState::NEVER && now_ms.saturating_sub(last) < PRUNE_EVERY_MS { + return Ok(false); + } + if state.running.swap(true, Ordering::AcqRel) { + return Ok(false); + } + let _running = Running(state); + let done = self.audit_prune_batches(now_ms.saturating_sub(keep_ms), max_batches).await?; + if !done.more { + state.last_done_ms.store(now_ms, Ordering::Release); + } + Ok(true) + } + + /// Commits a change that wrote an audit event, then lets the write start a pruning ([`Meta::prune_soon`]). + pub(crate) async fn commit_audited(&self, w: crate::WriteTx, now_ms: i64) -> Result<(), MetaError> { + w.commit().await?; + self.prune_soon(now_ms); + Ok(()) + } + + /// Starts [`Meta::audit_prune_if_due`] in the background, so the write that asked does not wait for it. + pub(crate) fn prune_soon(&self, now_ms: i64) { + if self.audit_retention_ms.is_none() { + return; + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { return }; + let meta = self.clone(); + runtime.spawn(async move { + if let Err(e) = meta.audit_prune_if_due(now_ms).await { + tracing::warn!(error = %e, "couldn't prune old audit events"); + } + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_action_has_a_unique_name_that_parses_back() { + let mut seen = std::collections::HashSet::new(); + for action in AuditAction::ALL { + assert!(seen.insert(action.as_str()), "{} twice", action.as_str()); + assert_eq!(AuditAction::parse(action.as_str()), Some(action)); + assert!(action.as_str().bytes().all(|b| b.is_ascii_lowercase() || b == b'_' || b == b'.')); + } + assert_eq!(seen.len(), 30); + assert_eq!(AuditAction::parse("auth.login "), None); + assert_eq!(AuditAction::parse("AUTH.LOGIN"), None); + } + + #[test] + fn a_filter_is_a_name_or_a_prefix_of_names() { + assert_eq!(ActionFilter::parse("auth.login"), Some(ActionFilter::Exact(AuditAction::AuthLogin))); + let auth = ActionFilter::parse("auth.").unwrap(); + assert_eq!(auth.actions().len(), 5); + assert!(auth.actions().iter().all(|a| a.as_str().starts_with("auth."))); + assert_eq!(ActionFilter::parse("admin.").unwrap().actions().len(), 11); + for bad in ["", "auth", "au.", "auth.%", "auth._", "nope.", "AUTH.", "auth.login'", "auth. ", "auth.logi", "'; DROP TABLE x; --"] { + assert_eq!(ActionFilter::parse(bad), None, "{bad:?}"); + } + } + + #[test] + fn a_platform_is_reduced_to_the_closed_list() { + assert_eq!(Platform::normalize(""), None); + assert_eq!(Platform::normalize("Windows"), Some(Platform::Windows)); + assert_eq!(Platform::normalize("macos"), Some(Platform::Macos)); + // Hostile text a device may legally send: 32 bytes of printable characters or less. + for hostile in [ + "10.0.0.1", + "secret.psd", + "fe80::1", + "\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}", + "windows 11", + "linux\u{7f}", + ] { + assert_eq!(Platform::normalize(hostile), Some(Platform::Other), "{hostile:?}"); + } + } + + #[test] + fn details_keep_only_the_fields_their_event_allows() { + let all = AuditDetails { + client_kind: Some(ClientKind::Agent), + platform: Some(Platform::Linux), + reason: Some(RevokeReason::Admin), + step: Some(Step::Password), + role_from: Some(Role::Member), + role_to: Some(Role::Admin), + quota_bytes: Some(None), + revoked: Some(3), + device: Some(DeviceId::from_uuid(Uuid::nil())), + two_factor_removed: true, + }; + let keys = |action: AuditAction| -> Vec { + let value: Value = serde_json::from_str(&all.to_json(action)).unwrap(); + let mut keys: Vec = value.as_object().unwrap().keys().cloned().collect(); + keys.sort(); + keys + }; + assert_eq!(keys(AuditAction::AuthLogin), ["client_kind", "platform"]); + assert_eq!(keys(AuditAction::AuthLoginFailed), ["client_kind", "step"]); + assert_eq!(keys(AuditAction::AuthRecoveryCodeUsed), ["client_kind"]); + assert_eq!(keys(AuditAction::AdminRoleChanged), ["role_from", "role_to"]); + assert_eq!(keys(AuditAction::AdminQuotaChanged), ["quota_bytes"]); + assert_eq!(keys(AuditAction::AdminUserEnabled), Vec::::new()); + assert_eq!(keys(AuditAction::CliDeviceRevoked), ["device", "reason"]); + assert_eq!(keys(AuditAction::JoinResetRedeemed), ["client_kind", "platform", "revoked", "two_factor_removed"]); + assert_eq!(keys(AuditAction::AccountTwoFactorEnabled), Vec::::new()); + assert_eq!(all.to_json(AuditAction::AdminQuotaChanged), r#"{"quota_bytes":null}"#); + assert_eq!(AuditDetails::revoked(2).to_json(AuditAction::AdminUserDisabled), r#"{"revoked":2}"#); + assert_eq!(AuditDetails::default().to_json(AuditAction::AdminInviteIssued), "{}"); + } + + #[test] + fn details_over_the_limit_lose_whole_fields_and_stay_valid_json() { + // Multi-byte characters, so that cutting bytes anywhere would break the text. + let long = "\u{20ac}".repeat(300); + let fields = vec![ + ("first", Value::from("kept")), + ("second", Value::from(long.clone())), + ("third", Value::from(long.clone())), + ("last", Value::from(long)), + ]; + let text = serialize_fields(fields.clone(), MAX_DETAILS_BYTES); + assert!(text.len() <= MAX_DETAILS_BYTES, "{}", text.len()); + let value: Value = serde_json::from_str(&text).unwrap(); + let object = value.as_object().unwrap(); + assert_eq!(object.get("truncated"), Some(&Value::Bool(true))); + assert_eq!(object.get("first"), Some(&Value::from("kept"))); + // The least important go first, and a field that stays is whole. + assert!(object.get("last").is_none()); + for (name, value) in &fields { + if let Some(kept) = object.get(*name) { + assert_eq!(kept, value); + } + } + // Under the limit nothing is dropped and nothing is flagged. + let small = serialize_fields(vec![("a", Value::from(1))], MAX_DETAILS_BYTES); + assert_eq!(small, r#"{"a":1}"#); + // Even one field over the limit leaves a valid object. + let one = serialize_fields(vec![("only", Value::from("\u{20ac}".repeat(1000)))], MAX_DETAILS_BYTES); + assert_eq!(one, r#"{"truncated":true}"#); + } + + #[test] + fn a_cursor_round_trips_and_refuses_other_bytes() { + let cursor = AuditCursor { at_ms: 1_700_000_000_123, id: Uuid::now_v7() }; + assert_eq!(AuditCursor::from_bytes(&cursor.to_bytes()), Some(cursor)); + assert_eq!(AuditCursor::from_bytes(&[]), None); + assert_eq!(AuditCursor::from_bytes(&cursor.to_bytes()[..23]), None); + let mut longer = cursor.to_bytes().to_vec(); + longer.push(0); + assert_eq!(AuditCursor::from_bytes(&longer), None); + assert_eq!(AuditCursor::from_bytes(&AuditCursor { at_ms: -1, id: Uuid::nil() }.to_bytes()), None); + } +} diff --git a/crates/meta/src/blobs.rs b/crates/meta/src/blobs.rs index 45fbd781..32bab986 100644 --- a/crates/meta/src/blobs.rs +++ b/crates/meta/src/blobs.rs @@ -11,10 +11,10 @@ use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId, SpaceId}; use sqlx::Row; use crate::codec::{hash_from, to_i64, to_u64, uuid_bytes}; -use crate::{Meta, MetaError}; +use crate::{Dialect, Meta, MetaError}; /// How a blob's kind is stored in the `kind` columns. -trait KindCode { +pub(crate) trait KindCode { fn code(self) -> i16; } @@ -41,7 +41,7 @@ async fn space_has(tx: &mut sqlx::AnyConnection, space: SpaceId, kind: BlobKind, const BATCH: usize = 500; /// `$first, $first+1, …` for `count` placeholders. -fn placeholders(first: usize, count: usize) -> String { +pub(crate) fn placeholders(first: usize, count: usize) -> String { (first..first + count).map(|n| format!("${n}")).collect::>().join(", ") } @@ -104,27 +104,42 @@ pub(crate) async fn record(tx: &mut sqlx::AnyConnection, space: SpaceId, blob: B charge(tx, space, size, cap).await } -/// Adds `size` to the space's stored bytes, refusing (`QuotaExceeded`) when that goes over `cap`. On Postgres the -/// update locks the space's row until the transaction ends, so two uploads to one space can't both slip under. +/// Adds `size` to the space's stored bytes, refusing (`QuotaExceeded`) when that goes over the space's limit: its own +/// quota (set by an admin), else `cap`. On Postgres the update locks the space's row until the transaction ends, so two +/// uploads to one space can't both slip under. async fn charge(tx: &mut sqlx::AnyConnection, space: SpaceId, size: i64, cap: Option) -> Result<(), MetaError> { - let total: i64 = sqlx::query("UPDATE spaces SET stored_bytes = stored_bytes + $1 WHERE id = $2 RETURNING stored_bytes") + let row = sqlx::query("UPDATE spaces SET stored_bytes = stored_bytes + $1 WHERE id = $2 RETURNING stored_bytes, quota_bytes") .bind(size) .bind(uuid_bytes(space.as_uuid())) .fetch_one(&mut *tx) - .await? - .try_get("stored_bytes")?; - let total = to_u64(total)?; - match cap { + .await?; + let total = to_u64(row.try_get("stored_bytes")?)?; + let quota: Option = row.try_get("quota_bytes")?; + match effective_quota(quota.map(to_u64).transpose()?, cap) { Some(limit) if total > limit => Err(MetaError::QuotaExceeded { used: total.saturating_sub(to_u64(size)?), limit }), _ => Ok(()), } } +/// The account's quota override, else the server-wide cap. +pub(crate) fn effective_quota(quota: Option, cap: Option) -> Option { + quota.or(cap) +} + /// The bytes `space` stores. async fn stored_bytes(conn: &mut sqlx::AnyConnection, space: SpaceId) -> Result { - let row = sqlx::query("SELECT stored_bytes FROM spaces WHERE id = $1").bind(uuid_bytes(space.as_uuid())).fetch_optional(&mut *conn).await?; + Ok(stored_and_quota(conn, space).await?.0) +} + +/// The bytes `space` stores, and the quota an admin set for it (`None`: the server-wide cap, if any, applies). +async fn stored_and_quota(conn: &mut sqlx::AnyConnection, space: SpaceId) -> Result<(u64, Option), MetaError> { + let row = sqlx::query("SELECT stored_bytes, quota_bytes FROM spaces WHERE id = $1") + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *conn) + .await?; let row = row.ok_or(MetaError::NotFound("space"))?; - to_u64(row.try_get("stored_bytes")?) + let quota: Option = row.try_get("quota_bytes")?; + Ok((to_u64(row.try_get("stored_bytes")?)?, quota.map(to_u64).transpose()?)) } /// The file size recorded for a manifest this space has, if it has it. @@ -157,19 +172,20 @@ impl Meta { stored_bytes(&mut conn, space).await } - /// Whether `space` has room for a blob of `stored_size` bytes under the cap ([`Meta::with_stored_bytes_cap`]): + /// Whether `space` has room for a blob of `stored_size` bytes under its limit: the quota an admin set for the + /// space, else the cap ([`Meta::with_stored_bytes_cap`]), else none (so a quota holds without a cap): /// `QuotaExceeded` if it doesn't have the blob yet and taking it would go over. A cheap look before the bytes are /// stored, so a space over its cap can't fill the disk with blobs that are then refused; recording the blob /// checks again, under the write lock. Two uploads under the cap can both pass this look and one then be refused /// after its bytes are stored: at most a few unrecorded blobs per account (bounded by its body slots), which GC /// (M4) sweeps. pub async fn check_room(&self, space: SpaceId, kind: BlobKind, id: &[u8; 32], stored_size: u64) -> Result<(), MetaError> { - let Some(limit) = self.stored_cap else { return Ok(()) }; let mut conn = self.pool.acquire().await?; + let (used, quota) = stored_and_quota(&mut conn, space).await?; + let Some(limit) = effective_quota(quota, self.stored_cap) else { return Ok(()) }; if space_has(&mut conn, space, kind, id).await? { return Ok(()); } - let used = stored_bytes(&mut conn, space).await?; if used.saturating_add(stored_size) > limit { Err(MetaError::QuotaExceeded { used, limit }) } else { Ok(()) } } @@ -250,6 +266,9 @@ impl Meta { let id = manifest.id(); let blob = Blob { kind: BlobKind::Manifest, id: id.as_bytes(), stored_size: to_i64(stored_size)?, file_size: Some(to_i64(manifest.size())?) }; record(&mut w.tx, space, blob, now_ms, self.stored_cap).await?; + // The index of the manifest's chunks is kept in the same transaction (see `index`): an accepted manifest is never + // without an index row. + crate::index::register(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; w.commit().await } diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index bd8911b9..6828d9a7 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -13,12 +13,16 @@ use std::collections::HashMap; +use craft_cloud_proto::conflict_name; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; -use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; +use craft_cloud_proto::{ + Conflict, ConflictKind, ConflictStamp, DeviceId, Epoch, ManifestId, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId, +}; use sqlx::Row; use crate::blobs::manifest_file_size; -use crate::codec::{flag, to_i64, uuid_bytes, uuid_from}; +use crate::codec::{flag, hash_from, to_i64, uuid_bytes, uuid_from}; +use crate::trash::TRASH_BELOW; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; @@ -29,6 +33,61 @@ pub struct CommitOutcome { pub epoch: Epoch, } +/// A successful restore of an old version: the version it made, the one that was current, and the space's new +/// sequence number and epoch (unchanged by a restore). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RestoreOutcome { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// A node that came back from the trash: where it sits now, whether it took a conflict name, and the space's new +/// sequence number and epoch (unchanged). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashRestoreOutcome { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + /// The file's new version (a copy of its last one); none for a folder. + pub version: Option, + pub seq: u64, + pub epoch: Epoch, +} + +/// The most conflict names tried for one restored node. +const MAX_CONFLICT_COUNTER: u32 = 100; + +fn rejected(kind: ConflictKind) -> MetaError { + MetaError::Rejected(vec![Conflict { op: 0, kind }]) +} + +/// `ms` since the Unix epoch as a UTC minute, for a conflict name (the server doesn't know a user's time zone). +fn conflict_stamp(ms: i64) -> ConflictStamp { + let days = ms.div_euclid(86_400_000); + let minutes = ms.rem_euclid(86_400_000) / 60_000; + // Days to a civil date in the proleptic Gregorian calendar, counted from 0000-03-01. + let z = days.saturating_add(719_468); + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let shifted_month = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * shifted_month + 2) / 5 + 1; + let month = if shifted_month < 10 { shifted_month + 3 } else { shifted_month - 9 }; + let year = year_of_era.saturating_add(era.saturating_mul(400)).saturating_add(i64::from(month <= 2)); + ConflictStamp { + year: u16::try_from(year).unwrap_or(0), + month: u8::try_from(month).unwrap_or(1), + day: u8::try_from(day).unwrap_or(1), + hour: u8::try_from(minutes / 60).unwrap_or(0), + minute: u8::try_from(minutes % 60).unwrap_or(0), + utc: true, + } +} + struct NodeRow { parent: Option, name: String, @@ -116,8 +175,12 @@ impl Ctx<'_> { if deleted { self.failed_restores.clear(); } - sqlx::query("UPDATE nodes SET deleted = $1, changed_seq = $2 WHERE space_id = $3 AND id = $4") + // The time in the trash starts at the change from live to deleted and ends when the node comes back: a later + // deletion starts it afresh, and a pending request to remove the node for good dies with the deletion it was for. + let deleted_ms = deleted.then_some(self.now_ms); + sqlx::query("UPDATE nodes SET deleted = $1, deleted_ms = $2, purge_requested_ms = NULL, changed_seq = $3 WHERE space_id = $4 AND id = $5") .bind(i64::from(deleted)) + .bind(deleted_ms) .bind(self.seq) .bind(self.space()) .bind(uuid_bytes(id.as_uuid())) @@ -344,6 +407,194 @@ impl Ctx<'_> { } } } + /// The first of 100 conflict names for `original` in `parent` that no live node has (the first has no counter, + /// then `, 2` to `, 100`), or `None`. + async fn free_conflict_name(&mut self, parent: NodeId, original: &Name, is_dir: bool, device: &str) -> Result, MetaError> { + let stamp = conflict_stamp(self.now_ms); + for counter in 1..=MAX_CONFLICT_COUNTER { + let candidate = conflict_name(original, is_dir, device, stamp, counter).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + if self.live_child(parent, &candidate).await?.is_none() { + return Ok(Some(candidate)); + } + } + Ok(None) + } + + /// Writes where a deleted node comes back and makes it live: one statement, one sequence number. + async fn revive(&mut self, id: NodeId, parent: NodeId, name: &Name) -> Result<(), MetaError> { + sqlx::query( + "UPDATE nodes SET parent = $1, name = $2, fold_key = $3, deleted = 0, deleted_ms = NULL, purge_requested_ms = NULL, changed_seq = $4 \ + WHERE space_id = $5 AND id = $6", + ) + .bind(uuid_bytes(parent.as_uuid())) + .bind(name.as_str().to_owned()) + .bind(name.fold_key().as_str().to_owned()) + .bind(self.seq) + .bind(self.space()) + .bind(uuid_bytes(id.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + /// Brings a node of the trash back, with the deleted folders above it, root-most first and with their IDs. Where + /// a deleted folder's name is taken by a live folder, that folder is used instead (the deleted one stays deleted) + /// and what is below it moves in. Where a name is taken by a file, or the node's own name is taken by anything, the + /// node comes back under a conflict name. Never overwrites, never creates a node. A conflict that can't be solved + /// is an `Err(MetaError::Rejected)`, and the caller drops the transaction. + async fn restore_from_trash(&mut self, node: NodeId, tree_cap: u64) -> Result { + let found = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await?; + let Some(found) = found else { return Err(MetaError::NotFound("node in the trash")) }; + // A node that was removed from the trash is gone from it, whatever the tombstone still holds. + if !flag(&found, "deleted")? || found.try_get::, _>("purge_requested_ms")?.is_some() { + return Err(MetaError::NotFound("node in the trash")); + } + let is_dir = flag(&found, "is_dir")?; + let device = sqlx::query("SELECT name FROM devices WHERE id = $1") + .bind(uuid_bytes(self.device.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .map(|row| row.try_get::("name")) + .transpose()? + .unwrap_or_default(); + let up = self.ancestors(node).await?; + let deleted_run: Vec<&Ancestor> = up.iter().take_while(|a| a.deleted).collect(); + let Some(base) = up.get(deleted_run.len()) else { + // The walk ended inside the deleted run: the depth bound (too deep) or a deleted root (damaged data). + return if up.last().is_some_and(|a| a.parent.is_some()) { + Err(rejected(ConflictKind::TooDeep)) + } else { + Err(MetaError::Corrupt("the root is deleted")) + }; + }; + let mut parent = base.id; + for (index, a) in deleted_run.iter().enumerate().rev() { + let own = Name::new(&a.name).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let taken = self.live_child(parent, &own).await?; + let is_target = index == 0; + if !is_target && let Some(existing) = taken { + // A live folder of that name stands in for the deleted one; a file there means this folder is renamed. + if self.node(existing).await?.is_some_and(|n| n.is_dir) { + parent = existing; + continue; + } + } + let final_name = match taken { + None => own, + Some(existing) => { + let folder = !is_target || is_dir; + match self.free_conflict_name(parent, &own, folder, &device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + } + } + }; + // A folder comes back where it did not stand before (below a live folder that has moved, or merged into + // another): it must not take the tree past its depth limit. A file may sit in the deepest folder. + if (!is_target || is_dir) + && let Err(kind) = self.check_placement(parent, None).await? + { + return Err(rejected(kind)); + } + self.revive(a.id, parent, &final_name).await?; + parent = a.id; + } + // A file comes back as a new version with the content of its last one. A machine that sent the delete and + // never heard the answer still names the old version in it, and so can't delete the restored file again: the + // server refuses that delete as a version mismatch, and the machine's engine already treats a node that came + // back at another version as one whose delete went through. + let version = if is_dir { None } else { Some(self.renew_version(node).await?) }; + if is_dir { + self.restore_below(node, tree_cap, &device).await?; + } + let placed = self.node(node).await?.ok_or(MetaError::Corrupt("a restored node vanished"))?; + let parent = placed.parent.ok_or(MetaError::Corrupt("a restored node has no parent"))?; + let original = up.first().map(|a| a.name.as_str()).unwrap_or_default(); + Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, version, seq: 0, epoch: self.epoch }) + } + + /// Brings back what is in the trash below the folder `top`, which is live again: its deleted children and theirs, + /// whenever they were deleted, at most `cap` of them, shallowest first (then newest deletion first, then by ID), so a + /// node never comes back without its folder. The rest stays in the trash; its folder is live or still in the trash + /// under a live one, so it is listed on its own and a further restore brings it back. A child keeps its name; one + /// whose name another child that came back took (a name deleted twice, the newest deletion first) gets a conflict + /// name. A removed node, and what is only below a removed folder, stays deleted. The work done row by row under the + /// space's write lock is bounded by `cap`. + async fn restore_below(&mut self, top: NodeId, cap: u64, device: &str) -> Result<(), MetaError> { + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let limit = i64::try_from(cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} SELECT n.id, n.parent, n.name, n.is_dir FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id \ + ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $4" + ))) + .bind(self.space()) + .bind(uuid_bytes(top.as_uuid())) + .bind(bound) + .bind(limit) + .fetch_all(&mut *self.conn) + .await?; + for row in &rows { + let id = NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?); + let parent = NodeId::from_uuid(uuid_from(&row.try_get::, _>("parent")?)?); + let is_dir = flag(row, "is_dir")?; + let own = Name::new(&row.try_get::("name")?).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let name = match self.live_child(parent, &own).await? { + None => own, + Some(existing) => match self.free_conflict_name(parent, &own, is_dir, device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + }, + }; + if is_dir && let Err(kind) = self.check_placement(parent, None).await? { + return Err(rejected(kind)); + } + self.revive(id, parent, &name).await?; + if !is_dir { + self.renew_version(id).await?; + } + } + Ok(()) + } + + /// Gives the file `node` a new current version with the manifest, modification time and executable bit of its last + /// one (no save record), after it. Returns the new version's ID. + async fn renew_version(&mut self, node: NodeId) -> Result { + let row = sqlx::query( + "SELECT n.version AS last, v.manifest, v.mtime_ms, v.exec FROM nodes n \ + JOIN versions v ON v.space_id = n.space_id AND v.id = n.version WHERE n.space_id = $1 AND n.id = $2", + ) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .ok_or(MetaError::Corrupt("a file has no current version"))?; + let last = VersionId::from_uuid(uuid_from(&row.try_get::, _>("last")?)?); + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&row.try_get::, _>("manifest")?)?), + mtime_ms: row.try_get("mtime_ms")?, + exec: flag(&row, "exec")?, + save: None, + }; + match self.insert_version(node, Some(last), &new).await? { + Ok(()) => {} + // The space no longer holds the manifest: the content is not there to bring back. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(rejected(kind)), + } + sqlx::query("UPDATE nodes SET version = $1 WHERE space_id = $2 AND id = $3") + .bind(uuid_bytes(new.id.as_uuid())) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(new.id) + } + /// `node` and its ancestors, nearest first; stops after `MAX_TREE_DEPTH + 2` rows, so a walk that did not /// reach the root (last parent is `Some`) is too deep. async fn ancestors(&mut self, node: NodeId) -> Result, MetaError> { @@ -417,6 +668,84 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } + /// Brings the node `node` of the trash back in `space`, as one change of the space (one sequence number, all or + /// nothing): see `Ctx::restore_from_trash` for the rules. A folder brings back what is in the trash below it too. `NotFound` for a node that is not in the trash (live, + /// unknown, another space's, or removed from the trash); `Rejected` (`NameTaken`) when no conflict name is free. + pub async fn restore_from_trash(&self, space: SpaceId, device: DeviceId, node: NodeId, now_ms: i64) -> Result { + let mut w = self.begin_write().await?; + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let outcome = ctx.restore_from_trash(node, self.trash_tree_cap).await?; + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(TrashRestoreOutcome { seq, ..outcome }) + } + + /// Makes `source`, an older version of the file `node`, its content again as a new version: an ordinary + /// `Update` after `expected`, which must still be the current version. The new version has a fresh ID, copies the + /// manifest, modification time and executable bit of `source`, and gets no save record (a save record names a + /// state of a live document, and a restore changes the file only). The epoch stays. Everything is checked under the + /// space's write lock, in the transaction that writes the version. + /// + /// `NotFound` for a node or source version the space does not have (a source of another file included) or whose + /// manifest the space no longer holds. `Invalid` for a directory, a deleted file, or a file that has a live + /// document. `Rejected` (a `VersionMismatch` for operation 0) when `expected` is no longer current. + pub async fn restore_version( + &self, + space: SpaceId, + device: DeviceId, + node: NodeId, + source: VersionId, + expected: VersionId, + now_ms: i64, + ) -> Result { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the restore with every other commit and save of the space. + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let row = ctx.node(node).await?.ok_or(MetaError::NotFound("node"))?; + if row.is_dir { + return Err(MetaError::Invalid("only files have versions to restore")); + } + if row.deleted { + return Err(MetaError::Invalid("the file is deleted: bring it back with POST /trash/{id}/restore first, then restore a version")); + } + let live = sqlx::query("SELECT 1 FROM live_docs WHERE space_id = $1 AND node = $2 LIMIT 1") + .bind(ctx.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await?; + if live.is_some() { + return Err(MetaError::Invalid("this file has a live document: restoring a version applies to plain file versions only")); + } + let found = sqlx::query("SELECT manifest, mtime_ms, exec FROM versions WHERE space_id = $1 AND id = $2 AND node = $3") + .bind(ctx.space()) + .bind(uuid_bytes(source.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await? + .ok_or(MetaError::NotFound("version"))?; + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&found.try_get::, _>("manifest")?)?), + mtime_ms: found.try_get("mtime_ms")?, + exec: flag(&found, "exec")?, + save: None, + }; + let version = new.id; + match ctx.apply(&Op::Update { node, expected, version: new }).await? { + Ok(()) => {} + // The space holds no manifest of that ID any more: the data is not there to restore. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(MetaError::Rejected(vec![Conflict { op: 0, kind }])), + } + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(RestoreOutcome { version, parent_version: expected, seq, epoch: info.epoch }) + } + /// Adds a save record to `node`'s version `version`, which went up without one (the app's record reached the /// agent only after the upload: it was saved while the agent was away). Appended only: a version that has a /// record keeps it, and the same call again changes nothing. When it is added, the node is in the change feed @@ -459,3 +788,25 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_millisecond_count_reads_as_a_utc_minute() { + let at = |ms: i64| { + let s = conflict_stamp(ms); + (s.year, s.month, s.day, s.hour, s.minute, s.utc) + }; + assert_eq!(at(0), (1970, 1, 1, 0, 0, true)); + assert_eq!(at(1_760_000_000_000), (2025, 10, 9, 8, 53, true)); + // A leap day, and the last minute of a year. + assert_eq!(at(1_709_164_800_000 + 59 * 60_000), (2024, 2, 29, 0, 59, true)); + assert_eq!(at(1_767_225_540_000), (2025, 12, 31, 23, 59, true)); + // Before the epoch and absurd values do not panic. + assert_eq!(at(-1).0, 1969); + let _ = conflict_stamp(i64::MAX); + let _ = conflict_stamp(i64::MIN); + } +} diff --git a/crates/meta/src/history.rs b/crates/meta/src/history.rs new file mode 100644 index 00000000..2c0dd98b --- /dev/null +++ b/crates/meta/src/history.rs @@ -0,0 +1,82 @@ +//! A file's version history: the versions of one file, newest first, a page at a time. +//! +//! Versions are never removed, so every row of `versions` for the node is reported. The `pruned` flag of the answer is +//! part of the contract for the day versions can be thinned; until then it is always false. + +use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; +use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; +use sqlx::Row; + +use crate::codec::{flag, to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::{Meta, MetaError}; + +/// One page of a file's history. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HistoryPage { + pub versions: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, +} + +impl Meta { + /// One page of the versions of the file `node` in `space`, ordered by `(seq, id)` descending. `after` is the + /// previous page's `next` (exclusive). `NotFound` when the space has no such node; `Invalid` for a directory or + /// a `limit` outside 1 to [`HISTORY_MAX_LIMIT`]. A deleted file still has a history. Each call reads one snapshot. + pub async fn history(&self, space: SpaceId, node: NodeId, after: Option, limit: u32) -> Result { + if limit == 0 || limit > HISTORY_MAX_LIMIT { + return Err(MetaError::Invalid("a history page has 1 to 200 versions")); + } + let mut tx = self.read_snapshot().await?; + let found = sqlx::query("SELECT is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("node"))?; + if flag(&found, "is_dir")? { + return Err(MetaError::Invalid("only files have a history")); + } + let (cursor_seq, cursor_version) = match after { + Some(cursor) => (to_i64(cursor.seq)?, uuid_bytes(cursor.version.as_uuid())), + // Above every sequence number and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The cursor is exclusive. The row-value + // comparison and the order match the index `versions_node_seq`, so the database seeks to the cursor and stops + // after the page. + let rows = sqlx::query( + "SELECT id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, \ + CAST(CASE WHEN save IS NULL THEN 0 ELSE 1 END AS BIGINT) AS has_save \ + FROM versions WHERE space_id = $1 AND node = $2 AND (seq, id) < ($3, $4) \ + ORDER BY seq DESC, id DESC LIMIT $5", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(cursor_seq) + .bind(cursor_version) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut versions = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent_version")?; + versions.push(HistoryVersion { + id: VersionId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent_version: parent.as_deref().map(uuid_from).transpose()?.map(VersionId::from_uuid), + epoch: Epoch::from_uuid(uuid_from(&row.try_get::, _>("epoch")?)?), + seq: to_u64(row.try_get("seq")?)?, + size: to_u64(row.try_get("size")?)?, + mtime_ms: row.try_get("mtime_ms")?, + device: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("device")?)?), + created_ms: row.try_get("created_ms")?, + save: flag(row, "has_save")?, + pruned: false, + }); + } + let next = if more { versions.last().map(|v| HistoryCursor { seq: v.seq, version: v.id }) } else { None }; + Ok(HistoryPage { versions, next }) + } +} diff --git a/crates/meta/src/index.rs b/crates/meta/src/index.rs new file mode 100644 index 00000000..8d8bcf95 --- /dev/null +++ b/crates/meta/src/index.rs @@ -0,0 +1,640 @@ +//! The reference index: which chunks each manifest names, kept in the database next to the manifests themselves. +//! +//! A manifest is a file's content: a list of chunks. The list lives in the manifest's object in the store, which is +//! the authority; this index is a copy of it that the database can query ("which chunks does this manifest need", +//! "which manifests use this chunk"). Because it is a copy it is only trusted when it is *complete*: every manifest has +//! a row in `manifest_index`, and its state is 1 only after the whole list of edges was compared, ordinal by ordinal, +//! with the manifest it was made from. A row count is never enough. +//! +//! The index is kept in the transaction that records a manifest (`Meta::add_manifest`): up to a budget of rows per +//! transaction. A longer manifest is recorded as being built, and the builder ([`Meta::index_complete`]) writes the +//! rest in further transactions of at most that many rows, so no one holds the write lock for a million rows. The same +//! builder indexes manifests that existed before the index did; it resumes where an interrupted run stopped. +//! +//! When the edges of a manifest do not match it, they are thrown away and written again, in batches of the same size +//! (state 2, "resetting", marks it): an index of a million edges is never deleted in one transaction, and a run that stops +//! between two batches is resumed by the next stage. Rows are compared from the lowest ordinal there is, so an edge at a +//! negative ordinal is found too. +//! +//! Nothing here frees anything. The index is read by the checker (`verify`) and by whatever later decides what is +//! garbage; until it says a manifest is complete, that manifest's chunks are unknown, and unknown means kept. + +use std::collections::HashMap; + +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use sqlx::Row; + +use crate::blobs::{KindCode, placeholders}; +use crate::codec::{hash_from, to_i64, to_u64}; +use crate::{Dialect, Meta, MetaError}; + +/// Edges one transaction writes by default (and how many rows a manifest may have to be indexed whole in the +/// transaction that records it). +pub const INDEX_BATCH_ROWS: usize = 5_000; + +/// Edges per `INSERT` statement. +const ROWS_PER_STATEMENT: usize = 500; + +/// How often a builder starts over for one manifest whose edges do not match it before giving up. +const REBUILDS: usize = 2; + +const BUILDING: i64 = 0; +const COMPLETE: i64 = 1; +/// The edges are being thrown away to start the index again, a batch of rows to a transaction. Nothing is written +/// and nothing is marked complete until the last edge is gone and the state is back to [`BUILDING`]. A run that stops +/// halfway leaves this marker, and the next one carries on from the rows that are left. +const RESETTING: i64 = 2; + +/// What the database records of a blob: its ID, the bytes it is stored as (the format byte included) and, for a +/// manifest, the size of its file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobRecord { + pub id: [u8; 32], + pub stored_size: u64, + pub file_size: Option, +} + +/// How far a manifest's index has come. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexState { + /// No row: the manifest was recorded before the index existed and nothing has indexed it yet. + Absent, + /// Some edges are written (`done` of `total`), or all of them but not yet compared with the manifest. + Building { done: u64, total: u64 }, + /// Every edge was compared with the manifest. + Complete { total: u64 }, +} + +/// What a comparison of a manifest's stored edges with the manifest found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeCheck { + /// The edges are exactly the manifest's chunks, in order, nothing more. + Exact, + /// The first place they differ. + Mismatch(EdgeProblem), +} + +/// Where and how stored edges differ from their manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EdgeProblem { + pub ordinal: u64, + pub kind: EdgeProblemKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeProblemKind { + /// The edge at the ordinal names another chunk than the manifest does there. + WrongChunk, + /// The manifest has a chunk at the ordinal that has no edge. + MissingEdge, + /// There is an edge at an ordinal the manifest has no chunk for. + ExtraEdge, + /// There is an edge at a negative ordinal, which no manifest has. The ordinal of the problem is its absolute value. + NegativeOrdinal, +} + +impl EdgeProblemKind { + pub fn as_str(self) -> &'static str { + match self { + EdgeProblemKind::WrongChunk => "wrong_chunk", + EdgeProblemKind::MissingEdge => "missing_edge", + EdgeProblemKind::ExtraEdge => "extra_edge", + EdgeProblemKind::NegativeOrdinal => "negative_ordinal", + } + } +} + +/// The result of checking a manifest's edges and, when they are exact, marking its index complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Validation { + Complete, + Mismatch(EdgeProblem), + /// Another builder changed the index meanwhile; look again. + Raced, +} + +/// What the database alone says about the index (the checker adds what only the store can tell). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IndexAudit { + /// Recorded chunks (`blobs` of kind chunk). + pub chunks: u64, + /// Stored bytes of chunks no manifest index edge names. + pub unreferenced_chunk_bytes: u64, + /// Manifests recorded (`blobs` of kind manifest), and how many of them are in each state of the index. + pub manifests: u64, + pub complete: u64, + pub building: u64, + pub absent: u64, + /// Rows of `manifest_index` whose manifest is not recorded. + pub index_without_manifest: Finding, + /// Manifests that have edges but no `manifest_index` row. + pub edges_without_index: Finding, + /// Complete manifests whose number of edges is not the number of chunks they were indexed with. + pub complete_with_wrong_count: Finding, + /// Edges that name a chunk that is not recorded: (manifest, chunk). + pub unknown_dependencies: Finding, + /// Versions and snapshots that name a manifest that is not recorded. + pub roots_without_manifest: Finding, + /// Manifests named by a version or snapshot whose index is not complete. + pub roots_not_indexed: Finding, + /// Recorded manifests that no version, snapshot or import snapshot names. These may be live-document assets, whose + /// references sit inside the documents (the server cannot read them); they are reported, never judged. + pub unrooted_manifests: Finding, + /// Recorded chunks no complete manifest names (only meaningful once every manifest is complete). + pub unreferenced_chunks: Finding, + /// Space memberships of a blob that is not recorded. + pub space_blobs_without_blob: Finding, +} + +/// A count and a few examples (hex IDs, `manifest` or `manifest/chunk`). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Finding { + pub count: u64, + pub examples: Vec, +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A row of `manifest_index`. +struct IndexRow { + chunk_count: i64, + indexed: i64, + state: i64, +} + +async fn read_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, for_update: bool) -> Result, MetaError> { + let sql = if for_update { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1 FOR UPDATE" + } else { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1" + }; + let row = sqlx::query(sql).bind(id.as_bytes().to_vec()).fetch_optional(&mut *conn).await?; + row.map(|r| Ok(IndexRow { chunk_count: r.try_get("chunk_count")?, indexed: r.try_get("indexed_ordinals")?, state: r.try_get("state")? })) + .transpose() +} + +/// The row of a manifest, made (as being built, nothing written) if there is none. On Postgres the row stays locked until the +/// transaction ends, so two builders of one manifest take turns. +async fn ensure_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64, for_update: bool) -> Result { + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, $2, 0, 0) ON CONFLICT DO NOTHING") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + read_state(conn, id, for_update).await?.ok_or(MetaError::Corrupt("an index row vanished")) +} + +/// Marks a manifest's index as starting again: the row takes the manifest's chunk count, writes nothing, and is not +/// complete (the edges are derived: the manifest is the authority). The edges themselves go in [`reset_step`]. +async fn begin_reset(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result<(), MetaError> { + sqlx::query("UPDATE manifest_index SET chunk_count = $1, indexed_ordinals = 0, state = $2 WHERE manifest = $3") + .bind(total) + .bind(RESETTING) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Throws away at most `budget` of the edges of a manifest whose index starts again, the lowest ordinals first (a +/// negative one included), so no transaction holds the write lock for the up to a million rows a manifest can have. +/// Returns whether none are left: then the row is building again. Fewer rows deleted than the budget means none are. +async fn reset_step(conn: &mut sqlx::AnyConnection, id: &ManifestId, budget: usize) -> Result { + let limit = i64::try_from(budget.max(1)).map_err(|_| MetaError::Invalid("batch too large"))?; + let deleted = sqlx::query( + "DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal IN (SELECT ordinal FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal LIMIT $2)", + ) + .bind(id.as_bytes().to_vec()) + .bind(limit) + .execute(&mut *conn) + .await? + .rows_affected(); + let finished = i64::try_from(deleted).is_ok_and(|n| n < limit); + if finished { + sqlx::query("UPDATE manifest_index SET state = $1 WHERE manifest = $2 AND state = $3") + .bind(BUILDING) + .bind(id.as_bytes().to_vec()) + .bind(RESETTING) + .execute(&mut *conn) + .await?; + } + Ok(finished) +} + +/// Writes the edges for the ordinals `from..to` of `manifest`, 500 to a statement. Edges already there are left alone +/// (the comparison catches one that is wrong). +async fn write_edges(conn: &mut sqlx::AnyConnection, manifest: &Manifest, from: usize, to: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let chunks = manifest.chunks().get(from..to).ok_or(MetaError::Corrupt("an index range outside its manifest"))?; + for (part_index, part) in chunks.chunks(ROWS_PER_STATEMENT).enumerate() { + let rows = (0..part.len()).map(|i| format!("({})", placeholders(i * 3 + 1, 3))).collect::>().join(", "); + let mut query = + sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES {rows} ON CONFLICT DO NOTHING"))); + for (i, chunk) in part.iter().enumerate() { + let ordinal = from + .checked_add(part_index.saturating_mul(ROWS_PER_STATEMENT)) + .and_then(|n| n.checked_add(i)) + .ok_or(MetaError::Corrupt("ordinal overflow"))?; + query = query.bind(id.as_bytes().to_vec()).bind(to_i64(ordinal as u64)?).bind(chunk.id.as_bytes().to_vec()); + } + query.execute(&mut *conn).await?; + } + Ok(()) +} + +/// Compares the stored edges of `manifest`, a page of `page` rows at a time, with its chunks. +async fn compare(conn: &mut sqlx::AnyConnection, manifest: &Manifest, page: usize) -> Result { + let id = manifest.id(); + let chunks = manifest.chunks(); + let limit = i64::try_from(page.max(1)).map_err(|_| MetaError::Invalid("page too large"))?; + // Pagination starts at the lowest number there is, so an edge at a negative ordinal is not skipped. + let (mut next, mut from): (usize, i64) = (0, i64::MIN); + loop { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 AND ordinal >= $2 ORDER BY ordinal LIMIT $3") + .bind(id.as_bytes().to_vec()) + .bind(from) + .bind(limit) + .fetch_all(&mut *conn) + .await?; + for row in &rows { + let raw: i64 = row.try_get("ordinal")?; + if raw < 0 { + // Rows come lowest first, so a negative ordinal is the first thing seen: no manifest has such an edge. + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: raw.unsigned_abs(), kind: EdgeProblemKind::NegativeOrdinal })); + } + let ordinal = to_u64(raw)?; + let chunk = hash_from(&row.try_get::, _>("chunk")?)?; + let expected = next as u64; + if ordinal > expected { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: expected, kind: EdgeProblemKind::MissingEdge })); + } + match chunks.get(next) { + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + Some(c) if *c.id.as_bytes() != chunk => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::WrongChunk })), + Some(_) => {} + } + next += 1; + from = match raw.checked_add(1) { + Some(n) => n, + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + }; + } + if rows.len() < page.max(1) { + break; + } + } + if next < chunks.len() { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: next as u64, kind: EdgeProblemKind::MissingEdge })); + } + Ok(EdgeCheck::Exact) +} + +/// Marks a manifest's index complete if it is still as it was when its edges were compared (`indexed_ordinals` and +/// `chunk_count` both `total`, state still 0). Returns whether it did. +async fn mark_complete(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result { + let done = sqlx::query("UPDATE manifest_index SET state = 1 WHERE manifest = $1 AND state = 0 AND chunk_count = $2 AND indexed_ordinals = $2") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + Ok(done.rows_affected() == 1) +} + +/// Writes the next batch of edges of `manifest` (at most `budget`) and moves the frontier. Returns the state after it, +/// and whether the index is still being thrown away to start again (then this step only deleted edges, at most +/// `budget` of them, and `done` says nothing about edges being written). +pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(IndexState, bool), MetaError> { + let id = manifest.id(); + let total = manifest.chunks().len(); + let total_i = to_i64(total as u64)?; + let row = ensure_state(conn, &id, total_i, pg).await?; + // A row that does not belong to this manifest's list starts again: the manifest is the authority. + let foreign = row.chunk_count != total_i || row.indexed < 0 || row.indexed > total_i; + if foreign { + begin_reset(conn, &id, total_i).await?; + } else if row.state == COMPLETE { + return Ok((IndexState::Complete { total: total as u64 }, false)); + } + if foreign || row.state == RESETTING { + let finished = reset_step(conn, &id, budget).await?; + return Ok((IndexState::Building { done: 0, total: total as u64 }, !finished)); + } + let start = usize::try_from(row.indexed).map_err(|_| MetaError::Corrupt("a negative index frontier"))?; + let end = start.saturating_add(budget.max(1)).min(total); + if end > start { + write_edges(conn, manifest, start, end).await?; + sqlx::query("UPDATE manifest_index SET indexed_ordinals = $1 WHERE manifest = $2 AND indexed_ordinals < $1") + .bind(to_i64(end as u64)?) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + } + Ok((IndexState::Building { done: end as u64, total: total as u64 }, false)) +} + +/// Called in the transaction that records a manifest: makes its index row and writes up to a budget of edges, and when +/// that is all of them compares them with the manifest and marks the index complete. A longer manifest is left being +/// built for [`Meta::index_complete`]. A row that already exists is not touched beyond that. +pub(crate) async fn register(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + if let (IndexState::Building { done, total: n }, false) = stage(conn, pg, manifest, budget).await? + && done >= n + && compare(conn, manifest, budget).await? == EdgeCheck::Exact + { + mark_complete(conn, &id, total).await?; + } + Ok(()) +} + +impl Meta { + /// The store with another budget of edges per index transaction (the default is [`INDEX_BATCH_ROWS`]). For tests. + pub fn with_index_batch_rows(mut self, rows: usize) -> Self { + self.index_rows = rows.max(1); + self + } + + /// Writes the next batch of `manifest`'s edges, at most the index budget, in one transaction, and says how far the + /// index has come. Call it again until it says all edges are written, then [`Meta::index_validate`]; or call + /// [`Meta::index_complete`], which does both. Each call is safe to repeat and to run beside commits and other builders. + pub async fn index_stage(&self, manifest: &Manifest) -> Result { + let mut w = self.begin_write().await?; + let (state, _) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + Ok(state) + } + + /// Compares every edge of `manifest` with it, in a read of one snapshot, and if they are exact marks its index + /// complete (provided every edge was written and nothing changed meanwhile). Nothing else marks an index complete. + pub async fn index_validate(&self, manifest: &Manifest) -> Result { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + let check = { + let mut tx = self.read_snapshot().await?; + let state = read_state(&mut tx, &id, false).await?; + let ready = state.is_some_and(|s| s.chunk_count == total && s.indexed == total && s.state != RESETTING); + let check = if ready { Some(compare(&mut tx, manifest, self.index_rows).await?) } else { None }; + tx.commit().await?; + check + }; + match check { + None => Ok(Validation::Raced), + Some(EdgeCheck::Mismatch(problem)) => Ok(Validation::Mismatch(problem)), + Some(EdgeCheck::Exact) => { + let mut w = self.begin_write().await?; + let marked = mark_complete(&mut w.tx, &id, total).await?; + w.commit().await?; + // Already complete (another builder was first) is as good. + if marked || matches!(self.index_state(&id).await?, IndexState::Complete { .. }) { + Ok(Validation::Complete) + } else { + Ok(Validation::Raced) + } + } + } + } + + /// Indexes `manifest` to the end: edges in batches of at most the index budget, each its own transaction, then the + /// comparison that marks it complete. Resumes where an earlier run stopped. Edges that do not match the manifest + /// are thrown away and written again (the manifest is the authority), at most twice; then it is `Corrupt`. + pub async fn index_complete(&self, manifest: &Manifest) -> Result<(), MetaError> { + let id = manifest.id(); + let total = manifest.chunks().len() as u64; + let mut rebuilds = 0; + for _ in 0..(REBUILDS + 1) * 4 { + loop { + let mut w = self.begin_write().await?; + let (state, resetting) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + match state { + IndexState::Complete { .. } => return Ok(()), + // Still throwing edges away: nothing is written yet, whatever the frontier says. + IndexState::Building { .. } if resetting => {} + IndexState::Building { done, .. } if done >= total => break, + IndexState::Building { .. } | IndexState::Absent => {} + } + } + match self.index_validate(manifest).await? { + Validation::Complete => return Ok(()), + Validation::Raced => {} + Validation::Mismatch(_) => { + if rebuilds >= REBUILDS { + return Err(MetaError::Corrupt("a manifest's index does not match it after being rebuilt")); + } + rebuilds += 1; + let mut w = self.begin_write().await?; + // Only a row that is not complete is rebuilt here: a complete one that is wrong is the checker's to report. + let row = read_state(&mut w.tx, &id, self.dialect == Dialect::Postgres).await?; + // The edges go in batches of the next stages, each its own transaction, and a stop between them resumes. + if row.is_some_and(|r| r.state != COMPLETE) { + begin_reset(&mut w.tx, &id, to_i64(total)?).await?; + } + w.commit().await?; + } + } + } + Err(MetaError::Corrupt("a manifest's index could not be completed")) + } + + /// How far `manifest`'s index has come. + pub async fn index_state(&self, manifest: &ManifestId) -> Result { + let mut conn = self.pool.acquire().await?; + Ok(match read_state(&mut conn, manifest, false).await? { + None => IndexState::Absent, + Some(r) if r.state == COMPLETE => IndexState::Complete { total: to_u64(r.chunk_count)? }, + Some(r) => IndexState::Building { done: to_u64(r.indexed)?, total: to_u64(r.chunk_count)? }, + }) + } + + /// Compares the edges stored for `manifest` with it, without changing anything. + pub async fn index_check(&self, manifest: &Manifest) -> Result { + let mut tx = self.read_snapshot().await?; + let check = compare(&mut tx, manifest, self.index_rows).await?; + tx.commit().await?; + Ok(check) + } + + /// The stored edges of a manifest, in order: `(ordinal, chunk)`. For tests and the checker. + pub async fn index_edges(&self, manifest: &ManifestId) -> Result, MetaError> { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal") + .bind(manifest.as_bytes().to_vec()) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok((to_u64(r.try_get("ordinal")?)?, ChunkId::from_bytes(hash_from(&r.try_get::, _>("chunk")?)?)))).collect() + } + + /// How many edges the index holds in all. + pub async fn index_edge_count(&self) -> Result { + let n: i64 = sqlx::query("SELECT COUNT(*) AS n FROM manifest_chunks").fetch_one(&self.pool).await?.try_get("n")?; + to_u64(n) + } + + /// Up to `limit` recorded manifests after `after` (in ID order) whose index is not complete: the work of a builder. + pub async fn manifests_to_index(&self, after: Option, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |m| m.as_bytes().to_vec()); + let rows = sqlx::query( + "SELECT b.id FROM blobs b LEFT JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = $1 AND b.id > $2 AND (i.manifest IS NULL OR i.state <> 1) \ + ORDER BY b.id LIMIT $3", + ) + .bind(BlobKind::Manifest.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok(ManifestId::from_bytes(hash_from(&r.try_get::, _>("id")?)?))).collect() + } + + /// Up to `limit` recorded blobs of `kind` after `after`, in ID order, with what the database says of their sizes. + pub async fn blob_records_after(&self, kind: BlobKind, after: Option<[u8; 32]>, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |id| id.to_vec()); + let rows = sqlx::query("SELECT id, stored_size, file_size FROM blobs WHERE kind = $1 AND id > $2 ORDER BY id LIMIT $3") + .bind(kind.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|r| { + let file_size: Option = r.try_get("file_size")?; + Ok(BlobRecord { + id: hash_from(&r.try_get::, _>("id")?)?, + stored_size: to_u64(r.try_get("stored_size")?)?, + file_size: file_size.map(to_u64).transpose()?, + }) + }) + .collect() + } + + /// Of `ids`, the recorded chunks and how many bytes each is stored as (the format byte included). + pub async fn recorded_chunk_sizes(&self, ids: &[ChunkId]) -> Result, MetaError> { + let mut found = HashMap::new(); + for part in ids.chunks(ROWS_PER_STATEMENT) { + let sql = format!("SELECT id, stored_size FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, part.len())); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(BlobKind::Chunk.code()); + for id in part { + query = query.bind(id.as_bytes().to_vec()); + } + for row in query.fetch_all(&self.pool).await? { + found.insert(ChunkId::from_bytes(hash_from(&row.try_get::, _>("id")?)?), to_u64(row.try_get("stored_size")?)?); + } + } + Ok(found) + } + + /// The recorded blobs among the IDs of one bounded store-list page, with the stored size the database records for each. + pub async fn recorded_blob_sizes(&self, kind: BlobKind, ids: &[[u8; 32]]) -> Result, MetaError> { + let mut found = std::collections::HashMap::new(); + if !ids.is_empty() { + let sql = format!("SELECT id, stored_size FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, ids.len())); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(kind.code()); + for id in ids { + query = query.bind(id.to_vec()); + } + for row in query.fetch_all(&self.pool).await? { + let size = u64::try_from(row.try_get::("stored_size")?).unwrap_or(0); + found.insert(hash_from(&row.try_get::, _>("id")?)?, size); + } + } + Ok(found) + } + + /// What the database says about the index, its roots and the blobs it names, in a read of one snapshot, with up to + /// `examples` examples of each problem. Changes nothing. + pub async fn index_audit(&self, examples: usize) -> Result { + let mut tx = self.read_snapshot().await?; + let mut audit = IndexAudit::default(); + audit.manifests = scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs WHERE kind = 1").await?; + audit.chunks = scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs WHERE kind = 0").await?; + audit.unreferenced_chunk_bytes = scalar( + &mut tx, + "SELECT CAST(COALESCE(SUM(b.stored_size), 0) AS BIGINT) AS n FROM blobs b WHERE b.kind = 0 AND NOT EXISTS (SELECT 1 FROM manifest_chunks mc WHERE mc.chunk = b.id)", + ) + .await?; + audit.complete = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state = 1").await?; + audit.building = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state <> 1").await?; + audit.absent = audit.manifests.saturating_sub(audit.complete).saturating_sub(audit.building); + // Every manifest a version, a live document's snapshot or its import snapshot names. + const ROOTS: &str = "(SELECT manifest AS a FROM versions UNION SELECT snapshot AS a FROM live_docs WHERE snapshot IS NOT NULL UNION SELECT import_snapshot AS a FROM live_docs WHERE import_snapshot IS NOT NULL) r"; + audit.index_without_manifest = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = i.manifest)", + ) + .await?; + audit.edges_without_index = finding( + &mut tx, + examples, + &["mc.manifest"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = mc.manifest)", + ) + .await?; + audit.complete_with_wrong_count = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE i.state = 1 AND i.chunk_count <> (SELECT COUNT(*) FROM manifest_chunks mc WHERE mc.manifest = i.manifest)", + ) + .await?; + audit.unknown_dependencies = finding( + &mut tx, + examples, + &["mc.manifest", "mc.chunk"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 0 AND b.id = mc.chunk)", + ) + .await?; + audit.roots_without_manifest = + finding(&mut tx, examples, &["r.a"], &format!("{ROOTS} WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a)")) + .await?; + audit.roots_not_indexed = finding( + &mut tx, + examples, + &["r.a"], + &format!( + "{ROOTS} WHERE EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a) AND NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = r.a AND i.state = 1)" + ), + ) + .await?; + audit.unrooted_manifests = + finding(&mut tx, examples, &["b.id"], &format!("blobs b WHERE b.kind = 1 AND NOT EXISTS (SELECT 1 FROM {ROOTS} WHERE r.a = b.id)")) + .await?; + audit.unreferenced_chunks = + finding(&mut tx, examples, &["b.id"], "blobs b WHERE b.kind = 0 AND NOT EXISTS (SELECT 1 FROM manifest_chunks mc WHERE mc.chunk = b.id)") + .await?; + audit.space_blobs_without_blob = + finding(&mut tx, examples, &["sb.id"], "space_blobs sb WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = sb.kind AND b.id = sb.id)") + .await?; + tx.commit().await?; + Ok(audit) + } +} + +/// A count query's number. +async fn scalar(conn: &mut sqlx::AnyConnection, sql: &'static str) -> Result { + to_u64(sqlx::query(sql).fetch_one(&mut *conn).await?.try_get("n")?) +} + +/// How many distinct `columns` the rows of `from_where` have, and the first few (hex, joined by `/`). Only constant +/// text goes into the SQL. +async fn finding(conn: &mut sqlx::AnyConnection, examples: usize, columns: &[&str], from_where: &str) -> Result { + let select = columns.iter().enumerate().map(|(i, c)| format!("{c} AS c{i}")).collect::>().join(", "); + let counted = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT COUNT(*) AS n FROM (SELECT DISTINCT {select} FROM {from_where}) q"))) + .fetch_one(&mut *conn) + .await?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT DISTINCT {select} FROM {from_where} ORDER BY 1 LIMIT {examples}"))) + .fetch_all(&mut *conn) + .await?; + let mut shown = Vec::with_capacity(rows.len()); + for row in &rows { + let mut parts = Vec::with_capacity(columns.len()); + for i in 0..columns.len() { + parts.push(hex(&row.try_get::, _>(format!("c{i}").as_str())?)); + } + shown.push(parts.join("/")); + } + Ok(Finding { count: to_u64(counted.try_get("n")?)?, examples: shown }) +} diff --git a/crates/meta/src/join.rs b/crates/meta/src/join.rs index 4a42a837..e5b6ca6a 100644 --- a/crates/meta/src/join.rs +++ b/crates/meta/src/join.rs @@ -7,8 +7,19 @@ //! Of two redeems of one code at the same moment, the second finds nothing ([`MetaError::InvalidCode`]): SQLite runs //! them one after the other, and on Postgres the second statement waits for the first's row lock and then sees the //! row gone or used. +//! - An invite row has a purpose: `invite` (a new computer for the account: the command line's `user invite` and the +//! admin API's invites) or `reset` (a password reset code made through the admin API, good for an hour). Redeeming +//! a reset code always sets a new password, signs out every device of the account and deletes its pairings, and +//! then, as every redeem does, creates the redeeming computer's device. +//! - A disabled account, or one marked for removal, redeems nothing ([`MetaError::AccountDisabled`]). That is decided +//! after the account's row is locked, so a disable that commits meanwhile is not followed by a device. //! - A pairing works only while the device that made it is signed in: revoking that device deletes it, and a redeem //! checks it too. +//! - An account with two-factor sign-in needs its second factor to redeem an invite or a reset code ([`Proof`]): the +//! factor is checked and used up in the same transaction as the invite and the device, a wrong one leaves the invite +//! as it was (its counters are committed on their own), and none is [`MetaError::TwoFactorRequired`] without using the +//! code up. A reset code the admin made with `reset_two_factor` takes the second factor away instead of asking for it. +//! A pairing needs none: only a signed-in device can make one. //! - Expired rows are deleted on the next write to their table (a new invite or a new pairing). use craft_cloud_proto::api::PairingState; @@ -17,9 +28,61 @@ use craft_cloud_proto::{DeviceId, NodeId, SpaceId}; use sqlx::Row; use uuid::Uuid; +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; use crate::codec::{uuid_bytes, uuid_from}; -use crate::users::{check_device_name, insert_device, insert_user}; -use crate::{Meta, MetaError, Role, UserId, valid_username}; +use crate::two_factor::{self, Proof}; +use crate::users::{DeviceClient, account_blocked, check_device_name, delete_join_codes, insert_device, insert_user, lock_user, revoke_devices}; +use crate::{Meta, MetaError, RevokeReason, Role, UserId, valid_username}; + +/// What a row of `invites` is for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Purpose { + /// Adds a computer to the account (and, for an account without a password, sets it). + Invite, + /// Sets a new password and signs the account's devices out. + Reset, +} + +impl Purpose { + pub(crate) fn as_str(self) -> &'static str { + match self { + Purpose::Invite => "invite", + Purpose::Reset => "reset", + } + } + + fn parse(s: &str) -> Result { + match s { + "invite" => Ok(Purpose::Invite), + "reset" => Ok(Purpose::Reset), + _ => Err(MetaError::Corrupt("unknown invite purpose")), + } + } +} + +/// Replaces `user`'s open invite or reset code with one under `code_hash`, in `tx`, after deleting every expired one. +/// `reset_two_factor`: redeeming a reset code also turns the account's two-factor sign-in off. +pub(crate) async fn put_code( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + code_hash: &[u8; 32], + purpose: Purpose, + reset_two_factor: bool, + now_ms: i64, + expires_ms: i64, +) -> Result<(), MetaError> { + sqlx::query("DELETE FROM invites WHERE expires_ms <= $1").bind(now_ms).execute(&mut **tx).await?; + sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("INSERT INTO invites (user_id, code_hash, expires_ms, purpose, reset_two_factor) VALUES ($1, $2, $3, $4, $5)") + .bind(uuid_bytes(&user.0)) + .bind(code_hash.to_vec()) + .bind(expires_ms) + .bind(purpose.as_str()) + .bind(i64::from(reset_two_factor)) + .execute(&mut **tx) + .await?; + Ok(()) +} /// The account an invite is for, from [`Meta::create_invite`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -31,10 +94,12 @@ pub struct InviteFor { } /// What [`Meta::redeem`] created: a device of `user`'s. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Redeemed { pub user: UserId, pub device: DeviceId, + /// The devices a reset code signed out (empty for every other code), for the caller to end their live connections. + pub revoked: Vec, } impl Meta { @@ -56,13 +121,8 @@ impl Meta { (insert_user(&mut w.tx, &username, "", Role::Member, now_ms, ids).await?.id, true) } }; - sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; - sqlx::query("INSERT INTO invites (user_id, code_hash, expires_ms) VALUES ($1, $2, $3)") - .bind(uuid_bytes(&user.0)) - .bind(code_hash.to_vec()) - .bind(expires_ms) - .execute(&mut *w.tx) - .await?; + // An open reset code of the account is replaced as well: the command line's invite is an invite. + put_code(&mut w.tx, user, code_hash, Purpose::Invite, false, now_ms, expires_ms).await?; w.commit().await?; Ok(InviteFor { user, new_account }) } @@ -86,6 +146,11 @@ impl Meta { let id = Uuid::now_v7(); let expires_ms = now_ms.checked_add(PAIRING_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; let mut w = self.begin_write().await?; + // Users first, as a sign-out does: a pairing is never made for a device a sign-out just revoked. + lock_user(&mut w.tx, user).await?; + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } sqlx::query("DELETE FROM pairings WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; let done = sqlx::query( @@ -139,23 +204,30 @@ impl Meta { w.commit().await } - /// Whether the invite under `code_hash` needs a new password (its account has none): `None` when no unexpired - /// invite has that hash (it may be a pairing's, or wrong). Lets the server answer `password_required` before it - /// spends time hashing a password. + /// Whether the invite under `code_hash` needs a new password (its account has none, or it is a reset code, which + /// always does): `None` when no unexpired invite has that hash (it may be a pairing's, or wrong). Lets the server + /// answer `password_required` before it spends time hashing a password. pub async fn invite_needs_password(&self, code_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { - let row = sqlx::query("SELECT u.password_hash FROM invites i JOIN users u ON u.id = i.user_id WHERE i.code_hash = $1 AND i.expires_ms > $2") - .bind(code_hash.to_vec()) - .bind(now_ms) - .fetch_optional(&self.pool) - .await?; - row.map(|row| row.try_get::("password_hash").map(|hash| hash.is_empty())).transpose().map_err(MetaError::from) + let row = sqlx::query( + "SELECT u.password_hash, i.purpose FROM invites i JOIN users u ON u.id = i.user_id WHERE i.code_hash = $1 AND i.expires_ms > $2", + ) + .bind(code_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + let purpose = Purpose::parse(&row.try_get::("purpose")?)?; + Ok(Some(purpose == Purpose::Reset || row.try_get::("password_hash")?.is_empty())) } /// Redeems the code under `code_hash` for a new device named `device_name` with `token_hash`, in one transaction: /// an unexpired invite (its row deleted) or an unused, unexpired pairing whose maker is still signed in (marked /// used by the new device). An invite to an account without a password needs `new_password_hash`, which becomes the /// account's ([`MetaError::PasswordRequired`] without it, and nothing changes); for any other code it is ignored. - /// Anything else is [`MetaError::InvalidCode`]. + /// A reset code always needs it: it becomes the account's password whatever the old one was, every device of the + /// account is signed out first ([`Redeemed::revoked`]) and its pairings are deleted, so the new device is the only + /// one signed in. A disabled account, or one marked for removal, is [`MetaError::AccountDisabled`] and nothing + /// changes. Anything else is [`MetaError::InvalidCode`]. pub async fn redeem( &self, code_hash: &[u8; 32], @@ -163,48 +235,180 @@ impl Meta { token_hash: &[u8; 32], new_password_hash: Option<&str>, now_ms: i64, + ) -> Result { + self.redeem_with(code_hash, device_name, &DeviceClient::default(), token_hash, new_password_hash, now_ms).await + } + + /// [`Meta::redeem`] for a device that says what client it is. An account with two-factor sign-in is + /// [`MetaError::TwoFactorRequired`] here (see [`Meta::redeem_with_second_factor`]). + pub async fn redeem_with( + &self, + code_hash: &[u8; 32], + device_name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + new_password_hash: Option<&str>, + now_ms: i64, + ) -> Result { + self.redeem_with_second_factor(code_hash, device_name, client, token_hash, new_password_hash, None, now_ms).await + } + + /// [`Meta::redeem_with`] with the second factor the person gave, for an invite or reset code of an account that has + /// two-factor sign-in. Under the account's lock, after the invite is found again and consumed, the account's + /// **current** state decides: with two-factor sign-in on, a reset code made with `reset_two_factor` takes it away and + /// asks for nothing; any other code needs `second_factor`. None is [`MetaError::TwoFactorRequired`]; a wrong one + /// [`MetaError::TwoFactorInvalid`], with the invite still there and the wrong-proof counters committed on their own; + /// a lock in force [`MetaError::TwoFactorLocked`]. A right one is used up in this transaction, with the invite and the + /// device. A pairing code never needs one, and a second factor given for an account that has none is ignored. + #[allow(clippy::too_many_arguments)] + pub async fn redeem_with_second_factor( + &self, + code_hash: &[u8; 32], + device_name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + new_password_hash: Option<&str>, + second_factor: Option<&Proof<'_>>, + now_ms: i64, ) -> Result { check_device_name(device_name)?; + client.check()?; let device = DeviceId::generate(); let mut w = self.begin_write().await?; - let invite = sqlx::query("DELETE FROM invites WHERE code_hash = $1 AND expires_ms > $2 RETURNING user_id") + // The account's row is locked before the invite is consumed (see `lock_user`): a password reset takes the + // user and then deletes the invite, so taking them the other way round could deadlock on Postgres. The invite + // is found with a plain read first, and consumed under the lock with its conditions checked again. + let found = sqlx::query("SELECT user_id, purpose, reset_two_factor FROM invites WHERE code_hash = $1 AND expires_ms > $2") .bind(code_hash.to_vec()) .bind(now_ms) .fetch_optional(&mut *w.tx) .await?; - let user = match invite { + let mut revoked = Vec::new(); + let mut two_factor_removed = false; + let (user, action) = match found { Some(row) => { let user = UserId(uuid_from(&row.try_get::, _>("user_id")?)?); + let purpose = Purpose::parse(&row.try_get::("purpose")?)?; + let reset_two_factor: i64 = row.try_get("reset_two_factor")?; + lock_user(&mut w.tx, user).await?; + let still_there = sqlx::query( + "SELECT 1 FROM invites WHERE code_hash = $1 AND user_id = $2 AND expires_ms > $3 AND purpose = $4 AND reset_two_factor = $5", + ) + .bind(code_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(purpose.as_str()) + .bind(reset_two_factor) + .fetch_optional(&mut *w.tx) + .await?; + if still_there.is_none() { + // Taken, replaced or deleted (a reset) while this waited for the lock. + return Err(MetaError::InvalidCode); + } + // Decided under the account's lock: a disable that committed while this waited is seen here, and one + // that follows waits for this transaction and then signs the new device out with the rest. + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } let current: String = sqlx::query("SELECT password_hash FROM users WHERE id = $1") .bind(uuid_bytes(&user.0)) .fetch_optional(&mut *w.tx) .await? .ok_or(MetaError::Corrupt("an invite's account is missing"))? .try_get("password_hash")?; - if current.is_empty() { - // Dropping the transaction puts the invite back. - let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; - sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2").bind(new).bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + // The account's two-factor state as it is under the lock now, whatever it was when the invite was made. + if two_factor::is_enabled(&mut w.tx, user).await? { + if purpose == Purpose::Reset && reset_two_factor == 1 { + two_factor::remove_in(&mut w.tx, user).await?; + two_factor_removed = true; + } else { + let Some(proof) = second_factor else { return Err(MetaError::TwoFactorRequired) }; + let row = two_factor::load_row(&mut w.tx, user).await?.ok_or(MetaError::Corrupt("two-factor row vanished under the lock"))?; + if let two_factor::Checked::Wrong = two_factor::check_proof(&mut w.tx, user, Some(device), false, &row, proof, now_ms).await? + { + // Nothing has been consumed yet: the wrong proof is counted and committed in this very + // transaction, under the account's lock, and the invite stays. + return Err(self.refuse_sign_in_proof(w, user, None, proof, now_ms, false).await); + } + } } - user + // Only now is the invite used up, with everything above decided under the lock. + let consumed = sqlx::query( + "DELETE FROM invites WHERE code_hash = $1 AND user_id = $2 AND expires_ms > $3 AND purpose = $4 AND reset_two_factor = $5", + ) + .bind(code_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(purpose.as_str()) + .bind(reset_two_factor) + .execute(&mut *w.tx) + .await?; + if consumed.rows_affected() == 0 { + return Err(MetaError::InvalidCode); + } + match purpose { + Purpose::Reset => { + // Dropping the transaction puts the code back. + let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; + sqlx::query("UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3") + .bind(new) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + // Every device goes, then the pairings: the new device is added after, so it is the only + // one signed in. + revoked = revoke_devices(&mut w.tx, user, None, RevokeReason::Reset, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + } + Purpose::Invite if current.is_empty() => { + // Dropping the transaction puts the invite back. + let new = new_password_hash.filter(|hash| !hash.is_empty()).ok_or(MetaError::PasswordRequired)?; + sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2") + .bind(new) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + } + Purpose::Invite => {} + } + (user, if purpose == Purpose::Reset { AuditAction::JoinResetRedeemed } else { AuditAction::JoinInviteRedeemed }) } None => { + // The same for a pairing: its account is locked before the device is added, so a reset that revokes + // the devices of the account cannot miss one that is being added. + let owner = + sqlx::query("SELECT user_id FROM pairings WHERE code_hash = $1").bind(code_hash.to_vec()).fetch_optional(&mut *w.tx).await?; + let Some(owner) = owner else { return Err(MetaError::InvalidCode) }; + let owner = UserId(uuid_from(&owner.try_get::, _>("user_id")?)?); + lock_user(&mut w.tx, owner).await?; + if account_blocked(&mut w.tx, owner).await? { + return Err(MetaError::AccountDisabled); + } let pairing = sqlx::query( "UPDATE pairings SET used_by = $1 \ - WHERE code_hash = $2 AND used_by IS NULL AND expires_ms > $3 \ + WHERE code_hash = $2 AND user_id = $3 AND used_by IS NULL AND expires_ms > $4 \ AND device_id IN (SELECT id FROM devices WHERE revoked = 0) RETURNING user_id", ) .bind(uuid_bytes(device.as_uuid())) .bind(code_hash.to_vec()) + .bind(uuid_bytes(&owner.0)) .bind(now_ms) .fetch_optional(&mut *w.tx) .await? .ok_or(MetaError::InvalidCode)?; - UserId(uuid_from(&pairing.try_get::, _>("user_id")?)?) + (UserId(uuid_from(&pairing.try_get::, _>("user_id")?)?), AuditAction::JoinPairingRedeemed) } }; - insert_device(&mut w.tx, device, user, device_name, token_hash, now_ms).await?; - w.commit().await?; - Ok(Redeemed { user, device }) + insert_device(&mut w.tx, device, user, device_name, client, token_hash, now_ms).await?; + let mut details = AuditDetails::client(client); + if action == AuditAction::JoinResetRedeemed { + details.revoked = Some(u32::try_from(revoked.len()).unwrap_or(u32::MAX)); + // The actor is the account that redeemed the code: the code does not say which admin made it. + details.two_factor_removed = two_factor_removed; + } + record_in(&mut w.tx, now_ms, NewEvent::new(action).by(user, Some(device)).on_device(device).with(details)).await?; + self.commit_audited(w, now_ms).await?; + Ok(Redeemed { user, device, revoked }) } } diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 1784df29..8fb0a1ad 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -7,14 +7,22 @@ //! row, so commits to one space run one at a time and commits to different spaces run in parallel. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod admin; +mod audit; mod blobs; mod codec; mod commit; +mod history; +mod index; mod join; mod live; mod schema; +mod trash; mod tree; +mod two_factor; +mod usage; mod users; +mod web; use std::str::FromStr; use std::sync::Arc; @@ -24,15 +32,33 @@ use sqlx::any::{AnyConnectOptions, AnyPoolOptions}; use sqlx::{AnyPool, ConnectOptions}; use tokio::sync::{Mutex, OwnedMutexGuard}; -pub use commit::CommitOutcome; +pub use admin::{AdminActor, AdminUser, AdminVia, Issued, MAX_SEARCH_BYTES, MIN_QUOTA_BYTES, RESET_TTL_MS, ServerCounts, UserPage}; +pub use audit::{ + ActionFilter, AuditAction, AuditCursor, AuditDetails, AuditEvent, AuditFailure, AuditOutcome, AuditQuery, ClientKind, Field, MAX_DETAILS_BYTES, + MAX_RETENTION_DAYS, MIN_RETENTION_DAYS, OPPORTUNISTIC_PRUNE_BATCHES, Origin, PRUNE_BATCH, PRUNE_EVERY_MS, Platform, Pruned, Step, TargetKind, +}; +pub use commit::{CommitOutcome, RestoreOutcome, TrashRestoreOutcome}; pub use craft_cloud_proto::BlobKind; +pub use history::HistoryPage; +pub use index::{BlobRecord, EdgeCheck, EdgeProblem, EdgeProblemKind, Finding, INDEX_BATCH_ROWS, IndexAudit, IndexState, Validation}; pub use join::{InviteFor, Redeemed}; pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, MAX_LIVE_DOCUMENTS_PER_SPACE, MAX_LIVE_PAGE, SnapshotRef, }; +pub use trash::{TrashEntry, TrashPage}; pub use tree::{MAX_PAGE_SAVE_BYTES, SpaceInfo}; -pub use users::{DeviceAuth, DeviceRecord, Role, UserId, UserRecord, valid_username}; +pub use two_factor::{ + CHALLENGE_TTL_MS, Completed, DeviceActor, LOCK_AFTER_FAILURES, LOCK_BASE_MS, LOCK_MAX_MS, MAX_CHALLENGE_ATTEMPTS, MAX_OPEN_CHALLENGES, + MAX_SETUP_ATTEMPTS, NewChallenge, Proof, RECOVERY_CODES, SETUP_TTL_MS, SignIn, StoredSecrets, TotpVerifier, TwoFactorInfo, UnreadableSecret, + retry_after_secs, +}; +pub use usage::{StorageAccount, StorageEstimate, StorageEstimateReason, StorageIndexCounts, StorageOverview}; +pub use users::{ + AccountRecord, CLIENT_KINDS, DeviceAuth, DeviceClient, DeviceRecord, MAX_CLIENT_FIELD_BYTES, RevokeReason, Role, UserId, UserRecord, + valid_username, +}; +pub use web::{NewWebSession, WebSession, WebSignIn}; #[derive(Debug, thiserror::Error)] pub enum MetaError { @@ -77,15 +103,60 @@ pub enum MetaError { /// An invite to an account without a password was redeemed without a new one; nothing was changed. #[error("this invite needs a new password for the account")] PasswordRequired, + /// The device that asked for a change was revoked while the request ran (it is no longer signed in); nothing was + /// changed. + #[error("this device is no longer signed in")] + DeviceRevoked, + /// What the request was based on changed while it ran (the password it checked is no longer the account's); + /// nothing was changed and the request may be made again. + #[error("{0}")] + Conflict(&'static str), + /// The admin who asked may not do this (their role changed while the request ran, or the rules about resetting + /// another admin); nothing was changed. + #[error("{0}")] + Forbidden(&'static str), + /// The change would leave the server without an administrator who can sign in; nothing was changed. + #[error("this would leave the server without an enabled administrator")] + LastAdmin, + /// The account is disabled or marked for removal. Said only to someone who proved they hold its password or a + /// code of it, never to a guesser. + #[error("this account is disabled")] + AccountDisabled, + /// The account has two-factor sign-in and the request carried no second factor; nothing was changed (an invite or + /// reset code is not used up). + #[error("this account needs its second factor")] + TwoFactorRequired, + /// A code or recovery code was wrong, or a code that was used before; its counters were committed and nothing else + /// was changed. + #[error("the second factor is wrong")] + TwoFactorInvalid, + /// Too many wrong second factors: every check is refused until `retry_after_ms` have passed. + #[error("the second factor is locked after too many wrong codes")] + TwoFactorLocked { retry_after_ms: i64 }, + /// A set-up of two-factor sign-in is open on another device. + #[error("two-factor set-up is open on another device")] + TwoFactorPending, + /// A sign-in challenge that is unknown, expired, used or no longer fit, never saying which. + #[error("this sign-in step doesn't work any more")] + ChallengeGone, #[error("invalid input: {0}")] Invalid(&'static str), #[error("stored data is damaged: {0}")] Corrupt(&'static str), + /// A database that was to exist already ([`Meta::open_existing`]) is not there or not ready; nothing was created. + #[error("{0}")] + Missing(&'static str), } /// How long a SQLite connection waits for a lock another connection or program holds before giving up. pub const SQLITE_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +/// The most deleted items below one folder that restoring or removing the folder from the trash takes with it in one +/// call, shallowest first. What is beyond stays in the trash and, once its folder is live or removed from the trash, +/// appears in the trash list on its own, to be restored or removed by a further call. The bound keeps the work done +/// under the space's write lock by one call small. +pub const MAX_TRASH_TREE_NODES: u64 = 10_000; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Dialect { Sqlite, @@ -113,12 +184,45 @@ pub struct Meta { live_held_documents_cap: u64, /// The most bytes all of one space's live documents may hold ([`Meta::with_live_bytes_cap`]). live_bytes_cap: u64, + /// How long audit events are kept when a write prunes them ([`Meta::with_audit_retention_days`]); `None`: a write never prunes. + audit_retention_ms: Option, + /// When the last complete pruning ran, and whether one runs now; shared by the clones. + prune_state: Arc, + /// Where the store reads the time of day for what it judges itself (a browser session's expiry inside an admin change); + /// `None`: the system clock ([`Meta::with_clock`]). + clock: Option i64 + Send + Sync>>, + /// Edges of the reference index one transaction writes ([`Meta::with_index_batch_rows`]). + index_rows: usize, + /// The most deleted items below one folder that restoring or removing the folder from the trash takes with it + /// ([`Meta::with_trash_tree_cap`]). + trash_tree_cap: u64, } impl Meta { + /// The configured stored-bytes cap for one space, if any. + pub fn stored_bytes_cap(&self) -> Option { + self.stored_cap + } /// Connects and brings the schema up to date. `url` is `sqlite:path/to/meta.db` (created if missing), /// `sqlite::memory:`, or `postgres://user:pass@host/db`. pub async fn open(url: &str) -> Result { + Self::start(url, false).await + } + + /// Connects to a database that already exists and creates nothing: a SQLite file that is not there is an error + /// (`mode=ro`, whatever options the URL had), not a new empty database, and a database whose schema is not the + /// current one is refused, not upgraded. For a command that only reads, so that a wrong path is reported instead of + /// being taken for an empty database that has no problems. `sqlite::memory:` has nothing to exist. + /// + /// A read-only SQLite connection never writes the database file or the WAL, and cannot checkpoint: closing the last + /// connection of a read-write one checkpoints the WAL of a crashed process into the database file and removes it, + /// which is a write a read-only command must not make. It still reads the WAL's committed data. Opening may leave + /// an empty `-wal` and a `-shm` file (SQLite's rebuilt index of the WAL, no data) when there were none. + pub async fn open_existing(url: &str) -> Result { + Self::start(url, true).await + } + + async fn start(url: &str, existing: bool) -> Result { sqlx::any::install_default_drivers(); let dialect = if url.starts_with("sqlite:") { Dialect::Sqlite @@ -128,6 +232,8 @@ impl Meta { return Err(MetaError::UnsupportedUrl); }; let url = match dialect { + Dialect::Sqlite if existing && url.contains(":memory:") => return Err(MetaError::Missing("an in-memory database has no existing data")), + Dialect::Sqlite if existing => format!("{}?mode=ro", url.split_once('?').map_or(url, |(path, _)| path)), Dialect::Sqlite if !url.contains('?') && !url.contains(":memory:") => format!("{url}?mode=rwc"), _ => url.to_owned(), }; @@ -154,10 +260,14 @@ impl Meta { Dialect::Postgres => pool_options, }; let pool = pool_options.connect_with(options).await?; - if dialect == Dialect::Sqlite && !url.contains(":memory:") { - sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + if existing { + schema::check_current(&pool, dialect).await?; + } else { + if dialect == Dialect::Sqlite && !url.contains(":memory:") { + sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + } + schema::migrate(&pool, dialect).await?; } - schema::migrate(&pool, dialect).await?; Ok(Self { pool, dialect, @@ -169,9 +279,34 @@ impl Meta { live_documents_cap: MAX_LIVE_DOCUMENTS_PER_SPACE, live_held_documents_cap: MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, live_bytes_cap: DEFAULT_LIVE_BYTES_PER_SPACE, + audit_retention_ms: None, + prune_state: Arc::new(audit::PruneState::new()), + clock: None, + index_rows: INDEX_BATCH_ROWS, + trash_tree_cap: MAX_TRASH_TREE_NODES, }) } + /// The store with another clock, for what it judges by the time of day itself: whether a browser session is still inside its + /// lifetimes when an admin change runs (read after the change's locks are taken). The server gives it the clock it injects + /// for tests; without one the system clock is read. The clock belongs to this handle and its clones. + pub fn with_clock(mut self, clock: Arc i64 + Send + Sync>) -> Self { + self.clock = Some(clock); + self + } + + /// Milliseconds since the Unix epoch, by the store's clock. + pub(crate) fn clock_ms(&self) -> i64 { + self.clock.as_ref().map_or_else(users::wall_clock_ms, |clock| clock()) + } + + /// The store with another cap on the deleted items below one folder that restoring or removing the folder from the + /// trash takes with it in one call (the default is [`MAX_TRASH_TREE_NODES`]); the rest stays in the trash. For tests. + pub fn with_trash_tree_cap(mut self, cap: u64) -> Self { + self.trash_tree_cap = cap; + self + } + /// The store with another cap on the bytes of updates one live document's log may hold (the default is /// [`MAX_LIVE_DOC_BYTES`]); an append over it is refused with [`MetaError::DocFull`]. For tests. pub fn with_live_doc_cap(mut self, cap: u64) -> Self { @@ -226,6 +361,14 @@ impl Meta { self } + /// `sqlite` or `postgres`, for the admin's server overview. + pub fn database_kind(&self) -> &'static str { + match self.dialect { + Dialect::Sqlite => "sqlite", + Dialect::Postgres => "postgres", + } + } + /// Whether a database connection is in use (a query under way, or a connection on its way back to the pool). /// A simulation on a simulated clock (`turmoil`) keeps time still while it is, since SQLite answers from a /// thread of its own, outside the simulation: otherwise how far simulated time got before an answer would depend diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index e4c5220d..ca43f09d 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -12,6 +12,14 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0004_join.sql"), include_str!("../migrations/sqlite/0005_live_documents.sql"), include_str!("../migrations/sqlite/0006_save_records.sql"), + include_str!("../migrations/sqlite/0007_devices_v2.sql"), + include_str!("../migrations/sqlite/0008_admin_resets.sql"), + include_str!("../migrations/sqlite/0009_audit_events.sql"), + include_str!("../migrations/sqlite/0010_two_factor.sql"), + include_str!("../migrations/sqlite/0011_web_sessions.sql"), + include_str!("../migrations/sqlite/0012_versions_node_seq.sql"), + include_str!("../migrations/sqlite/0013_trash.sql"), + include_str!("../migrations/sqlite/0014_manifest_index.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -20,12 +28,51 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0004_join.sql"), include_str!("../migrations/postgres/0005_live_documents.sql"), include_str!("../migrations/postgres/0006_save_records.sql"), + include_str!("../migrations/postgres/0007_devices_v2.sql"), + include_str!("../migrations/postgres/0008_admin_resets.sql"), + include_str!("../migrations/postgres/0009_audit_events.sql"), + include_str!("../migrations/postgres/0010_two_factor.sql"), + include_str!("../migrations/postgres/0011_web_sessions.sql"), + include_str!("../migrations/postgres/0012_versions_node_seq.sql"), + include_str!("../migrations/postgres/0013_trash.sql"), + include_str!("../migrations/postgres/0014_manifest_index.sql"), ]; /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one /// database. Every server must use the same key; no other code takes this lock. const MIGRATION_LOCK_KEY: i64 = 7240318150661; +/// Checks that the database has the current schema, and changes nothing: for a database that must exist already +/// ([`Meta::open_existing`](crate::Meta::open_existing)). +pub(crate) async fn check_current(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { + let migrations = match dialect { + Dialect::Sqlite => SQLITE, + Dialect::Postgres => POSTGRES, + }; + let known = i64::try_from(migrations.len()).map_err(|_| MetaError::Invalid("too many migrations"))?; + let table = match dialect { + Dialect::Sqlite => "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'", + Dialect::Postgres => { + "SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'schema_version'" + } + }; + let tables: i64 = sqlx::query(table).fetch_one(pool).await?.try_get("n")?; + if tables == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + let current: i64 = sqlx::query("SELECT COALESCE(MAX(version), 0) AS v FROM schema_version").fetch_one(pool).await?.try_get("v")?; + if current == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + if current > known { + return Err(MetaError::Corrupt("the database was created by a newer Craft Cloud")); + } + if current < known { + return Err(MetaError::Missing("the database schema is older than this program; start the server once (`serve`) to upgrade it")); + } + Ok(()) +} + pub(crate) async fn migrate(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { // One connection for the whole run. The Postgres advisory lock belongs to this session, so the lock, // the work and the unlock must all use it. diff --git a/crates/meta/src/trash.rs b/crates/meta/src/trash.rs new file mode 100644 index 00000000..f53c5868 --- /dev/null +++ b/crates/meta/src/trash.rs @@ -0,0 +1,173 @@ +//! The trash: the deleted nodes of a space, and the request to remove one of them for good. +//! +//! Deleting stays what it always was, a tombstone in the change feed. The trash is that same set of tombstones seen +//! with the time of the deletion. Bringing a node back is [`Meta::restore_from_trash`] in `commit.rs`, because it +//! shares the commit path's rules for names. Removing a node from the trash here only marks it: nothing is freed, +//! so versions, chunks and the tombstone stay as they were. +//! +//! A node is in the trash when it is deleted and its removal was not asked for. What is below a folder in the trash +//! is listed only through the folder (the folder's restore and removal take it along), whenever it was deleted. + +use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; +use craft_cloud_proto::limits::MAX_TREE_DEPTH; +use craft_cloud_proto::{NodeId, SpaceId}; +use sqlx::Row; + +use crate::codec::{flag, to_u64, uuid_bytes, uuid_from}; +use crate::tree::space_info; +use crate::{Dialect, Meta, MetaError}; + +/// A deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashEntry { + pub id: NodeId, + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + pub size: Option, + pub deleted_ms: i64, +} + +/// One page of the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashPage { + pub entries: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, + /// How many deleted nodes without a deletion time were found and stamped with `now` (a damaged database: the + /// migration and the commit path always set it). They are never treated as expired. + pub repaired: u64, +} + +/// The deleted nodes below the node `$2` of space `$1` that are in the trash, found through nodes that are in the trash +/// themselves (a folder that was removed from the trash hides what is below it); `depth` is 1 for a child. `$3` bounds +/// the depth. A statement continues after it with `$4` and up. +pub(crate) const TRASH_BELOW: &str = "WITH RECURSIVE sub(id, depth) AS (\ + SELECT id, 1 FROM nodes WHERE space_id = $1 AND parent = $2 AND deleted = 1 AND purge_requested_ms IS NULL \ + UNION ALL \ + SELECT n.id, sub.depth + 1 FROM nodes n JOIN sub ON n.parent = sub.id \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND sub.depth < $3) "; + +impl Meta { + /// One page of the trash of `space`: its deleted nodes that were not removed from the trash, newest deletion + /// first by `(deleted_ms, id)`, after `after` (exclusive). `Invalid` for a `limit` outside 1 to + /// [`TRASH_MAX_LIMIT`]. A deleted node without a deletion time gets `now_ms` first, so the trash never lists a + /// node as older than it can be shown to be. + pub async fn trash(&self, space: SpaceId, after: Option, limit: u32, now_ms: i64) -> Result { + if limit == 0 || limit > TRASH_MAX_LIMIT { + return Err(MetaError::Invalid("a trash page has 1 to 200 items")); + } + let repaired = self.stamp_unstamped(space, now_ms).await?; + let mut tx = self.read_snapshot().await?; + let (cursor_ms, cursor_node) = match after { + Some(cursor) => (cursor.deleted_ms, uuid_bytes(cursor.node.as_uuid())), + // Above every time and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The row-value comparison and the order + // match the index `nodes_trash`. + let rows = sqlx::query( + "SELECT n.id, n.parent, n.name, n.is_dir, n.deleted_ms, v.size AS v_size \ + FROM nodes n LEFT JOIN versions v ON v.space_id = n.space_id AND v.id = n.version \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND (n.deleted_ms, n.id) < ($2, $3) \ + AND NOT EXISTS (SELECT 1 FROM nodes p WHERE p.space_id = n.space_id AND p.id = n.parent AND p.deleted = 1 AND p.purge_requested_ms IS NULL) \ + ORDER BY n.deleted_ms DESC, n.id DESC LIMIT $4", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(cursor_ms) + .bind(cursor_node) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut entries = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent")?; + let size: Option = row.try_get("v_size")?; + entries.push(TrashEntry { + id: NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent: NodeId::from_uuid(uuid_from(&parent.ok_or(MetaError::Corrupt("a deleted node has no parent"))?)?), + name: row.try_get("name")?, + is_dir: flag(row, "is_dir")?, + size: size.map(to_u64).transpose()?, + deleted_ms: row.try_get("deleted_ms")?, + }); + } + let next = if more { entries.last().map(|e| TrashCursor { deleted_ms: e.deleted_ms, node: e.id }) } else { None }; + Ok(TrashPage { entries, next, repaired }) + } + + /// Gives every deleted node of `space` that has no deletion time `now_ms`; returns how many. Looks first without + /// the write lock, since there are none in a sound database. + async fn stamp_unstamped(&self, space: SpaceId, now_ms: i64) -> Result { + let space_key = uuid_bytes(space.as_uuid()); + let any = sqlx::query("SELECT 1 FROM nodes WHERE space_id = $1 AND deleted = 1 AND deleted_ms IS NULL LIMIT 1") + .bind(space_key.clone()) + .fetch_optional(&self.pool) + .await?; + if any.is_none() { + return Ok(0); + } + let mut w = self.begin_write().await?; + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let done = sqlx::query("UPDATE nodes SET deleted_ms = $1 WHERE space_id = $2 AND deleted = 1 AND deleted_ms IS NULL") + .bind(now_ms) + .bind(space_key) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(done.rows_affected()) + } + + /// Takes the deleted node `node` out of the trash: it is no longer listed and can't be restored from the trash. + /// Only marks it (`purge_requested_ms`): no version, chunk or tombstone is touched, so the change feed and the + /// sequence number stay as they are. Asking again changes nothing. `NotFound` when `node` is not a deleted node of + /// `space` in the trash. Removing a folder marks the nodes in the trash below it too, whenever they were deleted: at + /// most [`crate::MAX_TRASH_TREE_NODES`] of them (see [`Meta::with_trash_tree_cap`]), shallowest first, in the order of a + /// restore. The rest is no longer below a folder in the trash, so it is listed and can be removed by a further call. + pub async fn request_purge(&self, space: SpaceId, node: NodeId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the mark with commits and restores of the space. + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let row = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("node in the trash"))?; + if !flag(&row, "deleted")? { + return Err(MetaError::NotFound("node in the trash")); + } + if row.try_get::, _>("purge_requested_ms")?.is_some() { + return Ok(()); + } + if flag(&row, "is_dir")? { + // What is in the trash below the folder goes with it, so it does not stay hidden and unmarked: up to the cap, + // shallowest first (so a marked node's folder is always marked too). Marking is bounded by the cap under + // the write lock. What is left is listed once the folder is marked. + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let cap = i64::try_from(self.trash_tree_cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; + sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} UPDATE nodes SET purge_requested_ms = $4 WHERE space_id = $1 AND id IN ( SELECT n.id FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $5)" + ))) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(bound) + .bind(now_ms) + .bind(cap) + .execute(&mut *w.tx) + .await?; + } + sqlx::query("UPDATE nodes SET purge_requested_ms = $1 WHERE space_id = $2 AND id = $3") + .bind(now_ms) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(()) + } +} diff --git a/crates/meta/src/tree.rs b/crates/meta/src/tree.rs index c5b30d05..47f0f9a6 100644 --- a/crates/meta/src/tree.rs +++ b/crates/meta/src/tree.rs @@ -77,7 +77,7 @@ fn node_state(row: &AnyRow) -> Result { impl Meta { /// Starts a read transaction that sees one consistent snapshot. On Postgres the isolation level has to be /// set as the transaction's first statement. SQLite read transactions are already snapshots. - async fn read_snapshot(&self) -> Result, MetaError> { + pub(crate) async fn read_snapshot(&self) -> Result, MetaError> { let mut tx = self.pool.begin().await?; if self.dialect == crate::Dialect::Postgres { sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY").execute(&mut *tx).await?; diff --git a/crates/meta/src/two_factor.rs b/crates/meta/src/two_factor.rs new file mode 100644 index 00000000..88664d0e --- /dev/null +++ b/crates/meta/src/two_factor.rs @@ -0,0 +1,967 @@ +//! Two-factor sign-in: the TOTP secret (stored encrypted, the server's `secretbox` makes and opens the ciphertext), the +//! recovery codes (stored as BLAKE3 hashes), the lock after wrong codes, and the challenges of a password sign-in that +//! still needs its second factor. +//! +//! - **One transaction contract for everything that issues a device or consumes a proof.** The account's row is locked +//! first (`users::lock_user`, the order every other transaction on the account uses), then what the caller checked +//! before is read again under the lock (the device still signed in, the password still the one that was verified, +//! the account's current two-factor state, the challenge), and only then is a challenge or a device made. A second +//! factor and the challenge or invite it opens are used up in the same transaction as the device they let in: the +//! proof is never spent by one commit and followed by another. +//! - **Proofs are single use.** A TOTP code is accepted for a time step greater than the last accepted one +//! (`last_accepted_step`, updated with `WHERE last_accepted_step < step`), so two requests with one code can't both +//! succeed, and a second confirmation in the same 30-second step needs the next code. Sign-in and the admin's +//! confirmations share that counter. A recovery code is a row updated once (`WHERE used_ms IS NULL`). +//! - **Wrong proofs are counted in the transaction that refused them, under the account's lock.** Every check runs before +//! anything is consumed, so the refusing transaction commits only the counters (the invite stays, the challenge stays, +//! no device), and the next request, which waited for the lock, sees the attempts and the lock this one made. After +//! [`LOCK_AFTER_FAILURES`] wrong proofs in a +//! row the second factor is locked for [`LOCK_BASE_MS`], and each wrong proof after that doubles the time up to +//! [`LOCK_MAX_MS`]. A lock is never for good. While it lasts every check answers [`MetaError::TwoFactorLocked`] +//! without counting. +//! - **Lock order** inside a transaction: the account's row, then its two-factor row, then challenges (the failed-proof +//! write takes the last two in that order and never the account's row, so it can't deadlock with them). +//! - **Turning it on or off, and a reset, end what an earlier check let through**: `users.auth_gen` goes up, every +//! open challenge, invite and pairing of the account is deleted, and the devices are signed out (all of them for a +//! disable made by someone else, a reset or the command line; all but the asking device for the account's own). + +use craft_cloud_proto::DeviceId; +use sqlx::Row; + +use crate::audit::{AuditAction, AuditDetails, AuditFailure, NewEvent, Step as AuditStep, record_in}; +use crate::codec::{uuid_bytes, uuid_from}; +use crate::users::{DeviceClient, account_blocked, check_device_name, delete_join_codes, insert_device, lock_user, revoke_devices}; +use crate::{Meta, MetaError, RevokeReason, UserId, WriteTx}; + +/// How long a set-up stays open: ten minutes. +pub const SETUP_TTL_MS: i64 = 10 * 60 * 1000; +/// Wrong codes one set-up takes before it is used up. +pub const MAX_SETUP_ATTEMPTS: i64 = 5; +/// How long a challenge works: five minutes. +pub const CHALLENGE_TTL_MS: i64 = 5 * 60 * 1000; +/// Wrong codes one challenge takes before it is deleted. +pub const MAX_CHALLENGE_ATTEMPTS: i64 = 5; +/// Open challenges one account may have; making another replaces the oldest. +pub const MAX_OPEN_CHALLENGES: i64 = 5; +/// Wrong proofs in a row that lock the second factor. +pub const LOCK_AFTER_FAILURES: i64 = 10; +/// The first lock: 15 minutes. Every wrong proof after a lock ran out doubles it… +pub const LOCK_BASE_MS: i64 = 15 * 60 * 1000; +/// …up to 24 hours. +pub const LOCK_MAX_MS: i64 = 24 * 60 * 60 * 1000; +/// Recovery codes made at once. +pub const RECOVERY_CODES: usize = 10; + +/// A stored secret that the caller's key could not open (the wrong key, a damaged value, another account's). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnreadableSecret; + +/// Checks a TOTP code against an account's stored secret (the ciphertext, which only the caller can open): the time step +/// of the code, if it is right for a step in the window, else `None`. Called with the transaction open, on the +/// ciphertext that is current under the account's lock. +pub type TotpVerifier<'a> = dyn Fn(UserId, &[u8]) -> Result, UnreadableSecret> + Send + Sync + 'a; + +/// Rewrites one stored ciphertext of an account: the one to store in its place, or `None` to leave it ([`Meta::two_factor_rewrite`]). +pub type Rewrite<'a> = dyn Fn(UserId, &[u8]) -> Result>, UnreadableSecret> + Send + Sync + 'a; + +/// A second factor a caller presents. +pub enum Proof<'a> { + /// A TOTP code, as the check that knows it. + Totp(&'a TotpVerifier<'a>), + /// The BLAKE3 hash of a normalised recovery code. + Recovery([u8; 32]), +} + +/// The signed-in device that asks for a change to its account's two-factor sign-in, as the handler knew it when it +/// verified the password: all of it is checked again under the account's lock. +#[derive(Clone, PartialEq, Eq)] +pub struct DeviceActor { + pub user: UserId, + /// The device the request came with; it must still be signed in. + pub device: DeviceId, + /// The stored password hash the password was verified against; it must still be the account's. + pub password_hash: String, +} + +impl std::fmt::Debug for DeviceActor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceActor").field("user", &self.user).field("device", &self.device).field("password_hash", &"").finish() + } +} + +/// What `GET /account/two-factor` tells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TwoFactorInfo { + pub enabled_ms: Option, + pub recovery_codes_left: u32, + /// A set-up is open (not expired, not used up). + pub pending: bool, +} + +/// A new challenge for a password sign-in that needs a second factor. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewChallenge { + /// The BLAKE3 hash of the challenge token: all the server keeps of it. + pub id_hash: [u8; 32], + /// The client address the challenge was made for. Only an additional binding: behind a proxy that hides addresses + /// every client has the same one, so the secret token is what binds the challenge. + pub client_key: String, +} + +/// The result of a password sign-in that checked out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SignIn { + /// The account has no second factor: the device is made. + Device(DeviceId), + /// The account needs its second factor: the challenge is made and expires at this time. No device. + Challenge { expires_ms: i64 }, +} + +/// What [`Meta::complete_second_factor`] made. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Completed { + pub user: UserId, + pub device: DeviceId, +} + +/// A stored secret for the key tools: the ciphertexts of one account. Its `Debug` output shows only their lengths. +#[derive(Clone, PartialEq, Eq)] +pub struct StoredSecrets { + pub user: UserId, + /// The ciphertext of the active secret, if two-factor sign-in is on. + pub active: Option>, + /// The ciphertext of an open set-up's secret, if there is one that is not expired. + pub pending: Option>, +} + +impl std::fmt::Debug for StoredSecrets { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StoredSecrets") + .field("user", &self.user) + .field("active", &self.active.as_ref().map(|sealed| format!("<{} bytes>", sealed.len()))) + .field("pending", &self.pending.as_ref().map(|sealed| format!("<{} bytes>", sealed.len()))) + .finish() + } +} + +/// The row of `user_two_factor`. +pub(crate) struct TfRow { + enabled_ms: Option, + secret: Option>, + pending: Option>, + pending_device: Option>, + pending_expires_ms: Option, + last_step: i64, + locked_until_ms: Option, +} + +impl TfRow { + /// Two-factor sign-in is on. + pub(crate) fn active(&self) -> bool { + self.enabled_ms.is_some() && self.secret.is_some() + } +} + +pub(crate) async fn load_row(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result, MetaError> { + let row = sqlx::query( + "SELECT enabled_ms, secret_ciphertext, pending_ciphertext, pending_device, pending_expires_ms, last_accepted_step, \ + locked_until_ms FROM user_two_factor WHERE user_id = $1", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut **tx) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some(TfRow { + enabled_ms: row.try_get("enabled_ms")?, + secret: row.try_get("secret_ciphertext")?, + pending: row.try_get("pending_ciphertext")?, + pending_device: row.try_get("pending_device")?, + pending_expires_ms: row.try_get("pending_expires_ms")?, + last_step: row.try_get("last_accepted_step")?, + locked_until_ms: row.try_get("locked_until_ms")?, + })) +} + +/// Whether two-factor sign-in is on for `user`, read in `tx`. +pub(crate) async fn is_enabled(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + Ok(load_row(tx, user).await?.is_some_and(|row| row.active())) +} + +/// How long the second factor stays locked after the `failures`-th wrong proof in a row (`None`: no lock yet). +fn lock_for(failures: i64) -> Option { + if failures < LOCK_AFTER_FAILURES { + return None; + } + let doublings = u32::try_from(failures.saturating_sub(LOCK_AFTER_FAILURES)).unwrap_or(u32::MAX).min(16); + Some(LOCK_BASE_MS.saturating_mul(1i64 << doublings).min(LOCK_MAX_MS)) +} + +/// Seconds a client should wait before trying again, for a `Retry-After` header: rounded up, at least one. +pub fn retry_after_secs(retry_after_ms: i64) -> u32 { + u32::try_from((retry_after_ms.max(0) + 999) / 1000).unwrap_or(u32::MAX).max(1) +} + +pub(crate) enum Checked { + Accepted, + Wrong, +} + +/// Checks `proof` against the account's two-factor row `row` and, when it is right, uses it up and clears the wrong-proof +/// counters, all in `tx` (the account's row is locked). A lock that is in force is [`MetaError::TwoFactorLocked`] and +/// nothing is checked or counted. A recovery code that is used up writes `auth.recovery_code_used` in `tx` (a code that +/// is not used up writes nothing); `device` is the device it was used from, where there is one. +pub(crate) async fn check_proof( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + device: Option, + web: bool, + row: &TfRow, + proof: &Proof<'_>, + now_ms: i64, +) -> Result { + if let Some(until) = row.locked_until_ms + && until > now_ms + { + return Err(MetaError::TwoFactorLocked { retry_after_ms: until - now_ms }); + } + match proof { + Proof::Totp(verify) => { + let secret = row.secret.as_deref().ok_or(MetaError::Corrupt("a two-factor row has no secret"))?; + let step = verify(user, secret).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))?; + // A step that was accepted before, or an older one, is a replay. + let Some(step) = step.filter(|step| *step > row.last_step) else { return Ok(Checked::Wrong) }; + let done = sqlx::query( + "UPDATE user_two_factor SET last_accepted_step = $1, failed_attempts = 0, locked_until_ms = NULL \ + WHERE user_id = $2 AND last_accepted_step < $1", + ) + .bind(step) + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + Ok(if done.rows_affected() == 1 { Checked::Accepted } else { Checked::Wrong }) + } + Proof::Recovery(hash) => { + let done = sqlx::query("UPDATE user_recovery_codes SET used_ms = $1 WHERE user_id = $2 AND code_hash = $3 AND used_ms IS NULL") + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .bind(hash.to_vec()) + .execute(&mut **tx) + .await?; + if done.rows_affected() != 1 { + return Ok(Checked::Wrong); + } + sqlx::query("UPDATE user_two_factor SET failed_attempts = 0, locked_until_ms = NULL WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + let event = NewEvent::new(AuditAction::AuthRecoveryCodeUsed).by(user, device).on_user(user); + let event = if web { event.with(AuditDetails::web()) } else { event }; + record_in(tx, now_ms, event).await?; + Ok(Checked::Accepted) + } + } +} + +/// Deletes the account's two-factor secret, recovery codes and open challenges, and raises `auth_gen`, in `tx`. Whether +/// there was anything to delete is the answer. +pub(crate) async fn remove_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + sqlx::query("DELETE FROM user_recovery_codes WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + let removed = sqlx::query("DELETE FROM user_two_factor WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("UPDATE users SET auth_gen = auth_gen + 1 WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(removed.rows_affected() > 0) +} + +/// The account's two-factor sign-in removed by someone else (the command line, an admin): the secret, the codes and the +/// challenges go, every device is signed out ([`RevokeReason::TwoFactor`]) and the invite and pairings are deleted. The +/// devices signed out are returned, and the answer is `None` when the account had none (nothing is changed then). +pub(crate) async fn reset_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, now_ms: i64) -> Result>, MetaError> { + if load_row(tx, user).await?.is_none() { + return Ok(None); + } + remove_in(tx, user).await?; + let revoked = revoke_devices(tx, user, None, RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(tx, user).await?; + Ok(Some(revoked)) +} + +/// Checks the device and the password a handler verified against what is stored now, under the account's lock. +async fn check_actor(w: &mut WriteTx, actor: &DeviceActor) -> Result<(), MetaError> { + if !lock_user(&mut w.tx, actor.user).await? { + return Err(MetaError::DeviceRevoked); + } + // The device must still be signed in; the no-op update locks its row too, so a revoke running now waits. + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(actor.device.as_uuid())) + .bind(uuid_bytes(&actor.user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, actor.user).await? { + return Err(MetaError::DeviceRevoked); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2") + .bind(uuid_bytes(&actor.user.0)) + .bind(actor.password_hash.as_str()) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Err(MetaError::Conflict("the password changed while this request was running; try again")); + } + Ok(()) +} + +impl Meta { + /// Whether `user` has two-factor sign-in on. + pub async fn two_factor_enabled(&self, user: UserId) -> Result { + let row = sqlx::query("SELECT 1 FROM user_two_factor WHERE user_id = $1 AND enabled_ms IS NOT NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + Ok(row.is_some()) + } + + /// How many milliseconds `user`'s second factor stays locked after `now_ms` (`None`: it is not locked, or is off). A plain + /// read, for a caller that wants to say "locked" rather than "too many tries". + pub async fn two_factor_locked_for(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let row = sqlx::query("SELECT locked_until_ms FROM user_two_factor WHERE user_id = $1 AND enabled_ms IS NOT NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + let until: Option = row.map(|row| row.try_get("locked_until_ms")).transpose()?.flatten(); + Ok(until.filter(|until| *until > now_ms).map(|until| until - now_ms)) + } + + /// What `GET /account/two-factor` tells about `user`. + pub async fn two_factor_info(&self, user: UserId, now_ms: i64) -> Result { + let row = sqlx::query("SELECT enabled_ms, pending_ciphertext, pending_expires_ms FROM user_two_factor WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(TwoFactorInfo { enabled_ms: None, recovery_codes_left: 0, pending: false }) }; + let enabled_ms: Option = row.try_get("enabled_ms")?; + let pending = row.try_get::>, _>("pending_ciphertext")?.is_some() + && row.try_get::, _>("pending_expires_ms")?.is_some_and(|expires| expires > now_ms); + let left = if enabled_ms.is_some() { + let n: i64 = sqlx::query("SELECT COUNT(*) AS n FROM user_recovery_codes WHERE user_id = $1 AND used_ms IS NULL") + .bind(uuid_bytes(&user.0)) + .fetch_one(&self.pool) + .await? + .try_get("n")?; + u32::try_from(n).unwrap_or(u32::MAX) + } else { + 0 + }; + Ok(TwoFactorInfo { enabled_ms, recovery_codes_left: left, pending }) + } + + /// Starts a set-up for `actor`'s account: `pending_ciphertext` (the new secret, sealed by the caller) is kept for + /// [`SETUP_TTL_MS`] and bound to the asking device, and its expiry is returned. Only while two-factor sign-in is off + /// (`Conflict` otherwise). A set-up that is open on another device is [`MetaError::TwoFactorPending`]; the same + /// device may start again, which replaces the secret and clears its wrong codes. A device that was signed out is + /// [`MetaError::DeviceRevoked`], a password that changed `Conflict`. + pub async fn two_factor_begin_setup(&self, actor: &DeviceActor, pending_ciphertext: &[u8], now_ms: i64) -> Result { + let expires_ms = now_ms.checked_add(SETUP_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + if let Some(row) = load_row(&mut w.tx, actor.user).await? { + if row.active() { + return Err(MetaError::Conflict("two-factor sign-in is on already; turn it off first")); + } + let open = row.pending.is_some() && row.pending_expires_ms.is_some_and(|expires| expires > now_ms); + if open && row.pending_device.as_deref() != Some(actor.device.as_uuid().as_bytes().as_slice()) { + return Err(MetaError::TwoFactorPending); + } + } + sqlx::query( + "INSERT INTO user_two_factor (user_id, pending_ciphertext, pending_device, pending_expires_ms, pending_attempts) \ + VALUES ($1, $2, $3, $4, 0) \ + ON CONFLICT (user_id) DO UPDATE SET pending_ciphertext = $2, pending_device = $3, pending_expires_ms = $4, pending_attempts = 0", + ) + .bind(uuid_bytes(&actor.user.0)) + .bind(pending_ciphertext.to_vec()) + .bind(uuid_bytes(actor.device.as_uuid())) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(expires_ms) + } + + /// Turns two-factor sign-in on with the first code of the set-up that `device` of `user` started: the code must be + /// right for the open set-up's secret (at most [`MAX_SETUP_ATTEMPTS`] wrong ones, then the set-up is used up). In one + /// transaction the pending secret becomes the active one, the account's recovery codes become `recovery_hashes` + /// (`BLAKE3` hashes of the codes the caller shows once), `auth_gen` goes up, every other device is signed out + /// ([`RevokeReason::TwoFactor`]) and the account's invite, pairings and challenges are deleted. Two confirmations + /// at once turn it on once. Returns the devices signed out. No open set-up, one that was started on another device + /// (the code is then not tried) or one that ran out is `Conflict`; a signed-out device [`MetaError::DeviceRevoked`]; + /// a wrong code [`MetaError::TwoFactorInvalid`]. + pub async fn two_factor_confirm( + &self, + user: UserId, + device: DeviceId, + verify: &TotpVerifier<'_>, + recovery_hashes: &[[u8; 32]], + now_ms: i64, + ) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let row = load_row(&mut w.tx, user).await?; + let Some(row) = row.filter(|row| !row.active()) else { + return Err(MetaError::Conflict("there is no two-factor set-up to confirm: start one first")); + }; + let open = row.pending.is_some() && row.pending_expires_ms.is_some_and(|expires| expires > now_ms); + if !open { + return Err(MetaError::Conflict("the two-factor set-up ran out: start it again")); + } + if row.pending_device.as_deref() != Some(device.as_uuid().as_bytes().as_slice()) { + return Err(MetaError::Conflict("this two-factor set-up was started on another device: confirm it there, or start it again here")); + } + let pending = row.pending.as_deref().ok_or(MetaError::Corrupt("an open set-up has no secret"))?; + let step = verify(user, pending).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))?; + let Some(step) = step else { + // Nothing was changed yet: the wrong code is counted and committed in this transaction, under the lock. + count_wrong_setup_in(&mut w.tx, user, device).await?; + w.commit().await?; + return Err(MetaError::TwoFactorInvalid); + }; + let done = sqlx::query( + "UPDATE user_two_factor SET secret_ciphertext = pending_ciphertext, enabled_ms = $1, pending_ciphertext = NULL, pending_device = NULL, \ + pending_expires_ms = NULL, pending_attempts = 0, last_accepted_step = $2, failed_attempts = 0, locked_until_ms = NULL \ + WHERE user_id = $3 AND enabled_ms IS NULL AND pending_ciphertext IS NOT NULL", + ) + .bind(now_ms) + .bind(step) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() != 1 { + return Err(MetaError::Conflict("two-factor sign-in changed while this request was running; try again")); + } + replace_recovery_codes(&mut w.tx, user, recovery_hashes).await?; + sqlx::query("UPDATE users SET auth_gen = auth_gen + 1 WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + let revoked = revoke_devices(&mut w.tx, user, Some(device), RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AccountTwoFactorEnabled).by(user, Some(device)).on_user(user)).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Turns `actor`'s two-factor sign-in off with `proof` (a code or a recovery code): in one transaction the secret, + /// the codes and the challenges go, `auth_gen` goes up, every other device is signed out ([`RevokeReason::TwoFactor`]) + /// and the invite and pairings are deleted. Returns the devices signed out. Off already is `Conflict`; a wrong + /// proof [`MetaError::TwoFactorInvalid`] (counted, and it can lock); a lock in force [`MetaError::TwoFactorLocked`]. + pub async fn two_factor_disable(&self, actor: &DeviceActor, proof: &Proof<'_>, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + let row = load_row(&mut w.tx, actor.user).await?.filter(TfRow::active); + let Some(row) = row else { return Err(MetaError::Conflict("two-factor sign-in is not on")) }; + if let Checked::Wrong = check_proof(&mut w.tx, actor.user, Some(actor.device), false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, actor.user, None, now_ms, false).await); + } + remove_in(&mut w.tx, actor.user).await?; + let revoked = revoke_devices(&mut w.tx, actor.user, Some(actor.device), RevokeReason::TwoFactor, now_ms).await?; + delete_join_codes(&mut w.tx, actor.user).await?; + let event = NewEvent::new(AuditAction::AccountTwoFactorDisabled).by(actor.user, Some(actor.device)).on_user(actor.user); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Replaces `actor`'s recovery codes with `recovery_hashes` (the old ones stop working at once), after `proof`. Off + /// is `Conflict`; the proof's errors are as for [`Meta::two_factor_disable`]. + pub async fn two_factor_new_recovery_codes( + &self, + actor: &DeviceActor, + proof: &Proof<'_>, + recovery_hashes: &[[u8; 32]], + now_ms: i64, + ) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + check_actor(&mut w, actor).await?; + let row = load_row(&mut w.tx, actor.user).await?.filter(TfRow::active); + let Some(row) = row else { return Err(MetaError::Conflict("two-factor sign-in is not on")) }; + if let Checked::Wrong = check_proof(&mut w.tx, actor.user, Some(actor.device), false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, actor.user, None, now_ms, false).await); + } + replace_recovery_codes(&mut w.tx, actor.user, recovery_hashes).await?; + let event = NewEvent::new(AuditAction::AccountRecoveryCodesRegenerated).by(actor.user, Some(actor.device)).on_user(actor.user); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await + } + + /// The confirmation an admin gives for a change: when `user` has two-factor sign-in on, `proof` is checked and used up + /// (the time step or recovery code can't serve a second change), and the answer is `true`; when it is off, nothing is + /// checked and the answer is `false`. Whether it is on is decided under the account's lock. A missing proof for an + /// account that has it on is [`MetaError::TwoFactorInvalid`] (not counted: nothing was tried); the other errors are as + /// for [`Meta::two_factor_disable`]. + pub async fn confirm_second_factor(&self, user: UserId, proof: Option<&Proof<'_>>, now_ms: i64) -> Result { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let Some(row) = load_row(&mut w.tx, user).await?.filter(TfRow::active) else { return Ok(false) }; + let Some(proof) = proof else { return Err(MetaError::TwoFactorInvalid) }; + if let Checked::Wrong = check_proof(&mut w.tx, user, None, false, &row, proof, now_ms).await? { + return Err(self.refuse_proof(w, user, None, now_ms, false).await); + } + // The proof is used up here, in a transaction of its own that commits before the change it confirms: a recovery + // code used for a change that then fails is still used, and its event stays. + self.commit_audited(w, now_ms).await?; + Ok(true) + } + + /// Takes away `user`'s two-factor sign-in on the command line: the secret, the codes and the challenges go, the lock is + /// gone with the row, every device is signed out ([`RevokeReason::TwoFactor`]) and the invite and pairings are + /// deleted. Returns the devices signed out. `NotFound` for a user that isn't there, `Conflict` for one without + /// two-factor sign-in (nothing is signed out then). + pub async fn reset_two_factor(&self, user: UserId, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("user")); + } + let revoked = reset_in(&mut w.tx, user, now_ms).await?.ok_or(MetaError::Conflict("this account has no two-factor sign-in"))?; + let event = NewEvent::new(AuditAction::CliTwoFactorReset).on_user(user).with(AuditDetails::revoked(revoked.len())); + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// A password sign-in that checked out, as [`Meta::sign_in_device`], for an account that may have two-factor sign-in: + /// under the account's lock the password hash and `auth_gen` the caller read before verifying the password are + /// compared again, then the account's **current** state decides. Without two-factor sign-in the device is made + /// ([`SignIn::Device`]); with it only `challenge` is made ([`SignIn::Challenge`], good for [`CHALLENGE_TTL_MS`]; the + /// account's expired challenges are deleted and the oldest goes when it would have more than + /// [`MAX_OPEN_CHALLENGES`]) and no device. `None`, a disabled account and the errors are as for `sign_in_device`. + #[allow(clippy::too_many_arguments)] + pub async fn sign_in( + &self, + user: UserId, + checked: (&str, i64), + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + challenge: &NewChallenge, + now_ms: i64, + ) -> Result, MetaError> { + check_device_name(name)?; + client.check()?; + let (hash, auth_gen) = checked; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Ok(None); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + if is_enabled(&mut w.tx, user).await? { + let expires_ms = now_ms.checked_add(CHALLENGE_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + sqlx::query("DELETE FROM login_challenges WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; + let open: i64 = sqlx::query("SELECT COUNT(*) AS n FROM login_challenges WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_one(&mut *w.tx) + .await? + .try_get("n")?; + if open >= MAX_OPEN_CHALLENGES { + sqlx::query( + "DELETE FROM login_challenges WHERE id_hash IN \ + (SELECT id_hash FROM login_challenges WHERE user_id = $1 ORDER BY expires_ms ASC, id_hash ASC LIMIT $2)", + ) + .bind(uuid_bytes(&user.0)) + .bind(open - MAX_OPEN_CHALLENGES + 1) + .execute(&mut *w.tx) + .await?; + } + sqlx::query( + "INSERT INTO login_challenges (id_hash, user_id, kind, client_key, device_name, client_kind, platform, app_version, attempts, expires_ms) \ + VALUES ($1, $2, 'login', $3, $4, $5, $6, $7, 0, $8)", + ) + .bind(challenge.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(challenge.client_key.as_str()) + .bind(name) + .bind(client.kind.as_str()) + .bind(client.platform.as_str()) + .bind(client.app_version.as_str()) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + return Ok(Some(SignIn::Challenge { expires_ms })); + } + let device = DeviceId::generate(); + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; + // A sign-in that made a device is a login; a challenge (above) is not: nobody is signed in yet. + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(SignIn::Device(device))) + } + + /// The account a challenge was made for and its username, by a plain read: only so the caller can count the try against + /// the account before the real check. `None` for a challenge that is unknown, expired or used. Only the challenges of a + /// device sign-in count: a web sign-in's ([`Meta::web_challenge_owner`]) can never make a device. + pub async fn challenge_owner(&self, id_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT c.user_id, u.username FROM login_challenges c JOIN users u ON u.id = c.user_id WHERE c.id_hash = $1 AND c.kind = 'login' AND c.expires_ms > $2", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some((UserId(uuid_from(&row.try_get::, _>("user_id")?)?), row.try_get("username")?))) + } + + /// Finishes a sign-in with the second factor: the account is found by a plain read of the challenge, its row is locked + /// **before** the challenge or the factor is touched, and under the lock the challenge (still there, not expired) and + /// the account's **current** state (two-factor sign-in still on, not disabled) are checked again. Only then are the + /// proof and the challenge used up and the device made, in this one transaction. A challenge that is unknown, expired, + /// used or no longer fit is [`MetaError::ChallengeGone`], the same for every case; a wrong proof is + /// [`MetaError::TwoFactorInvalid`] after its counters were committed on their own (the challenge's attempts, deleted at + /// the fifth; the account's wrong proofs and the lock); a lock in force is [`MetaError::TwoFactorLocked`]. + pub async fn complete_second_factor( + &self, + challenge: &[u8; 32], + proof: &Proof<'_>, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result { + let Some((user, _)) = self.challenge_owner(challenge, now_ms).await? else { return Err(MetaError::ChallengeGone) }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + let found = sqlx::query( + "SELECT device_name, client_kind, platform, app_version, attempts, expires_ms FROM login_challenges WHERE id_hash = $1 AND user_id = $2 AND kind = 'login'", + ) + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(found) = found.filter(|row| row.try_get::("expires_ms").is_ok_and(|expires| expires > now_ms)) else { + return Err(MetaError::ChallengeGone); + }; + let attempts: i64 = found.try_get("attempts")?; + let row = load_row(&mut w.tx, user).await?.filter(TfRow::active); + let Some(row) = row else { + // Turned off (or reset) since: the challenge is of no use any more. + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + return Err(MetaError::ChallengeGone); + }; + if attempts >= MAX_CHALLENGE_ATTEMPTS { + return Err(MetaError::ChallengeGone); + } + let device = DeviceId::generate(); + if let Checked::Wrong = check_proof(&mut w.tx, user, Some(device), false, &row, proof, now_ms).await? { + return Err(self.refuse_sign_in_proof(w, user, Some(challenge), proof, now_ms, false).await); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + let used = sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND user_id = $2") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if used.rows_affected() != 1 { + return Err(MetaError::ChallengeGone); + } + let client = + DeviceClient { kind: found.try_get("client_kind")?, platform: found.try_get("platform")?, app_version: found.try_get("app_version")? }; + let name: String = found.try_get("device_name")?; + insert_device(&mut w.tx, device, user, &name, &client, token_hash, now_ms).await?; + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(&client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Completed { user, device }) + } + + /// `w` refused a wrong proof and has changed nothing (every caller checks the proof before it consumes anything): the wrong + /// proof's counters are written and **committed in `w` itself**, with the account's row still locked, and the answer is + /// [`MetaError::TwoFactorInvalid`] (or the error that stopped the counters being written). Counting under the same lock as + /// the check means the next request sees the attempts and the lock this one made: no more than five wrong proofs per + /// challenge, and the lock set by exactly the tenth. + pub(crate) async fn refuse_proof(&self, mut w: WriteTx, user: UserId, challenge: Option<&[u8; 32]>, now_ms: i64, web: bool) -> MetaError { + let counted = async { + let locked = count_wrong_in(&mut w.tx, user, challenge, now_ms).await?; + w.commit().await?; + Ok::(locked) + } + .await; + match counted { + Ok(locked) => { + if locked { + // After the counters and the lock are committed, like a failed sign-in's event: an error in the log + // is reported and changes neither the answer nor what was counted (failing closed here would let a + // fault in the log undo the lock). + if let Err(e) = self.audit_lock_set(user, now_ms, web).await { + tracing::warn!(error = %e, "couldn't write an audit event"); + } + } + MetaError::TwoFactorInvalid + } + Err(e) => e, + } + } + + /// `auth.two_factor_locked`, in a transaction of its own. Nobody proved who they were: no actor. + async fn audit_lock_set(&self, user: UserId, now_ms: i64, web: bool) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + let event = NewEvent::new(AuditAction::AuthTwoFactorLocked).on_user(user); + let event = if web { event.with(AuditDetails::web()) } else { event }; + record_in(&mut w.tx, now_ms, event).await?; + self.commit_audited(w, now_ms).await + } + + /// [`Meta::refuse_proof`] for a proof given at sign-in (the second step, or a join): after the counters are committed + /// and the answer is decided, the wrong proof is also counted in the audit log, `auth.second_factor_failed`, in a + /// transaction of its own and aggregated per account and minute. Writing it can fail without changing the answer. A + /// wrong proof on a signed-in route (turning it off, new recovery codes, an admin's confirmation) is not written one by + /// one: the lock they can cause is (`auth.two_factor_locked`). + pub(crate) async fn refuse_sign_in_proof( + &self, + w: WriteTx, + user: UserId, + challenge: Option<&[u8; 32]>, + proof: &Proof<'_>, + now_ms: i64, + web: bool, + ) -> MetaError { + let refused = self.refuse_proof(w, user, challenge, now_ms, web).await; + if matches!(refused, MetaError::TwoFactorInvalid) { + let step = match proof { + Proof::Totp(_) => AuditStep::Totp, + Proof::Recovery(_) => AuditStep::Recovery, + }; + let failure = AuditFailure { action: AuditAction::AuthSecondFactorFailed, target: Some(user), step }; + let result = if web { self.audit_record_web_failure(failure, now_ms).await } else { self.audit_record_failure(failure, now_ms).await }; + if let Err(e) = result { + tracing::warn!(error = %e, "couldn't write an audit event"); + } + } + refused + } + + /// The ciphertexts of every account's secrets (the active ones, and the open set-ups' that are not expired), for the + /// server to open them at start and for the key tools. + pub async fn two_factor_secrets(&self, now_ms: i64) -> Result, MetaError> { + let rows = sqlx::query( + "SELECT user_id, secret_ciphertext, pending_ciphertext, pending_expires_ms FROM user_two_factor \ + WHERE secret_ciphertext IS NOT NULL OR pending_ciphertext IS NOT NULL ORDER BY user_id", + ) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|row| { + let open = row.try_get::, _>("pending_expires_ms")?.is_some_and(|expires| expires > now_ms); + Ok(StoredSecrets { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + active: row.try_get("secret_ciphertext")?, + pending: if open { row.try_get("pending_ciphertext")? } else { None }, + }) + }) + .collect() + } + + /// Every account's ciphertexts, the open set-ups' and the expired ones too (a plain read, for the key tools). + pub async fn two_factor_all_secrets(&self) -> Result, MetaError> { + let rows = sqlx::query( + "SELECT user_id, secret_ciphertext, pending_ciphertext FROM user_two_factor \ + WHERE secret_ciphertext IS NOT NULL OR pending_ciphertext IS NOT NULL ORDER BY user_id", + ) + .fetch_all(&self.pool) + .await?; + rows.iter() + .map(|row| { + Ok(StoredSecrets { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + active: row.try_get("secret_ciphertext")?, + pending: row.try_get("pending_ciphertext")?, + }) + }) + .collect() + } + + /// Rewrites `user`'s ciphertexts, the active secret and the set-up's both, **read fresh under the account's lock** (a + /// confirmation that moved a set-up's secret to the active one meanwhile is seen): `rewrite` gets each and answers the + /// ciphertext to store in its place, or `None` to leave it. Returns how many were rewritten and how many left. An account + /// that is gone, or has none, is `(0, 0)`; a ciphertext `rewrite` can't handle is an error and changes nothing. + pub async fn two_factor_rewrite(&self, user: UserId, rewrite: &Rewrite<'_>) -> Result<(usize, usize), MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok((0, 0)); + } + let row = sqlx::query("SELECT secret_ciphertext, pending_ciphertext FROM user_two_factor WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(row) = row else { return Ok((0, 0)) }; + let (mut rewritten, mut kept) = (0, 0); + for (column, update) in [ + ("secret_ciphertext", "UPDATE user_two_factor SET secret_ciphertext = $1 WHERE user_id = $2"), + ("pending_ciphertext", "UPDATE user_two_factor SET pending_ciphertext = $1 WHERE user_id = $2"), + ] { + let Some(sealed) = row.try_get::>, _>(column)? else { continue }; + match rewrite(user, &sealed).map_err(|_| MetaError::Corrupt("a stored two-factor secret can't be read"))? { + Some(new) => { + sqlx::query(update).bind(new).bind(uuid_bytes(&user.0)).execute(&mut *w.tx).await?; + rewritten += 1; + } + None => kept += 1, + } + } + w.commit().await?; + Ok((rewritten, kept)) + } + + /// Replaces the ciphertext `old` of `user`'s active secret (`pending` false) or open set-up (`pending` true) by `new`, if + /// it still is `old`; `false` when it is not (changed meanwhile). For rewriting secrets under a new key. + pub async fn two_factor_replace_ciphertext(&self, user: UserId, pending: bool, old: &[u8], new: &[u8]) -> Result { + let sql = if pending { + "UPDATE user_two_factor SET pending_ciphertext = $1 WHERE user_id = $2 AND pending_ciphertext = $3" + } else { + "UPDATE user_two_factor SET secret_ciphertext = $1 WHERE user_id = $2 AND secret_ciphertext = $3" + }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(false); + } + let done = sqlx::query(sql).bind(new.to_vec()).bind(uuid_bytes(&user.0)).bind(old.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(done.rows_affected() == 1) + } +} + +/// Counts a wrong code against the open set-up of `device`, in `tx`; the fifth uses the set-up up. +async fn count_wrong_setup_in(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, device: DeviceId) -> Result<(), MetaError> { + sqlx::query( + "UPDATE user_two_factor SET pending_attempts = pending_attempts + 1 \ + WHERE user_id = $1 AND enabled_ms IS NULL AND pending_device = $2 AND pending_ciphertext IS NOT NULL", + ) + .bind(uuid_bytes(&user.0)) + .bind(uuid_bytes(device.as_uuid())) + .execute(&mut **tx) + .await?; + sqlx::query( + "UPDATE user_two_factor SET pending_ciphertext = NULL, pending_device = NULL, pending_expires_ms = NULL, pending_attempts = 0 \ + WHERE user_id = $1 AND enabled_ms IS NULL AND pending_attempts >= $2", + ) + .bind(uuid_bytes(&user.0)) + .bind(MAX_SETUP_ATTEMPTS) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Counts one wrong second factor of `user` in `tx` (the account's row is locked by it): the wrong proofs in a row go up, +/// and at [`LOCK_AFTER_FAILURES`] (and every one after) the lock is set or doubled; the challenge's attempts go up too, and +/// the challenge is deleted at [`MAX_CHALLENGE_ATTEMPTS`]. Lock order: the account's row, the two-factor row, the challenge. +/// Answers whether a lock was set or lengthened (for the caller to write its event once the counters are committed). +async fn count_wrong_in( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + challenge: Option<&[u8; 32]>, + now_ms: i64, +) -> Result { + let mut locked = false; + let counted = sqlx::query( + "UPDATE user_two_factor SET failed_attempts = failed_attempts + 1 WHERE user_id = $1 AND enabled_ms IS NOT NULL \ + RETURNING failed_attempts, locked_until_ms", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut **tx) + .await?; + if let Some(row) = counted + && let Some(duration) = lock_for(row.try_get("failed_attempts")?) + { + let until = now_ms.saturating_add(duration); + let until = row.try_get::, _>("locked_until_ms")?.map_or(until, |current| current.max(until)); + sqlx::query("UPDATE user_two_factor SET locked_until_ms = $1 WHERE user_id = $2") + .bind(until) + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + // Only here, where the lock is set or doubled, and never for a refusal while it is in force (`check_proof` stops + // those before anything is counted). + locked = true; + } + if let Some(challenge) = challenge { + sqlx::query("UPDATE login_challenges SET attempts = attempts + 1 WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND attempts >= $2") + .bind(challenge.to_vec()) + .bind(MAX_CHALLENGE_ATTEMPTS) + .execute(&mut **tx) + .await?; + } + Ok(locked) +} + +/// Makes `hashes` the account's recovery codes, all unused, in `tx` (the old ones, used or not, are gone). +async fn replace_recovery_codes(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId, hashes: &[[u8; 32]]) -> Result<(), MetaError> { + sqlx::query("DELETE FROM user_recovery_codes WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + for hash in hashes { + sqlx::query("INSERT INTO user_recovery_codes (user_id, code_hash, used_ms) VALUES ($1, $2, NULL)") + .bind(uuid_bytes(&user.0)) + .bind(hash.to_vec()) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lock_starts_at_ten_and_doubles_up_to_a_day() { + assert_eq!(lock_for(0), None); + assert_eq!(lock_for(9), None); + assert_eq!(lock_for(10), Some(15 * 60 * 1000)); + assert_eq!(lock_for(11), Some(30 * 60 * 1000)); + assert_eq!(lock_for(12), Some(60 * 60 * 1000)); + assert_eq!(lock_for(15), Some(8 * 60 * 60 * 1000), "8 hours"); + assert_eq!(lock_for(16), Some(16 * 60 * 60 * 1000), "16 hours"); + assert_eq!(lock_for(17), Some(LOCK_MAX_MS), "32 hours is over a day"); + assert_eq!(lock_for(i64::MAX), Some(LOCK_MAX_MS)); + } + + #[test] + fn sealed_secrets_show_only_their_length() { + let secrets = StoredSecrets { user: UserId(uuid::Uuid::nil()), active: Some(vec![7, 8, 9]), pending: None }; + let shown = format!("{secrets:?}"); + assert!(shown.contains("<3 bytes>") && !shown.contains("[7"), "{shown}"); + let device = DeviceActor { user: UserId(uuid::Uuid::nil()), device: DeviceId::generate(), password_hash: "$argon2id$secret".to_owned() }; + assert!(!format!("{device:?}").contains("argon2id")); + } + + #[test] + fn retry_after_rounds_up_and_is_never_zero() { + assert_eq!(retry_after_secs(0), 1); + assert_eq!(retry_after_secs(1), 1); + assert_eq!(retry_after_secs(1000), 1); + assert_eq!(retry_after_secs(1001), 2); + assert_eq!(retry_after_secs(-5), 1); + } +} diff --git a/crates/meta/src/usage.rs b/crates/meta/src/usage.rs new file mode 100644 index 00000000..995b6851 --- /dev/null +++ b/crates/meta/src/usage.rs @@ -0,0 +1,207 @@ +//! Read-only storage estimates for one space. + +use craft_cloud_proto::SpaceId; +use sqlx::Row; + +use crate::blobs::effective_quota; +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::{Meta, MetaError, UserId}; + +const STORAGE_ESTIMATE_SQL: &str = include_str!("usage_estimate.sql"); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageEstimateReason { + IndexIncomplete, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StorageEstimate { + pub stored_bytes: u64, + pub quota_bytes: Option, + pub trash_bytes: Option, + pub history_bytes: Option, + pub reason: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageAccount { + pub id: UserId, + pub username: String, + pub stored_bytes: u64, + pub quota_bytes: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageOverview { + pub accounts: u64, + pub stored_bytes: u64, + pub default_quota_bytes: Option, + pub accounts_with_override: u64, + pub accounts_near_quota: u64, + pub accounts_over_quota: u64, + pub top_accounts: Vec, + pub manifest_index: StorageIndexCounts, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StorageIndexCounts { + pub complete: u64, + pub building: u64, + pub not_indexed: u64, +} + +const STORAGE_OVERVIEW_SQL: &str = "WITH account_rows AS (\ + SELECT u.id, u.username, s.stored_bytes, s.quota_bytes, COALESCE(s.quota_bytes, $1) AS effective_quota \ + FROM users u JOIN spaces s ON s.owner = u.id WHERE u.removed_ms IS NULL\ +), account_totals AS (\ + SELECT COUNT(*) AS accounts, CAST(COALESCE(SUM(stored_bytes), 0) AS BIGINT) AS stored_bytes, \ + COALESCE(SUM(CASE WHEN quota_bytes IS NOT NULL THEN 1 ELSE 0 END), 0) AS accounts_with_override, \ + COALESCE(SUM(CASE WHEN effective_quota IS NOT NULL AND stored_bytes >= effective_quota - (effective_quota / 10) THEN 1 ELSE 0 END), 0) AS accounts_near_quota, \ + COALESCE(SUM(CASE WHEN effective_quota IS NOT NULL AND stored_bytes >= effective_quota THEN 1 ELSE 0 END), 0) AS accounts_over_quota \ + FROM account_rows\ +), index_counts AS (\ + SELECT COUNT(*) AS manifests, \ + COALESCE(SUM(CASE WHEN i.state = 1 THEN 1 ELSE 0 END), 0) AS complete, \ + COALESCE(SUM(CASE WHEN i.manifest IS NOT NULL AND i.state <> 1 THEN 1 ELSE 0 END), 0) AS building \ + FROM blobs b LEFT JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1\ +) SELECT account_totals.*, index_counts.manifests, index_counts.complete AS index_complete, index_counts.building AS index_building \ +FROM account_totals CROSS JOIN index_counts"; + +impl Meta { + /// Reads one consistent snapshot. Estimates are null while any manifest in the space has an incomplete index. + pub async fn storage_estimate(&self, space: SpaceId) -> Result { + let mut tx = self.read_snapshot().await?; + let row = sqlx::query(STORAGE_ESTIMATE_SQL) + .bind(uuid_bytes(space.as_uuid())) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("space"))?; + let estimate = self.storage_estimate_from_row(&row)?; + tx.commit().await?; + Ok(estimate) + } + + /// Reads an account's estimate in one snapshot. Removed and unknown accounts are indistinguishable. + pub async fn admin_storage_estimate(&self, user: UserId) -> Result { + let mut tx = self.read_snapshot().await?; + let row = sqlx::query("SELECT s.id FROM users u JOIN spaces s ON s.owner = u.id WHERE u.id = $1 AND u.removed_ms IS NULL") + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("user"))?; + let space = uuid_from(&row.try_get::, _>("id")?)?; + let row = sqlx::query(STORAGE_ESTIMATE_SQL).bind(uuid_bytes(&space)).fetch_optional(&mut *tx).await?.ok_or(MetaError::NotFound("user"))?; + let estimate = self.storage_estimate_from_row(&row)?; + tx.commit().await?; + Ok(estimate) + } + + /// Aggregate counts and the ten largest current accounts from one consistent snapshot. + pub async fn admin_storage_overview(&self) -> Result { + let mut tx = self.read_snapshot().await?; + let cap = self.stored_bytes_cap().map(i64::try_from).transpose().map_err(|_| MetaError::Invalid("number too large"))?; + let row = sqlx::query(STORAGE_OVERVIEW_SQL).bind(cap).fetch_one(&mut *tx).await?; + let top_rows = sqlx::query( + "SELECT u.id, u.username, s.stored_bytes, COALESCE(s.quota_bytes, $1) AS effective_quota \ + FROM users u JOIN spaces s ON s.owner = u.id WHERE u.removed_ms IS NULL \ + ORDER BY s.stored_bytes DESC, u.username ASC, u.id ASC LIMIT 10", + ) + .bind(cap) + .fetch_all(&mut *tx) + .await?; + let top_accounts = top_rows + .iter() + .map(|row| { + let quota: Option = row.try_get("effective_quota")?; + Ok(StorageAccount { + id: UserId(uuid_from(&row.try_get::, _>("id")?)?), + username: row.try_get("username")?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + quota_bytes: quota.map(to_u64).transpose()?, + }) + }) + .collect::, MetaError>>()?; + let manifests: u64 = to_u64(row.try_get("manifests")?)?; + let complete: u64 = to_u64(row.try_get("index_complete")?)?; + let building: u64 = to_u64(row.try_get("index_building")?)?; + let overview = StorageOverview { + accounts: to_u64(row.try_get("accounts")?)?, + stored_bytes: to_u64(row.try_get("stored_bytes")?)?, + default_quota_bytes: self.stored_bytes_cap(), + accounts_with_override: to_u64(row.try_get("accounts_with_override")?)?, + accounts_near_quota: to_u64(row.try_get("accounts_near_quota")?)?, + accounts_over_quota: to_u64(row.try_get("accounts_over_quota")?)?, + top_accounts, + manifest_index: StorageIndexCounts { complete, building, not_indexed: manifests.saturating_sub(complete).saturating_sub(building) }, + }; + tx.commit().await?; + Ok(overview) + } + + fn storage_estimate_from_row(&self, row: &sqlx::any::AnyRow) -> Result { + let stored_bytes = to_u64(row.try_get("stored_bytes")?)?; + let stored_quota: Option = row.try_get("quota_bytes")?; + let quota_bytes = effective_quota(stored_quota.map(to_u64).transpose()?, self.stored_bytes_cap()); + let trash_bytes: Option = row.try_get("trash_bytes")?; + let history_bytes: Option = row.try_get("history_bytes")?; + let (trash_bytes, history_bytes, reason) = match (trash_bytes, history_bytes) { + (Some(trash), Some(history)) => (Some(to_u64(trash)?), Some(to_u64(history)?), None), + (None, None) => (None, None, Some(StorageEstimateReason::IndexIncomplete)), + _ => return Err(MetaError::Corrupt("storage estimates have inconsistent index state")), + }; + Ok(StorageEstimate { stored_bytes, quota_bytes, trash_bytes, history_bytes, reason }) + } +} + +#[cfg(test)] +mod tests { + use craft_cloud_proto::{ChunkId, ChunkRef, ChunkerParamsId, FileHash, Manifest}; + use sqlx::Row; + + use crate::{Meta, Role}; + + #[tokio::test] + async fn any_incomplete_space_manifest_makes_both_estimates_null() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let owner = meta.create_user("ana", "$argon2id$stub", Role::Member, 1).await.unwrap(); + let chunk = ChunkId::from_bytes([1; 32]); + meta.add_chunk(owner.space, &chunk, 6, 1).await.unwrap(); + let manifest = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([2; 32]), vec![ChunkRef { id: chunk, len: 5 }]) + .unwrap(); + meta.add_manifest(owner.space, &manifest, 100, 1).await.unwrap(); + sqlx::query("UPDATE manifest_index SET state = 0 WHERE manifest = $1") + .bind(manifest.id().as_bytes().to_vec()) + .execute(&meta.pool) + .await + .unwrap(); + + let estimate = meta.storage_estimate(owner.space).await.unwrap(); + assert_eq!((estimate.trash_bytes, estimate.history_bytes), (None, None)); + assert_eq!(estimate.reason, Some(super::StorageEstimateReason::IndexIncomplete)); + } + + #[tokio::test] + async fn storage_estimate_plan_uses_reference_and_version_indexes() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let owner = meta.create_user("ana", "$argon2id$stub", Role::Member, 1).await.unwrap(); + let plan_sql = format!("EXPLAIN QUERY PLAN {}", super::STORAGE_ESTIMATE_SQL); + let rows = + sqlx::query(sqlx::AssertSqlSafe(plan_sql)).bind(crate::codec::uuid_bytes(owner.space.as_uuid())).fetch_all(&meta.pool).await.unwrap(); + let plan = rows.iter().map(|row| row.try_get::("detail").unwrap()).collect::>().join(" | "); + assert!(plan.contains("versions_node_seq"), "missing existing version index in plan: {plan}"); + assert!(plan.contains("manifest_chunks_by_chunk"), "missing existing reverse-reference index in plan: {plan}"); + println!("storage estimate SQLite plan: {plan}"); + } + + #[tokio::test] + async fn storage_overview_plan_uses_bounded_aggregate_scans() { + let meta = Meta::open("sqlite::memory:").await.unwrap(); + let plan_sql = format!("EXPLAIN QUERY PLAN {}", super::STORAGE_OVERVIEW_SQL); + let rows = sqlx::query(sqlx::AssertSqlSafe(plan_sql)).bind(Some(1_000_i64)).fetch_all(&meta.pool).await.unwrap(); + let plan = rows.iter().map(|row| row.try_get::("detail").unwrap()).collect::>().join(" | "); + assert!(plan.contains("SCAN u") || plan.contains("SEARCH u"), "missing account aggregate scan in plan: {plan}"); + assert!(plan.contains("b USING") && plan.contains("i USING"), "missing bounded manifest/index scans in plan: {plan}"); + println!("storage overview SQLite plan: {plan}"); + } +} diff --git a/crates/meta/src/usage_estimate.sql b/crates/meta/src/usage_estimate.sql new file mode 100644 index 00000000..80bc2682 --- /dev/null +++ b/crates/meta/src/usage_estimate.sql @@ -0,0 +1,50 @@ +WITH space_manifests AS ( + SELECT sb.id AS manifest FROM space_blobs sb WHERE sb.space_id = $1 AND sb.kind = 1 +), incomplete AS ( + SELECT 1 FROM space_manifests sm + LEFT JOIN manifest_index i ON i.manifest = sm.manifest + WHERE i.manifest IS NULL OR i.state <> 1 OR i.indexed_ordinals <> i.chunk_count + OR i.chunk_count <> (SELECT COUNT(*) FROM manifest_chunks mc WHERE mc.manifest = sm.manifest) + LIMIT 1 +), version_roles AS ( + SELECT v.manifest, + MAX(CASE WHEN n.deleted = 1 THEN 1 ELSE 0 END) AS has_deleted, + MAX(CASE WHEN n.deleted = 0 THEN 1 ELSE 0 END) AS has_live, + MAX(CASE WHEN n.deleted = 0 AND n.version = v.id THEN 1 ELSE 0 END) AS has_current, + MAX(CASE WHEN n.deleted = 0 AND n.version <> v.id THEN 1 ELSE 0 END) AS has_history, + MAX(CASE WHEN n.deleted = 0 AND v.save IS NOT NULL THEN 1 ELSE 0 END) AS has_saved + FROM nodes n JOIN versions v ON v.space_id = n.space_id AND v.node = n.id + WHERE n.space_id = $1 + GROUP BY v.manifest +), snapshot_manifests AS ( + SELECT snapshot AS manifest FROM live_docs WHERE space_id = $1 AND snapshot IS NOT NULL + UNION + SELECT import_snapshot AS manifest FROM live_docs WHERE space_id = $1 AND import_snapshot IS NOT NULL +), classified AS ( + SELECT sm.manifest, b.stored_size, + CASE WHEN vr.has_deleted = 1 AND vr.has_live = 0 AND snapshots.manifest IS NULL + THEN 1 ELSE 0 END AS trash_only, + CASE WHEN vr.has_history = 1 AND vr.has_deleted = 0 AND vr.has_current = 0 AND vr.has_saved = 0 AND snapshots.manifest IS NULL + THEN 1 ELSE 0 END AS history_only + FROM space_manifests sm JOIN blobs b ON b.kind = 1 AND b.id = sm.manifest + LEFT JOIN version_roles vr ON vr.manifest = sm.manifest + LEFT JOIN snapshot_manifests snapshots ON snapshots.manifest = sm.manifest +), eligible_chunks AS ( + SELECT sb.id, b.stored_size, + MIN(CASE WHEN c.trash_only = 1 THEN 1 ELSE 0 END) AS trash_only, + MIN(CASE WHEN c.history_only = 1 THEN 1 ELSE 0 END) AS history_only + FROM space_blobs sb JOIN blobs b ON b.kind = 0 AND b.id = sb.id + JOIN manifest_chunks mc ON mc.chunk = sb.id + JOIN space_manifests sm ON sm.manifest = mc.manifest + JOIN classified c ON c.manifest = sm.manifest + WHERE sb.space_id = $1 + GROUP BY sb.id, b.stored_size +) +SELECT s.stored_bytes, s.quota_bytes, + CASE WHEN EXISTS (SELECT 1 FROM incomplete) THEN NULL ELSE + CAST(COALESCE((SELECT SUM(stored_size) FROM classified WHERE trash_only = 1), 0) AS BIGINT) + + CAST(COALESCE((SELECT SUM(stored_size) FROM eligible_chunks WHERE trash_only = 1), 0) AS BIGINT) END AS trash_bytes, + CASE WHEN EXISTS (SELECT 1 FROM incomplete) THEN NULL ELSE + CAST(COALESCE((SELECT SUM(stored_size) FROM classified WHERE history_only = 1), 0) AS BIGINT) + + CAST(COALESCE((SELECT SUM(stored_size) FROM eligible_chunks WHERE history_only = 1), 0) AS BIGINT) END AS history_bytes +FROM spaces s WHERE s.id = $1 diff --git a/crates/meta/src/users.rs b/crates/meta/src/users.rs index 1d2b0255..9f02abcb 100644 --- a/crates/meta/src/users.rs +++ b/crates/meta/src/users.rs @@ -3,11 +3,16 @@ //! Passwords are hashed by the caller (Argon2id, in the server); this module only stores the hash string. An account //! an invite created has an empty one until the invite is redeemed: no password matches it, since it is no hash. //! Device tokens are random 256-bit values; only their hash is stored, and a device is found by that hash. +//! +//! A device also records what client made it ([`DeviceClient`], shown in the device list and never a basis for a +//! permission), and, once revoked, when and why ([`RevokeReason`]). The `revoked` flag is the one that is checked: +//! the flag, the time and the reason are always set together, in one transaction, and only by the first revoke. use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId}; use sqlx::Row; use uuid::Uuid; +use crate::audit::{AuditAction, AuditDetails, NewEvent, Origin, record_in}; use crate::codec::{flag, uuid_bytes, uuid_from}; use crate::{Meta, MetaError}; @@ -21,14 +26,14 @@ pub enum Role { } impl Role { - fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Role::Admin => "admin", Role::Member => "member", } } - fn parse(s: &str) -> Result { + pub(crate) fn parse(s: &str) -> Result { match s { "admin" => Ok(Role::Admin), "member" => Ok(Role::Member), @@ -44,6 +49,9 @@ pub struct UserRecord { pub password_hash: String, pub role: Role, pub space: SpaceId, + /// Counts the changes that end what a sign-in checked before them (a disable, an enable, a removal, a password + /// change or reset). Read together with the password hash, and handed back to [`Meta::sign_in_device`]. + pub auth_gen: i64, } impl std::fmt::Debug for UserRecord { @@ -54,10 +62,114 @@ impl std::fmt::Debug for UserRecord { .field("password_hash", &"") .field("role", &self.role) .field("space", &self.space) + .field("auth_gen", &self.auth_gen) .finish() } } +/// What `GET /account` tells about a user. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountRecord { + pub id: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + /// When the password last changed through the server (`None`: never, the password is still the first one). + pub password_changed_ms: Option, + /// Whether signing in asks for a second factor. + pub two_factor_enabled: bool, +} + +/// The client kinds a device may name when it signs in. Empty means the client didn't say. +pub const CLIENT_KINDS: [&str; 3] = ["agent", "desktop", "cli"]; +/// The longest platform or app version, in bytes. +pub const MAX_CLIENT_FIELD_BYTES: usize = 32; + +/// What a device says about the client it signed in with. Display only: no permission ever depends on it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DeviceClient { + /// One of [`CLIENT_KINDS`], or empty. + pub kind: String, + pub platform: String, + pub app_version: String, +} + +impl DeviceClient { + /// The client a sign-in names; a field left out is empty. An unknown kind, a field over + /// [`MAX_CLIENT_FIELD_BYTES`] or a control character is `Invalid`. + pub fn new(kind: Option<&str>, platform: Option<&str>, app_version: Option<&str>) -> Result { + let client = Self { + kind: kind.unwrap_or_default().to_owned(), + platform: platform.unwrap_or_default().to_owned(), + app_version: app_version.unwrap_or_default().to_owned(), + }; + client.check()?; + Ok(client) + } + + pub(crate) fn check(&self) -> Result<(), MetaError> { + if !self.kind.is_empty() && !CLIENT_KINDS.contains(&self.kind.as_str()) { + return Err(MetaError::Invalid("client_kind is agent, desktop or cli")); + } + for field in [&self.platform, &self.app_version] { + if field.len() > MAX_CLIENT_FIELD_BYTES || field.chars().any(char::is_control) { + return Err(MetaError::Invalid("platform and app_version are at most 32 bytes without control characters")); + } + } + Ok(()) + } +} + +/// Why a device was revoked, as stored in `devices.revoke_reason`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RevokeReason { + /// The user revoked this device (`DELETE /devices/{id}`). + User, + /// The user signed out every other device. + Others, + /// The user changed the password. + Password, + /// An admin revoked it, on the command line or through the admin API. + Admin, + /// The account was disabled or marked for removal by an admin. + Disabled, + /// Two-factor sign-in was set up or changed (reserved). + TwoFactor, + /// The device sat idle too long (reserved). + Idle, + /// An admin reset the password (on the command line), or a reset code the admin made was redeemed. + Reset, +} + +impl RevokeReason { + pub fn as_str(self) -> &'static str { + match self { + RevokeReason::User => "user", + RevokeReason::Others => "others", + RevokeReason::Password => "password", + RevokeReason::Admin => "admin", + RevokeReason::Disabled => "disabled", + RevokeReason::TwoFactor => "two_factor", + RevokeReason::Idle => "idle", + RevokeReason::Reset => "reset", + } + } + + fn parse(s: &str) -> Result { + Ok(match s { + "user" => RevokeReason::User, + "others" => RevokeReason::Others, + "password" => RevokeReason::Password, + "admin" => RevokeReason::Admin, + "disabled" => RevokeReason::Disabled, + "two_factor" => RevokeReason::TwoFactor, + "idle" => RevokeReason::Idle, + "reset" => RevokeReason::Reset, + _ => return Err(MetaError::Corrupt("unknown revoke reason")), + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DeviceRecord { pub id: DeviceId, @@ -65,6 +177,11 @@ pub struct DeviceRecord { pub created_ms: i64, pub last_used_ms: i64, pub revoked: bool, + /// What the device said about its client when it signed in (empty for devices from before it was recorded). + pub client: DeviceClient, + /// When it was revoked, and why; both `None` while it isn't (and for devices revoked before they were recorded). + pub revoked_ms: Option, + pub revoke_reason: Option, } /// Who a device token belongs to. @@ -129,7 +246,7 @@ impl Meta { pub async fn user_by_name(&self, username: &str) -> Result, MetaError> { let username = username.to_ascii_lowercase(); let row = sqlx::query( - "SELECT u.id, u.username, u.password_hash, u.role, s.id AS space FROM users u JOIN spaces s ON s.owner = u.id WHERE u.username = $1", + "SELECT u.id, u.username, u.password_hash, u.role, u.auth_gen, s.id AS space FROM users u JOIN spaces s ON s.owner = u.id WHERE u.username = $1", ) .bind(username.as_str()) .fetch_optional(&self.pool) @@ -141,38 +258,93 @@ impl Meta { password_hash: row.try_get("password_hash")?, role: Role::parse(&row.try_get::("role")?)?, space: SpaceId::from_uuid(uuid_from(&row.try_get::, _>("space")?)?), + auth_gen: row.try_get("auth_gen")?, })) } - pub async fn set_password_hash(&self, user: UserId, password_hash: &str) -> Result<(), MetaError> { - let w = self.begin_write().await?; - let mut w = w; - let done = sqlx::query("UPDATE users SET password_hash = $1 WHERE id = $2") - .bind(password_hash) - .bind(uuid_bytes(&user.0)) - .execute(&mut *w.tx) - .await?; - if done.rows_affected() == 0 { - return Err(MetaError::NotFound("user")); - } - w.commit().await + /// Registers a device for `user` under the hash of its token, with no word on its client. + pub async fn create_device(&self, user: UserId, name: &str, token_hash: &[u8; 32], now_ms: i64) -> Result { + self.create_device_with(user, name, &DeviceClient::default(), token_hash, now_ms).await } - /// Registers a device for `user` under the hash of its token. - pub async fn create_device(&self, user: UserId, name: &str, token_hash: &[u8; 32], now_ms: i64) -> Result { + /// [`Meta::create_device`] for a device that says what client it is. + pub async fn create_device_with( + &self, + user: UserId, + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result { check_device_name(name)?; + client.check()?; let device = DeviceId::generate(); let mut w = self.begin_write().await?; - insert_device(&mut w.tx, device, user, name, token_hash, now_ms).await?; + // The account first, as every transaction that disables it or revokes its devices takes it: a device is + // never added to an account that was disabled or marked for removal meanwhile. + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("user")); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; w.commit().await?; Ok(device) } - /// The device holding this token hash, if any and not revoked. + /// Registers a device for `user` after a password sign-in, if the password it was checked against is still the + /// account's (an account with two-factor sign-in is [`MetaError::TwoFactorRequired`] here: it gets a device only from + /// [`Meta::complete_second_factor`], and [`Meta::sign_in`] is the call that knows both): `checked` is the stored hash and the generation ([`UserRecord::password_hash`], [`UserRecord::auth_gen`]) the caller read + /// before it verified the password. The account is locked first and both compared again in the same transaction, so a + /// password change or reset, or a disable followed by an enable (which leaves the hash as it was), that committed + /// while the (slow) check ran is not followed by a device that check let in. `None`: the user is gone or + /// something changed meanwhile; the caller answers as for a wrong password. An account that is disabled or marked for removal is + /// [`MetaError::AccountDisabled`], which the caller may only say after the password was right. + pub async fn sign_in_device( + &self, + user: UserId, + checked: (&str, i64), + name: &str, + client: &DeviceClient, + token_hash: &[u8; 32], + now_ms: i64, + ) -> Result, MetaError> { + check_device_name(name)?; + client.check()?; + let (hash, auth_gen) = checked; + let device = DeviceId::generate(); + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() { + return Ok(None); + } + if account_blocked(&mut w.tx, user).await? { + return Err(MetaError::AccountDisabled); + } + if crate::two_factor::is_enabled(&mut w.tx, user).await? { + return Err(MetaError::TwoFactorRequired); + } + insert_device(&mut w.tx, device, user, name, client, token_hash, now_ms).await?; + let login = NewEvent::new(AuditAction::AuthLogin).by(user, Some(device)).on_device(device).with(AuditDetails::client(client)); + record_in(&mut w.tx, now_ms, login).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(device)) + } + + /// The device holding this token hash, if any, not revoked and of an account that isn't disabled. pub async fn device_by_token(&self, token_hash: &[u8; 32]) -> Result, MetaError> { let row = sqlx::query( "SELECT d.id, d.user_id, u.role, u.username, s.id AS space FROM devices d JOIN users u ON u.id = d.user_id JOIN spaces s ON s.owner = u.id \ - WHERE d.token_hash = $1 AND d.revoked = 0", + WHERE d.token_hash = $1 AND d.revoked = 0 AND u.disabled_ms IS NULL", ) .bind(token_hash.to_vec()) .fetch_optional(&self.pool) @@ -195,21 +367,34 @@ impl Meta { } pub async fn list_devices(&self, user: UserId) -> Result, MetaError> { - let rows = sqlx::query("SELECT id, name, created_ms, last_used_ms, revoked FROM devices WHERE user_id = $1 ORDER BY created_ms, id") - .bind(uuid_bytes(&user.0)) - .fetch_all(&self.pool) - .await?; - rows.iter() - .map(|row| { - Ok(DeviceRecord { - id: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), - name: row.try_get("name")?, - created_ms: row.try_get("created_ms")?, - last_used_ms: row.try_get("last_used_ms")?, - revoked: flag(row, "revoked")?, - }) - }) - .collect() + let rows = sqlx::query( + "SELECT id, name, created_ms, last_used_ms, revoked, client_kind, platform, app_version, revoked_ms, revoke_reason \ + FROM devices WHERE user_id = $1 ORDER BY created_ms, id", + ) + .bind(uuid_bytes(&user.0)) + .fetch_all(&self.pool) + .await?; + rows.iter().map(device_record).collect() + } + + /// What `GET /account` tells about `user`; `NotFound` for a user that isn't there. + pub async fn account(&self, user: UserId) -> Result { + let row = sqlx::query( + "SELECT username, role, created_ms, password_changed_ms, \ + (SELECT COUNT(*) FROM user_two_factor t WHERE t.user_id = users.id AND t.enabled_ms IS NOT NULL) AS two_factor FROM users WHERE id = $1", + ) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&self.pool) + .await? + .ok_or(MetaError::NotFound("user"))?; + Ok(AccountRecord { + id: user, + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + password_changed_ms: row.try_get("password_changed_ms")?, + two_factor_enabled: row.try_get::("two_factor")? > 0, + }) } /// Whether `device` exists and isn't revoked (a long-lived connection asks again now and then: the admin CLI may @@ -220,21 +405,269 @@ impl Meta { Ok(row.is_some()) } - /// Revokes one of `user`'s devices, and deletes the pairing it made, if any (its add-a-computer code dies with - /// it). Revoking another user's device is `NotFound`. + /// Revokes one of `user`'s devices as the user's own act ([`RevokeReason::User`]), by the system clock. See + /// [`Meta::revoke_device_as`]. pub async fn revoke_device(&self, user: UserId, device: DeviceId) -> Result<(), MetaError> { + self.revoke_device_as(user, device, RevokeReason::User, wall_clock_ms()).await + } + + /// Revokes one of `user`'s devices, and deletes the pairing it made, if any (its add-a-computer code dies with + /// it). Revoking another user's device is `NotFound`. A device that is revoked already stays as it is, with its + /// first time and reason, and the call succeeds. The audit event is `device.revoked` for the account's own act + /// ([`Origin::Account`]); [`Meta::revoke_device_by`] says which device asked, or that the command line did. + pub async fn revoke_device_as(&self, user: UserId, device: DeviceId, reason: RevokeReason, now_ms: i64) -> Result<(), MetaError> { + self.revoke_device_by(user, device, reason, Origin::Account, now_ms).await + } + + /// [`Meta::revoke_device_as`] with who asked. A device that was revoked already is left as it is and writes no event: + /// nothing changed. + pub async fn revoke_device_by(&self, user: UserId, device: DeviceId, reason: RevokeReason, origin: Origin, now_ms: i64) -> Result<(), MetaError> { let mut w = self.begin_write().await?; - let done = sqlx::query("UPDATE devices SET revoked = 1 WHERE id = $1 AND user_id = $2") + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::NotFound("device")); + } + let done = sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE id = $3 AND user_id = $4 AND revoked = 0") + .bind(now_ms) + .bind(reason.as_str()) .bind(uuid_bytes(device.as_uuid())) .bind(uuid_bytes(&user.0)) .execute(&mut *w.tx) .await?; if done.rows_affected() == 0 { - return Err(MetaError::NotFound("device")); + let owned = sqlx::query("SELECT 1 FROM devices WHERE id = $1 AND user_id = $2") + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + if owned.is_none() { + return Err(MetaError::NotFound("device")); + } } sqlx::query("DELETE FROM pairings WHERE device_id = $1").bind(uuid_bytes(device.as_uuid())).execute(&mut *w.tx).await?; - w.commit().await + if done.rows_affected() > 0 { + let details = AuditDetails { reason: Some(reason), device: Some(device), ..AuditDetails::default() }; + let event = match origin { + Origin::Account => NewEvent::new(AuditAction::DeviceRevoked).by(user, None).on_device(device), + Origin::Device(by) => NewEvent::new(AuditAction::DeviceRevoked).by(user, Some(by)).on_device(device), + // The command line acts as no account. The event is about the account, whose own activity then shows it. + Origin::Cli => NewEvent::new(AuditAction::CliDeviceRevoked).on_user(user), + }; + record_in(&mut w.tx, now_ms, event.with(details)).await?; + } + self.commit_audited(w, now_ms).await } + + /// Renames one of `user`'s devices that isn't revoked and returns it. Another user's, a revoked or a missing + /// device is `NotFound`, never a different answer, so nothing says which devices exist. + pub async fn rename_device(&self, user: UserId, device: DeviceId, name: &str) -> Result { + check_device_name(name)?; + let mut w = self.begin_write().await?; + let done = sqlx::query("UPDATE devices SET name = $1 WHERE id = $2 AND user_id = $3 AND revoked = 0") + .bind(name) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("device")); + } + let row = sqlx::query( + "SELECT id, name, created_ms, last_used_ms, revoked, client_kind, platform, app_version, revoked_ms, revoke_reason \ + FROM devices WHERE id = $1", + ) + .bind(uuid_bytes(device.as_uuid())) + .fetch_one(&mut *w.tx) + .await?; + let record = device_record(&row)?; + w.commit().await?; + Ok(record) + } + + /// Revokes every device of `user` but `keep` that isn't revoked (with `reason`), and deletes the pairings those + /// devices made, in one transaction. Returns the devices revoked, for the caller to end their live connections. + pub async fn revoke_other_devices(&self, user: UserId, keep: DeviceId, reason: RevokeReason, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(Vec::new()); + } + let revoked = revoke_devices(&mut w.tx, user, Some(keep), reason, now_ms).await?; + // One event with the count, however many devices went, and none when there was nobody to sign out. + if !revoked.is_empty() { + let details = + AuditDetails { reason: Some(reason), revoked: Some(u32::try_from(revoked.len()).unwrap_or(u32::MAX)), ..AuditDetails::default() }; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::DeviceRevokedOthers).by(user, Some(keep)).on_user(user).with(details)).await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// [`Meta::revoke_other_devices`] for every device of `user` (the admin's command line, which writes + /// `cli.devices_revoked_all` when there was a device to sign out). + pub async fn revoke_all_devices(&self, user: UserId, reason: RevokeReason, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(Vec::new()); + } + let revoked = revoke_devices(&mut w.tx, user, None, reason, now_ms).await?; + if !revoked.is_empty() { + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::CliDevicesRevokedAll).on_user(user).with(AuditDetails::revoked(revoked.len()))) + .await?; + } + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// Changes `user`'s password to `new_hash`, in one transaction: sets the hash and `password_changed_ms`, + /// revokes every device but `keep` ([`RevokeReason::Password`]), and deletes the account's open invite and all + /// its pairings (a code made before the change must not work after it). `current_hash` is the hash the caller + /// verified the current password against: if the password changed meanwhile, nothing changes and the answer is + /// `Conflict`; if `keep` was revoked meanwhile, it is `DeviceRevoked`. Returns the devices revoked. A failure at + /// any step changes nothing. + pub async fn change_password( + &self, + user: UserId, + current_hash: &str, + new_hash: &str, + keep: DeviceId, + now_ms: i64, + ) -> Result, MetaError> { + let mut w = self.begin_write().await?; + // The user's row first, as everywhere (see `lock_user`). + lock_user(&mut w.tx, user).await?; + // The caller's device must still be signed in: its Argon2 checks took a while, and a revoke since would + // otherwise leave a password changed by a device that is signed out. The no-op update locks the row on + // both databases, so a revoke running now waits for this transaction (and then finds the device kept). + let kept = sqlx::query("UPDATE devices SET revoked = 0 WHERE id = $1 AND user_id = $2 AND revoked = 0") + .bind(uuid_bytes(keep.as_uuid())) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if kept.rows_affected() == 0 || account_blocked(&mut w.tx, user).await? { + return Err(MetaError::DeviceRevoked); + } + let done = sqlx::query( + "UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3 AND password_hash = $4", + ) + .bind(new_hash) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .bind(current_hash) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::Conflict("the password changed while this request was running; try again")); + } + let revoked = revoke_devices(&mut w.tx, user, Some(keep), RevokeReason::Password, now_ms).await?; + delete_join_codes(&mut w.tx, user).await?; + // The devices signed out with it are counted in this one event, not in events of their own. + let changed = + NewEvent::new(AuditAction::AccountPasswordChanged).by(user, Some(keep)).on_user(user).with(AuditDetails::revoked(revoked.len())); + record_in(&mut w.tx, now_ms, changed).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } + + /// The admin's password reset on the command line (`cli.password_reset`): sets `new_hash` and `password_changed_ms`, deletes the account's open invite and + /// pairings, and, unless `keep_devices`, revokes every device ([`RevokeReason::Reset`]), all in one transaction. + /// `NotFound` for a user that isn't there. Returns the devices revoked. + pub async fn reset_password(&self, user: UserId, new_hash: &str, keep_devices: bool, now_ms: i64) -> Result, MetaError> { + let mut w = self.begin_write().await?; + lock_user(&mut w.tx, user).await?; + let done = sqlx::query("UPDATE users SET password_hash = $1, password_changed_ms = $2, auth_gen = auth_gen + 1 WHERE id = $3") + .bind(new_hash) + .bind(now_ms) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if done.rows_affected() == 0 { + return Err(MetaError::NotFound("user")); + } + let revoked = if keep_devices { Vec::new() } else { revoke_devices(&mut w.tx, user, None, RevokeReason::Reset, now_ms).await? }; + delete_join_codes(&mut w.tx, user).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::CliPasswordReset).on_user(user).with(AuditDetails::revoked(revoked.len()))).await?; + self.commit_audited(w, now_ms).await?; + Ok(revoked) + } +} + +/// Takes the row lock of `user` (a no-op update, so nothing changes), and says whether the user exists. Every +/// transaction that revokes devices or changes a password takes it first, then the rows of the devices, then those of +/// the pairings: one order for all of them, so two of them at once (a password change from one device, a sign-out of +/// the others from another, an admin reset) wait for each other on Postgres and never deadlock. SQLite writers are +/// serialised already. +pub(crate) async fn lock_user(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let done = sqlx::query("UPDATE users SET username = username WHERE id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(done.rows_affected() > 0) +} + +/// Whether `user` is disabled or marked for removal, read in `tx` after the user's row was locked. +pub(crate) async fn account_blocked(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let row = sqlx::query("SELECT disabled_ms, removed_ms FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut **tx).await?; + let Some(row) = row else { return Ok(false) }; + Ok(row.try_get::, _>("disabled_ms")?.is_some() || row.try_get::, _>("removed_ms")?.is_some()) +} + +fn device_record(row: &sqlx::any::AnyRow) -> Result { + Ok(DeviceRecord { + id: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + name: row.try_get("name")?, + created_ms: row.try_get("created_ms")?, + last_used_ms: row.try_get("last_used_ms")?, + revoked: flag(row, "revoked")?, + client: DeviceClient { kind: row.try_get("client_kind")?, platform: row.try_get("platform")?, app_version: row.try_get("app_version")? }, + revoked_ms: row.try_get("revoked_ms")?, + revoke_reason: row.try_get::, _>("revoke_reason")?.as_deref().map(RevokeReason::parse).transpose()?, + }) +} + +/// Revokes `user`'s devices that aren't revoked yet, except `keep`, and deletes the pairings of every revoked device +/// of theirs, in `tx`. Time and reason are set with the flag, and only here, so a device revoked before keeps its own. +pub(crate) async fn revoke_devices( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + keep: Option, + reason: RevokeReason, + now_ms: i64, +) -> Result, MetaError> { + let rows = + match keep { + Some(keep) => sqlx::query( + "UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE user_id = $3 AND revoked = 0 AND id <> $4 RETURNING id", + ) + .bind(now_ms) + .bind(reason.as_str()) + .bind(uuid_bytes(&user.0)) + .bind(uuid_bytes(keep.as_uuid())) + .fetch_all(&mut **tx) + .await?, + None => { + sqlx::query("UPDATE devices SET revoked = 1, revoked_ms = $1, revoke_reason = $2 WHERE user_id = $3 AND revoked = 0 RETURNING id") + .bind(now_ms) + .bind(reason.as_str()) + .bind(uuid_bytes(&user.0)) + .fetch_all(&mut **tx) + .await? + } + }; + sqlx::query("DELETE FROM pairings WHERE device_id IN (SELECT id FROM devices WHERE user_id = $1 AND revoked = 1)") + .bind(uuid_bytes(&user.0)) + .execute(&mut **tx) + .await?; + rows.iter().map(|row| Ok(DeviceId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?))).collect() +} + +/// Deletes `user`'s open invite, every pairing of theirs and every open sign-in challenge, in `tx`: a code or challenge +/// made before a password change, a reset or a disable must not work after it. +pub(crate) async fn delete_join_codes(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result<(), MetaError> { + sqlx::query("DELETE FROM invites WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM pairings WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + sqlx::query("DELETE FROM login_challenges WHERE user_id = $1").bind(uuid_bytes(&user.0)).execute(&mut **tx).await?; + Ok(()) +} + +/// The system clock, for the call that takes no time of its own. +pub(crate) fn wall_clock_ms() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)).unwrap_or(0) } /// The device name rule: 1–128 bytes without control characters. @@ -284,7 +717,7 @@ pub(crate) async fn insert_user( .bind(uuid_bytes(root.as_uuid())) .execute(&mut **tx) .await?; - Ok(UserRecord { id: user, username: username.to_owned(), password_hash: password_hash.to_owned(), role, space }) + Ok(UserRecord { id: user, username: username.to_owned(), password_hash: password_hash.to_owned(), role, space, auth_gen: 0 }) } /// Inserts a device (its name already checked) in `tx`. @@ -293,16 +726,23 @@ pub(crate) async fn insert_device( device: DeviceId, user: UserId, name: &str, + client: &DeviceClient, token_hash: &[u8; 32], now_ms: i64, ) -> Result<(), MetaError> { - sqlx::query("INSERT INTO devices (id, user_id, name, token_hash, created_ms, last_used_ms, revoked) VALUES ($1, $2, $3, $4, $5, $5, 0)") - .bind(uuid_bytes(device.as_uuid())) - .bind(uuid_bytes(&user.0)) - .bind(name) - .bind(token_hash.to_vec()) - .bind(now_ms) - .execute(&mut **tx) - .await?; + sqlx::query( + "INSERT INTO devices (id, user_id, name, token_hash, created_ms, last_used_ms, revoked, client_kind, platform, app_version) \ + VALUES ($1, $2, $3, $4, $5, $5, 0, $6, $7, $8)", + ) + .bind(uuid_bytes(device.as_uuid())) + .bind(uuid_bytes(&user.0)) + .bind(name) + .bind(token_hash.to_vec()) + .bind(now_ms) + .bind(client.kind.as_str()) + .bind(client.platform.as_str()) + .bind(client.app_version.as_str()) + .execute(&mut **tx) + .await?; Ok(()) } diff --git a/crates/meta/src/web.rs b/crates/meta/src/web.rs new file mode 100644 index 00000000..271ccc9a --- /dev/null +++ b/crates/meta/src/web.rs @@ -0,0 +1,347 @@ +//! Browser sessions of the web admin: the `web_sessions` table, signing in to one (with the second factor, when the +//! account has one), looking one up, logging out and pruning. +//! +//! - **Only a hash is stored.** The session token is 32 random bytes the server makes and puts in a cookie; the table keeps +//! its BLAKE3 hash (`id_hash`), like a device's token. A row holds no password, no token, no address and no user agent. +//! - **A session lives exactly as long as the account's `auth_gen` it was made under.** Every change that raises +//! `users.auth_gen` (a password change or reset, a disable, an enable, a removal, turning two-factor sign-in on or off, an +//! admin's reset of it) ends the account's sessions with it, with no code of its own in those changes. A session is +//! also over when the account is disabled or marked for removal, when it is logged out (`revoked_ms`), when it was +//! made longer ago than the absolute lifetime, and when it was not seen for longer than the idle lifetime. +//! - **Only administrators sign in.** The check is made under the account's lock, together with the password's +//! generation, so a demotion or a password change that ran while the (slow) password check did cannot be followed by a +//! session. A refusal is `None` for every reason (wrong state, not an admin, blocked), which the caller answers like a +//! wrong password: nothing says whether an account exists, is a member, or is disabled. +//! - **The second factor is the device sign-in's, not a copy of it.** A web sign-in's challenge is a row of +//! `login_challenges` of its own kind (`web`), counted, locked and used up by the same code (`two_factor::check_proof`, +//! `refuse_sign_in_proof`), and can never be completed into a device (nor a device sign-in's into a session). +//! - **Pruning is bounded.** [`Meta::web_sessions_prune`] deletes at most a given number of expired or revoked rows at a +//! time, so the server can run it after a sign-in without stalling it, and the command line loops until it is done. + +use craft_cloud_proto::DeviceId; +use sqlx::Row; + +use crate::audit::{AuditAction, AuditDetails, NewEvent, record_in}; +use crate::codec::{to_u64, uuid_bytes, uuid_from}; +use crate::two_factor::{ + CHALLENGE_TTL_MS, Checked, MAX_CHALLENGE_ATTEMPTS, MAX_OPEN_CHALLENGES, NewChallenge, Proof, TfRow, check_proof, is_enabled, load_row, +}; +use crate::users::{account_blocked, lock_user}; +use crate::{Meta, MetaError, Role, UserId, WriteTx}; + +/// A session to make: the hash of its token and how long it may live in all. +#[derive(Clone, PartialEq, Eq)] +pub struct NewWebSession { + /// The BLAKE3 hash of the session token: all the server keeps of it. + pub id_hash: [u8; 32], + /// The most the session may live from now, whatever happens: its absolute lifetime. + pub absolute_ms: i64, +} + +impl std::fmt::Debug for NewWebSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NewWebSession").field("id_hash", &"").field("absolute_ms", &self.absolute_ms).finish() + } +} + +/// The result of a password sign-in to the web admin that checked out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WebSignIn { + /// The account has no second factor: the session is made. + Session, + /// The account needs its second factor: only the challenge is made, good until this time. + Challenge { expires_ms: i64 }, +} + +/// A session that is valid now, with the account it belongs to as the database has it now. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WebSession { + pub user: UserId, + pub username: String, + pub role: Role, + pub created_ms: i64, + pub last_seen_ms: i64, + pub absolute_expires_ms: i64, + /// When a second factor was given at sign-in; `None` for an account without one. + pub mfa_verified_ms: Option, +} + +/// The SQL that says a session is over, for `$1` the time now and `$2` the oldest `last_seen_ms` that is still idle-valid. +const OVER: &str = "revoked_ms IS NOT NULL OR absolute_expires_ms <= $1 OR last_seen_ms <= $2"; + +/// Inserts the session of `user` in `tx` (the account's row locked), under the account's current generation. +async fn insert_session( + tx: &mut sqlx::Transaction<'static, sqlx::Any>, + user: UserId, + session: &NewWebSession, + mfa_verified_ms: Option, + now_ms: i64, +) -> Result<(), MetaError> { + let absolute = now_ms.checked_add(session.absolute_ms).ok_or(MetaError::Invalid("the session's lifetime is too long"))?; + let generation: i64 = + sqlx::query("SELECT auth_gen FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_one(&mut **tx).await?.try_get("auth_gen")?; + sqlx::query( + "INSERT INTO web_sessions (id_hash, user_id, auth_gen, created_ms, last_seen_ms, absolute_expires_ms, revoked_ms, mfa_verified_ms) \ + VALUES ($1, $2, $3, $4, $4, $5, NULL, $6)", + ) + .bind(session.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(generation) + .bind(now_ms) + .bind(absolute) + .bind(mfa_verified_ms) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn is_admin(tx: &mut sqlx::Transaction<'static, sqlx::Any>, user: UserId) -> Result { + let row = sqlx::query("SELECT role FROM users WHERE id = $1").bind(uuid_bytes(&user.0)).fetch_optional(&mut **tx).await?; + match row { + Some(row) => Ok(Role::parse(&row.try_get::("role")?)? == Role::Admin), + None => Ok(false), + } +} + +impl Meta { + /// A password sign-in to the web admin that checked out: under the account's lock the password hash and `auth_gen` the + /// caller read before it verified the password are compared again, and the account must still be an administrator, not + /// disabled and not marked for removal. Then, without two-factor sign-in, the session is made and `auth.login` is + /// written (no device acts); with it only a challenge of kind `web` is made (the account's expired challenges are + /// deleted and the oldest goes when it would have more than [`MAX_OPEN_CHALLENGES`]) and no session. `None` for + /// everything that is not a sign-in, whatever the reason. + pub async fn web_sign_in( + &self, + user: UserId, + checked: (&str, i64), + challenge: &NewChallenge, + session: &NewWebSession, + now_ms: i64, + ) -> Result, MetaError> { + let (hash, auth_gen) = checked; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Ok(None); + } + let same = sqlx::query("SELECT 1 FROM users WHERE id = $1 AND password_hash = $2 AND auth_gen = $3") + .bind(uuid_bytes(&user.0)) + .bind(hash) + .bind(auth_gen) + .fetch_optional(&mut *w.tx) + .await?; + if same.is_none() || account_blocked(&mut w.tx, user).await? || !is_admin(&mut w.tx, user).await? { + return Ok(None); + } + if is_enabled(&mut w.tx, user).await? { + let expires_ms = now_ms.checked_add(CHALLENGE_TTL_MS).ok_or(MetaError::Invalid("the clock is out of range"))?; + sqlx::query("DELETE FROM login_challenges WHERE expires_ms <= $1").bind(now_ms).execute(&mut *w.tx).await?; + let open: i64 = sqlx::query("SELECT COUNT(*) AS n FROM login_challenges WHERE user_id = $1") + .bind(uuid_bytes(&user.0)) + .fetch_one(&mut *w.tx) + .await? + .try_get("n")?; + if open >= MAX_OPEN_CHALLENGES { + sqlx::query( + "DELETE FROM login_challenges WHERE id_hash IN \ + (SELECT id_hash FROM login_challenges WHERE user_id = $1 ORDER BY expires_ms ASC, id_hash ASC LIMIT $2)", + ) + .bind(uuid_bytes(&user.0)) + .bind(open - MAX_OPEN_CHALLENGES + 1) + .execute(&mut *w.tx) + .await?; + } + sqlx::query( + "INSERT INTO login_challenges (id_hash, user_id, kind, client_key, device_name, attempts, expires_ms) \ + VALUES ($1, $2, 'web', $3, '', 0, $4)", + ) + .bind(challenge.id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(challenge.client_key.as_str()) + .bind(expires_ms) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + return Ok(Some(WebSignIn::Challenge { expires_ms })); + } + insert_session(&mut w.tx, user, session, None, now_ms).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AuthLogin).by(user, None::).with(AuditDetails::web())).await?; + self.commit_audited(w, now_ms).await?; + Ok(Some(WebSignIn::Session)) + } + + /// The account a web challenge was made for and its username, by a plain read, so the caller can count the try against + /// the account before the real check. `None` for a challenge that is unknown, expired, used, or a device sign-in's. + pub async fn web_challenge_owner(&self, id_hash: &[u8; 32], now_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT c.user_id, u.username FROM login_challenges c JOIN users u ON u.id = c.user_id \ + WHERE c.id_hash = $1 AND c.kind = 'web' AND c.expires_ms > $2", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some((UserId(uuid_from(&row.try_get::, _>("user_id")?)?), row.try_get("username")?))) + } + + /// Finishes a web sign-in with the second factor, as [`Meta::complete_second_factor`] finishes a device's: the account's + /// row is locked before the challenge or the factor is touched, the challenge and the account's current state (still an + /// administrator, two-factor sign-in still on, not disabled) are checked again under the lock, and only then are the + /// proof and the challenge used up and the session made, in this one transaction. A challenge that is unknown, + /// expired, used or no longer fit is [`MetaError::ChallengeGone`], the same for every case, an account that is no longer + /// an administrator or is blocked too; a wrong proof is [`MetaError::TwoFactorInvalid`] after its counters were + /// committed on their own; a lock in force is [`MetaError::TwoFactorLocked`]. + pub async fn web_complete_second_factor( + &self, + challenge: &[u8; 32], + proof: &Proof<'_>, + session: &NewWebSession, + now_ms: i64, + ) -> Result { + let Some((user, _)) = self.web_challenge_owner(challenge, now_ms).await? else { return Err(MetaError::ChallengeGone) }; + let mut w = self.begin_write().await?; + if !lock_user(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + let found = sqlx::query("SELECT attempts, expires_ms FROM login_challenges WHERE id_hash = $1 AND user_id = $2 AND kind = 'web'") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .fetch_optional(&mut *w.tx) + .await?; + let Some(found) = found.filter(|row| row.try_get::("expires_ms").is_ok_and(|expires| expires > now_ms)) else { + return Err(MetaError::ChallengeGone); + }; + let attempts: i64 = found.try_get("attempts")?; + let Some(row) = load_row(&mut w.tx, user).await?.filter(TfRow::active) else { + // Turned off (or reset) since: the challenge is of no use any more. + sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1").bind(challenge.to_vec()).execute(&mut *w.tx).await?; + w.commit().await?; + return Err(MetaError::ChallengeGone); + }; + if attempts >= MAX_CHALLENGE_ATTEMPTS || account_blocked(&mut w.tx, user).await? || !is_admin(&mut w.tx, user).await? { + return Err(MetaError::ChallengeGone); + } + if let Checked::Wrong = check_proof(&mut w.tx, user, None, true, &row, proof, now_ms).await? { + return Err(self.refuse_sign_in_proof(w, user, Some(challenge), proof, now_ms, true).await); + } + let used = sqlx::query("DELETE FROM login_challenges WHERE id_hash = $1 AND user_id = $2") + .bind(challenge.to_vec()) + .bind(uuid_bytes(&user.0)) + .execute(&mut *w.tx) + .await?; + if used.rows_affected() != 1 { + return Err(MetaError::ChallengeGone); + } + insert_session(&mut w.tx, user, session, Some(now_ms), now_ms).await?; + record_in(&mut w.tx, now_ms, NewEvent::new(AuditAction::AuthLogin).by(user, None::).with(AuditDetails::web())).await?; + self.commit_audited(w, now_ms).await?; + Ok(user) + } + + /// The session under this token hash, if it is valid at `now_ms`: not logged out, younger than its absolute lifetime, + /// seen within `idle_ms`, made under the account's current `auth_gen`, and of an account that is neither disabled nor + /// marked for removal. The account's role is the one the database has now. + pub async fn web_session(&self, id_hash: &[u8; 32], now_ms: i64, idle_ms: i64) -> Result, MetaError> { + let row = sqlx::query( + "SELECT s.user_id, s.created_ms, s.last_seen_ms, s.absolute_expires_ms, s.mfa_verified_ms, u.username, u.role \ + FROM web_sessions s JOIN users u ON u.id = s.user_id \ + WHERE s.id_hash = $1 AND s.revoked_ms IS NULL AND s.absolute_expires_ms > $2 AND s.last_seen_ms > $3 \ + AND s.auth_gen = u.auth_gen AND u.disabled_ms IS NULL AND u.removed_ms IS NULL", + ) + .bind(id_hash.to_vec()) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_optional(&self.pool) + .await?; + let Some(row) = row else { return Ok(None) }; + Ok(Some(WebSession { + user: UserId(uuid_from(&row.try_get::, _>("user_id")?)?), + username: row.try_get("username")?, + role: Role::parse(&row.try_get::("role")?)?, + created_ms: row.try_get("created_ms")?, + last_seen_ms: row.try_get("last_seen_ms")?, + absolute_expires_ms: row.try_get("absolute_expires_ms")?, + mfa_verified_ms: row.try_get("mfa_verified_ms")?, + })) + } + + /// Records that the session was just used (its idle time starts again). Never moves the time back. + pub async fn web_session_seen(&self, id_hash: &[u8; 32], now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + sqlx::query("UPDATE web_sessions SET last_seen_ms = $1 WHERE id_hash = $2 AND last_seen_ms < $1 AND revoked_ms IS NULL") + .bind(now_ms) + .bind(id_hash.to_vec()) + .execute(&mut *w.tx) + .await?; + w.commit().await + } + + /// Logs the session out. One that is gone or over already is left as it is, and the call succeeds. + pub async fn web_logout(&self, id_hash: &[u8; 32], now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + sqlx::query("UPDATE web_sessions SET revoked_ms = $1 WHERE id_hash = $2 AND revoked_ms IS NULL") + .bind(now_ms) + .bind(id_hash.to_vec()) + .execute(&mut *w.tx) + .await?; + w.commit().await + } + + /// Deletes at most `max_rows` sessions that are over (logged out, past their absolute lifetime, or not seen for `idle_ms`) + /// and says how many went. Rows that are valid are never touched. The rows of sessions that an account change ended + /// (`auth_gen`) are deleted when their other lifetimes run out, which is at most the absolute lifetime. + pub async fn web_sessions_prune(&self, now_ms: i64, idle_ms: i64, max_rows: u32) -> Result { + let mut w = self.begin_write().await?; + let done = sqlx::query(sqlx::AssertSqlSafe(format!( + "DELETE FROM web_sessions WHERE id_hash IN (SELECT id_hash FROM web_sessions WHERE {OVER} LIMIT $3)" + ))) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .bind(i64::from(max_rows)) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(done.rows_affected()) + } + + /// How many sessions [`Meta::web_sessions_prune`] would delete now (the command line's dry run). + pub async fn web_sessions_count_over(&self, now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT COUNT(*) AS n FROM web_sessions WHERE {OVER}"))) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_one(&self.pool) + .await?; + to_u64(row.try_get("n")?) + } + + /// How many sessions the account has that are valid at `now_ms`, for tests and the overview. + pub async fn web_sessions_open(&self, user: UserId, now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) AS n FROM web_sessions s JOIN users u ON u.id = s.user_id \ + WHERE s.user_id = $1 AND s.revoked_ms IS NULL AND s.absolute_expires_ms > $2 AND s.last_seen_ms > $3 AND s.auth_gen = u.auth_gen", + ) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_one(&self.pool) + .await?; + to_u64(row.try_get("n")?) + } +} + +/// Whether the session `id_hash` of `user` is valid at `now_ms`, read inside a change (see `begin_admin`) under the account's +/// lock: not logged out, under the account's current generation, inside its absolute lifetime and used within `idle_ms`. A +/// request that began while the session was valid (a slow body, a wait for a lock or a password check) therefore cannot change +/// anything after the session ended. The account's disabled and removed marks need no look: they raise `auth_gen`. +pub(crate) async fn session_is_live(w: &mut WriteTx, user: UserId, id_hash: &[u8; 32], now_ms: i64, idle_ms: i64) -> Result { + let row = sqlx::query( + "SELECT 1 FROM web_sessions s JOIN users u ON u.id = s.user_id WHERE s.id_hash = $1 AND s.user_id = $2 AND s.revoked_ms IS NULL AND s.auth_gen = u.auth_gen AND s.absolute_expires_ms > $3 AND s.last_seen_ms > $4", + ) + .bind(id_hash.to_vec()) + .bind(uuid_bytes(&user.0)) + .bind(now_ms) + .bind(now_ms.saturating_sub(idle_ms)) + .fetch_optional(&mut *w.tx) + .await?; + Ok(row.is_some()) +} diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index d875fc30..01ac060e 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -3,7 +3,10 @@ //! test creates and drops its own database. CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS`, which turns a missing //! `CRAFT_CLOUD_TEST_PG` into a failure instead of a skip. -use craft_cloud_meta::{BlobKind, InviteFor, Meta, MetaError, Redeemed, Role, UserRecord}; +use craft_cloud_meta::{ + AdminActor, AdminUser, AdminVia, BlobKind, DeviceActor, DeviceClient, DeviceRecord, InviteFor, MIN_QUOTA_BYTES, Meta, MetaError, NewChallenge, + NewWebSession, Proof, RECOVERY_CODES, RESET_TTL_MS, Redeemed, RevokeReason, Role, SignIn, UnreadableSecret, UserId, UserRecord, WebSignIn, +}; use craft_cloud_proto::api::PairingState; use craft_cloud_proto::join::{Code, INVITE_CHARS, INVITE_TTL_MS, PAIRING_CHARS, PAIRING_TTL_MS}; use craft_cloud_proto::limits::MAX_PAGE_NODES; @@ -90,6 +93,14 @@ on_every_backend!( a_late_save_record_is_appended_once, manifests_need_their_chunks_and_are_per_space, stored_bytes_are_counted_once_per_space_and_capped, + account_usage_estimates_history_and_trash_without_adding_them, + account_usage_excludes_chunks_shared_with_current_version, + account_usage_deduplicates_per_space_and_counts_pending_purge, + admin_storage_overview_is_aggregated_and_orders_ties, + admin_storage_overview_handles_an_empty_server, + admin_storage_estimates_hide_removed_and_unknown_accounts, + account_usage_excludes_live_document_snapshot_blobs, + account_usage_leaves_saved_versions_out_of_history, concurrent_chunk_batches_charge_a_shared_chunk_once, concurrent_signups_with_one_name_give_username_taken, an_offline_batch_creates_a_folder_and_files_in_it, @@ -104,6 +115,32 @@ on_every_backend!( ids_are_never_reused, the_change_feed_and_epochs, concurrent_commits_on_one_file_let_exactly_one_win, + history_pages_newest_first_for_own_files_only, + a_restore_is_a_new_version_after_the_head, + a_restore_refuses_what_it_can_not_restore, + concurrent_restores_let_exactly_one_win, + a_restore_and_a_late_save_record_both_apply, + deleting_stamps_the_time_once_and_coming_back_clears_it, + the_trash_pages_newest_first_for_its_own_space, + a_restore_brings_the_deleted_folders_back_with_their_ids, + a_restore_merges_into_live_folders_and_renames_around_taken_names, + a_restore_with_no_free_name_changes_nothing, + the_trash_lists_a_folder_without_what_is_below_it, + a_folder_restore_brings_back_everything_in_the_trash_below_it, + a_folder_restore_gives_children_of_one_name_a_conflict_name, + removing_a_folder_from_the_trash_takes_what_is_below_it, + a_folder_restore_takes_at_most_the_cap_below_it, + a_folder_restore_leaves_the_rest_listed_through_its_own_folder, + removing_a_folder_takes_at_most_the_cap_below_it, + only_nodes_in_the_trash_can_be_restored_or_removed, + removing_from_the_trash_frees_nothing, + a_trash_restore_and_a_commit_for_the_same_name_agree, + a_trash_restore_and_an_edit_of_the_file_agree, + a_restored_file_gets_a_new_version_with_its_old_content, + a_restore_does_not_exceed_the_depth_limit, + a_new_manifest_has_exact_edges_complete_in_its_own_transaction, + a_long_manifest_is_indexed_in_batches_of_the_budget, + the_index_stays_exact_through_every_write_path, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -139,6 +176,71 @@ on_every_backend!( concurrent_redeems_of_one_code_give_one_device, an_invite_to_an_account_with_a_password_needs_none, an_invite_to_a_new_account_needs_a_password, + a_device_records_its_client, + only_own_live_devices_are_renamed, + every_revoke_path_sets_flag_time_and_reason_together, + revoking_others_keeps_the_caller_and_ends_their_pairings, + a_password_change_is_one_step, + an_admin_reset_signs_out_every_device_unless_kept, + concurrent_password_changes_and_sign_outs_never_fail_the_database, + a_sign_in_checked_against_an_old_password_creates_no_device, + redeems_sign_ins_and_resets_at_the_same_moment_leave_no_live_device, + the_admin_list_pages_searches_and_counts, + a_sign_in_checked_before_a_disable_and_an_enable_creates_no_device, + an_admin_makes_an_account_with_a_role_and_an_invite, + an_admin_changes_a_role_and_a_quota, + the_acting_admins_state_is_checked_in_the_transaction, + disabling_signs_everything_out_and_enabling_brings_back_nothing, + issuing_access_while_an_account_is_disabled_leaves_no_usable_token, + invites_and_reset_codes_replace_each_other, + a_reset_code_sets_a_password_and_signs_everything_out, + a_quota_overrides_the_server_wide_cap, + an_admin_lists_and_revokes_a_users_devices, + two_demotions_at_once_keep_an_admin, + two_self_demotions_at_once_keep_an_admin, + two_disables_at_once_keep_an_admin, + a_demotion_and_a_disable_at_once_keep_an_admin, + a_removal_and_a_demotion_at_once_keep_an_admin, + a_self_demotion_and_a_disable_of_the_other_at_once_keep_an_admin, + every_operation_writes_exactly_its_events, + a_refused_change_writes_no_event, + failed_sign_ins_count_up_in_one_row_per_account_and_minute, + concurrent_failures_with_one_key_make_one_row_with_every_count, + audit_lists_filter_and_page_by_cursor, + pruning_works_in_batches_and_keeps_its_time_until_done, + a_removed_accounts_events_stay, + no_event_holds_a_name_or_a_secret, + set_up_confirmed_and_listed, + a_set_up_belongs_to_one_device_expires_and_takes_five_tries, + changes_of_the_settings_recheck_the_device_and_the_password, + two_confirmations_at_once_turn_it_on_once, + a_time_step_is_accepted_once_and_only_later_ones_after_it, + recovery_codes_work_once_and_new_ones_replace_them, + wrong_proofs_lock_for_a_while_double_and_reset_on_success, + wrong_proofs_at_the_same_moment_all_count, + a_challenge_is_made_counted_used_once_and_carries_the_device, + an_account_keeps_five_open_challenges_and_expired_ones_go, + a_challenge_dies_with_every_change_of_the_account, + two_completions_of_one_challenge_give_one_device, + completions_and_changes_of_the_account_at_the_same_moment_leave_no_stray_device, + turning_two_factor_on_and_off_at_the_same_moment_as_a_sign_in, + an_invite_to_a_protected_account_needs_the_second_factor_in_the_same_transaction, + a_reset_code_for_a_protected_account_needs_the_factor_unless_it_turns_it_off, + admins_two_factor_is_checked_in_the_change_and_resets_need_it, + the_command_line_resets_two_factor_and_the_account_shows_it, + secrets_are_listed_and_replaced_only_if_unchanged, + parallel_wrong_proofs_on_one_challenge_count_exactly_five, + parallel_wrong_proofs_lock_for_exactly_fifteen_minutes, + two_factor_changes_write_their_events, + two_factor_sign_in_writes_login_and_counts_failures, + the_lock_writes_one_event_when_it_is_set, + a_join_writes_its_two_factor_events, + resets_and_confirmations_write_their_events, + web_sessions_are_made_looked_up_expire_and_log_out, + a_web_session_ends_with_every_change_of_the_accounts_generation, + a_web_sign_in_with_two_factor_needs_its_own_challenge, + web_sessions_are_pruned_in_bounded_batches, + a_web_actor_changes_accounts_until_its_session_ends, ); // ---- helpers ---- @@ -291,7 +393,7 @@ async fn users_and_devices(meta: &Meta) { assert_eq!(meta.user_by_name("ana").await.unwrap(), Some(ana.clone())); assert_eq!(meta.user_by_name("bob").await.unwrap(), None); - meta.set_password_hash(ana.id, "$argon2id$new").await.unwrap(); + meta.reset_password(ana.id, "$argon2id$new", true, NOW + 1).await.unwrap(); assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); let token = [7u8; 32]; @@ -416,6 +518,242 @@ async fn stored_bytes_are_counted_once_per_space_and_capped(meta: &Meta) { assert!(!meta.has_manifest(ana.space, &m2.id()).await.unwrap()); } +async fn account_usage_estimates_history_and_trash_without_adding_them(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let empty = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((empty.stored_bytes, empty.trash_bytes, empty.history_bytes), (0, Some(0), Some(0))); + assert_eq!(empty.reason, None); + let _asset_manifest = manifest(meta, &ana, 3).await; + let assets = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((assets.trash_bytes, assets.history_bytes), (Some(0), Some(0)), "unrooted app assets are not freeable"); + + let (old, current) = (manifest(meta, &ana, 4).await, manifest(meta, &ana, 6).await); + let node = NodeId::generate(); + let first = version(old); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("file.txt"), version: first.clone() }], NOW) + .await + .unwrap(); + let live = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!((live.trash_bytes, live.history_bytes), (Some(0), Some(0))); + + let latest = version(current); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: latest.clone() }], NOW).await.unwrap(); + let historical = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(historical.history_bytes, Some(105), "old chunk's 5 stored bytes plus its 100-byte manifest"); + assert_eq!(historical.trash_bytes, Some(0)); + assert_eq!(historical.stored_bytes, meta.stored_bytes(ana.space).await.unwrap()); + + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(latest.id) }], NOW).await.unwrap(); + let trashed = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(trashed.trash_bytes, Some(212), "both versions' chunks and manifests belong to trash"); + assert_eq!(trashed.history_bytes, Some(0), "trashed node history is not history estimate"); + assert_eq!(trashed.stored_bytes, historical.stored_bytes, "labeled estimates never add to stored bytes"); +} + +async fn admin_storage_overview_is_aggregated_and_orders_ties(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let mut accounts = Vec::new(); + for index in 0..13 { + accounts.push(user(meta, &format!("a{index:02}")).await); + } + let cap = 10_000_000; + let quota = 12_000_000; + meta.admin_update_user(&actor, accounts[0].id, None, Some(Some(quota)), NOW).await.unwrap(); + for (index, stored) in [quota, 8_999_999, 9_000_000, cap, 10, 10, 10, 10, 10, 10, 10, 10, 10].into_iter().enumerate() { + meta.add_chunk(accounts[index].space, &ChunkId::from_bytes([index as u8 + 1; 32]), stored, NOW).await.unwrap(); + } + + let report = meta.clone().with_stored_bytes_cap(Some(cap)).admin_storage_overview().await.unwrap(); + assert_eq!(report.accounts, 14); + assert_eq!(report.stored_bytes, 40_000_089); + assert_eq!(report.default_quota_bytes, Some(cap)); + assert_eq!(report.accounts_with_override, 1); + assert_eq!(report.accounts_near_quota, 3, "exactly 90% counts as near"); + assert_eq!(report.accounts_over_quota, 2, "exactly 100% counts as over"); + assert_eq!( + report.top_accounts.iter().map(|account| account.username.as_str()).collect::>(), + ["a00", "a03", "a02", "a01", "a04", "a05", "a06", "a07", "a08", "a09"] + ); + assert_eq!(report.top_accounts[0].quota_bytes, Some(quota), "account override beats cap"); + let capped = meta.clone().with_stored_bytes_cap(Some(cap)); + assert_eq!(capped.storage_estimate(accounts[0].space).await.unwrap().quota_bytes, Some(quota)); + assert_eq!(capped.storage_estimate(accounts[1].space).await.unwrap().quota_bytes, Some(cap)); + assert_eq!(meta.storage_estimate(accounts[1].space).await.unwrap().quota_bytes, None); + assert_eq!(report.manifest_index.complete, 0); + assert_eq!(report.manifest_index.building, 0); + assert_eq!(report.manifest_index.not_indexed, 0); +} + +async fn admin_storage_overview_handles_an_empty_server(meta: &Meta) { + let report = meta.admin_storage_overview().await.unwrap(); + assert_eq!(report.accounts, 0); + assert_eq!(report.stored_bytes, 0); + assert_eq!(report.accounts_with_override, 0); + assert_eq!(report.accounts_near_quota, 0); + assert_eq!(report.accounts_over_quota, 0); + assert!(report.top_accounts.is_empty()); + assert_eq!(report.manifest_index.complete, 0); + assert_eq!(report.manifest_index.building, 0); + assert_eq!(report.manifest_index.not_indexed, 0); +} + +async fn admin_storage_estimates_hide_removed_and_unknown_accounts(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let target = user(meta, "target").await; + let estimate = meta.admin_storage_estimate(target.id).await.unwrap(); + assert_eq!(estimate.quota_bytes, None); + assert_eq!(estimate.stored_bytes, 0); + assert!(matches!(meta.admin_storage_estimate(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound("user")))); + meta.admin_remove_user(&actor, target.id, NOW).await.unwrap(); + assert!(matches!(meta.admin_storage_estimate(target.id).await, Err(MetaError::NotFound("user")))); +} + +async fn account_usage_excludes_chunks_shared_with_current_version(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + let old = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([4; 32]), vec![ChunkRef { id: shared, len: 5 }]).unwrap(); + let current = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([5; 32]), vec![ChunkRef { id: shared, len: 5 }]).unwrap(); + meta.add_manifest(ana.space, &old, 100, NOW).await.unwrap(); + meta.add_manifest(ana.space, ¤t, 100, NOW).await.unwrap(); + let node = NodeId::generate(); + let first = version(old.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("shared.txt"), version: first.clone() }], NOW) + .await + .unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: version(current.id()) }], NOW).await.unwrap(); + + assert_eq!(meta.storage_estimate(ana.space).await.unwrap().history_bytes, Some(100), "the 6-byte chunk also belongs to the current version"); +} + +async fn account_usage_deduplicates_per_space_and_counts_pending_purge(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let (ana_device, bob_device) = (device(meta, &ana).await, device(meta, &bob).await); + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + meta.add_chunk(bob.space, &shared, 999, NOW).await.unwrap(); + + let make_manifest = |file_hash| { + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes(file_hash), vec![ChunkRef { id: shared, len: 5 }]).unwrap() + }; + let (trash_manifest, live_manifest, bob_manifest) = (make_manifest([1; 32]), make_manifest([2; 32]), make_manifest([3; 32])); + for (owner, manifest) in [(&ana, &trash_manifest), (&ana, &live_manifest), (&bob, &bob_manifest)] { + meta.add_manifest(owner.space, manifest, 100, NOW).await.unwrap(); + } + + let (ana_trash, ana_live) = (NodeId::generate(), NodeId::generate()); + let (trash_version, live_version) = (version(trash_manifest.id()), version(live_manifest.id())); + meta.commit( + ana.space, + ana_device, + &[ + Op::Create { node: ana_trash, parent: root(meta, &ana).await, name: name("trash.txt"), version: trash_version.clone() }, + Op::Create { node: ana_live, parent: root(meta, &ana).await, name: name("live.txt"), version: live_version }, + Op::Delete { node: ana_trash, expected: Some(trash_version.id) }, + ], + NOW, + ) + .await + .unwrap(); + let estimate = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(estimate.trash_bytes, Some(100), "shared chunk is also named by a live manifest"); + assert_eq!(estimate.history_bytes, Some(0)); + meta.request_purge(ana.space, ana_trash, NOW).await.unwrap(); + assert_eq!(meta.storage_estimate(ana.space).await.unwrap().trash_bytes, Some(100), "pending purge still stores trash blobs"); + + let bob_trash = NodeId::generate(); + let bob_version = version(bob_manifest.id()); + meta.commit( + bob.space, + bob_device, + &[ + Op::Create { node: bob_trash, parent: root(meta, &bob).await, name: name("shared.txt"), version: bob_version.clone() }, + Op::Delete { node: bob_trash, expected: Some(bob_version.id) }, + ], + NOW, + ) + .await + .unwrap(); + assert_eq!(meta.storage_estimate(bob.space).await.unwrap().trash_bytes, Some(106), "shared global chunk counts once in Bob's space"); +} + +async fn account_usage_excludes_live_document_snapshot_blobs(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let old = manifest(meta, &ana, 4).await; + let current = manifest(meta, &ana, 6).await; + let (node, first) = (NodeId::generate(), version(old)); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("document.kitdemo"), version: first.clone() }], NOW) + .await + .unwrap(); + let doc = craft_cloud_proto::live::node_doc(ana.space, node); + meta.create_live_document(ana.space, doc, "document:wordcraft", node, &old, NOW).await.unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node, expected: first.id, version: version(current) }], NOW).await.unwrap(); + + let estimate = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(estimate.history_bytes, Some(0), "old version's manifest and chunk remain named by the live-document snapshot"); +} + +async fn account_usage_leaves_saved_versions_out_of_history(meta: &Meta) { + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let shared = random_chunk(); + meta.add_chunk(ana.space, &shared, 6, NOW).await.unwrap(); + let make_manifest = |file_hash| { + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes(file_hash), vec![ChunkRef { id: shared, len: 5 }]).unwrap() + }; + let (saved_old, unsaved_old) = (make_manifest([7; 32]), make_manifest([8; 32])); + for m in [&saved_old, &unsaved_old] { + meta.add_manifest(ana.space, m, 100, NOW).await.unwrap(); + } + let current = manifest(meta, &ana, 6).await; + + // v1 (saved) and v2 (not saved) name one chunk; v3 is current. The saved version's manifest and the chunk it shares + // with unsaved history stay out of the estimate: only v2's own 100-byte manifest is history. + let (node, v1) = (NodeId::generate(), version(saved_old.id())); + meta.commit(ana.space, dev, &[Op::Create { node, parent: root(meta, &ana).await, name: name("saved.kitdemo"), version: v1.clone() }], NOW) + .await + .unwrap(); + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(ana.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(ana.space, node, v1.id, &record).await.unwrap(); + let v2 = version(unsaved_old.id()); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v1.id, version: v2.clone() }], NOW).await.unwrap(); + let v3 = version(current); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v2.id, version: v3.clone() }], NOW).await.unwrap(); + assert_eq!( + meta.storage_estimate(ana.space).await.unwrap().history_bytes, + Some(100), + "the saved version's manifest and the chunk it shares with unsaved history are kept" + ); + + // A saved version whose chunk nothing else names contributes nothing; an unsaved old version still counts in full. + let (exclusive_saved, exclusive_unsaved) = (manifest(meta, &ana, 4).await, manifest(meta, &ana, 4).await); + let (other, w1) = (NodeId::generate(), version(exclusive_saved)); + meta.commit(ana.space, dev, &[Op::Create { node: other, parent: root(meta, &ana).await, name: name("other.kitdemo"), version: w1.clone() }], NOW) + .await + .unwrap(); + let other_record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(ana.space, other), frontiers: vec![(1, 5)] }; + meta.attach_save(ana.space, other, w1.id, &other_record).await.unwrap(); + let w2 = version(exclusive_unsaved); + meta.commit(ana.space, dev, &[Op::Update { node: other, expected: w1.id, version: w2.clone() }], NOW).await.unwrap(); + meta.commit(ana.space, dev, &[Op::Update { node: other, expected: w2.id, version: version(manifest(meta, &ana, 6).await) }], NOW).await.unwrap(); + assert_eq!( + meta.storage_estimate(ana.space).await.unwrap().history_bytes, + Some(205), + "unsaved 100 + 105 for the second file's unsaved old version" + ); + + // In the trash a saved version still counts: purging the node removes it with the rest. + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(v3.id) }], NOW).await.unwrap(); + let trashed = meta.storage_estimate(ana.space).await.unwrap(); + assert_eq!(trashed.history_bytes, Some(105)); + assert_eq!(trashed.trash_bytes, Some(313), "three manifests, the shared 6-byte chunk and the current version's 7-byte chunk"); +} + /// Review finding 3 of M3c: two batches racing with an overlapping chunk charge it once (on Postgres both used to /// see it as new before inserting). SQLite runs one write at a time, so there it pins the result. async fn concurrent_chunk_batches_charge_a_shared_chunk_once(meta: &Meta) { @@ -1771,6 +2109,12 @@ async fn upgrading_counts_what_each_space_already_stores() { // The database as it was before stored bytes were counted. use sqlx::Connection; let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + // Before the audit log. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE spaces DROP COLUMN stored_bytes").execute(&mut conn).await.unwrap(); // And before live documents of files. sqlx::query("ALTER TABLE spaces DROP COLUMN live_bytes").execute(&mut conn).await.unwrap(); @@ -1782,7 +2126,27 @@ async fn upgrading_counts_what_each_space_already_stores() { sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); // And before save records. + // And before the reference index. + sqlx::query("DROP TABLE manifest_chunks").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE manifest_index").execute(&mut conn).await.unwrap(); + // And before the trash. + sqlx::query("DROP INDEX nodes_trash").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN deleted_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN purge_requested_ms").execute(&mut conn).await.unwrap(); + // And before the history index. + sqlx::query("DROP INDEX versions_node_seq").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE versions DROP COLUMN save").execute(&mut conn).await.unwrap(); + // And before the admin API (its `invites` column went with the table). + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + // And before device sessions. + for column in ["client_kind", "platform", "app_version", "revoked_ms", "revoke_reason"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE devices DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + for column in ["password_changed_ms", "disabled_ms"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE users DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } sqlx::query("DELETE FROM schema_version WHERE version >= 2").execute(&mut conn).await.unwrap(); } let meta = Meta::open(&url).await.unwrap(); @@ -1807,5 +2171,5692 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 6); + assert_eq!(versions, 14); +} + +// ---- device sessions (migration 0007) ---- + +/// A device's record, read back from the list. +async fn record(meta: &Meta, user: &UserRecord, id: DeviceId) -> DeviceRecord { + meta.list_devices(user.id).await.unwrap().into_iter().find(|d| d.id == id).unwrap() +} + +/// The three columns of a revoke always agree: all set or none. +fn assert_revoke_columns_agree(devices: &[DeviceRecord]) { + for d in devices { + assert_eq!((d.revoked, d.revoked_ms.is_some(), d.revoke_reason.is_some()), (d.revoked, d.revoked, d.revoked), "{d:?}"); + } +} + +fn sorted(mut ids: Vec) -> Vec { + ids.sort_by_key(|id| *id.as_uuid()); + ids +} + +async fn a_device_records_its_client(meta: &Meta) { + let ana = user(meta, "ana").await; + let plain = device(meta, &ana).await; + let client = DeviceClient::new(Some("agent"), Some("windows x86_64"), Some("0.19.0")).unwrap(); + let agent = meta.create_device_with(ana.id, "studio", &client, &token(), NOW + 1).await.unwrap(); + assert_eq!(record(meta, &ana, agent).await.client, client); + let old = record(meta, &ana, plain).await; + assert_eq!((old.client, old.revoked_ms, old.revoke_reason), (DeviceClient::default(), None, None), "no word on the client: empty"); + + // What a sign-in may say: a known kind or none, short free text without control characters. + assert_eq!(DeviceClient::new(None, None, None).unwrap(), DeviceClient::default()); + assert!(DeviceClient::new(Some(""), Some(&"x".repeat(32)), Some(&"y".repeat(32))).is_ok()); + let long = "x".repeat(33); + for (kind, platform, version) in [ + (Some("browser"), None, None), + (Some("AGENT"), None, None), + (None, Some(long.as_str()), None), + (None, None, Some(long.as_str())), + (None, Some("a\nb"), None), + (None, None, Some("1.0\u{7}")), + ] { + assert!(matches!(DeviceClient::new(kind, platform, version), Err(MetaError::Invalid(_))), "{kind:?} {platform:?} {version:?}"); + } + // The store checks too, for a caller that built the value by hand: nothing is created. + let bad = DeviceClient { kind: "browser".into(), ..DeviceClient::default() }; + assert!(matches!(meta.create_device_with(ana.id, "x", &bad, &token(), NOW).await, Err(MetaError::Invalid(_)))); + assert_eq!(meta.list_devices(ana.id).await.unwrap().len(), 2); + + // A join says it too. + let (_, _, pairing, _) = paired(meta, "bob").await; + let desktop = DeviceClient::new(Some("desktop"), Some("macos"), Some("1.2")).unwrap(); + let joined = meta.redeem_with(&pairing.hash(), "mac", &desktop, &token(), None, NOW + 1).await.unwrap(); + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + assert_eq!(record(meta, &bob, joined.device).await.client, desktop); +} + +async fn only_own_live_devices_are_renamed(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let (mine, theirs) = (device(meta, &ana).await, device(meta, &bob).await); + let renamed = meta.rename_device(ana.id, mine, "studio pc").await.unwrap(); + assert_eq!((renamed.id, renamed.name.as_str()), (mine, "studio pc")); + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); + // Another account's device, and one that isn't there, are one answer. + assert!(matches!(meta.rename_device(ana.id, theirs, "mine now").await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.rename_device(ana.id, DeviceId::generate(), "x").await, Err(MetaError::NotFound(_)))); + assert_eq!(record(meta, &bob, theirs).await.name, "laptop"); + // The name rule holds. + for bad in ["", "tab\there", &"n".repeat(129)] { + assert!(matches!(meta.rename_device(ana.id, mine, bad).await, Err(MetaError::Invalid(_))), "{bad:?}"); + } + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); + // A revoked device keeps its name. + meta.revoke_device_as(ana.id, mine, RevokeReason::User, NOW + 1).await.unwrap(); + assert!(matches!(meta.rename_device(ana.id, mine, "again").await, Err(MetaError::NotFound(_)))); + assert_eq!(record(meta, &ana, mine).await.name, "studio pc"); +} + +async fn every_revoke_path_sets_flag_time_and_reason_together(meta: &Meta) { + // One device revoked by its user, and revoked again later: the first time and reason stay. + let ana = user(meta, "ana").await; + let (a1, a2, a3, a4) = (device(meta, &ana).await, device(meta, &ana).await, device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_device_as(ana.id, a1, RevokeReason::User, NOW + 10).await.unwrap(); + let first = record(meta, &ana, a1).await; + assert_eq!((first.revoked, first.revoked_ms, first.revoke_reason), (true, Some(NOW + 10), Some(RevokeReason::User))); + meta.revoke_device_as(ana.id, a1, RevokeReason::Admin, NOW + 20).await.unwrap(); + assert_eq!(record(meta, &ana, a1).await, first, "a second revoke changes nothing"); + assert!(matches!(meta.revoke_device_as(user(meta, "bob").await.id, a1, RevokeReason::User, NOW).await, Err(MetaError::NotFound(_)))); + // The others, but one: the device revoked before keeps its own reason, the kept one stays. + let revoked = meta.revoke_other_devices(ana.id, a2, RevokeReason::Others, NOW + 30).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![a3, a4]), "only the live others"); + for id in [a3, a4] { + let d = record(meta, &ana, id).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 30), Some(RevokeReason::Others))); + } + assert_eq!(record(meta, &ana, a1).await, first); + let kept = record(meta, &ana, a2).await; + assert_eq!((kept.revoked, kept.revoked_ms, kept.revoke_reason), (false, None, None)); + assert!(meta.revoke_other_devices(ana.id, a2, RevokeReason::Others, NOW + 40).await.unwrap().is_empty(), "nobody left to sign out"); + // Every device (the admin's command). + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + let (b1, b2) = (device(meta, &bob).await, device(meta, &bob).await); + let revoked = meta.revoke_all_devices(bob.id, RevokeReason::Admin, NOW + 50).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![b1, b2])); + assert!(meta.revoke_all_devices(bob.id, RevokeReason::Admin, NOW + 60).await.unwrap().is_empty()); + for id in [b1, b2] { + let d = record(meta, &bob, id).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 50), Some(RevokeReason::Admin))); + } + // A password change and a reset. + let cy = user(meta, "cy").await; + let (c1, c2) = (device(meta, &cy).await, device(meta, &cy).await); + meta.change_password(cy.id, "$argon2id$stub", "$argon2id$new", c1, NOW + 70).await.unwrap(); + let d = record(meta, &cy, c2).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 70), Some(RevokeReason::Password))); + let dee = user(meta, "dee").await; + let d1 = device(meta, &dee).await; + meta.reset_password(dee.id, "$argon2id$new", false, NOW + 80).await.unwrap(); + let d = record(meta, &dee, d1).await; + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 80), Some(RevokeReason::Reset))); + // The plain call (no clock of its own) is the user's act, with a time. + let eve = user(meta, "eve").await; + let e1 = device(meta, &eve).await; + meta.revoke_device(eve.id, e1).await.unwrap(); + let d = record(meta, &eve, e1).await; + assert_eq!((d.revoked, d.revoke_reason), (true, Some(RevokeReason::User))); + assert!(d.revoked_ms.is_some_and(|ms| ms > NOW)); + for u in [&ana, &bob, &cy, &dee, &eve] { + assert_revoke_columns_agree(&meta.list_devices(u.id).await.unwrap()); + } +} + +async fn revoking_others_keeps_the_caller_and_ends_their_pairings(meta: &Meta) { + let (ana, keep, keep_pairing, keep_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_id = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + let revoked = meta.revoke_other_devices(ana.id, keep, RevokeReason::Others, NOW + 1).await.unwrap(); + assert_eq!(revoked, vec![other]); + assert!(meta.device_active(keep).await.unwrap() && !meta.device_active(other).await.unwrap()); + assert_eq!(meta.pairing_status(ana.id, keep_id, NOW + 2).await.unwrap(), Some((PairingState::Waiting, None)), "the caller's own stays"); + assert_eq!(meta.pairing_status(ana.id, other_id, NOW + 2).await.unwrap(), None, "a revoked device's pairing dies"); + redeem(meta, &keep_pairing, "pc", NOW + 3).await.unwrap(); + // Another account is never touched. + assert!(meta.device_active(bobs).await.unwrap()); + assert!(meta.pairing_status(bob.id, bobs_pairing, NOW + 2).await.unwrap().is_some()); +} + +async fn a_password_change_is_one_step(meta: &Meta) { + let (ana, keep, _, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_pairing = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + let bobs_invite = code(INVITE_CHARS); + meta.create_invite("bob", &bobs_invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + + let account = meta.account(ana.id).await.unwrap(); + assert_eq!((account.username.as_str(), account.role, account.created_ms, account.password_changed_ms), ("ana", Role::Member, NOW, None)); + assert!(matches!(meta.account(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound(_)))); + + // A password that is not the one the caller verified (it changed meanwhile): nothing at all changes. + let stale = meta.change_password(ana.id, "$argon2id$stale", "$argon2id$new", keep, NOW + 5).await; + assert!(matches!(stale, Err(MetaError::Conflict(_))), "{stale:?}"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + assert!(meta.device_active(keep).await.unwrap() && meta.device_active(other).await.unwrap()); + assert!(meta.pairing_status(ana.id, pairing_id, NOW + 6).await.unwrap().is_some()); + assert!(meta.pairing_status(ana.id, other_pairing, NOW + 6).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 6).await.unwrap(), Some(false)); + + // The change: new hash and time, the other device signed out, invite and pairings gone, the caller stays. + let revoked = meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 7).await.unwrap(); + assert_eq!(revoked, vec![other]); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 7)); + assert!(meta.device_active(keep).await.unwrap() && !meta.device_active(other).await.unwrap()); + let d = record(meta, &ana, other).await; + assert_eq!((d.revoked_ms, d.revoke_reason), (Some(NOW + 7), Some(RevokeReason::Password))); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 8).await.unwrap(), None); + assert_eq!(meta.pairing_status(ana.id, other_pairing, NOW + 8).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 8).await.unwrap(), None); + assert!(matches!(redeem(meta, &invite, "pc", NOW + 8).await, Err(MetaError::InvalidCode))); + // The old hash no longer matches, so the same change again (a replayed request) does nothing. + assert!(matches!(meta.change_password(ana.id, "$argon2id$stub", "$argon2id$third", keep, NOW + 9).await, Err(MetaError::Conflict(_)))); + // A caller that was signed out meanwhile changes nothing either. + assert!( + matches!(meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", other, NOW + 9).await, Err(MetaError::DeviceRevoked)), + "revoked" + ); + assert!( + matches!( + meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", DeviceId::generate(), NOW + 9).await, + Err(MetaError::DeviceRevoked) + ), + "unknown" + ); + let bobs_device = device(meta, &user(meta, "cy").await).await; + assert!( + matches!(meta.change_password(ana.id, "$argon2id$new", "$argon2id$third", bobs_device, NOW + 9).await, Err(MetaError::DeviceRevoked)), + "another account's" + ); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$new"); + // Another account is never touched. + assert_eq!(meta.user_by_name("bob").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert!(meta.device_active(bobs).await.unwrap()); + assert!(meta.pairing_status(bob.id, bobs_pairing, NOW + 8).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&bobs_invite.hash(), NOW + 8).await.unwrap(), Some(false)); +} + +async fn an_admin_reset_signs_out_every_device_unless_kept(meta: &Meta) { + let (ana, maker, _, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let revoked = meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![maker, other])); + assert!(!meta.device_active(maker).await.unwrap() && !meta.device_active(other).await.unwrap()); + assert_eq!(record(meta, &ana, other).await.revoke_reason, Some(RevokeReason::Reset)); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$reset"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 1)); + + // `--keep-devices`: the devices stay, the password changes, and codes made before it are still deleted. + let (bob, bobs, _, bobs_pairing) = paired(meta, "bob").await; + assert!(meta.reset_password(bob.id, "$argon2id$reset", true, NOW + 3).await.unwrap().is_empty()); + assert!(meta.device_active(bobs).await.unwrap()); + assert_eq!(record(meta, &bob, bobs).await.revoke_reason, None); + assert_eq!(meta.pairing_status(bob.id, bobs_pairing, NOW + 4).await.unwrap(), None); + assert_eq!(meta.user_by_name("bob").await.unwrap().unwrap().password_hash, "$argon2id$reset"); + + // A user that isn't there changes nothing. + assert!(matches!(meta.reset_password(UserId(uuid::Uuid::now_v7()), "$argon2id$x", false, NOW).await, Err(MetaError::NotFound(_)))); +} + +/// A password change from one device, a sign-out of the others from another and an admin reset, at the same moment +/// and over and over: each ends as a success or as a conflict or a signed-out caller, never as a database error (on +/// Postgres, two transactions taking the same rows in different orders would deadlock and one would fail). +async fn concurrent_password_changes_and_sign_outs_never_fail_the_database(meta: &Meta) { + for round in 0..15 { + let ana = user(meta, &format!("ana{round}")).await; + let (a, b) = (device(meta, &ana).await, device(meta, &ana).await); + let mut tasks = Vec::new(); + for job in 0..6 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.change_password(ana.id, "$argon2id$stub", "$argon2id$one", a, NOW + 1).await.map(|_| ()), + 1 => meta.change_password(ana.id, "$argon2id$stub", "$argon2id$two", b, NOW + 1).await.map(|_| ()), + 2 => meta.revoke_other_devices(ana.id, b, RevokeReason::Others, NOW + 1).await.map(|_| ()), + 3 => meta.revoke_device_as(ana.id, a, RevokeReason::User, NOW + 1).await, + 4 => meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.map(|_| ()), + _ => meta.revoke_all_devices(ana.id, RevokeReason::Admin, NOW + 1).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!(matches!(result, Ok(()) | Err(MetaError::Conflict(_) | MetaError::DeviceRevoked)), "round {round}: {result:?}"); + } + assert_revoke_columns_agree(&meta.list_devices(ana.id).await.unwrap()); + } +} + +/// A device is created after a password sign-in only if the password that was checked is still the account's. +async fn a_sign_in_checked_against_an_old_password_creates_no_device(meta: &Meta) { + let ana = user(meta, "ana").await; + let client = DeviceClient::default(); + let first = meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "pc", &client, &token(), NOW).await.unwrap(); + assert!(first.is_some_and(|d| d != DeviceId::generate())); + assert!(meta.sign_in_device(ana.id, ("$argon2id$other", 0), "pc", &client, &token(), NOW).await.unwrap().is_none(), "not the stored hash"); + assert!( + meta.sign_in_device(UserId(uuid::Uuid::now_v7()), ("$argon2id$stub", 0), "pc", &client, &token(), NOW).await.unwrap().is_none(), + "no such user" + ); + assert!(matches!(meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "", &client, &token(), NOW).await, Err(MetaError::Invalid(_)))); + // The password was checked, then reset: the account's devices are signed out and the late sign-in adds none. + meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 1).await.unwrap(); + assert!(meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "late", &client, &token(), NOW + 2).await.unwrap().is_none()); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert_eq!(devices.len(), 1); + assert!(devices.iter().all(|d| d.revoked)); + // With the new hash it works. + assert!(meta.sign_in_device(ana.id, ("$argon2id$reset", gen_of(meta, "ana").await), "new", &client, &token(), NOW + 3).await.unwrap().is_some()); +} + +/// Redeeming an invite or a pairing, a password sign-in and a reset at the same moment, over and over: no database +/// error (on Postgres, the invite and the user taken in different orders would deadlock), and once the reset is +/// done no device of the account is signed in, whichever of them came first. +async fn redeems_sign_ins_and_resets_at_the_same_moment_leave_no_live_device(meta: &Meta) { + for round in 0..15 { + let (ana, maker, pairing, _) = paired(meta, &format!("ana{round}")).await; + let invite = code(INVITE_CHARS); + meta.create_invite(&format!("ana{round}"), &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..5 { + let (meta, invite, pairing) = (meta.clone(), invite.hash(), pairing.hash()); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.redeem(&invite, "invited", &token(), None, NOW + 1).await.map(|_| ()), + 1 => meta.redeem(&pairing, "paired", &token(), None, NOW + 1).await.map(|_| ()), + 2 => { + meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "signed in", &DeviceClient::default(), &token(), NOW + 1).await.map(|_| ()) + } + 3 => meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 1).await.map(|_| ()), + _ => meta.reset_password(ana.id, "$argon2id$reset", false, NOW + 2).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!(matches!(result, Ok(()) | Err(MetaError::InvalidCode | MetaError::NotFound(_))), "round {round}: {result:?}"); + } + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| d.revoked), "round {round}: {devices:?}"); + assert_revoke_columns_agree(&devices); + } +} + +// ---- the migration from 0006 to 0007 ---- + +/// Drops what migration 0007 added and forgets it ran, which leaves the database as 0006 left it (with its rows). +async fn back_to_0006(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + // What migrations 0010, 0009 and 0008 added goes first: they run again with 0007 on the next open. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN purpose").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + for column in ["client_kind", "platform", "app_version", "revoked_ms", "revoke_reason"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE devices DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + for column in ["password_changed_ms", "disabled_ms"] { + sqlx::query(sqlx::AssertSqlSafe(format!("ALTER TABLE users DROP COLUMN {column}"))).execute(&mut conn).await.unwrap(); + } + sqlx::query("DELETE FROM schema_version WHERE version >= 7").execute(&mut conn).await.unwrap(); +} + +async fn schema_version(url: &str) -> i64 { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap() +} + +/// Existing users and devices (live and revoked) survive the migration, read with empty client fields and no revoke +/// time or reason, still work (a live one is found by its token, a revoked one is not), and take the new columns; +/// a database newer than the program is refused. +async fn migrating_from_0006_keeps_the_rows(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, live, revoked, old_token) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + let live = meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + let revoked = device(&meta, &ana).await; + meta.revoke_device(ana.id, revoked).await.unwrap(); + (ana, live, revoked, old_token) + }; + back_to_0006(url).await; + assert_eq!(schema_version(url).await, 6); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14); + let old = record(&meta, &ana, live).await; + assert_eq!( + (old.name.as_str(), old.revoked, old.client, old.revoked_ms, old.revoke_reason), + ("old pc", false, DeviceClient::default(), None, None) + ); + let gone = record(&meta, &ana, revoked).await; + assert_eq!((gone.revoked, gone.client, gone.revoked_ms, gone.revoke_reason), (true, DeviceClient::default(), None, None)); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.device), Some(live)); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + // The new columns take values. + let client = DeviceClient::new(Some("cli"), Some("linux"), Some("1")).unwrap(); + let fresh = meta.create_device_with(ana.id, "new", &client, &token(), NOW + 1).await.unwrap(); + assert_eq!(record(&meta, &ana, fresh).await.client, client); + meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", fresh, NOW + 2).await.unwrap(); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 2)); + assert_eq!(record(&meta, &ana, live).await.revoke_reason, Some(RevokeReason::Password)); + + // A database from a newer Craft Cloud is still refused. + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (15)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0006_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0006_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0006_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0006_keeps_the_rows(&url).await; +} + +// ---- a failure in the middle of a transaction ---- + +/// A trigger that makes every delete from `pairings` fail, which is the step after the password and the devices +/// were already written in `change_password` and `reset_password`. Test only: the migrations carry no trigger. +async fn break_pairing_deletes(url: &str, postgres: bool) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + let statements: &[&str] = if postgres { + &[ + "CREATE FUNCTION cc_test_boom() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$ LANGUAGE plpgsql", + "CREATE TRIGGER cc_test_boom BEFORE DELETE ON pairings FOR EACH STATEMENT EXECUTE FUNCTION cc_test_boom()", + ] + } else { + &["CREATE TRIGGER cc_test_boom BEFORE DELETE ON pairings BEGIN SELECT RAISE(ABORT, 'boom'); END"] + }; + for statement in statements { + sqlx::query(sqlx::AssertSqlSafe((*statement).to_owned())).execute(&mut conn).await.unwrap(); + } +} + +async fn mend_pairing_deletes(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("DROP TRIGGER cc_test_boom ON pairings").execute(&mut conn).await.ok(); + sqlx::query("DROP TRIGGER cc_test_boom").execute(&mut conn).await.ok(); + sqlx::query("DROP FUNCTION cc_test_boom()").execute(&mut conn).await.ok(); +} + +/// A step that fails after others were written undoes them all: the hash, the time, the devices, the pairings and +/// the invite are as they were, for a password change and for an admin reset; once the step works, the same call +/// goes through. +async fn a_failure_in_the_middle_changes_nothing(url: &str, postgres: bool) { + let meta = Meta::open(url).await.unwrap(); + let (ana, keep, _, pairing_id) = paired(&meta, "ana").await; + let other = device(&meta, &ana).await; + let other_pairing = meta.create_pairing(ana.id, other, &code(PAIRING_CHARS).hash(), NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + break_pairing_deletes(url, postgres).await; + + let invite_hash = invite.hash(); + let unchanged = || async { + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, None); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| !d.revoked && d.revoked_ms.is_none() && d.revoke_reason.is_none()), "{devices:?}"); + assert!(meta.pairing_status(ana.id, pairing_id, NOW + 1).await.unwrap().is_some()); + assert!(meta.pairing_status(ana.id, other_pairing, NOW + 1).await.unwrap().is_some()); + assert_eq!(meta.invite_needs_password(&invite_hash, NOW + 1).await.unwrap(), Some(false)); + }; + let failed = meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + let failed = meta.reset_password(ana.id, "$argon2id$new", false, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + let failed = meta.revoke_other_devices(ana.id, keep, RevokeReason::Others, NOW + 2).await; + assert!(matches!(failed, Err(MetaError::Db(_))), "{failed:?}"); + unchanged().await; + + mend_pairing_deletes(url).await; + assert_eq!(meta.change_password(ana.id, "$argon2id$stub", "$argon2id$new", keep, NOW + 3).await.unwrap(), vec![other]); +} + +#[tokio::test] +async fn a_failure_in_the_middle_changes_nothing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_failure_in_the_middle_changes_nothing(&format!("sqlite:{}", dir.path().join("meta.db").display()), false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failure_in_the_middle_changes_nothing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_failure_in_the_middle_changes_nothing(&url, true).await; +} + +// ---- the admin API (migration 0008) ---- + +const ADMIN_HASH: &str = "$argon2id$admin"; + +/// An admin account with a device, and the actor an admin request on its behalf would carry. +async fn admin_actor(meta: &Meta, username: &str) -> (UserRecord, AdminActor) { + let u = meta.create_user(username, ADMIN_HASH, Role::Admin, NOW).await.unwrap(); + let device = device(meta, &u).await; + let actor = AdminActor { user: u.id, via: AdminVia::Device(device), password_hash: ADMIN_HASH.to_owned(), two_factor: false }; + (u, actor) +} + +/// The device a device-borne actor acts with. +fn device_of(actor: &AdminActor) -> DeviceId { + match actor.via { + AdminVia::Device(device) => device, + AdminVia::Web { .. } => panic!("a device actor was expected"), + } +} + +/// The generation a sign-in would read for `username` now. +async fn gen_of(meta: &Meta, username: &str) -> i64 { + meta.user_by_name(username).await.unwrap().unwrap().auth_gen +} + +async fn admin_view(meta: &Meta, user: UserId) -> AdminUser { + meta.admin_user(user, NOW).await.unwrap().unwrap() +} + +async fn users_page(meta: &Meta, after: Option<&str>, search: Option<&str>, limit: u32) -> (Vec, Option) { + let page = meta.admin_list_users(after, search, limit, NOW).await.unwrap(); + (page.users.into_iter().map(|u| u.username).collect(), page.next_after) +} + +async fn the_admin_list_pages_searches_and_counts(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for name in ["ana", "bob", "a_b", "axb", "carol"] { + user(meta, name).await; + } + // By username, in pages; the last page has no continuation. + let (first, next) = users_page(meta, None, None, 3).await; + assert_eq!(first.len(), 3); + let (second, next2) = users_page(meta, next.as_deref(), None, 3).await; + assert_eq!(second.len(), 3); + assert_eq!(next2, None); + let mut all: Vec<_> = first.into_iter().chain(second).collect(); + let mut sorted_all = all.clone(); + sorted_all.sort(); + assert_eq!(all, sorted_all, "pages follow one order"); + all.dedup(); + assert_eq!(all.len(), 6, "no account twice, none missing"); + assert_eq!(users_page(meta, None, None, 6).await.1, None, "exactly a full page has no continuation"); + // A search finds a substring, ignores case, and takes `%` and `_` literally. + assert_eq!(users_page(meta, None, Some("A_B"), 50).await.0, vec!["a_b"]); + assert_eq!(users_page(meta, None, Some("a"), 50).await.0, vec!["a_b", "ana", "axb", "carol"]); + assert_eq!(users_page(meta, None, Some("%"), 50).await.0, Vec::::new()); + assert_eq!(users_page(meta, None, Some("_"), 50).await.0, vec!["a_b"]); + assert_eq!(users_page(meta, None, Some("zzz"), 50).await.0, Vec::::new()); + // A search pages too. + let (one, next) = users_page(meta, None, Some("a"), 1).await; + assert_eq!(one, vec!["a_b"]); + assert_eq!(users_page(meta, next.as_deref(), Some("a"), 2).await.0, vec!["ana", "axb"]); + assert!(matches!(meta.admin_list_users(None, Some(&"x".repeat(65)), 10, NOW).await, Err(MetaError::Invalid(_)))); + meta.admin_list_users(None, Some(&"x".repeat(64)), 10, NOW).await.unwrap(); + + // What a row says. + let ana = meta.user_by_name("ana").await.unwrap().unwrap(); + let dev = device(meta, &ana).await; + meta.add_chunk(ana.space, &random_chunk(), 70, NOW).await.unwrap(); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let row = admin_view(meta, ana.id).await; + assert_eq!((row.role, row.devices_active, row.stored_bytes, row.quota_bytes, row.invite_pending), (Role::Member, 1, 70, None, true)); + meta.revoke_device(ana.id, dev).await.unwrap(); + assert_eq!(admin_view(meta, ana.id).await.devices_active, 0); + // An expired invite is not pending. + assert!(!meta.admin_user(ana.id, NOW + INVITE_TTL_MS).await.unwrap().unwrap().invite_pending); + // A reset code is not an invite. + let bob = meta.user_by_name("bob").await.unwrap().unwrap(); + meta.admin_issue_reset(&actor, bob.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + assert!(!admin_view(meta, bob.id).await.invite_pending); + + // The counters: accounts, live devices, bytes. + let counts = meta.admin_server_counts().await.unwrap(); + assert_eq!((counts.users, counts.devices_active, counts.stored_bytes, counts.live_bytes), (6, 1, 70, 0), "only root's device is live"); + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.admin_server_counts().await.unwrap().users, 5); + assert!(!users_page(meta, None, None, 50).await.0.contains(&"ana".to_owned()), "a removed account is not listed"); + assert!(meta.admin_user(ana.id, NOW).await.unwrap().is_some(), "but it can still be looked up"); +} + +async fn an_admin_makes_an_account_with_a_role_and_an_invite(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let invite = code(INVITE_CHARS); + let (made, issued) = meta.admin_create_user(&actor, "Newbie", Role::Admin, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + assert_eq!((made.username.as_str(), made.role, made.devices_active, made.invite_pending), ("newbie", Role::Admin, 0, true)); + assert_eq!(issued.expires_ms, NOW + INVITE_TTL_MS); + // A name that exists, in any case, any role, is taken, and nothing is changed. + for name in ["newbie", "NEWBIE", "root"] { + let again = meta.admin_create_user(&actor, name, Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await; + assert!(matches!(again, Err(MetaError::UsernameTaken)), "{name}: {again:?}"); + } + assert!(matches!( + meta.admin_create_user(&actor, "bad name", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::Invalid(_)) + )); + // It has no password: the invite asks for one, and sets it. The account keeps its role and has its own space. + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(true)); + assert!(matches!(meta.redeem(&invite.hash(), "pc", &token(), None, NOW + 1).await, Err(MetaError::PasswordRequired))); + let joined = meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$chosen"), NOW + 2).await.unwrap(); + assert_eq!(joined.user, made.id); + let record = meta.user_by_name("newbie").await.unwrap().unwrap(); + assert_eq!((record.role, record.password_hash.as_str()), (Role::Admin, "$argon2id$chosen")); + assert!(meta.space_info(record.space).await.is_ok()); + assert!(!admin_view(meta, made.id).await.invite_pending); +} + +async fn an_admin_changes_a_role_and_a_quota(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + // Nothing to change is invalid. + assert!(matches!(meta.admin_update_user(&actor, ana.id, None, None, NOW).await, Err(MetaError::Invalid(_)))); + let promoted = meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + assert_eq!((promoted.role, promoted.quota_bytes), (Role::Admin, None)); + // Role and quota together; a missing quota leaves it, `Some(None)` removes it. + let both = meta.admin_update_user(&actor, ana.id, Some(Role::Member), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert_eq!((both.role, both.quota_bytes), (Role::Member, Some(MIN_QUOTA_BYTES))); + let role_only = meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await.unwrap(); + assert_eq!(role_only.quota_bytes, Some(MIN_QUOTA_BYTES), "a missing quota leaves it"); + let cleared = meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + assert_eq!(cleared.quota_bytes, None); + // Bounds: a chunk up to i64::MAX. + for bad in [0, 1, MIN_QUOTA_BYTES - 1, i64::MAX as u64 + 1, u64::MAX] { + let result = meta.admin_update_user(&actor, ana.id, None, Some(Some(bad)), NOW).await; + assert!(matches!(result, Err(MetaError::Invalid(_))), "{bad}: {result:?}"); + } + for good in [MIN_QUOTA_BYTES, i64::MAX as u64] { + assert_eq!(meta.admin_update_user(&actor, ana.id, None, Some(Some(good)), NOW).await.unwrap().quota_bytes, Some(good)); + } + // Unknown users, and accounts marked for removal. + assert!(matches!(meta.admin_update_user(&actor, UserId(uuid::Uuid::now_v7()), Some(Role::Admin), None, NOW).await, Err(MetaError::NotFound(_)))); + // The last administrator can't be demoted, even by themselves; with another one they can. + let result = meta.admin_update_user(&actor, root.id, Some(Role::Member), None, NOW).await; + assert!(matches!(result, Err(MetaError::LastAdmin)), "{result:?}"); + assert_eq!(admin_view(meta, root.id).await.role, Role::Admin, "and nothing was changed"); + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + meta.admin_update_user(&actor, root.id, Some(Role::Member), None, NOW).await.unwrap(); + // The demoted admin's next change is refused: they are no admin any more. + let result = meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + let ana_actor = + AdminActor { user: ana.id, via: AdminVia::Device(device(meta, &ana).await), password_hash: "$argon2id$stub".to_owned(), two_factor: false }; + meta.admin_remove_user(&ana_actor, root.id, NOW).await.unwrap(); + let result = meta.admin_update_user(&ana_actor, root.id, Some(Role::Admin), None, NOW).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "{result:?}"); +} + +/// The acting admin's state is read again inside each change (the role and the password check the handler did can be +/// stale by then): a demotion, a sign-out, a password change or a disable since lets nothing through, and nothing is +/// changed. +async fn the_acting_admins_state_is_checked_in_the_transaction(meta: &Meta) { + let (_, boss) = admin_actor(meta, "boss").await; + let victim = user(meta, "victim").await; + let issue = |actor: AdminActor| { + let meta = meta.clone(); + async move { + meta.admin_create_user( + &actor, + &format!("made{}", uuid::Uuid::now_v7().simple()), + Role::Member, + &code(INVITE_CHARS).hash(), + NOW, + INVITE_TTL_MS, + ) + .await + .map(|_| ()) + } + }; + let users_before = meta.admin_server_counts().await.unwrap().users; + + // A member (a demoted admin) is forbidden. + let (demoted, demoted_actor) = admin_actor(meta, "demoted").await; + meta.admin_update_user(&boss, demoted.id, Some(Role::Member), None, NOW).await.unwrap(); + assert!(matches!(issue(demoted_actor).await, Err(MetaError::Forbidden(_)))); + // A signed-out device is unauthorized. + let (signed_out, signed_out_actor) = admin_actor(meta, "signedout").await; + meta.revoke_device(signed_out.id, device_of(&signed_out_actor)).await.unwrap(); + assert!(matches!(issue(signed_out_actor).await, Err(MetaError::DeviceRevoked))); + // A password changed since it was checked is a conflict. + let (changed, changed_actor) = admin_actor(meta, "changed").await; + meta.reset_password(changed.id, "$argon2id$other", true, NOW).await.unwrap(); + assert!(matches!(issue(changed_actor).await, Err(MetaError::Conflict(_)))); + // A disabled admin is unauthorized (their devices are revoked, and the check says it even for a device kept). + let (off, off_actor) = admin_actor(meta, "off").await; + meta.admin_disable_user(&boss, off.id, NOW).await.unwrap(); + assert!(matches!(issue(off_actor).await, Err(MetaError::DeviceRevoked))); + // Another account's device is not the actor's. + let borrowed = AdminActor { via: AdminVia::Device(device(meta, &victim).await), ..boss.clone() }; + assert!(matches!(issue(borrowed).await, Err(MetaError::DeviceRevoked))); + // A member who is no admin at all. + let member_actor = AdminActor { + user: victim.id, + via: AdminVia::Device(device(meta, &victim).await), + password_hash: "$argon2id$stub".to_owned(), + two_factor: false, + }; + assert!(matches!(meta.admin_disable_user(&member_actor, demoted.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert_eq!(meta.admin_server_counts().await.unwrap().users, users_before + 4, "only the four admins made here, no account from a refused change"); + // Every kind of change checks, not only the first: one of each against a stale actor. + let (stale_user, stale_actor) = admin_actor(meta, "stale").await; + meta.admin_update_user(&boss, stale_user.id, Some(Role::Member), None, NOW).await.unwrap(); + assert!(matches!(meta.admin_update_user(&stale_actor, victim.id, Some(Role::Admin), None, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_disable_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_enable_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_remove_user(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + let hash = code(INVITE_CHARS).hash(); + assert!(matches!(meta.admin_issue_invite(&stale_actor, victim.id, &hash, NOW, INVITE_TTL_MS).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_revoke_invite(&stale_actor, victim.id, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(matches!(meta.admin_issue_reset(&stale_actor, victim.id, &hash, false, NOW, RESET_TTL_MS).await, Err(MetaError::Forbidden(_)))); + let victims_device = device(meta, &victim).await; + assert!(matches!(meta.admin_revoke_device(&stale_actor, victim.id, victims_device, NOW).await, Err(MetaError::Forbidden(_)))); + assert!(meta.device_active(victims_device).await.unwrap()); + assert_eq!(admin_view(meta, victim.id).await.role, Role::Member); +} + +/// A disabled account has nothing: its devices are signed out (with the reason), its codes gone, and no token, sign-in, +/// join or pairing works; enabling it brings back the account but never a device. +async fn disabling_signs_everything_out_and_enabling_brings_back_nothing(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (ana, maker, pairing, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let other_token = token(); + let third = meta.create_device(ana.id, "third", &other_token, NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + + // Not oneself. + assert!(matches!(meta.admin_disable_user(&actor, root.id, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.admin_remove_user(&actor, root.id, NOW).await, Err(MetaError::Invalid(_)))); + let revoked = meta.admin_disable_user(&actor, ana.id, NOW + 1).await.unwrap(); + assert_eq!(sorted(revoked), sorted(vec![maker, other, third])); + let view = admin_view(meta, ana.id).await; + assert_eq!((view.disabled_ms, view.devices_active, view.invite_pending), (Some(NOW + 1), 0, false)); + assert_eq!(meta.device_by_token(&other_token).await.unwrap(), None); + for d in meta.list_devices(ana.id).await.unwrap() { + assert_eq!((d.revoked, d.revoked_ms, d.revoke_reason), (true, Some(NOW + 1), Some(RevokeReason::Disabled))); + } + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None); + // Disabling again is harmless. + assert!(meta.admin_disable_user(&actor, ana.id, NOW + 5).await.unwrap().is_empty()); + assert_eq!(admin_view(meta, ana.id).await.disabled_ms, Some(NOW + 1), "the first time stays"); + + // Nothing gets in: a sign-in that checked the right password, a device, a code the command line makes, a pairing. + let sign_in = meta.sign_in_device(ana.id, ("$argon2id$stub", gen_of(meta, "ana").await), "pc", &DeviceClient::default(), &token(), NOW + 3).await; + assert!(matches!(sign_in, Err(MetaError::AccountDisabled)), "{sign_in:?}"); + assert!(matches!(meta.create_device(ana.id, "pc", &token(), NOW + 3).await, Err(MetaError::AccountDisabled))); + let late_invite = code(INVITE_CHARS); + meta.create_invite("ana", &late_invite.hash(), NOW + 3, INVITE_TTL_MS).await.unwrap(); + assert!(matches!(redeem(meta, &late_invite, "pc", NOW + 4).await, Err(MetaError::AccountDisabled))); + assert!(matches!(redeem(meta, &pairing, "pc", NOW + 4).await, Err(MetaError::InvalidCode)), "its pairing is gone"); + assert!(meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 4).await.is_err()); + assert_eq!(meta.list_devices(ana.id).await.unwrap().len(), 3, "no device was made"); + + // Enabling: sign-in works again; what was signed out stays signed out. + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(admin_view(meta, ana.id).await.disabled_ms, None); + assert!(meta.list_devices(ana.id).await.unwrap().iter().all(|d| d.revoked)); + assert_eq!(meta.device_by_token(&other_token).await.unwrap(), None); + assert!( + meta.sign_in_device(ana.id, ("$argon2id$stub", gen_of(meta, "ana").await), "pc", &DeviceClient::default(), &token(), NOW + 6) + .await + .unwrap() + .is_some() + ); + // The code made while the account was disabled works now, as an invite made for an enabled account would. + redeem(meta, &late_invite, "pc2", NOW + 7).await.unwrap(); + + // Marked for removal: disabled, not listed, signed out, can't be enabled, and the name stays taken. + let (bob, bobs, _, _) = paired(meta, "bob").await; + assert_eq!(meta.admin_remove_user(&actor, bob.id, NOW + 8).await.unwrap(), vec![bobs]); + let removed = admin_view(meta, bob.id).await; + assert_eq!((removed.disabled_ms, removed.devices_active), (Some(NOW + 8), 0)); + assert!(matches!(meta.admin_enable_user(&actor, bob.id, NOW).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.create_user("bob", "x", Role::Member, NOW).await, Err(MetaError::UsernameTaken))); + assert!(matches!( + meta.admin_create_user(&actor, "bob", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::UsernameTaken) + )); + assert!(matches!( + meta.admin_issue_reset(&actor, bob.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await, + Err(MetaError::Conflict(_)) + )); + let sign_in = meta.sign_in_device(bob.id, ("$argon2id$stub", gen_of(meta, "bob").await), "pc", &DeviceClient::default(), &token(), NOW + 9).await; + assert!(matches!(sign_in, Err(MetaError::AccountDisabled)), "{sign_in:?}"); + assert!(meta.admin_remove_user(&actor, bob.id, NOW + 10).await.unwrap().is_empty(), "removing again is harmless"); + assert_eq!(meta.admin_server_counts().await.unwrap().users, 2); +} + +/// A sign-in, a join with an invite, a join with a pairing, a new pairing and a plain device, all at the moment an +/// admin disables the account: each either finishes before and is signed out by the disable, or finds the account +/// disabled. Either way no device that is signed in is left, and enabling afterwards brings none back. +async fn issuing_access_while_an_account_is_disabled_leaves_no_usable_token(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for round in 0..12 { + let name = format!("ana{round}"); + let (ana, maker, pairing, _) = paired(meta, &name).await; + let invite = code(INVITE_CHARS); + meta.create_invite(&name, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..6 { + let (meta, actor, invite, pairing) = (meta.clone(), actor.clone(), invite.hash(), pairing.hash()); + tasks.push(tokio::spawn(async move { + match job { + 0 => meta.redeem(&invite, "invited", &token(), None, NOW + 1).await.map(|_| ()), + 1 => meta.redeem(&pairing, "paired", &token(), None, NOW + 1).await.map(|_| ()), + 2 => { + meta.sign_in_device(ana.id, ("$argon2id$stub", 0), "signed in", &DeviceClient::default(), &token(), NOW + 1).await.map(|_| ()) + } + 3 => meta.create_pairing(ana.id, maker, &code(PAIRING_CHARS).hash(), NOW + 1).await.map(|_| ()), + 4 => meta.create_device(ana.id, "plain", &token(), NOW + 1).await.map(|_| ()), + _ => meta.admin_disable_user(&actor, ana.id, NOW + 2).await.map(|_| ()), + } + })); + } + for task in tasks { + let result = task.await.unwrap(); + assert!( + matches!( + result, + Ok(()) | Err(MetaError::InvalidCode | MetaError::NotFound(_) | MetaError::AccountDisabled | MetaError::DeviceRevoked) + ), + "round {round}: {result:?}" + ); + } + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().all(|d| d.revoked), "round {round}: {devices:?}"); + assert_revoke_columns_agree(&devices); + } +} + +/// A sign-in reads the hash and generation, verifies the password (slowly), and asks to create the device. A disable and +/// an enable in between leave the hash as it was but not the generation: the device is not created. So are a password +/// change and a removal. The interleaving is played out by calling the API in that order. +async fn a_sign_in_checked_before_a_disable_and_an_enable_creates_no_device(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let client = DeviceClient::default(); + for (name, id) in [("ana", ana.id), ("bob", bob.id)] { + let read = meta.user_by_name(name).await.unwrap().unwrap(); + if name == "ana" { + meta.admin_disable_user(&actor, id, NOW).await.unwrap(); + meta.admin_enable_user(&actor, id, NOW).await.unwrap(); + } else { + meta.admin_remove_user(&actor, id, NOW).await.unwrap(); + } + let late = meta.sign_in_device(id, (&read.password_hash, read.auth_gen), "pc", &client, &token(), NOW + 1).await; + match name { + "ana" => assert!(matches!(late, Ok(None)), "{late:?}"), + _ => assert!(matches!(late, Ok(None) | Err(MetaError::AccountDisabled)), "{late:?}"), + } + assert!(meta.list_devices(id).await.unwrap().is_empty(), "{name}: no device was made"); + } + // A sign-in that read the state after the enable goes through. + let fresh = meta.user_by_name("ana").await.unwrap().unwrap(); + assert!(fresh.auth_gen >= 2); + assert!(meta.sign_in_device(ana.id, (&fresh.password_hash, fresh.auth_gen), "pc", &client, &token(), NOW + 2).await.unwrap().is_some()); + // A password change moves the generation too. + let before = gen_of(meta, "ana").await; + meta.reset_password(ana.id, "$argon2id$new", true, NOW + 3).await.unwrap(); + assert!(gen_of(meta, "ana").await > before); +} + +async fn invites_and_reset_codes_replace_each_other(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (fresh, _) = meta.admin_create_user(&actor, "fresh", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let ana = user(meta, "ana").await; + let (first, second, reset) = (code(INVITE_CHARS), code(INVITE_CHARS), code(INVITE_CHARS)); + + // An invite only for an account without a password; it replaces the open one, and can be revoked. + meta.admin_issue_invite(&actor, fresh.id, &first.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.admin_issue_invite(&actor, fresh.id, &second.hash(), NOW + 1, INVITE_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&first.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), Some(true)); + let conflict = meta.admin_issue_invite(&actor, ana.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await; + assert!(matches!(conflict, Err(MetaError::Conflict(_))), "an account with a password gets none: {conflict:?}"); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), Some(true)); + meta.admin_revoke_invite(&actor, fresh.id, NOW).await.unwrap(); + assert_eq!(meta.invite_needs_password(&second.hash(), NOW + 2).await.unwrap(), None); + assert!(matches!(meta.admin_revoke_invite(&actor, fresh.id, NOW).await, Err(MetaError::NotFound(_))), "nothing left to revoke"); + assert!(matches!(meta.admin_revoke_invite(&actor, UserId(uuid::Uuid::now_v7()), NOW).await, Err(MetaError::NotFound(_)))); + + // A reset code replaces an invite and the other way round, and the account has one row either way. + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW + 1, RESET_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 2).await.unwrap(), None); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 2).await.unwrap(), Some(true), "a reset always needs a password, though ana has one"); + let invite_again = code(INVITE_CHARS); + meta.create_invite("ana", &invite_again.hash(), NOW + 3, INVITE_TTL_MS).await.unwrap(); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 4).await.unwrap(), None, "the command line's invite replaces a reset code"); + assert_eq!(meta.invite_needs_password(&invite_again.hash(), NOW + 4).await.unwrap(), Some(false)); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW + 5, RESET_TTL_MS).await.unwrap(); + meta.admin_revoke_invite(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 6).await.unwrap(), None, "a revoke ends a reset code too"); + + // Not oneself, and not another admin. + assert!(matches!(meta.admin_issue_reset(&actor, root.id, &reset.hash(), false, NOW, RESET_TTL_MS).await, Err(MetaError::Invalid(_)))); + let (boss, _) = admin_actor(meta, "boss").await; + let result = meta.admin_issue_reset(&actor, boss.id, &reset.hash(), false, NOW, RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + assert_eq!(meta.invite_needs_password(&reset.hash(), NOW + 6).await.unwrap(), None); + // Not a disabled account. + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(matches!(meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await, Err(MetaError::Conflict(_)))); + assert!(matches!( + meta.admin_issue_reset(&actor, UserId(uuid::Uuid::now_v7()), &reset.hash(), false, NOW, RESET_TTL_MS).await, + Err(MetaError::NotFound(_)) + )); +} + +async fn a_reset_code_sets_a_password_and_signs_everything_out(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let (ana, maker, pairing, pairing_id) = paired(meta, "ana").await; + let other = device(meta, &ana).await; + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + // Nothing changes until it is redeemed. + assert!(meta.device_active(maker).await.unwrap()); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub"); + + // A reset without a new password is refused, and the code stays. + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), None, NOW + 1).await, Err(MetaError::PasswordRequired))); + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), Some(""), NOW + 1).await, Err(MetaError::PasswordRequired))); + assert!(meta.device_active(maker).await.unwrap()); + // With one: the password changes (though the account had one), every device is signed out, the pairings are gone, + // and the redeeming computer is the one device signed in. + let new_token = token(); + let joined = meta.redeem(&reset.hash(), "recovery pc", &new_token, Some("$argon2id$fresh"), NOW + 2).await.unwrap(); + assert_eq!(sorted(joined.revoked.clone()), sorted(vec![maker, other])); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$fresh"); + assert_eq!(meta.account(ana.id).await.unwrap().password_changed_ms, Some(NOW + 2)); + let devices = meta.list_devices(ana.id).await.unwrap(); + for d in &devices { + if d.id == joined.device { + assert!(!d.revoked); + } else { + assert_eq!((d.revoked, d.revoke_reason), (true, Some(RevokeReason::Reset))); + } + } + assert_revoke_columns_agree(&devices); + assert_eq!(meta.device_by_token(&new_token).await.unwrap().map(|a| a.device), Some(joined.device)); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 3).await.unwrap(), None); + assert!(matches!(redeem(meta, &pairing, "late", NOW + 3).await, Err(MetaError::InvalidCode))); + // Once only. + assert!(matches!(meta.redeem(&reset.hash(), "again", &token(), Some("$argon2id$x"), NOW + 4).await, Err(MetaError::InvalidCode))); + + // It lasts an hour. + let late = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &late.hash(), false, NOW + 10, RESET_TTL_MS).await.unwrap(); + let result = meta.redeem(&late.hash(), "late", &token(), Some("$argon2id$x"), NOW + 10 + RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::InvalidCode)), "{result:?}"); + assert_eq!(RESET_TTL_MS, 3_600_000); + meta.redeem(&late.hash(), "in time", &token(), Some("$argon2id$x"), NOW + 10 + RESET_TTL_MS - 1).await.unwrap(); + + // Disabling the account deletes the reset code it still held. + let disabled_code = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &disabled_code.hash(), false, NOW + 20, RESET_TTL_MS).await.unwrap(); + meta.admin_disable_user(&actor, ana.id, NOW + 21).await.unwrap(); + assert_eq!(meta.invite_needs_password(&disabled_code.hash(), NOW + 22).await.unwrap(), None, "disabling deletes it"); +} + +/// The quota of a space is its limit when set, with or without the server-wide cap, in the look before the bytes are +/// stored and in the count that follows. +async fn a_quota_overrides_the_server_wide_cap(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let quota = MIN_QUOTA_BYTES; + meta.admin_update_user(&actor, ana.id, None, Some(Some(quota)), NOW).await.unwrap(); + let (c1, c2, c3) = (random_chunk(), random_chunk(), random_chunk()); + + // No cap at all: the quota still holds, and a space without one is unlimited. + meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota).await.unwrap(); + let over = meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota + 1).await; + assert!(matches!(over, Err(MetaError::QuotaExceeded { used: 0, limit }) if limit == quota), "{over:?}"); + meta.check_room(bob.space, BlobKind::Chunk, c1.as_bytes(), quota * 10).await.unwrap(); + meta.add_chunk(ana.space, &c1, quota - 10, NOW).await.unwrap(); + let over = meta.add_chunk(ana.space, &c2, 11, NOW).await; + assert!(matches!(over, Err(MetaError::QuotaExceeded { limit, .. }) if limit == quota), "{over:?}"); + assert_eq!(meta.stored_bytes(ana.space).await.unwrap(), quota - 10, "nothing was recorded"); + assert!(matches!(meta.add_chunks(ana.space, &[(c2, 6), (c3, 6)], NOW).await, Err(MetaError::QuotaExceeded { .. }))); + meta.add_chunks(ana.space, &[(c2, 5), (c3, 5)], NOW).await.unwrap(); + assert_eq!(meta.stored_bytes(ana.space).await.unwrap(), quota); + meta.add_chunk(bob.space, &c1, quota * 3, NOW).await.unwrap(); + // What the space has already costs nothing to look at. + meta.check_room(ana.space, BlobKind::Chunk, c1.as_bytes(), quota * 5).await.unwrap(); + + // A cap larger than the quota: the quota wins. A cap smaller than the quota: the quota wins, too (it is the + // account's limit, whichever way it differs). + let big_cap = meta.clone().with_stored_bytes_cap(Some(quota * 100)); + assert!(matches!(big_cap.check_room(ana.space, BlobKind::Chunk, random_chunk().as_bytes(), 1).await, Err(MetaError::QuotaExceeded { .. }))); + assert!(matches!(big_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + let small_cap = meta.clone().with_stored_bytes_cap(Some(100)); + meta.admin_update_user(&actor, bob.id, None, Some(Some(quota * 10)), NOW).await.unwrap(); + small_cap.check_room(bob.space, BlobKind::Chunk, random_chunk().as_bytes(), quota).await.unwrap(); + small_cap.add_chunk(bob.space, &random_chunk(), quota, NOW).await.unwrap(); + // Removing the quota gives the cap back; a quota under the use deletes nothing and refuses new bytes. + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + big_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await.unwrap(); + assert!(matches!(small_cap.add_chunk(ana.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + meta.admin_update_user(&actor, bob.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert!(meta.stored_bytes(bob.space).await.unwrap() > MIN_QUOTA_BYTES); + assert!(matches!(meta.add_chunk(bob.space, &random_chunk(), 1, NOW).await, Err(MetaError::QuotaExceeded { .. }))); +} + +async fn an_admin_lists_and_revokes_a_users_devices(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let (ana, maker, _, pairing_id) = paired(meta, "ana").await; + let bob = user(meta, "bob").await; + let bobs = device(meta, &bob).await; + let other = device(meta, &ana).await; + meta.revoke_device(ana.id, other).await.unwrap(); + // Every device, revoked ones with their reason. + let listed = meta.admin_devices(ana.id).await.unwrap(); + assert_eq!(listed.len(), 2); + assert_eq!(listed.iter().find(|d| d.id == other).unwrap().revoke_reason, Some(RevokeReason::User)); + assert!(matches!(meta.admin_devices(UserId(uuid::Uuid::now_v7())).await, Err(MetaError::NotFound(_)))); + + meta.admin_revoke_device(&actor, ana.id, maker, NOW + 1).await.unwrap(); + let revoked = record(meta, &ana, maker).await; + assert_eq!((revoked.revoked, revoked.revoked_ms, revoked.revoke_reason), (true, Some(NOW + 1), Some(RevokeReason::Admin))); + assert_eq!(meta.pairing_status(ana.id, pairing_id, NOW + 2).await.unwrap(), None, "its pairing dies with it"); + // Again: the first time and reason stay. Another user's device, an unknown device or user: not found, nothing changed. + meta.admin_revoke_device(&actor, ana.id, maker, NOW + 5).await.unwrap(); + assert_eq!(record(meta, &ana, maker).await.revoked_ms, Some(NOW + 1)); + assert_eq!(record(meta, &ana, other).await.revoke_reason, Some(RevokeReason::User)); + assert!(matches!(meta.admin_revoke_device(&actor, ana.id, bobs, NOW).await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.admin_revoke_device(&actor, ana.id, DeviceId::generate(), NOW).await, Err(MetaError::NotFound(_)))); + assert!(matches!(meta.admin_revoke_device(&actor, UserId(uuid::Uuid::now_v7()), bobs, NOW).await, Err(MetaError::NotFound(_)))); + assert!(meta.device_active(bobs).await.unwrap()); +} + +// Two admins at the same moment, each taking the other (or oneself) away: whatever the order, one enabled admin is +// left, and no statement fails inside the database (on Postgres, rows taken in different orders would deadlock). + +#[derive(Clone, Copy, Debug)] +enum Act { + Demote(usize), + Disable(usize), + Remove(usize), +} + +/// Runs `acts` (who acts, what) at the same moment on a survivor and a new admin; exactly one enabled admin must be +/// left, and the second change of two must fail with a reason the rules give. Returns the admin left. +async fn race_two_admins(meta: &Meta, round: usize, survivor: (UserRecord, AdminActor), acts: [(usize, Act); 2]) -> (UserRecord, AdminActor) { + let fresh = admin_actor(meta, &format!("fresh{round}")).await; + let both = [survivor, fresh]; + let mut tasks = Vec::new(); + for (who, act) in acts { + let (meta, actor) = (meta.clone(), both[who].1.clone()); + let target = both[match act { + Act::Demote(t) | Act::Disable(t) | Act::Remove(t) => t, + }] + .0 + .id; + tasks.push(tokio::spawn(async move { + match act { + Act::Demote(_) => meta.admin_update_user(&actor, target, Some(Role::Member), None, NOW).await.map(|_| ()), + Act::Disable(_) => meta.admin_disable_user(&actor, target, NOW).await.map(|_| ()), + Act::Remove(_) => meta.admin_remove_user(&actor, target, NOW).await.map(|_| ()), + } + })); + } + for (task, (_, act)) in tasks.into_iter().zip(acts) { + let result = task.await.unwrap(); + assert!( + matches!(result, Ok(()) | Err(MetaError::LastAdmin | MetaError::Forbidden(_) | MetaError::DeviceRevoked | MetaError::Invalid(_))), + "round {round}, {act:?}: {result:?}" + ); + } + let page = meta.admin_list_users(None, None, 200, NOW).await.unwrap(); + let left: Vec<_> = page.users.iter().filter(|u| u.role == Role::Admin && u.disabled_ms.is_none()).collect(); + assert_eq!(left.len(), 1, "round {round}, {acts:?}: {left:?}"); + let left = left[0].id; + let [a, b] = both; + if a.0.id == left { a } else { b } +} + +async fn two_admins_racing_keep_one(meta: &Meta, combos: &[[(usize, Act); 2]]) { + let mut survivor = admin_actor(meta, "first").await; + for round in 0..8 { + for (i, combo) in combos.iter().enumerate() { + survivor = race_two_admins(meta, round * 10 + i, survivor, *combo).await; + } + } +} + +async fn two_demotions_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(1)), (1, Act::Demote(0))]]).await; +} + +/// Each admin steps down at the same moment: their rows are different, so only the `last_admin` lock keeps one of them. +async fn two_self_demotions_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(0)), (1, Act::Demote(1))]]).await; +} + +async fn two_disables_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Disable(1)), (1, Act::Disable(0))]]).await; +} + +async fn a_demotion_and_a_disable_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(1)), (1, Act::Disable(0))], [(0, Act::Disable(1)), (1, Act::Demote(0))]]).await; +} + +async fn a_removal_and_a_demotion_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Remove(1)), (1, Act::Demote(0))], [(0, Act::Demote(1)), (1, Act::Remove(0))]]).await; +} + +async fn a_self_demotion_and_a_disable_of_the_other_at_once_keep_an_admin(meta: &Meta) { + two_admins_racing_keep_one(meta, &[[(0, Act::Demote(0)), (0, Act::Disable(1))], [(1, Act::Demote(1)), (1, Act::Disable(0))]]).await; +} + +// ---- the migration from 0007 to 0008 ---- + +/// An account, its invite and the audit log survive the two-factor migration (0010, which runs after the audit log's 0009). +async fn migrating_from_0009_keeps_the_rows(url: &str) { + use sqlx::{Connection, Row}; + sqlx::any::install_default_drivers(); + let (ana, invite, audit_rows) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + let events = all_events(&meta).await; + assert!(!events.is_empty(), "the audit log has rows to keep"); + (ana, invite, events.len()) + }; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 10").execute(&mut conn).await.unwrap(); + drop(conn); + assert_eq!(schema_version(url).await, 9, "the audit log's migration stays"); + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14); + assert_eq!(all_events(&meta).await.len(), audit_rows, "the audit rows are kept"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().id, ana.id); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(false)); + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + let row = sqlx::query("SELECT reset_two_factor FROM invites").fetch_one(&mut conn).await.unwrap(); + assert_eq!(row.try_get::("reset_two_factor").unwrap(), 0); +} + +#[tokio::test] +async fn migrating_from_0009_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0009_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0009_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0009_keeps_the_rows(&url).await; +} + +/// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. +async fn file_with_versions(meta: &Meta, u: &UserRecord, dev: DeviceId, n: usize) -> (NodeId, Vec) { + let r = root(meta, u).await; + let m = manifest(meta, u, 3).await; + let node = NodeId::generate(); + let mut versions = vec![version(m)]; + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name("history.txt"), version: versions[0].clone() }], NOW).await.unwrap(); + for i in 1..n { + let m = manifest(meta, u, 3 + i as u32).await; + let next = NewVersion { mtime_ms: NOW + i as i64, ..version(m) }; + let expected = versions[i - 1].id; + meta.commit(u.space, dev, &[Op::Update { node, expected, version: next.clone() }], NOW + i as i64).await.unwrap(); + versions.push(next); + } + (node, versions) +} + +async fn file_named(meta: &Meta, u: &UserRecord, dev: DeviceId, file_name: &str) -> (NodeId, NewVersion) { + let r = root(meta, u).await; + let node = NodeId::generate(); + let v = version(manifest(meta, u, 5).await); + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name(file_name), version: v.clone() }], NOW).await.unwrap(); + (node, v) +} + +fn refused_restore(result: Result) -> Vec<(usize, ConflictKind)> { + match result { + Err(MetaError::Rejected(list)) => list.into_iter().map(|c| (c.op, c.kind)).collect(), + other => panic!("expected a refused restore, got {other:?}"), + } +} + +/// The history comes newest first in pages that follow each other without gaps or repeats, also across versions that +/// share one sequence number (a batch of updates); only files of the caller's own space have one; a deleted file keeps +/// its history; `pruned` is false and `save` tells whether a record hangs on the version. +async fn history_pages_newest_first_for_own_files_only(meta: &Meta) { + use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, mut versions) = file_with_versions(meta, &u, dev, 4).await; + // Three more versions in one commit: they share a sequence number, so the order inside it is by ID. + let (mut ops, mut prev) = (Vec::new(), versions[3].id); + for i in 0..3 { + let next = version(manifest(meta, &u, 20 + i).await); + ops.push(Op::Update { node, expected: prev, version: next.clone() }); + prev = next.id; + versions.push(next); + } + meta.commit(u.space, dev, &ops, NOW + 100).await.unwrap(); + let head = prev; + let epoch = meta.space_info(u.space).await.unwrap().epoch; + + let all = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap(); + assert_eq!(all.next, None); + assert_eq!(all.versions.len(), 7); + let key = |v: &HistoryVersion| (v.seq, *v.id.as_uuid().as_bytes()); + assert!(all.versions.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (seq, id)"); + assert!(all.versions.iter().all(|v| !v.pruned && !v.save && v.device == dev && v.epoch == epoch)); + assert_eq!(all.versions.iter().filter(|v| v.seq == 5).count(), 3, "the batch shares one sequence number"); + let oldest = all.versions.last().unwrap(); + assert_eq!((oldest.id, oldest.parent_version, oldest.seq, oldest.size), (versions[0].id, None, 1, 3)); + let second = &all.versions[all.versions.len() - 2]; + assert_eq!((second.id, second.parent_version, second.mtime_ms, second.size), (versions[1].id, Some(versions[0].id), NOW + 1, 4)); + + for limit in [1u32, 2, 3] { + let (mut seen, mut after, mut pages) = (Vec::new(), None, 0); + loop { + let page = meta.history(u.space, node, after, limit).await.unwrap(); + assert!(page.versions.len() <= limit as usize); + seen.extend(page.versions.iter().cloned()); + pages += 1; + match page.next { + Some(next) => { + let last = page.versions.last().unwrap(); + assert_eq!(next, HistoryCursor { seq: last.seq, version: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.versions, "limit {limit}: every version once, in order"); + assert_eq!(pages, 7usize.div_ceil(limit as usize)); + } + + // The limit is 1 to 200; a cursor past the end gives an empty last page. + assert!(matches!(meta.history(u.space, node, None, 0).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT + 1).await, Err(MetaError::Invalid(_)))); + let past = meta.history(u.space, node, Some(HistoryCursor { seq: 0, version: VersionId::generate() }), 10).await.unwrap(); + assert_eq!((past.versions.len(), past.next), (0, None)); + + // A record hanging on a version shows as `save`. + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + let with_save: Vec<_> = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.into_iter().filter(|v| v.save).collect(); + assert_eq!(with_save.iter().map(|v| v.id).collect::>(), vec![versions[1].id]); + + // Folders have no history, another account's node is not found, and a deleted file keeps its history. + let r = root(meta, &u).await; + assert!(matches!(meta.history(u.space, r, None, 10).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, NodeId::generate(), None, 10).await, Err(MetaError::NotFound(_)))); + let bob = user(meta, "bob").await; + assert!(matches!(meta.history(bob.space, node, None, 10).await, Err(MetaError::NotFound(_)))); + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(head) }], NOW + 200).await.unwrap(); + assert_eq!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.len(), 7); +} + +/// A restore is an ordinary update after the head: a fresh version with the old content (manifest, size, mtime), the +/// head as its parent, the next sequence number, the same epoch and no save record; it shows in the change feed, and +/// the old version stays where it was. +async fn a_restore_is_a_new_version_after_the_head(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let other = meta.create_device(u.id, "phone", &[9u8; 32], NOW).await.unwrap(); + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[0].id, &record).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_version(u.space, other, node, versions[0].id, versions[2].id, NOW + 50).await.unwrap(); + assert_eq!((done.parent_version, done.epoch, done.seq), (versions[2].id, before.epoch, before.seq + 1)); + assert!(!versions.iter().any(|v| v.id == done.version), "a fresh version ID"); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch)); + + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!( + (f.version, f.manifest, f.mtime_ms, f.device, f.save.clone()), + (done.version, versions[0].manifest, versions[0].mtime_ms, other, None) + ); + + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 4); + assert_eq!(history[0].id, done.version); + assert_eq!((history[0].parent_version, history[0].size, history[0].save, history[0].seq), (Some(versions[2].id), 3, false, done.seq)); + assert!(history.iter().any(|v| v.id == versions[0].id && v.save), "the old version and its record stay"); + + // The restored content is the head now: a stale `expected` is refused and changes nothing. + let stale = meta.restore_version(u.space, dev, node, versions[1].id, versions[2].id, NOW + 60).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(done.version) })]); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, after.seq); + // Restoring the content that is already current is allowed: it is one more version. + let again = meta.restore_version(u.space, dev, node, done.version, done.version, NOW + 70).await.unwrap(); + assert_eq!(again.seq, after.seq + 1); +} + +/// What a restore turns down, each leaving the space as it was: a folder, a deleted file, a file with a live document, +/// a node or version that is not there, a version of another file, and anything of another account. +async fn a_restore_refuses_what_it_can_not_restore(meta: &Meta) { + let u = user(meta, "ana").await; + let (space, dev) = (u.space, device(meta, &u).await); + let (file, versions) = file_with_versions(meta, &u, dev, 2).await; + let (second, second_version) = file_named(meta, &u, dev, "second.txt").await; + let r = root(meta, &u).await; + let seq = meta.space_info(space).await.unwrap().seq; + let (old, head) = (versions[0].id, versions[1].id); + let restore = |node: NodeId, source: VersionId, expected: VersionId| { + let meta = meta.clone(); + async move { meta.restore_version(space, dev, node, source, expected, NOW + 9).await } + }; + + assert!(matches!(restore(r, old, head).await, Err(MetaError::Invalid(_))), "a folder"); + assert!(matches!(restore(NodeId::generate(), old, head).await, Err(MetaError::NotFound(_))), "no such node"); + assert!(matches!(restore(file, VersionId::generate(), head).await, Err(MetaError::NotFound(_))), "no such version"); + assert!(matches!(restore(file, second_version.id, head).await, Err(MetaError::NotFound(_))), "a version of another file"); + // The source is checked before the head: a missing source is not a conflict. + assert!(matches!(restore(file, VersionId::generate(), VersionId::generate()).await, Err(MetaError::NotFound(_)))); + assert_eq!(refused_restore(restore(file, old, VersionId::generate()).await).len(), 1, "a head that is not current"); + + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + assert!(matches!(meta.restore_version(bob.space, bob_device, file, old, head, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + + // A file with a live document. + let doc = craft_cloud_proto::live::node_doc(space, second); + meta.create_live_doc(space, doc, "document:kitdemo", Some(second), NOW).await.unwrap(); + assert!(matches!(restore(second, second_version.id, second_version.id).await, Err(MetaError::Invalid(_))), "a live document"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq, "nothing changed so far"); + + // A deleted file. + meta.commit(space, dev, &[Op::Delete { node: file, expected: Some(head) }], NOW + 5).await.unwrap(); + assert!(matches!(restore(file, old, head).await, Err(MetaError::Invalid(_))), "a deleted file"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq + 1, "only the delete"); + let feed = meta.changes(space, seq, None, MAX_PAGE_NODES).await.unwrap(); + assert!(feed.nodes.len() == 1 && find(&feed.nodes, file).deleted, "still deleted"); +} + +/// Several restores of different old versions against one head: exactly one applies, the rest are refused with a +/// version mismatch, and the space moves one step. +async fn concurrent_restores_let_exactly_one_win(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let head = versions[2].id; + let before = meta.space_info(u.space).await.unwrap().seq; + let mut tasks = Vec::new(); + for i in 0..8 { + let (meta, space, source) = (meta.clone(), u.space, versions[i % 2].id); + tasks.push(tokio::spawn(async move { meta.restore_version(space, dev, node, source, head, NOW + 20).await })); + } + let mut wins = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => wins += 1, + Err(MetaError::Rejected(list)) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + Err(other) => panic!("unexpected error {other:?}"), + } + } + assert_eq!(wins, 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before + 1); +} + +/// A restore and a late save record for the head it replaces run together: both apply under the space's one lock, the +/// record stays on the version it was meant for, and the restored version never gets a record. +async fn a_restore_and_a_late_save_record_both_apply(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let (head, old) = (versions[2].id, versions[0].id); + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let save = tokio::spawn(async move { m1.attach_save(space, node, head, &record).await }); + let restore = tokio::spawn(async move { m2.restore_version(space, dev, node, old, head, NOW + 30).await }); + save.await.unwrap().unwrap(); + let restored = restore.await.unwrap().unwrap(); + let history = meta.history(space, node, None, 50).await.unwrap().versions; + assert_eq!(history.len(), 4); + for v in &history { + assert_eq!(v.save, v.id == head, "only the old head has the record: {}", v.id); + } + assert_eq!(history[0].id, restored.version); + // The create, two updates, the record and the restore. + assert_eq!(meta.space_info(space).await.unwrap().seq, 5); +} + +/// The database at the newest version when the code knows it: the version table holds one row per migration, and a +/// database written by a newer build is refused rather than half-read. +#[tokio::test] +async fn the_schema_version_is_recorded_and_a_newer_database_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 14); + // Opening again changes nothing. + drop(Meta::open(&url).await.unwrap()); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(rows, 14); + sqlx::query("INSERT INTO schema_version (version) VALUES (15)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(&url).await, Err(MetaError::Corrupt(_))), "a newer database is refused"); +} + +/// A history page is read through the index on `(space_id, node, seq, id)`, not by scanning the versions table. +#[tokio::test] +async fn a_history_page_uses_the_node_index_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let plan = sqlx::query( + "EXPLAIN QUERY PLAN SELECT id FROM versions WHERE space_id = ?1 AND node = ?2 AND (seq, id) < (?3, ?4) ORDER BY seq DESC, id DESC LIMIT ?5", + ) + .bind(vec![0u8; 16]) + .bind(vec![0u8; 16]) + .bind(i64::MAX) + .bind(vec![0xffu8; 16]) + .bind(51i64) + .fetch_all(&mut conn) + .await + .unwrap(); + use sqlx::Row; + let detail: Vec = plan.iter().map(|r| r.get::("detail")).collect(); + assert!(detail.iter().any(|d| d.contains("versions_node_seq")), "{detail:?}"); + assert!(!detail.iter().any(|d| d.contains("TEMP B-TREE")), "no sort: {detail:?}"); +} + +// ---- the trash ---- + +/// A folder `name` in `parent`: its node. +async fn folder(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, folder_name: &str, now: i64) -> NodeId { + let node = NodeId::generate(); + meta.commit(u.space, dev, &[Op::Mkdir { node, parent, name: name(folder_name) }], now).await.unwrap(); + node +} + +/// A file `name` in `parent` with one version: its node and version. +async fn file_in(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, file_name: &str, now: i64) -> (NodeId, NewVersion) { + let node = NodeId::generate(); + let v = version(manifest(meta, u, 6).await); + meta.commit(u.space, dev, &[Op::Create { node, parent, name: name(file_name), version: v.clone() }], now).await.unwrap(); + (node, v) +} + +async fn delete_file(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, current: VersionId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(current) }], now).await.unwrap(); +} + +async fn delete_folder(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: None }], now).await.unwrap(); +} + +async fn trash_ids(meta: &Meta, u: &UserRecord) -> Vec { + meta.trash(u.space, None, 200, NOW).await.unwrap().entries.iter().map(|e| e.id).collect() +} + +/// The name a restore gives a node whose name is taken, in the minute of `NOW` (2025-10-09 08:53 UTC), from a device +/// called `laptop`. +fn conflict_of(original: &str, is_dir: bool, counter: u32) -> String { + let stamp = craft_cloud_proto::ConflictStamp { year: 2025, month: 10, day: 9, hour: 8, minute: 53, utc: true }; + craft_cloud_proto::conflict_name(&name(original), is_dir, "laptop", stamp, counter).unwrap().as_str().to_owned() +} + +async fn live_name(meta: &Meta, u: &UserRecord, node: NodeId) -> (Option, String, bool) { + // The feed from the start, which has deleted nodes too. + let feed = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + let n = find(&feed.nodes, node); + (n.parent, n.name.as_ref().map(|n| n.as_str().to_owned()).unwrap_or_default(), n.deleted) +} + +/// The time in the trash starts when a live node is deleted, not when a deleted one is deleted again; coming back +/// clears it, by an edit or by a restore from the trash, and the next deletion starts afresh. +async fn deleting_stamps_the_time_once_and_coming_back_clears_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (file, v) = file_in(meta, &u, dev, r, "stamp.txt", NOW).await; + assert!(trash_ids(meta, &u).await.is_empty()); + + delete_file(meta, &u, dev, file, v.id, NOW + 1_000).await; + let page = meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms, page.repaired), (1, NOW + 1_000, 0)); + // Deleting a deleted file again is no change: the time stays, and the file is not stamped twice. + delete_file(meta, &u, dev, file, v.id, NOW + 5_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 1_000); + + // An edit brings it back and leaves the trash; a later deletion starts the time again. + let edited = version(manifest(meta, &u, 7).await); + meta.commit(u.space, dev, &[Op::Update { node: file, expected: v.id, version: edited.clone() }], NOW + 6_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "an edit brings the file back"); + delete_file(meta, &u, dev, file, edited.id, NOW + 7_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 7_000); + + // A restore from the trash clears it too, and the next deletion starts afresh again. + let restored = meta.restore_from_trash(u.space, dev, file, NOW + 8_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + // The restore made a new current version: the next delete names it. + delete_file(meta, &u, dev, file, restored.version.unwrap(), NOW + 8_500).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 8_500); +} + +/// The trash lists a space's deleted files and folders, newest deletion first by `(deleted_ms, id)`, in pages without +/// gaps or repeats (also for nodes deleted in one instant), with sizes of files; it lists nothing of another space and +/// nothing that was removed from the trash. +async fn the_trash_pages_newest_first_for_its_own_space(meta: &Meta) { + use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut made = Vec::new(); + for i in 0..5 { + let (node, v) = file_in(meta, &u, dev, r, &format!("f{i}.txt"), NOW).await; + // The last three go in one instant. + delete_file(meta, &u, dev, node, v.id, NOW + 1_000 + i64::from(i.min(2)) * 10).await; + made.push(node); + } + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + delete_folder(meta, &u, dev, dir, NOW + 5_000).await; + + let all = meta.trash(u.space, None, TRASH_MAX_LIMIT, NOW).await.unwrap(); + assert_eq!((all.entries.len(), all.next), (6, None)); + assert_eq!(all.entries[0].id, dir); + assert!(all.entries[0].is_dir && all.entries[0].size.is_none() && all.entries[0].parent == r); + assert!(all.entries[1..].iter().all(|e| !e.is_dir && e.size == Some(6))); + let key = |e: &craft_cloud_meta::TrashEntry| (e.deleted_ms, *e.id.as_uuid().as_bytes()); + assert!(all.entries.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (deleted_ms, id)"); + for limit in [1u32, 2, 4] { + let (mut seen, mut after) = (Vec::new(), None); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize); + seen.extend(page.entries.iter().cloned()); + match page.next { + Some(next) => { + let last = page.entries.last().unwrap(); + assert_eq!(next, TrashCursor { deleted_ms: last.deleted_ms, node: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.entries, "limit {limit}"); + } + assert!(matches!(meta.trash(u.space, None, 0, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.trash(u.space, None, TRASH_MAX_LIMIT + 1, NOW).await, Err(MetaError::Invalid(_)))); + + // Another space sees none of it. A node removed from the trash is not listed any more. + let bob = user(meta, "bob").await; + assert!(meta.trash(bob.space, None, 10, NOW).await.unwrap().entries.is_empty()); + meta.request_purge(u.space, dir, NOW + 6_000).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await.len(), 5); + assert!(!trash_ids(meta, &u).await.contains(&dir)); +} + +/// A file in deleted folders comes back with the folders, every node keeping its ID, as one change of the space. +async fn a_restore_brings_the_deleted_folders_back_with_their_ids(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (f, v) = file_in(meta, &u, dev, b, "f.txt", NOW).await; + let (sibling, sv) = file_in(meta, &u, dev, b, "sibling.txt", NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 1).await; + delete_file(meta, &u, dev, sibling, sv.id, NOW + 2).await; + delete_folder(meta, &u, dev, b, NOW + 3).await; + delete_folder(meta, &u, dev, a, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed), (f, b, "f.txt", false)); + assert_eq!((done.seq, done.epoch), (before.seq + 1, before.epoch)); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch), "one change of the space"); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), false)); + assert_eq!(live_name(meta, &u, b).await, (Some(a), "b".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(b), "f.txt".to_owned(), false)); + assert_eq!(trash_ids(meta, &u).await, vec![sibling], "only the node that was asked for and its folders come back"); + + // The change feed reports exactly the nodes that came back, at the new sequence number, as live. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(a, false), (b, false), (f, false)]; + expected.sort(); + assert_eq!(changed, expected); + let NodeKind::File(state) = &find(&feed.nodes, f).kind else { panic!("a file") }; + assert_eq!(state.manifest, v.manifest, "the file's content is what it was"); + assert_eq!(Some(state.version), done.version, "as a new version after the last one"); + assert_ne!(state.version, v.id); +} + +/// A name taken by a file or by anything at the node itself gives a conflict name (never an overwrite); a deleted +/// folder whose name is held by a live folder is merged into it; every name tried is free, and a folder that is renamed +/// is a folder. +async fn a_restore_merges_into_live_folders_and_renames_around_taken_names(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + + // The node's own name is taken: a conflict name, and the live file is untouched. + let (f, fv) = file_in(meta, &u, dev, r, "plan.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (squatter, _) = file_in(meta, &u, dev, r, "PLAN.txt", NOW + 2).await; + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("plan.txt", false, 1))); + assert_eq!(live_name(meta, &u, squatter).await, (Some(r), "PLAN.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(r), conflict_of("plan.txt", false, 1), false)); + + // A second restore of the same name in the same minute takes the next counter. + let (g, gv) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 3).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 4).await; + let (_h, _) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 5).await; + let (_i, _) = file_in(meta, &u, dev, r, &conflict_of("plan2.txt", false, 1), NOW + 6).await; + let done = meta.restore_from_trash(u.space, dev, g, NOW).await.unwrap(); + assert_eq!(done.name, conflict_of("plan2.txt", false, 2)); + + // A deleted folder held by a live folder of the same name: the live folder takes what is restored. + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (x, xv) = file_in(meta, &u, dev, b, "x.txt", NOW).await; + delete_file(meta, &u, dev, x, xv.id, NOW + 10).await; + delete_folder(meta, &u, dev, b, NOW + 11).await; + delete_folder(meta, &u, dev, a, NOW + 12).await; + let live_a = folder(meta, &u, dev, r, "a", NOW + 13).await; + let done = meta.restore_from_trash(u.space, dev, x, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.as_str()), (false, "x.txt")); + assert_eq!(live_name(meta, &u, b).await, (Some(live_a), "b".to_owned(), false), "b moved into the live folder"); + assert_eq!(live_name(meta, &u, x).await, (Some(b), "x.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), true), "the deleted a stays deleted"); + assert_eq!(trash_ids(meta, &u).await, vec![a]); + + // A deleted folder held by a file: the folder comes back under a conflict name, with its content below it. + let c = folder(meta, &u, dev, r, "c", NOW).await; + let (y, yv) = file_in(meta, &u, dev, c, "y.txt", NOW).await; + delete_file(meta, &u, dev, y, yv.id, NOW + 20).await; + delete_folder(meta, &u, dev, c, NOW + 21).await; + let (_c_file, _) = file_in(meta, &u, dev, r, "c", NOW + 22).await; + let done = meta.restore_from_trash(u.space, dev, y, NOW).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (c, "y.txt")); + assert_eq!(live_name(meta, &u, c).await, (Some(r), conflict_of("c", true, 1), false)); + + // A deleted folder restored itself, its name held by a live folder: no merge for the node asked for. + let d = folder(meta, &u, dev, r, "d", NOW).await; + delete_folder(meta, &u, dev, d, NOW + 30).await; + let live_d = folder(meta, &u, dev, r, "d", NOW + 31).await; + let done = meta.restore_from_trash(u.space, dev, d, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("d", true, 1))); + assert_eq!(live_name(meta, &u, live_d).await.1, "d"); +} + +/// A folder in the trash is listed alone: what is in the trash below it is listed only through it, however long before +/// it was deleted, and pages never split or repeat the top items. +async fn the_trash_lists_a_folder_without_what_is_below_it(meta: &Meta) { + use craft_cloud_proto::api::TrashCursor; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut tops = Vec::new(); + for i in 0..7i64 { + let dir = folder(meta, &u, dev, r, &format!("d{i}"), NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub, "deep.txt", NOW).await; + // The child goes long before its folder, the deep file with the folders. + delete_file(meta, &u, dev, child, cv.id, NOW + 100 * i).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, sub, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, dir, NOW + 1_000 + 100 * i).await; + tops.push(dir); + } + for i in 0..3 { + let (f, v) = file_in(meta, &u, dev, r, &format!("loose{i}.txt"), NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 500 + i64::from(i)).await; + tops.push(f); + } + tops.sort(); + for limit in [1u32, 2, 3, 200] { + let (mut seen, mut after) = (Vec::new(), None::); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize && !page.entries.is_empty(), "limit {limit}"); + seen.extend(page.entries.iter().map(|e| e.id)); + match page.next { + Some(next) => { + // A page is full whenever another follows: the hidden nodes take no room in it. + assert_eq!(page.entries.len(), limit as usize, "limit {limit}"); + after = Some(next); + } + None => break, + } + } + seen.sort(); + assert_eq!(seen, tops, "every top item exactly once, limit {limit}"); + } +} + +/// Restoring a folder brings back all that is in the trash below it, whenever it was deleted, as one change of the space +/// with each node revived like a single restore; a node removed from the trash, and what hangs from it, stays deleted. +async fn a_folder_restore_brings_back_everything_in_the_trash_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir1", NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let (early, ev) = file_in(meta, &u, dev, dir, "early.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let sub2 = folder(meta, &u, dev, sub, "sub2", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub2, "deep.txt", NOW).await; + let gone = folder(meta, &u, dev, dir, "gone", NOW).await; + let (hung, hv) = file_in(meta, &u, dev, gone, "hung.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, dir, "kept.txt", NOW).await; + // `early` was deleted on its own long before; `gone` and `kept` are removed from the trash; the rest goes with + // the folder. + delete_file(meta, &u, dev, early, ev.id, NOW + 10).await; + delete_file(meta, &u, dev, hung, hv.id, NOW + 1_000).await; + delete_folder(meta, &u, dev, gone, NOW + 1_001).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1_002).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_003).await; + delete_folder(meta, &u, dev, sub2, NOW + 1_004).await; + delete_folder(meta, &u, dev, sub, NOW + 1_005).await; + delete_file(meta, &u, dev, child, cv.id, NOW + 1_006).await; + delete_folder(meta, &u, dev, dir, NOW + 1_007).await; + meta.request_purge(u.space, kept, NOW + 2_000).await.unwrap(); + meta.request_purge(u.space, gone, NOW + 2_001).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![dir], "the folder alone"); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, dir, NOW + 3_000).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed, done.version), (dir, r, "dir1", false, None)); + assert_eq!(done.seq, before.seq + 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq + 1, "one change of the space"); + assert!(trash_ids(meta, &u).await.is_empty(), "nothing is left in the trash: the removed nodes are no longer listed"); + for (node, parent, node_name) in + [(dir, r, "dir1"), (child, dir, "child.txt"), (early, dir, "early.txt"), (sub, dir, "sub"), (sub2, sub, "sub2"), (deep, sub2, "deep.txt")] + { + assert_eq!(live_name(meta, &u, node).await, (Some(parent), node_name.to_owned(), false), "{node_name}"); + } + for node in [gone, hung, kept] { + assert!(live_name(meta, &u, node).await.2, "a removed node stays deleted"); + } + + // The feed reports exactly the nodes that came back, each at the new sequence number; the files as new versions. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(dir, false), (child, false), (early, false), (sub, false), (sub2, false), (deep, false)]; + expected.sort(); + assert_eq!(changed, expected); + for (node, old) in [(child, &cv), (early, &ev), (deep, &dv)] { + let NodeKind::File(state) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!(state.manifest, old.manifest, "the content is what it was"); + assert_ne!(state.version, old.id, "as a new version"); + } +} + +/// Two deleted children of one name come back together: the newest deletion keeps the name, the other gets a conflict +/// name, and the folder itself is renamed only where its own name is taken. +async fn a_folder_restore_gives_children_of_one_name_a_conflict_name(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (old, ov) = file_in(meta, &u, dev, dir, "a.txt", NOW).await; + delete_file(meta, &u, dev, old, ov.id, NOW + 1).await; + let (new, nv) = file_in(meta, &u, dev, dir, "a.txt", NOW + 2).await; + delete_file(meta, &u, dev, new, nv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let (_squatter, _) = file_in(meta, &u, dev, r, "dir", NOW + 5).await; + + let done = meta.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("dir", true, 1))); + assert_eq!(live_name(meta, &u, new).await, (Some(dir), "a.txt".to_owned(), false), "the newest deletion keeps the name"); + assert_eq!(live_name(meta, &u, old).await, (Some(dir), conflict_of("a.txt", false, 1), false)); +} + +/// Removing a folder from the trash removes what is in the trash below it too, so nothing stays listed or restorable +/// alone (up to the cap, see below). +async fn removing_a_folder_from_the_trash_takes_what_is_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (c, cv) = file_in(meta, &u, dev, dir, "c.txt", NOW).await; + let s = folder(meta, &u, dev, dir, "s", NOW).await; + let (d, dv) = file_in(meta, &u, dev, s, "d.txt", NOW).await; + delete_file(meta, &u, dev, c, cv.id, NOW + 1).await; + delete_file(meta, &u, dev, d, dv.id, NOW + 2).await; + delete_folder(meta, &u, dev, s, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + meta.request_purge(u.space, dir, NOW + 5).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + for node in [dir, c, s, d] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "{node}"); + assert!(live_name(meta, &u, node).await.2); + } +} + +/// A folder `name` in the root with `count` files, the file `i` deleted at `base + 1 + i` and the folder at `base + 20`: +/// the folder and its files, the newest deletion last. +async fn folder_with_deleted_files(meta: &Meta, u: &UserRecord, dev: DeviceId, folder_name: &str, count: i64, base: i64) -> (NodeId, Vec) { + let r = root(meta, u).await; + let dir = folder(meta, u, dev, r, folder_name, base).await; + let mut files = Vec::new(); + for i in 0..count { + let (f, v) = file_in(meta, u, dev, dir, &format!("f{i}.txt"), base).await; + delete_file(meta, u, dev, f, v.id, base + 1 + i).await; + files.push(f); + } + delete_folder(meta, u, dev, dir, base + 20).await; + (dir, files) +} + +fn sorted_nodes(mut ids: Vec) -> Vec { + ids.sort(); + ids +} + +/// A folder brings back at most the cap of what is in the trash below it, the newest deletions first among equals in +/// depth; the rest stays in the trash, is listed on its own (its folder is live) and comes back by a further restore. A +/// folder with exactly the cap below it brings everything. +async fn a_folder_restore_takes_at_most_the_cap_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let (exact, exact_files) = folder_with_deleted_files(meta, &u, dev, "exact", 3, NOW + 100).await; + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![dir, exact])); + + let before = meta.space_info(u.space).await.unwrap(); + let done = small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.node, done.seq), (dir, before.seq + 1), "one change of the space"); + assert!(!live_name(meta, &u, dir).await.2); + for f in &files[2..] { + assert!(!live_name(meta, &u, *f).await.2, "the three newest deletions came back"); + } + for f in &files[..2] { + assert!(live_name(meta, &u, *f).await.2, "the oldest two stay in the trash"); + } + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0], files[1]]), "the rest is listed on its own"); + small.restore_from_trash(u.space, dev, files[1], NOW).await.unwrap(); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0]])); + + small.restore_from_trash(u.space, dev, exact, NOW).await.unwrap(); + for f in &exact_files { + assert!(!live_name(meta, &u, *f).await.2, "exactly the cap brings everything"); + } + assert_eq!(trash_ids(meta, &u).await, vec![files[0]]); +} + +/// Of the nodes below a folder the shallow ones come first, and what is left is listed only if its own folder is not in +/// the trash: a left-over folder is listed, what is in it is hidden behind it until it is restored. +async fn a_folder_restore_leaves_the_rest_listed_through_its_own_folder(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(1); + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let a = folder(meta, &u, dev, dir, "a", NOW).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW).await; + let (b, bv) = file_in(meta, &u, dev, dir, "b.txt", NOW).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 1).await; + delete_folder(meta, &u, dev, a, NOW + 2).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + assert_eq!(trash_ids(meta, &u).await, vec![dir]); + + small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert!(!live_name(meta, &u, dir).await.2 && !live_name(meta, &u, b).await.2, "the folder and its newest child"); + assert!(live_name(meta, &u, a).await.2 && live_name(meta, &u, a1).await.2); + assert_eq!(trash_ids(meta, &u).await, vec![a], "the folder is listed, what is in it is not"); + small.restore_from_trash(u.space, dev, a, NOW).await.unwrap(); + assert!(!live_name(meta, &u, a).await.2 && !live_name(meta, &u, a1).await.2); + assert!(trash_ids(meta, &u).await.is_empty()); +} + +/// Removing a folder marks it and at most the cap below it, shallowest first; the rest is listed on its own (its folder +/// is no longer in the trash) and goes by a further removal. +async fn removing_a_folder_takes_at_most_the_cap_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let before = meta.space_info(u.space).await.unwrap(); + small.request_purge(u.space, dir, NOW + 50).await.unwrap(); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![files[0], files[1]])); + for f in &files[2..] { + assert!(matches!(meta.restore_from_trash(u.space, dev, *f, NOW).await, Err(MetaError::NotFound(_))), "removed with the folder"); + } + small.request_purge(u.space, files[0], NOW + 51).await.unwrap(); + small.request_purge(u.space, files[1], NOW + 51).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + + // A left-over folder is listed and what is in it stays hidden behind it. + let tree = folder(meta, &u, dev, r, "tree", NOW + 100).await; + let a = folder(meta, &u, dev, tree, "a", NOW + 100).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW + 100).await; + let (b, bv) = file_in(meta, &u, dev, tree, "b.txt", NOW + 100).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 101).await; + delete_folder(meta, &u, dev, a, NOW + 102).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 103).await; + delete_folder(meta, &u, dev, tree, NOW + 104).await; + meta.clone().with_trash_tree_cap(1).request_purge(u.space, tree, NOW + 105).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![a], "only the left-over folder is listed"); + meta.clone().with_trash_tree_cap(1).request_purge(u.space, a, NOW + 106).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert!(matches!(meta.restore_from_trash(u.space, dev, a1, NOW).await, Err(MetaError::NotFound(_)))); +} + +/// When every conflict name is taken too, the restore is refused as a conflict and the space is exactly as it was: no +/// sequence number, no revived node, the node still in the trash. +async fn a_restore_with_no_free_name_changes_nothing(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "busy.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + // The name itself and its 100 conflict names, in one batch. + let m = manifest(meta, &u, 1).await; + let mut names = vec!["busy.txt".to_owned()]; + names.extend((1..=100).map(|n| conflict_of("busy.txt", false, n))); + let ops: Vec = names.iter().map(|n| Op::Create { node: NodeId::generate(), parent: r, name: name(n), version: version(m) }).collect(); + meta.commit(u.space, dev, &ops, NOW + 2).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let tree_before = full_tree(meta, u.space).await.nodes.len(); + + let refused = meta.restore_from_trash(u.space, dev, f, NOW).await; + let list = refused_restore(refused); + assert!(matches!(list[0].1, ConflictKind::NameTaken { .. }), "{list:?}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, vec![f]); + assert_eq!(full_tree(meta, u.space).await.nodes.len(), tree_before); + assert!(meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap().nodes.is_empty()); +} + +/// What is not in the trash can't be restored or removed from it: a live node, an unknown one, another space's, one +/// already removed from the trash. +async fn only_nodes_in_the_trash_can_be_restored_or_removed(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (live, _) = file_in(meta, &u, dev, r, "live.txt", NOW).await; + let (gone, gv) = file_in(meta, &u, dev, r, "gone.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, r, "kept.txt", NOW).await; + delete_file(meta, &u, dev, gone, gv.id, NOW + 1).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1).await; + meta.request_purge(u.space, gone, NOW + 2).await.unwrap(); + let seq = meta.space_info(u.space).await.unwrap().seq; + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + + for node in [live, NodeId::generate(), r, gone] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "restore {node}"); + } + for node in [live, NodeId::generate(), r] { + assert!(matches!(meta.request_purge(u.space, node, NOW).await, Err(MetaError::NotFound(_))), "remove {node}"); + } + assert!(matches!(meta.restore_from_trash(bob.space, bob_device, kept, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + assert!(matches!(meta.request_purge(bob.space, kept, NOW).await, Err(MetaError::NotFound(_)))); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, seq, "nothing changed"); + assert_eq!(trash_ids(meta, &u).await, vec![kept]); +} + +/// Removing from the trash only marks the node: no version, manifest or tombstone changes, the change feed and the +/// sequence number stay, the history is still readable; asking twice changes nothing; an edit that brings the node back +/// ends the mark, and the next deletion lists it again. +async fn removing_from_the_trash_frees_nothing(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + delete_file(meta, &u, dev, node, versions[2].id, NOW + 100).await; + let before = meta.space_info(u.space).await.unwrap(); + let feed_before = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + + meta.request_purge(u.space, node, NOW + 200).await.unwrap(); + meta.request_purge(u.space, node, NOW + 300).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "it left the trash"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(), feed_before, "the tombstone is as it was"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3, "every version is still there"); + for v in &versions { + assert!(meta.has_manifest(u.space, &v.manifest).await.unwrap(), "every manifest is still the space's"); + } + + // An edit still brings the file back (an edit beats a deletion), which ends the mark. + let edit = version(manifest(meta, &u, 9).await); + meta.commit(u.space, dev, &[Op::Update { node, expected: versions[2].id, version: edit.clone() }], NOW + 400).await.unwrap(); + assert!(!live_name(meta, &u, node).await.2); + delete_file(meta, &u, dev, node, edit.id, NOW + 500).await; + let page = meta.trash(u.space, None, 10, NOW + 600).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms), (1, NOW + 500)); +} + +/// A restore and a commit that wants the same name run together: both are applied in some order and the result is +/// consistent, never two live nodes with one name. +async fn a_trash_restore_and_a_commit_for_the_same_name_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + for round in 0..4 { + let file_name = format!("race{round}.txt"); + let (f, fv) = file_in(meta, &u, dev, r, &file_name, NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let rival = NodeId::generate(); + let rival_op = Op::Create { node: rival, parent: r, name: name(&file_name), version: version(manifest(meta, &u, 2).await) }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let create = tokio::spawn(async move { m2.commit(space, dev, &[rival_op], NOW + 2).await }); + let restored = restore.await.unwrap().unwrap(); + match create.await.unwrap() { + // The commit came first: it holds the name, and the restore went around it. + Ok(_) => { + assert!(restored.renamed, "round {round}"); + assert_eq!(live_name(meta, &u, rival).await.1, file_name); + } + // The restore came first: the commit finds the name taken. + Err(MetaError::Rejected(list)) => { + assert!(matches!(list[0].kind, ConflictKind::NameTaken { .. }), "round {round}"); + assert!(!restored.renamed); + assert_eq!(live_name(meta, &u, f).await.1, file_name); + } + Err(other) => panic!("unexpected error {other:?}"), + } + } + // No two live nodes share a name (the unique index would have refused it; this says what is seen). + let tree = full_tree(meta, u.space).await; + let mut names: Vec = tree.nodes.iter().filter_map(|n| n.name.as_ref().map(|n| n.fold_key().as_str().to_owned())).collect(); + let total = names.len(); + names.sort(); + names.dedup(); + assert_eq!(names.len(), total); +} + +/// A restore and an edit of the same deleted file run together: whichever is first, the file ends live with its versions +/// intact. If the edit is first it brings the file back and the restore finds nothing in the trash; if the restore is +/// first it makes a new current version, so the edit, made on the old one, is refused as a stale update. +async fn a_trash_restore_and_an_edit_of_the_file_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "both.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let edit = version(manifest(meta, &u, 4).await); + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let edit_op = Op::Update { node: f, expected: fv.id, version: edit.clone() }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let update = tokio::spawn(async move { m2.commit(space, dev, &[edit_op], NOW + 2).await }); + let restored = restore.await.unwrap(); + let updated = update.await.unwrap(); + match (&restored, &updated) { + // The edit came first and brought the file back: the restore finds nothing in the trash. + (Err(MetaError::NotFound(_)), Ok(_)) => {} + // The restore came first and made a new current version: the edit, made on the old one, is a stale update. + (Ok(_), Err(MetaError::Rejected(list))) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + other => panic!("one of them must win: {other:?}"), + } + assert_eq!(live_name(meta, &u, f).await, (Some(r), "both.txt".to_owned(), false)); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.history(u.space, f, None, 10).await.unwrap().versions.len(), 2); +} + +// ---- the trash: the migration and a damaged database ---- + +/// A raw connection to the database `url`, for what the library never does (reading and breaking columns). +async fn raw(url: &str) -> sqlx::AnyConnection { + use sqlx::Connection; + sqlx::any::install_default_drivers(); + sqlx::AnyConnection::connect(url).await.unwrap() +} + +/// Removes the last three migrations and their version rows, so older migration tests can start from the version they name. +async fn drop_history_trash_and_manifest_index(conn: &mut sqlx::AnyConnection) { + for statement in [ + "DROP TABLE manifest_chunks", + "DROP TABLE manifest_index", + "DROP INDEX nodes_trash", + "ALTER TABLE nodes DROP COLUMN deleted_ms", + "ALTER TABLE nodes DROP COLUMN purge_requested_ms", + "DROP INDEX versions_node_seq", + "DELETE FROM schema_version WHERE version >= 12", + ] { + sqlx::query(statement).execute(&mut *conn).await.unwrap(); + } +} + +/// Removes the trash and manifest-index migrations but retains the version-history migration. +async fn drop_trash_and_manifest_index(conn: &mut sqlx::AnyConnection) { + for statement in [ + "DROP TABLE manifest_chunks", + "DROP TABLE manifest_index", + "DROP INDEX nodes_trash", + "ALTER TABLE nodes DROP COLUMN deleted_ms", + "ALTER TABLE nodes DROP COLUMN purge_requested_ms", + "DELETE FROM schema_version WHERE version >= 13", + ] { + sqlx::query(statement).execute(&mut *conn).await.unwrap(); + } +} + +/// Puts the database back to the state before the trash migration: no columns, no index, one version fewer. +async fn undo_the_trash_migration(url: &str) { + let mut conn = raw(url).await; + drop_trash_and_manifest_index(&mut conn).await; +} + +fn wall_clock_ms() -> i64 { + i64::try_from(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()).unwrap() +} + +/// Nodes deleted before the trash existed start their time in the trash at the migration, once: live nodes get no +/// time, and opening the database again does not stamp them again. +async fn the_trash_migration_dates_old_deletions_at_the_migration(url: &str) { + let (ana, file, folder_node) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "old.txt", NOW).await; + let (_kept, _) = file_in(&meta, &ana, dev, r, "kept.txt", NOW).await; + let dir = folder(&meta, &ana, dev, r, "olddir", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + delete_folder(&meta, &ana, dev, dir, NOW + 2).await; + (ana, file, dir) + }; + undo_the_trash_migration(url).await; + let before = wall_clock_ms(); + let meta = Meta::open(url).await.unwrap(); + let after = wall_clock_ms(); + let page = meta.trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(page.repaired, 0, "the migration stamped them, nothing was left to repair"); + let mut ids: Vec<_> = page.entries.iter().map(|e| e.id).collect(); + ids.sort(); + let mut expected = vec![file, folder_node]; + expected.sort(); + assert_eq!(ids, expected, "the live file is not in the trash"); + let stamps: Vec = page.entries.iter().map(|e| e.deleted_ms).collect(); + assert!(stamps.iter().all(|t| (before - 1_000..=after + 1_000).contains(t)), "{stamps:?} not in {before}..{after}"); + assert_eq!(stamps[0], stamps[1], "one baseline for all of them"); + // Another open changes nothing. + drop(meta); + let again = Meta::open(url).await.unwrap().trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(again.entries, page.entries); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 14); + let live_with_time: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM nodes WHERE deleted = 0 AND (deleted_ms IS NOT NULL OR purge_requested_ms IS NOT NULL)") + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(live_with_time, 0); +} + +#[tokio::test] +async fn the_trash_migration_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_trash_migration_dates_old_deletions_at_the_migration(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_trash_migration_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_trash_migration_dates_old_deletions_at_the_migration(&url).await; +} + +/// Drops what migration 0008 added and forgets it ran, which leaves the database as 0007 left it (with its rows). +async fn back_to_0007(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + // The audit log (migration 0009) and two-factor sign-in (migration 0010) go first: they run again with 0008 on the next open. + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN purpose").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE spaces DROP COLUMN quota_bytes").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN removed_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE users DROP COLUMN auth_gen").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 8").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the admin API keeps its accounts, devices and invites: an old invite reads as an +/// invite and still works, no space has a quota and no user a removal mark, and the new calls work on it. +async fn migrating_from_0007_keeps_the_rows(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, old_token, invite) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + meta.add_chunk(ana.space, &random_chunk(), 42, NOW).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + (ana, old_token, invite) + }; + back_to_0007(url).await; + assert_eq!(schema_version(url).await, 7); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(ana.id)); + let row = admin_view(&meta, ana.id).await; + assert_eq!((row.disabled_ms, row.quota_bytes, row.stored_bytes, row.devices_active, row.invite_pending), (None, None, 42, 1, true)); + assert_eq!(meta.invite_needs_password(&invite.hash(), NOW + 1).await.unwrap(), Some(false), "an old invite is an invite"); + let (_, actor) = admin_actor(&meta, "root").await; + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(meta.device_by_token(&old_token).await.unwrap(), None); + + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (15)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0007_keeps_the_rows_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0007_keeps_the_rows(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0007_keeps_the_rows_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0007_keeps_the_rows(&url).await; +} + +/// A deleted node without a deletion time (it can't happen through this code) is stamped with the time of the read +/// and reported, never listed as expired; the stamp stays. +async fn a_deleted_node_without_a_time_is_stamped_when_read(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "damaged.txt", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + let mut conn = raw(url).await; + sqlx::query("UPDATE nodes SET deleted_ms = NULL WHERE deleted = 1").execute(&mut conn).await.unwrap(); + let page = meta.trash(ana.space, None, 10, NOW + 77).await.unwrap(); + assert_eq!((page.repaired, page.entries.len(), page.entries[0].deleted_ms), (1, 1, NOW + 77)); + let later = meta.trash(ana.space, None, 10, NOW + 99).await.unwrap(); + assert_eq!((later.repaired, later.entries[0].deleted_ms), (0, NOW + 77)); +} + +#[tokio::test] +async fn a_damaged_deletion_time_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_deleted_node_without_a_time_is_stamped_when_read(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_damaged_deletion_time_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_deleted_node_without_a_time_is_stamped_when_read(&url).await; +} + +// ---- the audit log (migration 0009) ---- + +use craft_cloud_meta::{ + ActionFilter, AuditAction, AuditCursor, AuditEvent, AuditFailure, AuditOutcome, AuditQuery, Origin, PRUNE_BATCH, PRUNE_EVERY_MS, Step, +}; + +/// Every event, newest first. +async fn all_events(meta: &Meta) -> Vec { + meta.audit_list(&AuditQuery { limit: 1000, ..AuditQuery::default() }).await.unwrap() +} + +/// Remembers which events a test has looked at, and hands out what was written since, oldest first. +struct Seen<'a> { + meta: &'a Meta, + ids: std::collections::HashSet, +} + +impl<'a> Seen<'a> { + async fn new(meta: &'a Meta) -> Self { + let mut seen = Self { meta, ids: Default::default() }; + seen.fresh().await; + seen + } + + async fn fresh(&mut self) -> Vec { + let Self { meta, ids } = self; + let mut fresh: Vec = all_events(meta).await.into_iter().filter(|e| ids.insert(e.id)).collect(); + fresh.reverse(); + fresh + } + + /// The names of the events written since the last look, sorted (two events of one change have no order to rely on). + async fn actions(&mut self) -> Vec<&'static str> { + let mut names: Vec<&'static str> = self.fresh().await.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + names + } + + /// The one event written since the last look. + async fn one(&mut self) -> AuditEvent { + let mut fresh = self.fresh().await; + assert_eq!(fresh.len(), 1, "{:?}", fresh.iter().map(|e| e.action.as_str()).collect::>()); + fresh.remove(0) + } +} + +fn details(e: &AuditEvent) -> serde_json::Value { + serde_json::from_str(&e.details).unwrap() +} + +/// An event of the day-to-day, shorthand for the matrix below. +fn json(text: &str) -> serde_json::Value { + serde_json::from_str(text).unwrap() +} + +/// Every operation of the matrix in `docs/design/sync.md` writes exactly the events it names, with the actor, target and +/// typed details it names; a repeat or a no-op writes none; a bulk sign-out writes one event with the count. Creating +/// an account or a device by the test seams (`create_user`, `create_device`) writes none. +async fn every_operation_writes_exactly_its_events(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let mut seen = Seen::new(meta).await; + assert!(all_events(meta).await.is_empty(), "the test seams write no events"); + + // A sign-in: the account, the new device, and the client reduced to the closed lists. + let client = DeviceClient::new(Some("agent"), Some("Linux"), Some("1.2.3")).unwrap(); + let dev = meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &client, &token(), NOW).await.unwrap().unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.count), + (AuditAction::AuthLogin, AuditOutcome::Success, Some(ana.id), Some(dev), 1) + ); + assert_eq!((e.target_kind.as_str(), e.target_id), ("device", Some(*dev.as_uuid()))); + assert_eq!(details(&e), json(r#"{"client_kind":"agent","platform":"linux"}"#)); + assert_eq!(e.at_ms, NOW); + // A sign-in that is refused writes nothing. + assert!(meta.sign_in_device(ana.id, ("$argon2id$other", 0), "pc", &client, &token(), NOW).await.unwrap().is_none()); + assert!(seen.actions().await.is_empty()); + + // Signing out one device: from a device, then the same again (nothing changed), then as the account without a device. + let (d2, d3) = (device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_device_by(ana.id, d2, RevokeReason::User, Origin::Device(dev), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::DeviceRevoked, Some(ana.id), Some(dev), Some(*d2.as_uuid()))); + assert_eq!(details(&e), json(r#"{"reason":"user"}"#), "the device is the event's target"); + meta.revoke_device_by(ana.id, d2, RevokeReason::User, Origin::Device(dev), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "a device that is out already: no change, no event"); + meta.revoke_device(ana.id, d3).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::DeviceRevoked, Some(ana.id), None)); + + // Signing out the others: one event with the count; with nobody to sign out, none. + let (_e1, _e2) = (device(meta, &ana).await, device(meta, &ana).await); + meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_device, e.target_id), (AuditAction::DeviceRevokedOthers, Some(dev), Some(ana.id.0))); + assert_eq!(details(&e), json(r#"{"reason":"others","revoked":2}"#)); + meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // A password change: one event, the devices signed out with it are only in `revoked`. + let (_f1, _f2) = (device(meta, &ana).await, device(meta, &ana).await); + meta.change_password(ana.id, &ana.password_hash, "$argon2id$new", dev, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::AccountPasswordChanged, Some(ana.id), Some(dev))); + assert_eq!(details(&e), json(r#"{"revoked":2}"#)); + // A change that fails writes nothing: the password it checked is not the account's. + assert!(matches!(meta.change_password(ana.id, "$argon2id$wrong", "$argon2id$x", dev, NOW).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); + + // Accounts: created with a role; the invite that goes with it is not an event of its own. + let invite = code(INVITE_CHARS); + let (newbie, _) = meta.admin_create_user(&actor, "newbie", Role::Member, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AdminUserCreated, Some(root.id), Some(device_of(&actor)), Some(newbie.id.0)) + ); + assert_eq!(details(&e), json(r#"{"role_to":"member"}"#)); + // Invites: issued, revoked (a second revoke has nothing to revoke). + meta.admin_issue_invite(&actor, newbie.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.invite_issued"]); + meta.admin_revoke_invite(&actor, newbie.id, NOW).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.invite_revoked"]); + assert!(matches!(meta.admin_revoke_invite(&actor, newbie.id, NOW).await, Err(MetaError::NotFound(_)))); + assert!(seen.actions().await.is_empty()); + + // A role and a quota in one request: one event for each, in the same transaction. The same again changes nothing. + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + let both = seen.fresh().await; + let mut names: Vec<&str> = both.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["admin.quota_changed", "admin.role_changed"]); + for e in &both { + assert_eq!((e.actor_user, e.target_id), (Some(root.id), Some(ana.id.0))); + match e.action { + AuditAction::AdminRoleChanged => assert_eq!(details(e), json(r#"{"role_from":"member","role_to":"admin"}"#)), + _ => assert_eq!(details(e), json(&format!(r#"{{"quota_bytes":{MIN_QUOTA_BYTES}}}"#))), + } + } + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "the role and quota it has already: no change, no event"); + meta.admin_update_user(&actor, ana.id, Some(Role::Member), None, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, details(&e)), (AuditAction::AdminRoleChanged, json(r#"{"role_from":"admin","role_to":"member"}"#))); + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, details(&e)), (AuditAction::AdminQuotaChanged, json(r#"{"quota_bytes":null}"#)), "removing the quota says so"); + meta.admin_update_user(&actor, ana.id, None, Some(None), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // A reset code, and its redemption: the devices it signs out are counted in the one event. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let active = meta.list_devices(ana.id).await.unwrap().iter().filter(|d| !d.revoked).count(); + let redeemed = meta.redeem_with(&reset.hash(), "recovery", &client, &token(), Some("$argon2id$reset"), NOW).await.unwrap(); + assert_eq!(redeemed.revoked.len(), active); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::JoinResetRedeemed, Some(ana.id), Some(redeemed.device), Some(*redeemed.device.as_uuid())) + ); + assert_eq!(details(&e), json(&format!(r#"{{"client_kind":"agent","platform":"linux","revoked":{active}}}"#))); + + // An invite redeemed with a password for the new account, and a pairing redeemed from its device. + meta.admin_issue_invite(&actor, newbie.id, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap(); + seen.fresh().await; + let joined = meta.redeem(&invite.hash(), "newbie pc", &token(), Some("$argon2id$newbie"), NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinInviteRedeemed, Some(newbie.id), Some(joined.device))); + assert_eq!(details(&e), json("{}"), "no client named: no client in the event"); + let pairing = code(PAIRING_CHARS); + meta.create_pairing(newbie.id, joined.device, &pairing.hash(), NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "making a code is not an event"); + let paired = meta.redeem(&pairing.hash(), "second pc", &token(), None, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinPairingRedeemed, Some(newbie.id), Some(paired.device))); + // A code that does not work writes nothing. + assert!(matches!(meta.redeem(&pairing.hash(), "again", &token(), None, NOW).await, Err(MetaError::InvalidCode))); + assert!(seen.actions().await.is_empty()); + + // Disable (the devices go with it), a second disable, enable, a second enable. + let (_g1, _g2) = (device(meta, &ana).await, device(meta, &ana).await); + let signed_out = meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap().len(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.target_id), (AuditAction::AdminUserDisabled, Some(root.id), Some(ana.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{signed_out}}}"#))); + meta.admin_disable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty(), "disabled already: no change, no event"); + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.user_enabled"]); + meta.admin_enable_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // An admin signs out one device: about the account, naming the device; again, nothing. + let anas = device(meta, &ana).await; + meta.admin_revoke_device(&actor, ana.id, anas, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_kind.as_str(), e.target_id), + (AuditAction::AdminDeviceRevoked, Some(root.id), Some(device_of(&actor)), "user", Some(ana.id.0)) + ); + assert_eq!(details(&e), json(&format!(r#"{{"device":"{anas}","reason":"admin"}}"#))); + meta.admin_revoke_device(&actor, ana.id, anas, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // Marked for removal, once. + let _ = device(meta, &ana).await; + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.target_id), (AuditAction::AdminUserRemoved, Some(ana.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":1}"#)); + meta.admin_remove_user(&actor, ana.id, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + + // The command line acts as no account. + let cara = user(meta, "cara").await; + let (_h1, _h2) = (device(meta, &cara).await, device(meta, &cara).await); + meta.reset_password(cara.id, "$argon2id$cli", false, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliPasswordReset, None, None, Some(cara.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":2}"#)); + let one = device(meta, &cara).await; + meta.revoke_device_by(cara.id, one, RevokeReason::Admin, Origin::Cli, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliDeviceRevoked, None, None, Some(cara.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"device":"{one}","reason":"admin"}}"#))); + meta.revoke_device_by(cara.id, one, RevokeReason::Admin, Origin::Cli, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + let (_i1, _i2, _i3) = (device(meta, &cara).await, device(meta, &cara).await, device(meta, &cara).await); + meta.revoke_all_devices(cara.id, RevokeReason::Admin, NOW).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.target_id), (AuditAction::CliDevicesRevokedAll, None, Some(cara.id.0))); + assert_eq!(details(&e), json(r#"{"revoked":3}"#)); + meta.revoke_all_devices(cara.id, RevokeReason::Admin, NOW).await.unwrap(); + assert!(seen.actions().await.is_empty()); + // A reset that keeps the devices counts none. + meta.reset_password(cara.id, "$argon2id$cli2", true, NOW).await.unwrap(); + assert_eq!(details(&seen.one().await), json(r#"{"revoked":0}"#)); +} + +/// A change that fails, or is refused, leaves no event behind, and the last administrator stays one. +async fn a_refused_change_writes_no_event(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + let mut seen = Seen::new(meta).await; + // The only administrator cannot demote or disable themselves out of the job, and the role stays. + assert!(matches!(meta.admin_update_user(&actor, actor.user, Some(Role::Member), None, NOW).await, Err(MetaError::LastAdmin))); + assert!(matches!(meta.admin_disable_user(&actor, actor.user, NOW).await, Err(MetaError::Invalid(_)))); + // An actor whose role is gone, one whose password changed since, and a target that is not there. + let stale = AdminActor { password_hash: "$argon2id$old".to_owned(), ..actor.clone() }; + assert!(matches!(meta.admin_disable_user(&stale, ana.id, NOW).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.admin_update_user(&actor, UserId(uuid::Uuid::now_v7()), Some(Role::Admin), None, NOW).await, Err(MetaError::NotFound(_)))); + // A reset code without the new password, a name that is taken, a device of someone else. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&actor, ana.id, &reset.hash(), false, NOW, RESET_TTL_MS).await.unwrap(); + seen.fresh().await; + assert!(matches!(meta.redeem(&reset.hash(), "pc", &token(), None, NOW).await, Err(MetaError::PasswordRequired))); + assert!(matches!( + meta.admin_create_user(&actor, "ana", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await, + Err(MetaError::UsernameTaken) + )); + assert!(matches!(meta.revoke_device_as(ana.id, DeviceId::generate(), RevokeReason::User, NOW).await, Err(MetaError::NotFound(_)))); + assert!(seen.actions().await.is_empty()); + assert_eq!(meta.account(actor.user).await.unwrap().role, Role::Admin); +} + +/// A sign-in step that failed is one row per account, action, client kind and minute, whose count goes up and whose identity, time and +/// details stay those of the first try; names no account has share one row per minute; an event that is not a failed step +/// cannot be aggregated. +async fn failed_sign_ins_count_up_in_one_row_per_account_and_minute(meta: &Meta) { + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + let minute = NOW - NOW % 60_000; + let login = |target: Option| AuditFailure { action: AuditAction::AuthLoginFailed, target, step: Step::Password }; + meta.audit_record_failure(login(Some(ana.id)), minute + 1_000).await.unwrap(); + let first = all_events(meta).await; + assert_eq!(first.len(), 1); + let first = first[0].clone(); + assert_eq!( + (first.action, first.outcome, first.count, first.actor_user, first.target_id), + (AuditAction::AuthLoginFailed, AuditOutcome::Failure, 1, None, Some(ana.id.0)) + ); + assert_eq!((first.target_kind.as_str(), details(&first)), ("user", json(r#"{"step":"password"}"#))); + meta.audit_record_web_failure(login(Some(ana.id)), minute + 1_000).await.unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 2, "device and web failure aggregates stay separate"); + assert!(rows.iter().any(|event| details(event) == json(r#"{"client_kind":"web","step":"password"}"#))); + // The same minute: the count goes up, nothing else changes. + meta.audit_record_failure(login(Some(ana.id)), minute + 59_000).await.unwrap(); + meta.audit_record_failure(login(Some(ana.id)), minute + 59_999).await.unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 2); + let device_row = rows.iter().find(|event| event.id == first.id).unwrap(); + assert_eq!((device_row.id, device_row.at_ms, device_row.count, &device_row.details), (first.id, minute + 1_000, 3, &first.details)); + // The next minute is a new row; another account, another action, another row. + meta.audit_record_failure(login(Some(ana.id)), minute + 60_000).await.unwrap(); + meta.audit_record_failure(login(Some(bob.id)), minute + 1_000).await.unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthSecondFactorFailed, target: Some(ana.id), step: Step::Totp }, minute + 1_000) + .await + .unwrap(); + let rows = all_events(meta).await; + assert_eq!(rows.len(), 5); + let mut counts: Vec<(&str, u64)> = rows.iter().map(|e| (e.action.as_str(), e.count)).collect(); + counts.sort_unstable(); + assert_eq!( + counts, + [("auth.login_failed", 1), ("auth.login_failed", 1), ("auth.login_failed", 1), ("auth.login_failed", 3), ("auth.second_factor_failed", 1)] + ); + // Names nobody has: one row a minute whatever was typed, with no account in it. + for _ in 0..4 { + meta.audit_record_failure(login(None), minute + 2_000).await.unwrap(); + } + meta.audit_record_failure(login(None), minute + 62_000).await.unwrap(); + let unknown: Vec = all_events(meta).await.into_iter().filter(|e| e.target_id.is_none()).collect(); + assert_eq!(unknown.len(), 2); + assert!(unknown.iter().all(|e| e.actor_user.is_none() && e.target_kind.is_empty() && e.actor_device.is_none())); + let mut counts: Vec = unknown.iter().map(|e| e.count).collect(); + counts.sort_unstable(); + assert_eq!(counts, [1, 4]); + // Only the failed steps are aggregated. + for action in [AuditAction::AuthLogin, AuditAction::AdminUserCreated, AuditAction::CliPasswordReset] { + assert!(matches!( + meta.audit_record_failure(AuditFailure { action, target: None, step: Step::Password }, NOW).await, + Err(MetaError::Invalid(_)) + )); + } + // Other events have no key, so any number of them is allowed. + let (_, actor) = admin_actor(meta, "root").await; + for i in 0..3 { + meta.admin_update_user(&actor, ana.id, Some(if i % 2 == 0 { Role::Admin } else { Role::Member }), None, NOW).await.unwrap(); + } + assert_eq!(all_events(meta).await.iter().filter(|e| e.action == AuditAction::AdminRoleChanged).count(), 3); +} + +/// Many tries at one moment on one key: exactly one row, counting every one, and the first row's identity and time stay. +/// Run on both databases (on Postgres the collisions really are concurrent). +async fn concurrent_failures_with_one_key_make_one_row_with_every_count(meta: &Meta) { + let ana = user(meta, "ana").await; + let minute = NOW - NOW % 60_000; + const TRIES: u64 = 24; + for round in 0..3i64 { + let at = minute + round * 60_000; + let tries: Vec<_> = (0..TRIES) + .map(|i| { + let meta = meta.clone(); + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }; + tokio::spawn(async move { meta.audit_record_failure(failure, at + i as i64).await }) + }) + .collect(); + for t in tries { + t.await.unwrap().unwrap(); + } + } + let rows = all_events(meta).await; + assert_eq!(rows.len(), 3, "one row for each minute"); + assert!(rows.iter().all(|e| e.count == TRIES), "{:?}", rows.iter().map(|e| e.count).collect::>()); + // Each row kept the time of the try that made it: one of the first moment's tries, within its minute. + for (round, row) in rows.iter().rev().enumerate() { + let start = minute + round as i64 * 60_000; + assert!((start..start + TRIES as i64).contains(&row.at_ms), "{} not in {}..", row.at_ms, start); + } +} + +/// Lists: newest first in `(at_ms, id)` order, filters by account and by name or group of names, `before_ms` and an end +/// time, and a cursor that is exclusive so that events with one timestamp are neither repeated nor lost. +async fn audit_lists_filter_and_page_by_cursor(meta: &Meta) { + let (root, actor) = admin_actor(meta, "root").await; + let (ana, bob) = (user(meta, "ana").await, user(meta, "bob").await); + // Seven events at one moment. + for i in 0..7 { + meta.admin_update_user(&actor, ana.id, Some(if i % 2 == 0 { Role::Admin } else { Role::Member }), None, NOW).await.unwrap(); + } + meta.sign_in_device(bob.id, (&bob.password_hash, bob.auth_gen), "pc", &DeviceClient::default(), &token(), NOW + 5).await.unwrap().unwrap(); + let minute = NOW - NOW % 60_000 + 60_000; + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(bob.id), step: Step::Password }, minute) + .await + .unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }, minute + 60_000) + .await + .unwrap(); + + let all = all_events(meta).await; + assert_eq!(all.len(), 10); + let order: Vec<_> = all.iter().map(|e| (e.at_ms, e.id)).collect(); + assert!(order.windows(2).all(|w| w[0] > w[1]), "newest first, by (at_ms, id) descending"); + + // Pages of three through the cursor of the last event of each: all ten once, in order. + let mut paged = Vec::new(); + let mut after: Option = None; + for _ in 0..10 { + let page = meta.audit_list(&AuditQuery { after, limit: 3, ..AuditQuery::default() }).await.unwrap(); + let Some(last) = page.last() else { break }; + after = Some(last.cursor()); + paged.extend(page); + } + assert_eq!(paged, all); + // A cursor behind everything gives nothing; a cursor at an event never gives that event. + assert!( + meta.audit_list(&AuditQuery { after: Some(AuditCursor { at_ms: 0, id: uuid::Uuid::nil() }), limit: 10, ..AuditQuery::default() }) + .await + .unwrap() + .is_empty() + ); + let sevens: Vec<&AuditEvent> = all.iter().filter(|e| e.at_ms == NOW).collect(); + assert_eq!(sevens.len(), 7); + let page = meta.audit_list(&AuditQuery { after: Some(sevens[2].cursor()), limit: 100, ..AuditQuery::default() }).await.unwrap(); + assert_eq!(page.len(), 4, "the four behind the third of the seven that share a time"); + + // Times: before is exclusive, the end time inclusive. + let before = |ms| AuditQuery { before_ms: Some(ms), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&before(NOW)).await.unwrap().len(), 0); + assert_eq!(meta.audit_list(&before(NOW + 1)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&AuditQuery { until_ms: Some(NOW), limit: 100, ..AuditQuery::default() }).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&AuditQuery { until_ms: Some(NOW - 1), limit: 100, ..AuditQuery::default() }).await.unwrap().len(), 0); + + // Accounts: the actor or the target. + let by_user = |u: UserId| AuditQuery { user: Some(u), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&by_user(ana.id)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_user(root.id)).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_user(bob.id)).await.unwrap().len(), 2, "his sign-in, and the guess about his password"); + assert_eq!(meta.audit_list(&by_user(UserId(uuid::Uuid::now_v7()))).await.unwrap().len(), 0); + + // Names: exact, or a group. + let by_action = |text: &str| AuditQuery { action: ActionFilter::parse(text), limit: 100, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&by_action("admin.role_changed")).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_action("auth.login")).await.unwrap().len(), 1, "exact: not auth.login_failed"); + assert_eq!(meta.audit_list(&by_action("auth.")).await.unwrap().len(), 3); + assert_eq!(meta.audit_list(&by_action("admin.")).await.unwrap().len(), 7); + assert_eq!(meta.audit_list(&by_action("cli.")).await.unwrap().len(), 0); + // Both at once, and with a page. + let both = AuditQuery { user: Some(bob.id), action: ActionFilter::parse("auth."), limit: 1, ..AuditQuery::default() }; + assert_eq!(meta.audit_list(&both).await.unwrap().len(), 1); +} + +/// Pruning deletes what is older than the limit and nothing else, a batch at a time; a pruning that leaves a backlog +/// does not count as done, so the next write goes on; one that finished holds off the next for an hour; an error is not +/// done either. +async fn pruning_works_in_batches_and_keeps_its_time_until_done(meta: &Meta) { + let ana = user(meta, "ana").await; + let day = 24 * 60 * 60 * 1000i64; + let old = NOW - 3 * day; + // More old events than one batch holds, each in a minute of its own. + let n = PRUNE_BATCH + 100; + let writers: Vec<_> = (0..8) + .map(|w| { + let meta = meta.clone(); + tokio::spawn(async move { + let mut i = w; + while i < n { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old + i * 60_000).await.unwrap(); + i += 8; + } + }) + }) + .collect(); + for w in writers { + w.await.unwrap(); + } + // Recent events, which pruning leaves. + for i in 0..3 { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }; + meta.audit_record_failure(failure, NOW - day / 2 + i * 60_000).await.unwrap(); + } + let limit = NOW - day; + assert_eq!(meta.audit_count_before(limit).await.unwrap(), n as u64); + assert_eq!(meta.audit_count_before(old).await.unwrap(), 0, "the limit is exclusive"); + + // Not configured: a write prunes nothing. + assert!(!meta.audit_prune_if_due(NOW).await.unwrap()); + // A second handle that prunes when asked. The writes below go through `meta`, which never starts a pruning of its own. + let pruner = meta.clone().with_audit_retention_days(1); + // One batch, then the rest: the first leaves a backlog and so does not hold off the next. + assert!(pruner.audit_prune_if_due_in(NOW, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 100, "one batch of {PRUNE_BATCH}"); + assert!(pruner.audit_prune_if_due_in(NOW + 1, 1).await.unwrap(), "a backlog is not done"); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 0); + assert_eq!(all_events(meta).await.len(), 3, "the recent events stay"); + // Done: the next hour is left alone, then it runs again. + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old).await.unwrap(); + assert!(!pruner.audit_prune_if_due_in(NOW + 2, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 1); + assert!(pruner.audit_prune_if_due_in(NOW + 1 + PRUNE_EVERY_MS, 1).await.unwrap()); + assert_eq!(meta.audit_count_before(limit).await.unwrap(), 0); + + // `audit_prune` itself takes everything before the limit in batches, and reports the count. + for i in 0..(PRUNE_BATCH + 5) { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + meta.audit_record_failure(failure, old + i * 60_000).await.unwrap(); + } + assert_eq!(meta.audit_prune(limit).await.unwrap(), (PRUNE_BATCH + 5) as u64); + assert_eq!(meta.audit_prune(limit).await.unwrap(), 0); + assert_eq!(all_events(meta).await.len(), 3); +} + +/// Events outlive the account they are about: marking it for removal deletes nothing, and the account's ID still finds them. +async fn a_removed_accounts_events_stay(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let ana = user(meta, "ana").await; + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await.unwrap(); + meta.admin_remove_user(&actor, ana.id, NOW + 1).await.unwrap(); + let mine = meta.audit_list(&AuditQuery { user: Some(ana.id), limit: 100, ..AuditQuery::default() }).await.unwrap(); + let mut names: Vec<&str> = mine.iter().map(|e| e.action.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["admin.user_removed", "auth.login", "auth.login_failed"]); +} + +/// Nothing a client typed or a secret is in any event, whatever the account, the device or the client called itself. +async fn no_event_holds_a_name_or_a_secret(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + let hash = "$argon2id$SECRETHASH"; + let zelda = meta.create_user("zelda", hash, Role::Member, NOW).await.unwrap(); + // A platform may be any printable text of 32 bytes: here an address, a file name, and 32 bytes of multi-byte characters. + let hostile = + ["10.0.0.1", "secret.psd", "fe80::1%eth0", "\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}\u{20ac}", "windows 11"]; + let mut clock = NOW; + for platform in hostile { + let client = DeviceClient::new(Some("desktop"), Some(platform), Some("9.9.9-secretbuild")).unwrap(); + clock += 1; + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &client, &token(), clock).await.unwrap().unwrap(); + let e = all_events(meta).await.remove(0); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"other"}"#), "{platform:?}"); + } + let known = DeviceClient::new(Some("cli"), Some("MacOS"), None).unwrap(); + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &known, &token(), NOW + 100).await.unwrap().unwrap(); + assert_eq!(details(&all_events(meta).await.remove(0)), json(r#"{"client_kind":"cli","platform":"macos"}"#)); + let no_platform = DeviceClient::new(Some("agent"), None, None).unwrap(); + meta.sign_in_device(zelda.id, (hash, 0), "zelda-private-laptop", &no_platform, &token(), NOW + 200).await.unwrap().unwrap(); + assert_eq!(details(&all_events(meta).await.remove(0)), json(r#"{"client_kind":"agent"}"#), "no platform: no field"); + // A few more changes, and the same scan over everything written. + meta.change_password(zelda.id, hash, "$argon2id$NEWSECRETHASH", meta.list_devices(zelda.id).await.unwrap()[0].id, NOW).await.unwrap(); + meta.admin_update_user(&actor, zelda.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }, NOW).await.unwrap(); + let events = all_events(meta).await; + assert!(events.len() > 8); + let shown = format!("{events:?}"); + for secret in ["zelda", "SECRETHASH", "secretbuild", "10.0.0.1", "secret.psd", "fe80", "127.0.0.1", "\u{20ac}", "argon2", "laptop", "ZELDA"] { + assert!(!shown.contains(secret), "{secret:?} in {shown}"); + } + // Every `details` is an object of the allowed words and numbers only. + for e in &events { + let value = details(e); + let object = value.as_object().unwrap(); + for key in object.keys() { + assert!( + ["client_kind", "platform", "reason", "step", "role_from", "role_to", "quota_bytes", "revoked", "device", "truncated"] + .contains(&key.as_str()), + "{key}" + ); + } + assert!(e.details.len() <= 2048); + } +} + +/// The log is append-only in the code: the sources of this crate update and delete in `audit_events` in exactly the two +/// places the design names (the counter of an aggregated failure, the pruning). +#[test] +fn the_sources_change_audit_rows_only_to_count_and_to_prune() { + fn sources(dir: &std::path::Path, out: &mut Vec<(std::path::PathBuf, String)>) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + sources(&path, out); + } else if path.extension().is_some_and(|e| e == "rs") { + let text = std::fs::read_to_string(&path).unwrap(); + out.push((path, text)); + } + } + } + let mut files = Vec::new(); + sources(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), &mut files); + assert!(files.len() > 5, "the sources were found"); + let count = |needle: &str| -> Vec { + files + .iter() + .flat_map(|(path, text)| { + // Comments may name statements, only code counts. + text.lines() + .filter(|l| !l.trim_start().starts_with("//") && l.contains(needle)) + .map(move |_| path.file_name().unwrap().to_string_lossy().into_owned()) + }) + .collect() + }; + assert!(count("UPDATE audit_events").is_empty(), "no statement updates the log"); + assert!(count("audit_events SET").is_empty()); + assert_eq!(count("DELETE FROM audit_events"), ["audit.rs"], "exactly one deletion, the pruning"); + assert_eq!(count("DO UPDATE SET agg_count = audit_events.agg_count + 1"), ["audit.rs"], "exactly one update, the counter"); + assert!(count("TRUNCATE").is_empty() && count("DROP TABLE audit").is_empty()); +} + +// ---- an error in the log ---- + +/// Makes every use of the log fail (the table is renamed away), which stands for any error of the database in the log +/// and needs no code in the product to cause it. +async fn break_audit(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("ALTER TABLE audit_events RENAME TO audit_events_away").execute(&mut conn).await.unwrap(); +} + +async fn repair_audit(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("ALTER TABLE audit_events_away RENAME TO audit_events").execute(&mut conn).await.unwrap(); +} + +/// A change whose event cannot be written is not made (fail closed), whichever it is; a failed sign-in step's event is +/// only reported to its caller, which ignores it; pruning that fails is not done, so the next write tries again. +async fn an_error_in_the_log_fails_every_change_and_changes_nothing(url: &str) { + sqlx::any::install_default_drivers(); + let meta = Meta::open(url).await.unwrap(); + let pruner = meta.clone().with_audit_retention_days(1); + let (_, actor) = admin_actor(&meta, "root").await; + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let others = [device(&meta, &ana).await, device(&meta, &ana).await]; + let invite = code(INVITE_CHARS); + let fresh = meta.admin_create_user(&actor, "fresh", Role::Member, &invite.hash(), NOW, INVITE_TTL_MS).await.unwrap().0; + let events_before = all_events(&meta).await.len(); + let active = |meta: &Meta, user: UserId| { + let meta = meta.clone(); + async move { meta.list_devices(user).await.unwrap().iter().filter(|d| !d.revoked).count() } + }; + assert_eq!(active(&meta, ana.id).await, 3); + + break_audit(url).await; + let is_db = |r: Result<(), MetaError>| assert!(matches!(r, Err(MetaError::Db(_))), "{r:?}"); + is_db(meta.sign_in_device(ana.id, (&ana.password_hash, 0), "new", &DeviceClient::default(), &token(), NOW).await.map(|_| ())); + is_db(meta.revoke_device_by(ana.id, others[0], RevokeReason::User, Origin::Device(dev), NOW).await); + is_db(meta.revoke_other_devices(ana.id, dev, RevokeReason::Others, NOW).await.map(|_| ())); + is_db(meta.change_password(ana.id, &ana.password_hash, "$argon2id$new", dev, NOW).await.map(|_| ())); + is_db(meta.reset_password(ana.id, "$argon2id$cli", false, NOW).await.map(|_| ())); + is_db(meta.revoke_all_devices(ana.id, RevokeReason::Admin, NOW).await.map(|_| ())); + is_db(meta.admin_update_user(&actor, ana.id, Some(Role::Admin), Some(Some(MIN_QUOTA_BYTES)), NOW).await.map(|_| ())); + is_db(meta.admin_disable_user(&actor, ana.id, NOW).await.map(|_| ())); + is_db(meta.admin_remove_user(&actor, ana.id, NOW).await.map(|_| ())); + is_db(meta.admin_issue_reset(&actor, ana.id, &code(INVITE_CHARS).hash(), false, NOW, RESET_TTL_MS).await.map(|_| ())); + is_db(meta.admin_revoke_device(&actor, ana.id, dev, NOW).await); + is_db(meta.admin_issue_invite(&actor, fresh.id, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.map(|_| ())); + is_db(meta.admin_revoke_invite(&actor, fresh.id, NOW).await); + is_db(meta.admin_create_user(&actor, "nobody", Role::Member, &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.map(|_| ())); + is_db(meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$x"), NOW).await.map(|_| ())); + // A sign-in step that failed has no change to undo: the error is the caller's, and nothing else is touched. + is_db(meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await); + assert!(pruner.audit_prune_if_due(NOW).await.is_err(), "pruning with the log broken is an error"); + repair_audit(url).await; + + // Nothing was changed by any of it. + assert_eq!(all_events(&meta).await.len(), events_before, "no event, no change"); + assert_eq!(active(&meta, ana.id).await, 3, "no device was signed out or added"); + let row = meta.admin_user(ana.id, NOW).await.unwrap().unwrap(); + assert_eq!((row.role, row.disabled_ms, row.quota_bytes), (Role::Member, None, None)); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, ana.password_hash); + assert!(meta.user_by_name("nobody").await.unwrap().is_none(), "the account was not made"); + assert!(meta.admin_user(fresh.id, NOW).await.unwrap().unwrap().invite_pending, "the invite is still there"); + // And the same changes work once the log does, the invite included. + meta.redeem(&invite.hash(), "pc", &token(), Some("$argon2id$x"), NOW).await.unwrap(); + meta.admin_update_user(&actor, ana.id, Some(Role::Admin), None, NOW).await.unwrap(); + // A pruning that failed left no time behind: the next write's goes ahead at once. + assert!(pruner.audit_prune_if_due(NOW + 1).await.unwrap()); +} + +#[tokio::test] +async fn an_error_in_the_log_fails_every_change_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_error_in_the_log_fails_every_change_and_changes_nothing(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_error_in_the_log_fails_every_change_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_error_in_the_log_fails_every_change_and_changes_nothing(&url).await; +} + +/// A write prunes old events after its commit, in the background, when the store is told how long to keep them. +#[tokio::test] +async fn a_write_prunes_old_events_after_its_commit() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + let plain = Meta::open(&url).await.unwrap(); + let ana = user(&plain, "ana").await; + let day = 24 * 60 * 60 * 1000i64; + for i in 0..4 { + let failure = AuditFailure { action: AuditAction::AuthLoginFailed, target: None, step: Step::Password }; + plain.audit_record_failure(failure, NOW - 40 * day + i * 60_000).await.unwrap(); + } + // Without a retention a write leaves them. + plain.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert_eq!(plain.audit_count_before(NOW - 30 * day).await.unwrap(), 4); + // With one, the next write starts the pruning. + let meta = plain.clone().with_audit_retention_days(30); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW + 1).await.unwrap().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while meta.audit_count_before(NOW - 30 * day).await.unwrap() > 0 { + assert!(std::time::Instant::now() < deadline, "the old events were not pruned"); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!(all_events(&meta).await.len(), 2, "the two sign-ins stay"); +} + +// ---- the migration from 0008 to 0009 (and on to 0010) ---- + +/// Drops what migrations 0009 (the audit log) and 0010 (two-factor sign-in) added and forgets they ran, which leaves the database as 0008 left it (with its rows). +async fn back_to_0008(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE audit_events").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE login_challenges").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_recovery_codes").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE user_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE invites DROP COLUMN reset_two_factor").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 9").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the audit log keeps its accounts and devices, gets an empty log that works, and a +/// database newer than the program is still refused. +async fn migrating_from_0008_adds_the_audit_log(url: &str) { + sqlx::any::install_default_drivers(); + let (ana, old_token) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let old_token = token(); + meta.create_device(ana.id, "old pc", &old_token, NOW).await.unwrap(); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + assert_eq!(all_events(&meta).await.len(), 1); + (ana, old_token) + }; + back_to_0008(url).await; + assert_eq!(schema_version(url).await, 8); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14, "all migrations after the audit log run"); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(ana.id)); + assert!(all_events(&meta).await.is_empty(), "a new log is empty"); + meta.sign_in_device(ana.id, (&ana.password_hash, ana.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + meta.audit_record_failure(AuditFailure { action: AuditAction::AuthLoginFailed, target: Some(ana.id), step: Step::Password }, NOW).await.unwrap(); + assert_eq!(all_events(&meta).await.len(), 2); + + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (15)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0008_adds_the_audit_log_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0008_adds_the_audit_log(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0008_adds_the_audit_log_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0008_adds_the_audit_log(&url).await; +} + +// ---- two-factor sign-in (migration 0010) ---- + +type Check = Box Result, UnreadableSecret> + Send + Sync>; + +/// A TOTP check that says every code is right for time step `step` (the crypto is the server's; here only the order of +/// operations and what the database does with the answer are under test). +fn right(step: i64) -> Check { + Box::new(move |_, _| Ok(Some(step))) +} + +/// A TOTP check that says every code is wrong. +fn wrong() -> Check { + Box::new(|_, _| Ok(None)) +} + +fn totp(check: &Check) -> Proof<'_> { + Proof::Totp(&**check) +} + +/// The admin's confirmation with a TOTP check that says `step` (`None`: a wrong code). +async fn confirm_with(meta: &Meta, user: UserId, step: Option, now: i64) -> Result { + let check = step.map_or_else(wrong, right); + meta.confirm_second_factor(user, Some(&totp(&check)), now).await +} + +async fn redeem_with_factor(meta: &Meta, hash: &[u8; 32], proof: Option<&Proof<'_>>, now: i64) -> Result { + meta.redeem_with_second_factor(hash, "new pc", &DeviceClient::default(), &token(), None, proof, now).await.map(|r| r.device) +} + +const FIRST_STEP: i64 = 1_000; +const SETUP_AT: i64 = NOW + 1_000; +const MINUTE: i64 = 60_000; + +/// An account with a password, two devices (`first` asked for the set-up and stays signed in, `second` is signed out by +/// turning it on), and two-factor sign-in on from [`SETUP_AT`] (its first code was step [`FIRST_STEP`]). +struct Protected { + user: UserRecord, + first: DeviceId, + second: DeviceId, + codes: Vec, +} + +impl Protected { + fn actor(&self) -> DeviceActor { + DeviceActor { user: self.user.id, device: self.first, password_hash: "$argon2id$stub".to_owned() } + } + + fn hashes(&self) -> Vec<[u8; 32]> { + self.codes.iter().map(Code::hash).collect() + } +} + +async fn protected(meta: &Meta, username: &str) -> Protected { + let user = user(meta, username).await; + let first = device(meta, &user).await; + let second = device(meta, &user).await; + let codes: Vec = (0..10).map(|_| code(INVITE_CHARS)).collect(); + let me = Protected { user, first, second, codes }; + meta.two_factor_begin_setup(&me.actor(), b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + let revoked = meta.two_factor_confirm(me.user.id, me.first, &*right(FIRST_STEP), &me.hashes(), SETUP_AT + 1).await.unwrap(); + assert_eq!(revoked, vec![me.second]); + me +} + +fn new_challenge() -> NewChallenge { + NewChallenge { id_hash: token(), client_key: "ip:203.0.113.7".to_owned() } +} + +async fn live_devices(meta: &Meta, user: &UserRecord) -> Vec { + meta.list_devices(user.id).await.unwrap().into_iter().filter(|d| !d.revoked).map(|d| d.id).collect() +} + +async fn set_up_confirmed_and_listed(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, second) = (device(meta, &ana).await, device(meta, &ana).await); + let info = meta.two_factor_info(ana.id, SETUP_AT).await.unwrap(); + assert_eq!((info.enabled_ms, info.recovery_codes_left, info.pending), (None, 0, false)); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), NOW, INVITE_TTL_MS).await.unwrap(); + let pairing = code(PAIRING_CHARS); + meta.create_pairing(ana.id, second, &pairing.hash(), NOW).await.unwrap(); + + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let expires = meta.two_factor_begin_setup(&actor, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + assert_eq!(expires, SETUP_AT + 10 * MINUTE); + let info = meta.two_factor_info(ana.id, SETUP_AT + 1).await.unwrap(); + assert_eq!((info.enabled_ms, info.pending), (None, true), "a set-up is not two-factor sign-in"); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + // A password sign-in still gives a device while it is only set up. + let generation = gen_of(meta, "ana").await; + let signed = + meta.sign_in(ana.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Device(_)))); + let extra = live_devices(meta, &ana).await.len(); + assert_eq!(extra, 3); + + // A wrong first code is refused and the set-up stays. + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 2).await, Err(MetaError::TwoFactorInvalid))); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap()); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let before = gen_of(meta, "ana").await; + let revoked = meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 3).await.unwrap(); + assert_eq!(revoked.len(), 2, "every other device is signed out"); + assert_eq!(live_devices(meta, &ana).await, vec![first]); + let devices = meta.list_devices(ana.id).await.unwrap(); + assert!(devices.iter().filter(|d| d.revoked).all(|d| d.revoke_reason == Some(RevokeReason::TwoFactor))); + assert_revoke_columns_agree(&devices); + assert_eq!(gen_of(meta, "ana").await, before + 1, "what a sign-in checked before no longer counts"); + assert!(!meta.revoke_invite("ana").await.unwrap(), "the invite is gone"); + assert!(matches!(meta.redeem(&pairing.hash(), "x", &token(), None, SETUP_AT + 4).await, Err(MetaError::InvalidCode)), "pairings are deleted"); + let info = meta.two_factor_info(ana.id, SETUP_AT + 5).await.unwrap(); + assert_eq!((info.enabled_ms, info.recovery_codes_left, info.pending), (Some(SETUP_AT + 3), 10, false)); + assert!(meta.two_factor_enabled(ana.id).await.unwrap()); + assert!(meta.account(ana.id).await.unwrap().two_factor_enabled); + // It is on: a set-up is a conflict, and so is a second confirmation. + assert!(matches!(meta.two_factor_begin_setup(&actor, b"other", SETUP_AT + 6).await, Err(MetaError::Conflict(_)))); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP + 5), &hashes, SETUP_AT + 7).await, Err(MetaError::Conflict(_)))); +} + +async fn a_set_up_belongs_to_one_device_expires_and_takes_five_tries(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, second) = (device(meta, &ana).await, device(meta, &ana).await); + let actor = |device| DeviceActor { user: ana.id, device, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor(first), b"one", SETUP_AT).await.unwrap(); + // Open on the first device: the second may not start, nor confirm (and its code is not tried at all). + assert!(matches!(meta.two_factor_begin_setup(&actor(second), b"two", SETUP_AT + 1).await, Err(MetaError::TwoFactorPending))); + assert!(matches!(meta.two_factor_confirm(ana.id, second, &*right(FIRST_STEP), &[], SETUP_AT + 1).await, Err(MetaError::Conflict(_)))); + // The first may start again; that replaces the secret and gives it five new tries. + for round in 0..2 { + meta.two_factor_begin_setup(&actor(first), b"again", SETUP_AT + 2).await.unwrap(); + for _ in 0..4 { + assert!( + matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 3).await, Err(MetaError::TwoFactorInvalid)), + "round {round}" + ); + } + assert!(meta.two_factor_info(ana.id, SETUP_AT + 3).await.unwrap().pending, "four wrong codes leave it open"); + } + // The fifth wrong code uses it up: even the right code then is a conflict, and the set-up has to start again. + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &[], SETUP_AT + 3).await, Err(MetaError::TwoFactorInvalid))); + assert!(!meta.two_factor_info(ana.id, SETUP_AT + 3).await.unwrap().pending); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &[], SETUP_AT + 4).await, Err(MetaError::Conflict(_)))); + // After the ten minutes it ran out; another device may start one. + meta.two_factor_begin_setup(&actor(first), b"fresh", SETUP_AT + 5).await.unwrap(); + assert!(matches!( + meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &[], SETUP_AT + 5 + 10 * MINUTE).await, + Err(MetaError::Conflict(_)) + )); + assert!(!meta.two_factor_info(ana.id, SETUP_AT + 5 + 10 * MINUTE).await.unwrap().pending); + meta.two_factor_begin_setup(&actor(second), b"second's", SETUP_AT + 6 + 10 * MINUTE).await.unwrap(); + assert!(!meta.two_factor_enabled(ana.id).await.unwrap(), "nothing was ever turned on"); +} + +/// Set-up, confirm, disable and new recovery codes check, under the account's lock, that the device is still signed in and +/// that the password that was verified is still the account's. +async fn changes_of_the_settings_recheck_the_device_and_the_password(meta: &Meta) { + let me = protected(meta, "ana").await; + let revoked_actor = DeviceActor { device: me.second, ..me.actor() }; + let stale_actor = DeviceActor { password_hash: "$argon2id$other".to_owned(), ..me.actor() }; + let hashes = me.hashes(); + for (name, actor, expect_gone) in [("revoked device", &revoked_actor, true), ("changed password", &stale_actor, false)] { + let results = [ + meta.two_factor_begin_setup(actor, b"x", SETUP_AT + 10).await.map(|_| ()), + meta.two_factor_disable(actor, &Proof::Recovery(hashes[0]), SETUP_AT + 10).await.map(|_| ()), + meta.two_factor_new_recovery_codes(actor, &totp(&right(FIRST_STEP + 50)), &hashes, SETUP_AT + 10).await, + ]; + for result in results { + if expect_gone { + assert!(matches!(result, Err(MetaError::DeviceRevoked)), "{name}: {result:?}"); + } else { + assert!(matches!(result, Err(MetaError::Conflict(_))), "{name}: {result:?}"); + } + } + } + // Confirming from a device that was signed out is refused too. + assert!(matches!(meta.two_factor_confirm(me.user.id, me.second, &*right(FIRST_STEP), &[], SETUP_AT + 11).await, Err(MetaError::DeviceRevoked))); + // Nothing changed: it is still on, with all ten codes, and no proof was spent. + let info = meta.two_factor_info(me.user.id, SETUP_AT + 12).await.unwrap(); + assert_eq!((info.enabled_ms.is_some(), info.recovery_codes_left), (true, 10)); + // A disabled account's device is gone as well. + meta.reset_password(me.user.id, "$argon2id$stub", false, SETUP_AT + 13).await.unwrap(); + assert!(matches!(meta.two_factor_begin_setup(&me.actor(), b"x", SETUP_AT + 14).await, Err(MetaError::DeviceRevoked))); +} + +async fn two_confirmations_at_once_turn_it_on_once(meta: &Meta) { + for round in 0..8 { + let ana = user(meta, &format!("ana{round}")).await; + let first = device(meta, &ana).await; + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"secret", SETUP_AT).await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..4 { + let (meta, id) = (meta.clone(), ana.id); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + tasks.push(tokio::spawn(async move { meta.two_factor_confirm(id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 1).await })); + } + let mut turned_on = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => turned_on += 1, + Err(MetaError::Conflict(_)) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(turned_on, 1, "round {round}"); + assert_eq!(meta.two_factor_info(ana.id, SETUP_AT + 2).await.unwrap().recovery_codes_left, 10, "one set of codes, round {round}"); + } +} + +/// The step of a code is accepted once, and only a later one after it. The admin's confirmation and the sign-in share it. +async fn a_time_step_is_accepted_once_and_only_later_ones_after_it(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let at = SETUP_AT + 100; + // The first code of the set-up was step 1000: it can't be used again, nor an earlier step. + for step in [FIRST_STEP, FIRST_STEP - 1, 0] { + assert!(matches!(confirm_with(meta, id, Some(step), at).await, Err(MetaError::TwoFactorInvalid)), "step {step}"); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), at).await, Ok(true))); + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), at).await, Err(MetaError::TwoFactorInvalid)), "the same step twice"); + assert!(matches!(confirm_with(meta, id, None, at).await, Err(MetaError::TwoFactorInvalid))); + // Eight requests with one code at once: exactly one gets through. + for round in 0..6 { + let step = FIRST_STEP + 10 + round; + let mut tasks = Vec::new(); + for _ in 0..8 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, Some(step), at).await })); + } + let mut accepted = 0; + for task in tasks { + match task.await.unwrap() { + Ok(true) => accepted += 1, + Err(MetaError::TwoFactorInvalid | MetaError::TwoFactorLocked { .. }) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(accepted, 1, "round {round}: one time step, one use"); + } + // An account without two-factor sign-in asks for nothing, and a missing proof for one with it is refused unspent. + let bob = user(meta, "bob").await; + assert!(matches!(meta.confirm_second_factor(bob.id, None, at).await, Ok(false))); + assert!(matches!(confirm_with(meta, bob.id, Some(1), at).await, Ok(false))); + assert!(matches!(meta.confirm_second_factor(id, None, at).await, Err(MetaError::TwoFactorInvalid))); +} + +async fn recovery_codes_work_once_and_new_ones_replace_them(meta: &Meta) { + let me = protected(meta, "ana").await; + let (id, hashes) = (me.user.id, me.hashes()); + let at = SETUP_AT + 100; + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[0])), at).await, Ok(true))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[0])), at).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(code(INVITE_CHARS).hash())), at).await, Err(MetaError::TwoFactorInvalid))); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 9); + // The same code at the same moment: one wins. + let mut tasks = Vec::new(); + for _ in 0..6 { + let (meta, hash) = (meta.clone(), hashes[1]); + tasks.push(tokio::spawn(async move { meta.confirm_second_factor(id, Some(&Proof::Recovery(hash)), at).await })); + } + let mut accepted = 0; + for task in tasks { + if matches!(task.await.unwrap(), Ok(true)) { + accepted += 1; + } + } + assert_eq!(accepted, 1); + // New codes: the old ones, used or not, are dead, and the new ones all live. + let new: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 1)), &new, at).await.unwrap(); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 10); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(hashes[5])), at).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(new[3])), at).await, Ok(true))); + // A repeated code can't be stored (the primary key), and then nothing changes: the old set is still there. + let again = vec![new[0]; 2]; + assert!(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 2)), &again, at).await.is_err()); + assert_eq!(meta.two_factor_info(id, at).await.unwrap().recovery_codes_left, 9); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(new[7])), at).await, Ok(true))); + // A wrong proof for new codes changes none. + assert!(matches!(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&wrong()), &new, at).await, Err(MetaError::TwoFactorInvalid))); +} + +/// Ten wrong proofs in a row lock the second factor for 15 minutes; each wrong one after that doubles it, to 24 hours; a +/// right one resets all of it. While locked even a right proof is refused and nothing is counted. +async fn wrong_proofs_lock_for_a_while_double_and_reset_on_success(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let mut now = SETUP_AT + 100; + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}"); + } + // The tenth locks. + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1_000).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 15 * MINUTE - 1_000), "{locked:?}"); + // Every path refuses while it lasts: recovery codes, disable, new codes, a challenge. + let actor = me.actor(); + assert!(matches!(meta.confirm_second_factor(id, Some(&Proof::Recovery(me.hashes()[0])), now + 2).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(matches!(meta.two_factor_disable(&actor, &Proof::Recovery(me.hashes()[1]), now + 2).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(matches!( + meta.two_factor_new_recovery_codes(&actor, &totp(&right(5)), &me.hashes(), now + 2).await, + Err(MetaError::TwoFactorLocked { .. }) + )); + let generation = gen_of(meta, "ana").await; + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, now + 2).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[2]), &token(), now + 3).await, + Err(MetaError::TwoFactorLocked { .. }) + )); + assert_eq!(meta.two_factor_info(id, now + 2).await.unwrap().recovery_codes_left, 10, "a refused check uses nothing up"); + // Over, a wrong proof locks again for twice as long: 30 minutes, then 60. + now += 15 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 30 * MINUTE - 1), "{locked:?}"); + now += 30 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!( + confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await, + Err(MetaError::TwoFactorLocked { retry_after_ms }) if retry_after_ms == 60 * MINUTE - 1 + )); + // A right proof after it ran out resets the counters: nine more wrong ones do not lock. + now += 60 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), now).await, Ok(true))); + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "after the reset, wrong proof {n}"); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 2), now).await, Ok(true)), "nine are not ten"); + // The command line lifts any lock. + for _ in 0..10 { + let _ = confirm_with(meta, id, None, now).await; + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 3), now + 1).await, Err(MetaError::TwoFactorLocked { .. }))); + meta.reset_two_factor(id, now + 2).await.unwrap(); + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 3), now + 3).await, Ok(false)), "off, and nothing is locked"); +} + +/// The counters of wrong proofs are written outside the transaction that refused them, so requests at the same moment all +/// count: twenty wrong proofs lock it. +async fn wrong_proofs_at_the_same_moment_all_count(meta: &Meta) { + let me = protected(meta, "ana").await; + let (id, now) = (me.user.id, SETUP_AT + 100); + let mut tasks = Vec::new(); + for _ in 0..12 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, None, now).await })); + } + for task in tasks { + assert!(matches!(task.await.unwrap(), Err(MetaError::TwoFactorInvalid | MetaError::TwoFactorLocked { .. }))); + } + assert!(matches!(confirm_with(meta, id, Some(FIRST_STEP + 1), now + 1).await, Err(MetaError::TwoFactorLocked { .. }))); +} + +async fn a_challenge_is_made_counted_used_once_and_carries_the_device(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let generation = gen_of(meta, "ana").await; + let client = DeviceClient::new(Some("desktop"), Some("linux"), Some("1.2")).unwrap(); + let chal = new_challenge(); + let made = meta.sign_in(id, ("$argon2id$stub", generation), "work laptop", &client, &token(), &chal, SETUP_AT + 10).await.unwrap(); + assert_eq!(made, Some(SignIn::Challenge { expires_ms: SETUP_AT + 10 + 5 * MINUTE })); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "no device without the second factor"); + // The old call, for a caller that doesn't know, refuses an account that has two-factor sign-in. + assert!(matches!( + meta.sign_in_device(id, ("$argon2id$stub", generation), "x", &client, &token(), SETUP_AT + 10).await, + Err(MetaError::TwoFactorRequired) + )); + assert_eq!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), Some((id, "ana".to_owned()))); + // Four wrong codes are counted, the challenge stays, and the right one then works (a fifth wrong would delete it). + for _ in 0..4 { + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), SETUP_AT + 12).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + } + let tok = token(); + let done = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &tok, SETUP_AT + 13).await.unwrap(); + assert_eq!(done.user, id); + let auth = meta.device_by_token(&tok).await.unwrap().unwrap(); + assert_eq!(auth.device, done.device); + let made = meta.list_devices(id).await.unwrap().into_iter().find(|d| d.id == done.device).unwrap(); + assert_eq!((made.name.as_str(), made.client), ("work laptop", client.clone()), "the device is what the first call said"); + // Once: the same challenge again is gone, for a right proof as for a wrong one. + for proof in [totp(&right(FIRST_STEP + 2)), totp(&wrong())] { + assert!(matches!(meta.complete_second_factor(&chal.id_hash, &proof, &token(), SETUP_AT + 14).await, Err(MetaError::ChallengeGone))); + } + assert!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 14).await.unwrap().is_none()); + + // Five wrong codes delete the challenge: the right one is too late. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 20).await.unwrap(); + for _ in 0..5 { + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), SETUP_AT + 21).await, + Err(MetaError::TwoFactorInvalid) + )); + } + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 22).await, + Err(MetaError::ChallengeGone) + )); + // One that ran out is as gone, and an unknown one. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 30).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 30 + 5 * MINUTE).await, + Err(MetaError::ChallengeGone) + )); + assert!(matches!( + meta.complete_second_factor(&token(), &totp(&right(FIRST_STEP + 3)), &token(), SETUP_AT + 30).await, + Err(MetaError::ChallengeGone) + )); + // The recovery code finishes a sign-in too, once. + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 40).await.unwrap(); + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[4]), &token(), SETUP_AT + 41).await.unwrap(); + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, SETUP_AT + 42).await.unwrap(); + assert!(matches!( + meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[4]), &token(), SETUP_AT + 43).await, + Err(MetaError::TwoFactorInvalid) + )); +} + +/// Each account keeps at most five open challenges (the oldest goes), and expired ones are deleted by the next sign-in. +async fn an_account_keeps_five_open_challenges_and_expired_ones_go(meta: &Meta) { + let me = protected(meta, "ana").await; + let generation = gen_of(meta, "ana").await; + let mut made = Vec::new(); + for n in 0..7 { + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10 + n).await.unwrap(); + made.push(chal); + } + for (n, chal) in made.iter().enumerate() { + let open = meta.challenge_owner(&chal.id_hash, SETUP_AT + 20).await.unwrap().is_some(); + assert_eq!(open, n >= 2, "challenge {n}: the two oldest were replaced"); + } + // The next sign-in after they ran out deletes them (nothing else would). + let later = SETUP_AT + 10 + 5 * MINUTE + 100; + let fresh = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &fresh, later).await.unwrap(); + assert!(meta.challenge_owner(&fresh.id_hash, later).await.unwrap().is_some()); + // A stale password hash makes neither a challenge nor a device. + let stale = new_challenge(); + assert!( + meta.sign_in(me.user.id, ("$argon2id$old", generation), "pc", &DeviceClient::default(), &token(), &stale, later).await.unwrap().is_none() + ); + assert!( + meta.sign_in(me.user.id, ("$argon2id$stub", generation - 1), "pc", &DeviceClient::default(), &token(), &stale, later) + .await + .unwrap() + .is_none() + ); + assert!(meta.challenge_owner(&stale.id_hash, later).await.unwrap().is_none()); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first]); +} + +/// A challenge made before a password change, a reset, a turn-off, a reset of the second factor or a disable gives nothing after. +async fn a_challenge_dies_with_every_change_of_the_account(meta: &Meta) { + let (_, actor) = admin_actor(meta, "root").await; + for round in 0..6 { + let me = protected(meta, &format!("ana{round}")).await; + let generation = gen_of(meta, &format!("ana{round}")).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + let at = SETUP_AT + 11; + match round { + 0 => drop(meta.change_password(me.user.id, "$argon2id$stub", "$argon2id$new", me.first, at).await.unwrap()), + 1 => drop(meta.reset_password(me.user.id, "$argon2id$new", true, at).await.unwrap()), + 2 => drop(meta.two_factor_disable(&me.actor(), &Proof::Recovery(me.hashes()[0]), at).await.unwrap()), + 3 => drop(meta.reset_two_factor(me.user.id, at).await.unwrap()), + 4 => drop(meta.admin_disable_user(&actor, me.user.id, at).await.unwrap()), + _ => drop(meta.admin_reset_two_factor(&actor, me.user.id, at).await.unwrap()), + } + assert!(meta.challenge_owner(&chal.id_hash, at).await.unwrap().is_none(), "round {round}"); + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[1]), &token(), at + 1).await; + assert!(matches!(result, Err(MetaError::ChallengeGone)), "round {round}: {result:?}"); + assert!(live_devices(meta, &me.user).await.iter().all(|d| *d == me.first), "round {round}: no new device"); + } +} + +async fn two_completions_of_one_challenge_give_one_device(meta: &Meta) { + for round in 0..8 { + let me = protected(meta, &format!("ana{round}")).await; + let generation = gen_of(meta, &format!("ana{round}")).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + let mut tasks = Vec::new(); + for job in 0..6 { + let (meta, hash, recovery) = (meta.clone(), chal.id_hash, me.hashes()[job % 2]); + tasks.push(tokio::spawn(async move { + // The same TOTP step for some, two recovery codes for the others: a code is no way past the challenge. + if job % 2 == 0 { + meta.complete_second_factor(&hash, &totp(&right(FIRST_STEP + 1)), &token(), SETUP_AT + 11).await + } else { + meta.complete_second_factor(&hash, &Proof::Recovery(recovery), &token(), SETUP_AT + 11).await + } + })); + } + let mut done = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => done += 1, + Err(MetaError::ChallengeGone | MetaError::TwoFactorInvalid) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!(done, 1, "round {round}"); + assert_eq!(live_devices(meta, &me.user).await.len(), 2, "round {round}: the first device and one more"); + } +} + +/// Completing a sign-in and a password change, a reset, a turn-off or the command line's reset at the same moment, over and +/// over: no database error, and once the change is done no device of the account but the one that asked is signed in, +/// whichever came first. +async fn completions_and_changes_of_the_account_at_the_same_moment_leave_no_stray_device(meta: &Meta) { + for round in 0..12 { + let name = format!("ana{round}"); + let me = protected(meta, &name).await; + let generation = gen_of(meta, &name).await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10).await.unwrap(); + // A password change keeps the device that asked; a reset of the second factor or of the password signs out all. + let all_gone = round % 3 != 0; + let (completing, changing) = { + let (m1, m2) = (meta.clone(), meta.clone()); + let (hash, id, first, recovery) = (chal.id_hash, me.user.id, me.first, me.hashes()); + let completing = + tokio::spawn( + async move { m1.complete_second_factor(&hash, &Proof::Recovery(recovery[0]), &token(), SETUP_AT + 11).await.map(|_| ()) }, + ); + let changing = tokio::spawn(async move { + match round % 3 { + 0 => m2.change_password(id, "$argon2id$stub", "$argon2id$new", first, SETUP_AT + 12).await.map(|_| ()), + 1 => m2.reset_two_factor(id, SETUP_AT + 12).await.map(|_| ()), + _ => m2.reset_password(id, "$argon2id$new", false, SETUP_AT + 12).await.map(|_| ()), + } + }); + (completing, changing) + }; + let (completed, changed) = (completing.await.unwrap(), changing.await.unwrap()); + assert!(matches!(completed, Ok(()) | Err(MetaError::ChallengeGone)), "round {round}: {completed:?}"); + changed.unwrap_or_else(|e| panic!("round {round}: {e:?}")); + let live = live_devices(meta, &me.user).await; + if all_gone { + assert!(live.is_empty(), "round {round}: {live:?}"); + } else { + assert!(live.iter().all(|d| *d == me.first), "round {round}: {live:?}"); + } + assert_revoke_columns_agree(&meta.list_devices(me.user.id).await.unwrap()); + } +} + +/// Turning two-factor sign-in on or off while a password sign-in is under way: a device made without a second factor +/// before it is signed out by the turn-on, and one checked before it makes no device at all. +async fn turning_two_factor_on_and_off_at_the_same_moment_as_a_sign_in(meta: &Meta) { + for round in 0..8 { + let name = format!("on{round}"); + let ana = user(meta, &name).await; + let first = device(meta, &ana).await; + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"cipher", SETUP_AT).await.unwrap(); + let generation = gen_of(meta, &name).await; + let (m1, m2) = (meta.clone(), meta.clone()); + let id = ana.id; + let signing = tokio::spawn(async move { + m1.sign_in(id, ("$argon2id$stub", generation), "late", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 5).await + }); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let enabling = tokio::spawn(async move { m2.two_factor_confirm(id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 5).await }); + let (signed, enabled) = (signing.await.unwrap(), enabling.await.unwrap()); + enabled.unwrap(); + assert!(matches!(signed, Ok(None | Some(SignIn::Device(_)))), "round {round}: a sign-in that checked before the turn-on makes no challenge"); + assert_eq!(live_devices(meta, &ana).await, vec![first], "round {round}: only the device that turned it on"); + + // And off: a challenge made under it works once, but a device it made is signed out by the turn-off. + let generation = gen_of(meta, &name).await; + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "x", &DeviceClient::default(), &token(), &chal, SETUP_AT + 6).await.unwrap(); + let (m1, m2) = (meta.clone(), meta.clone()); + let (hash, other_code) = (chal.id_hash, code(INVITE_CHARS).hash()); + let completing = + tokio::spawn(async move { m1.complete_second_factor(&hash, &Proof::Recovery(other_code), &token(), SETUP_AT + 7).await.map(|_| ()) }); + let actor = DeviceActor { user: id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let disabling = tokio::spawn(async move { m2.two_factor_disable(&actor, &totp(&right(FIRST_STEP + 1)), SETUP_AT + 7).await.map(|_| ()) }); + let completed = completing.await.unwrap(); + disabling.await.unwrap().unwrap(); + assert!(matches!(completed, Err(MetaError::TwoFactorInvalid | MetaError::ChallengeGone)), "round {round}: {completed:?}"); + assert_eq!(live_devices(meta, &ana).await, vec![first]); + } +} + +async fn an_invite_to_a_protected_account_needs_the_second_factor_in_the_same_transaction(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), SETUP_AT + 10, INVITE_TTL_MS).await.unwrap(); + // No second factor: refused and the invite is kept (not used up), and nothing is counted. + assert!(matches!(redeem_with_factor(meta, &invite.hash(), None, SETUP_AT + 11).await, Err(MetaError::TwoFactorRequired))); + let old_call = meta.redeem_with(&invite.hash(), "x", &DeviceClient::default(), &token(), None, SETUP_AT + 11).await; + assert!(matches!(old_call, Err(MetaError::TwoFactorRequired))); + // Wrong factors: refused, the invite stays, and nine of them leave the counters just short of the lock. + for n in 1..=9 { + let result = redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), SETUP_AT + 12).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "wrong factor {n}: {result:?}"); + } + assert_eq!(meta.invite_needs_password(&invite.hash(), SETUP_AT + 12).await.unwrap(), Some(false), "the invite is still there"); + // The tenth locks, with the invite still there, and a right one is refused while it lasts. + assert!(matches!(redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), SETUP_AT + 12).await, Err(MetaError::TwoFactorInvalid))); + let locked = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP + 1))), SETUP_AT + 13).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + // A step that was used is wrong, even after the lock. + let after = SETUP_AT + 12 + 15 * MINUTE + 1; + let replay = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP))), after).await; + assert!(matches!(replay, Err(MetaError::TwoFactorInvalid)), "the set-up's own step: {replay:?}"); + assert_eq!(meta.invite_needs_password(&invite.hash(), after).await.unwrap(), Some(false), "still there"); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "no device was made by any of it"); + let _ = id; + + // A fresh account: a new step is right, and the invite, the step and the device go together. + let bea = protected(meta, "bea").await; + let invite = code(INVITE_CHARS); + meta.create_invite("bea", &invite.hash(), SETUP_AT + 10, INVITE_TTL_MS).await.unwrap(); + let device = redeem_with_factor(meta, &invite.hash(), Some(&totp(&right(FIRST_STEP + 1))), SETUP_AT + 20).await.unwrap(); + assert_eq!(live_devices(meta, &bea.user).await.len(), 2); + assert!(live_devices(meta, &bea.user).await.contains(&device)); + assert_eq!(meta.invite_needs_password(&invite.hash(), SETUP_AT + 21).await.unwrap(), None, "used up with the device"); + // Nobody else can use that step now (sign-in and join share the counter). + assert!(matches!(confirm_with(meta, bea.user.id, Some(FIRST_STEP + 1), SETUP_AT + 22).await, Err(MetaError::TwoFactorInvalid))); + // A pairing needs no second factor: only a signed-in device can make one. + let pairing = code(PAIRING_CHARS); + meta.create_pairing(bea.user.id, bea.first, &pairing.hash(), SETUP_AT + 30).await.unwrap(); + meta.redeem_with(&pairing.hash(), "paired", &DeviceClient::default(), &token(), None, SETUP_AT + 31).await.unwrap(); +} + +async fn a_reset_code_for_a_protected_account_needs_the_factor_unless_it_turns_it_off(meta: &Meta) { + let (_, admin) = admin_actor(meta, "root").await; + let admin = AdminActor { two_factor: false, ..admin }; + let me = protected(meta, "ana").await; + let (id, stale) = (me.user.id, me.hashes()); + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), false, SETUP_AT + 10, RESET_TTL_MS).await.unwrap(); + assert!(matches!( + meta.redeem_with(&reset.hash(), "pc", &DeviceClient::default(), &token(), Some("$argon2id$fresh"), SETUP_AT + 11).await, + Err(MetaError::TwoFactorRequired) + )); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$stub", "nothing changed"); + assert!(matches!( + meta.redeem_with_second_factor( + &reset.hash(), + "pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$fresh"), + Some(&totp(&wrong())), + SETUP_AT + 11 + ) + .await, + Err(MetaError::TwoFactorInvalid) + )); + assert_eq!(live_devices(meta, &me.user).await, vec![me.first], "the failed redeem signed nothing out"); + let first_done = meta + .redeem_with_second_factor( + &reset.hash(), + "pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$fresh"), + Some(&Proof::Recovery(stale[0])), + SETUP_AT + 12, + ) + .await + .unwrap(); + let done = &first_done; + assert_eq!(done.revoked, vec![me.first]); + assert!(meta.two_factor_enabled(id).await.unwrap(), "the code did not turn two-factor sign-in off"); + assert_eq!(meta.user_by_name("ana").await.unwrap().unwrap().password_hash, "$argon2id$fresh"); + + // A code made to turn it off asks for no second factor, turns it off and signs everything out. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), true, SETUP_AT + 20, RESET_TTL_MS).await.unwrap(); + let done = meta.redeem_with(&reset.hash(), "pc2", &DeviceClient::default(), &token(), Some("$argon2id$newer"), SETUP_AT + 21).await.unwrap(); + assert_eq!(done.revoked, vec![first_done.device], "the computer of the first redeem is signed out with the rest"); + assert!(!meta.two_factor_enabled(id).await.unwrap()); + assert_eq!(meta.two_factor_info(id, SETUP_AT + 22).await.unwrap().recovery_codes_left, 0); + assert_eq!(live_devices(meta, &me.user).await, vec![done.device], "the redeeming computer is the only one signed in"); + // An account without two-factor sign-in redeems a reset code as it always did, whatever the flag. + let plain = user(meta, "plain").await; + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, plain.id, &reset.hash(), false, SETUP_AT + 30, RESET_TTL_MS).await.unwrap(); + meta.redeem_with(&reset.hash(), "pc", &DeviceClient::default(), &token(), Some("$argon2id$fresh"), SETUP_AT + 31).await.unwrap(); +} + +/// The admin's own two-factor state is compared again in every change, and resetting someone's two-factor sign-in, or another +/// admin's password, needs the acting admin's own. +async fn admins_two_factor_is_checked_in_the_change_and_resets_need_it(meta: &Meta) { + let (root, plain_actor) = admin_actor(meta, "root").await; + let (other, _) = admin_actor(meta, "other").await; + let member = protected(meta, "member").await; + let (boss_admin, boss_actor) = { + let (u, a) = admin_actor(meta, "boss").await; + (u, a) + }; + // Turn two-factor sign-in on for the acting admin "boss" (device `device_of(&boss_actor)`). + let actor_for = DeviceActor { user: boss_admin.id, device: device_of(&boss_actor), password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&actor_for, b"cipher", SETUP_AT).await.unwrap(); + meta.two_factor_confirm(boss_admin.id, device_of(&boss_actor), &*right(FIRST_STEP), &[code(INVITE_CHARS).hash()], SETUP_AT + 1).await.unwrap(); + // A confirmation that was given when it was off, and a change after it turned on, conflict (and the other way round). + let hash = code(INVITE_CHARS).hash(); + let result = meta.admin_issue_invite(&boss_actor, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "{result:?}"); + let with_factor = AdminActor { two_factor: true, ..boss_actor.clone() }; + let result = meta.admin_issue_invite(&with_factor, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "it has a password: {result:?}"); + let result = + meta.admin_issue_invite(&AdminActor { two_factor: true, ..plain_actor.clone() }, member.user.id, &hash, SETUP_AT + 2, INVITE_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "root has none: {result:?}"); + + // Resetting a member's two-factor sign-in: without the admin's own it works for a member, not for an admin. + let result = meta.admin_reset_two_factor(&plain_actor, other.id, SETUP_AT + 3).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "another admin, no two-factor of one's own: {result:?}"); + let result = meta.admin_issue_reset(&plain_actor, other.id, &hash, false, SETUP_AT + 3, RESET_TTL_MS).await; + assert!(matches!(result, Err(MetaError::Forbidden(_))), "{result:?}"); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, root.id, SETUP_AT + 3).await, Err(MetaError::Invalid(_))), "not one's own"); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, UserId(uuid::Uuid::now_v7()), SETUP_AT + 3).await, Err(MetaError::NotFound(_)))); + // An admin with it may reset another admin's password, and that admin's two-factor sign-in (once they have one). + meta.admin_issue_reset(&with_factor, other.id, &hash, false, SETUP_AT + 4, RESET_TTL_MS).await.unwrap(); + let result = meta.admin_reset_two_factor(&with_factor, other.id, SETUP_AT + 4).await; + assert!(matches!(result, Err(MetaError::Conflict(_))), "the other admin has none: {result:?}"); + // A member's: everything of it goes, every device is signed out, the lock goes with it. + let revoked = meta.admin_reset_two_factor(&plain_actor, member.user.id, SETUP_AT + 5).await.unwrap(); + assert_eq!(revoked, vec![member.first]); + assert!(!meta.two_factor_enabled(member.user.id).await.unwrap()); + assert!(live_devices(meta, &member.user).await.is_empty()); + let devices = meta.list_devices(member.user.id).await.unwrap(); + assert!(devices.iter().any(|d| d.revoke_reason == Some(RevokeReason::TwoFactor))); + assert!(matches!(meta.admin_reset_two_factor(&plain_actor, member.user.id, SETUP_AT + 6).await, Err(MetaError::Conflict(_))), "twice"); + let row = admin_view(meta, boss_admin.id).await; + assert!(row.two_factor_enabled); + assert!(!admin_view(meta, member.user.id).await.two_factor_enabled); +} + +async fn the_command_line_resets_two_factor_and_the_account_shows_it(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + assert!(meta.account(id).await.unwrap().two_factor_enabled); + assert!(admin_view(meta, id).await.two_factor_enabled); + let generation = gen_of(meta, "ana").await; + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 5).await.unwrap(); + meta.create_invite("ana", &code(INVITE_CHARS).hash(), SETUP_AT + 5, INVITE_TTL_MS).await.unwrap(); + let revoked = meta.reset_two_factor(id, SETUP_AT + 6).await.unwrap(); + assert_eq!(revoked, vec![me.first]); + assert!(!meta.account(id).await.unwrap().two_factor_enabled); + assert!(!admin_view(meta, id).await.two_factor_enabled); + assert!(!meta.revoke_invite("ana").await.unwrap()); + assert!(matches!(meta.reset_two_factor(id, SETUP_AT + 7).await, Err(MetaError::Conflict(_))), "nothing to reset: nothing is signed out"); + assert!(matches!(meta.reset_two_factor(UserId(uuid::Uuid::now_v7()), SETUP_AT + 7).await, Err(MetaError::NotFound(_)))); + // After it a password sign-in gives a device again, and the account can set it up anew. + let generation = gen_of(meta, "ana").await; + let signed = + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &new_challenge(), SETUP_AT + 8).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Device(_)))); +} + +/// The key tools read the ciphertexts and replace one only if it is still the one they read. +async fn secrets_are_listed_and_replaced_only_if_unchanged(meta: &Meta) { + let me = protected(meta, "ana").await; + let other = user(meta, "bob").await; + let dev = device(meta, &other).await; + let actor = DeviceActor { user: other.id, device: dev, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&actor, b"cipher-pending", SETUP_AT).await.unwrap(); + let listed = meta.two_factor_secrets(SETUP_AT + 1).await.unwrap(); + assert_eq!(listed.len(), 2); + let ana = listed.iter().find(|s| s.user == me.user.id).unwrap(); + assert_eq!((ana.active.as_deref(), ana.pending.as_deref()), (Some(b"cipher-of-the-secret".as_slice()), None)); + let bob = listed.iter().find(|s| s.user == other.id).unwrap(); + assert_eq!((bob.active.as_deref(), bob.pending.as_deref()), (None, Some(b"cipher-pending".as_slice()))); + let later = meta.two_factor_secrets(SETUP_AT + 11 * MINUTE).await.unwrap(); + assert!(later.iter().find(|s| s.user == other.id).is_some_and(|s| s.pending.is_none()), "an expired set-up is not listed"); + assert!(meta.two_factor_replace_ciphertext(me.user.id, false, b"cipher-of-the-secret", b"new-cipher").await.unwrap()); + assert!( + !meta.two_factor_replace_ciphertext(me.user.id, false, b"cipher-of-the-secret", b"newer").await.unwrap(), + "no longer the one that was read" + ); + assert!(meta.two_factor_replace_ciphertext(other.id, true, b"cipher-pending", b"new-pending").await.unwrap()); + let listed = meta.two_factor_secrets(SETUP_AT + 1).await.unwrap(); + assert_eq!(listed.iter().find(|s| s.user == me.user.id).unwrap().active.as_deref(), Some(b"new-cipher".as_slice())); + // A secret that isn't the account's is never touched. + assert!(!meta.two_factor_replace_ciphertext(UserId(uuid::Uuid::now_v7()), false, b"x", b"y").await.unwrap()); +} + +/// Wrong proofs are counted under the lock that checked them: of many at once on one challenge exactly five are counted (the +/// fifth deletes it, the rest find it gone), and they all count against the account's ten. +async fn parallel_wrong_proofs_on_one_challenge_count_exactly_five(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let generation = gen_of(meta, "ana").await; + for round in 0..4 { + let chal = new_challenge(); + meta.sign_in(id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, SETUP_AT + 10 + round).await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..14 { + let (meta, hash) = (meta.clone(), chal.id_hash); + tasks.push(tokio::spawn(async move { meta.complete_second_factor(&hash, &totp(&wrong()), &token(), SETUP_AT + 20).await.map(|_| ()) })); + } + let (mut counted, mut gone) = (0, 0); + for task in tasks { + match task.await.unwrap() { + Err(MetaError::TwoFactorInvalid) => counted += 1, + Err(MetaError::ChallengeGone) => gone += 1, + Err(MetaError::TwoFactorLocked { .. }) => {} + other => panic!("round {round}: {other:?}"), + } + } + assert!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 21).await.unwrap().is_none(), "round {round}: five wrong proofs delete it"); + // Locked rounds stop counting; until then exactly five per challenge, never six. + if round == 0 { + assert_eq!((counted, gone), (5, 9), "round {round}"); + } + assert!(counted <= 5, "round {round}: {counted} proofs were checked against one challenge"); + if meta.two_factor_locked_for(id, SETUP_AT + 21).await.unwrap().is_some() { + break; + } + } +} + +/// Many wrong proofs at once, crossing ten: the lock is set by exactly the tenth, for exactly 15 minutes, and the ones after it +/// find it locked and count nothing (so it never jumps to 30). +async fn parallel_wrong_proofs_lock_for_exactly_fifteen_minutes(meta: &Meta) { + for round in 0..6 { + let me = protected(meta, &format!("ana{round}")).await; + let (id, now) = (me.user.id, SETUP_AT + 100); + let mut tasks = Vec::new(); + for _ in 0..24 { + let meta = meta.clone(); + tasks.push(tokio::spawn(async move { confirm_with(&meta, id, None, now).await })); + } + let (mut wrong_ones, mut locked) = (0, 0); + for task in tasks { + match task.await.unwrap() { + Err(MetaError::TwoFactorInvalid) => wrong_ones += 1, + Err(MetaError::TwoFactorLocked { .. }) => locked += 1, + other => panic!("round {round}: {other:?}"), + } + } + assert_eq!((wrong_ones, locked), (10, 14), "round {round}"); + assert_eq!(meta.two_factor_locked_for(id, now).await.unwrap(), Some(15 * MINUTE), "round {round}: 15 minutes, not 30"); + } +} + +// ---- the audit log of two-factor sign-in ---- + +/// The events of `action` in the log, oldest first (an aggregated row counts as one). +async fn events_of(meta: &Meta, action: AuditAction) -> Vec { + let mut events: Vec = all_events(meta).await.into_iter().filter(|e| e.action == action).collect(); + events.reverse(); + events +} + +/// Turning it on, new recovery codes and turning it off each write their event in their own transaction, as the account's own +/// act from the device that asked; a wrong proof on these routes writes nothing, and a recovery code that is used up writes +/// `auth.recovery_code_used` with the change it served. +async fn two_factor_changes_write_their_events(meta: &Meta) { + let ana = user(meta, "ana").await; + let (first, _second) = (device(meta, &ana).await, device(meta, &ana).await); + let actor = DeviceActor { user: ana.id, device: first, password_hash: "$argon2id$stub".to_owned() }; + let mut seen = Seen::new(meta).await; + let hashes: Vec<[u8; 32]> = (0..10).map(|_| code(INVITE_CHARS).hash()).collect(); + + meta.two_factor_begin_setup(&actor, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + assert!(seen.actions().await.is_empty(), "a set-up is not a change"); + assert!(matches!(meta.two_factor_confirm(ana.id, first, &*wrong(), &hashes, SETUP_AT + 1).await, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "a wrong first code writes nothing"); + meta.two_factor_confirm(ana.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 2).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.at_ms), + (AuditAction::AccountTwoFactorEnabled, AuditOutcome::Success, Some(ana.id), Some(first), Some(ana.id.0), SETUP_AT + 2) + ); + assert_eq!(e.target_kind.as_str(), "user"); + assert_eq!(details(&e), json("{}")); + + // New recovery codes: a wrong code writes nothing (the lock it can cause would), a right one the event. + let result = meta.two_factor_new_recovery_codes(&actor, &totp(&wrong()), &hashes, SETUP_AT + 3).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "a wrong proof on a signed-in route is not written one by one"); + meta.two_factor_new_recovery_codes(&actor, &totp(&right(FIRST_STEP + 1)), &hashes, SETUP_AT + 4).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AccountRecoveryCodesRegenerated, Some(ana.id), Some(first), Some(ana.id.0)) + ); + assert_eq!(details(&e), json("{}")); + + // Turning it off with a recovery code: the change and the code's use, from the same device. + meta.two_factor_disable(&actor, &Proof::Recovery(hashes[0]), SETUP_AT + 5).await.unwrap(); + assert_eq!(seen.actions().await, ["account.two_factor_disabled", "auth.recovery_code_used"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device, used.target_id, used.at_ms), (Some(ana.id), Some(first), Some(ana.id.0), SETUP_AT + 5)); + assert_eq!(details(used), json("{}"), "never the code"); + let off = &events_of(meta, AuditAction::AccountTwoFactorDisabled).await[0]; + assert_eq!((off.actor_user, off.actor_device), (Some(ana.id), Some(first))); + // Off: nothing to turn off, nothing written. + assert!(matches!(meta.two_factor_disable(&actor, &totp(&right(FIRST_STEP + 2)), SETUP_AT + 6).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); +} + +/// A sign-in that made a device is `auth.login` whichever way it went (no second factor, a code, a recovery code); the +/// challenge writes nothing; a wrong second step is one aggregated `auth.second_factor_failed` row per account and minute, +/// written after the answer is decided. +async fn two_factor_sign_in_writes_login_and_counts_failures(meta: &Meta) { + let me = protected(meta, "ana").await; + let plain = user(meta, "bob").await; + let mut seen = Seen::new(meta).await; + let client = DeviceClient::new(Some("desktop"), Some("Windows"), Some("1.0")).unwrap(); + let now = SETUP_AT + 100; + + // No two-factor sign-in: a login at once. + let generation = gen_of(meta, "bob").await; + let signed = meta.sign_in(plain.id, ("$argon2id$stub", generation), "pc", &client, &token(), &new_challenge(), now).await.unwrap(); + let Some(SignIn::Device(bobs)) = signed else { panic!("{signed:?}") }; + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthLogin, AuditOutcome::Success, Some(plain.id), Some(bobs), Some(*bobs.as_uuid())) + ); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"windows"}"#)); + + // Two-factor sign-in: the challenge is no login. + let generation = gen_of(meta, "ana").await; + let chal = new_challenge(); + let signed = meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now).await.unwrap(); + assert!(matches!(signed, Some(SignIn::Challenge { .. }))); + assert!(seen.actions().await.is_empty(), "no one is signed in yet"); + // A wrong code: one failure row, no login, the actor unknown (nobody proved who they were). + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), now + 1).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.count), + (AuditAction::AuthSecondFactorFailed, AuditOutcome::Failure, None, None, Some(me.user.id.0), 1) + ); + assert_eq!(details(&e), json(r#"{"step":"totp"}"#)); + // More in the same minute make the same row count up (a recovery code that is wrong too). + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(code(INVITE_CHARS).hash()), &token(), now + 2).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert!(seen.actions().await.is_empty(), "the row was written already; only its count goes up"); + let rows = events_of(meta, AuditAction::AuthSecondFactorFailed).await; + assert_eq!((rows.len(), rows[0].count), (1, 2)); + // A right code: the login of the new device. + let done = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), now + 3).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthLogin, Some(me.user.id), Some(done.device), Some(*done.device.as_uuid())) + ); + assert_eq!(details(&e), json(r#"{"client_kind":"desktop","platform":"windows"}"#)); + + // A recovery code: the login and the code's use, the device the same. + let chal = new_challenge(); + let generation = gen_of(meta, "ana").await; + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now + 4).await.unwrap(); + let done = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[0]), &token(), now + 5).await.unwrap(); + assert_eq!(seen.actions().await, ["auth.login", "auth.recovery_code_used"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device, used.target_id), (Some(me.user.id), Some(done.device), Some(me.user.id.0))); + // The same code again is a wrong recovery code: a failure, not a use. + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &client, &token(), &chal, now + 6).await.unwrap(); + let result = meta.complete_second_factor(&chal.id_hash, &Proof::Recovery(me.hashes()[0]), &token(), now + 7).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid))); + assert_eq!(events_of(meta, AuditAction::AuthRecoveryCodeUsed).await.len(), 1); + assert_eq!(events_of(meta, AuditAction::AuthSecondFactorFailed).await[0].count, 3); +} + +/// The lock writes `auth.two_factor_locked` once, when it is set or lengthened, and never for a refusal while it holds; wrong +/// proofs on a signed-in route or an admin's confirmation count towards it without an event of their own. +async fn the_lock_writes_one_event_when_it_is_set(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let mut seen = Seen::new(meta).await; + let mut now = SETUP_AT + 100; + for n in 1..=9 { + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}"); + } + assert!(seen.actions().await.is_empty(), "nine wrong proofs are not written one by one"); + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id, e.at_ms), + (AuditAction::AuthTwoFactorLocked, AuditOutcome::Success, None, None, Some(id.0), now) + ); + assert_eq!(details(&e), json("{}")); + // Refused while it holds, by every path: no further event. + for offset in 1..4 { + let locked = confirm_with(meta, id, Some(FIRST_STEP + 1), now + offset).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + } + assert!(matches!(meta.two_factor_disable(&me.actor(), &Proof::Recovery(me.hashes()[1]), now + 5).await, Err(MetaError::TwoFactorLocked { .. }))); + assert!(seen.actions().await.is_empty(), "a refusal under the lock writes nothing"); + // Over, one more wrong proof sets a longer lock: a new event. + now += 15 * MINUTE + 1; + assert!(matches!(confirm_with(meta, id, None, now).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!((e.action, e.at_ms), (AuditAction::AuthTwoFactorLocked, now)); + assert_eq!(events_of(meta, AuditAction::AuthTwoFactorLocked).await.len(), 2); + assert!(events_of(meta, AuditAction::AuthSecondFactorFailed).await.is_empty(), "these are not sign-in steps"); +} + +/// The second factor at a join: a wrong one is one failure row and the invite stays; a recovery code used for it is written +/// with the redemption; a reset code made with `reset_two_factor` takes it away and the redemption says so, by the account that +/// redeemed it (no admin acts at that moment). +async fn a_join_writes_its_two_factor_events(meta: &Meta) { + let me = protected(meta, "ana").await; + let id = me.user.id; + let (_, admin) = admin_actor(meta, "root").await; + let mut seen = Seen::new(meta).await; + let now = SETUP_AT + 100; + + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + assert!(matches!(redeem_with_factor(meta, &invite.hash(), None, now + 1).await, Err(MetaError::TwoFactorRequired))); + assert!(seen.actions().await.is_empty(), "asking for the factor writes nothing"); + assert!(matches!(redeem_with_factor(meta, &invite.hash(), Some(&totp(&wrong())), now + 2).await, Err(MetaError::TwoFactorInvalid))); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.target_id, e.count), + (AuditAction::AuthSecondFactorFailed, AuditOutcome::Failure, None, Some(id.0), 1) + ); + assert_eq!(details(&e), json(r#"{"step":"totp"}"#)); + // With a recovery code: the redemption and the code's use, by the new device. + let device = redeem_with_factor(meta, &invite.hash(), Some(&Proof::Recovery(me.hashes()[0])), now + 3).await.unwrap(); + assert_eq!(seen.actions().await, ["auth.recovery_code_used", "join.invite_redeemed"]); + let used = &events_of(meta, AuditAction::AuthRecoveryCodeUsed).await[0]; + assert_eq!((used.actor_user, used.actor_device), (Some(id), Some(device))); + + // A reset code without the removal of two-factor: the right code is needed; the event has no such detail. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), false, now + 4, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let redeemed = meta + .redeem_with_second_factor( + &reset.hash(), + "new pc", + &DeviceClient::default(), + &token(), + Some("$argon2id$new"), + Some(&totp(&right(FIRST_STEP + 1))), + now + 5, + ) + .await + .unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device), (AuditAction::JoinResetRedeemed, Some(id), Some(redeemed.device))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{}}}"#, redeemed.revoked.len())), "two-factor sign-in is still on"); + assert!(meta.two_factor_enabled(id).await.unwrap()); + + // A reset code made with `reset_two_factor`: it takes the second factor away and the event says so. + let reset = code(INVITE_CHARS); + meta.admin_issue_reset(&admin, id, &reset.hash(), true, now + 6, RESET_TTL_MS).await.unwrap(); + assert_eq!(seen.actions().await, ["admin.password_reset_issued"]); + let redeemed = meta + .redeem_with_second_factor(&reset.hash(), "newer pc", &DeviceClient::default(), &token(), Some("$argon2id$newer"), None, now + 7) + .await + .unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::JoinResetRedeemed, Some(id), Some(redeemed.device), Some(*redeemed.device.as_uuid())) + ); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{},"two_factor_removed":true}}"#, redeemed.revoked.len()))); + assert!(!meta.two_factor_enabled(id).await.unwrap()); + assert!(events_of(meta, AuditAction::AdminTwoFactorReset).await.is_empty(), "no admin is made up for it"); +} + +/// An admin's reset of another account's two-factor sign-in and the command line's each write their event; the admin's +/// confirmation by recovery code writes the code's use, for the admin's account, even if the change it confirms fails. +async fn resets_and_confirmations_write_their_events(meta: &Meta) { + let (_, admin) = admin_actor(meta, "root").await; + let member = protected(meta, "member").await; + let other = protected(meta, "other").await; + let mut seen = Seen::new(meta).await; + let now = SETUP_AT + 100; + + let revoked = meta.admin_reset_two_factor(&admin, member.user.id, now).await.unwrap(); + let e = seen.one().await; + assert_eq!( + (e.action, e.outcome, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AdminTwoFactorReset, AuditOutcome::Success, Some(admin.user), Some(device_of(&admin)), Some(member.user.id.0)) + ); + assert_eq!(details(&e), json("{}")); + assert_eq!(revoked.len(), 1); + assert!(matches!(meta.admin_reset_two_factor(&admin, member.user.id, now + 1).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty(), "nothing to reset: nothing written"); + + let revoked = meta.reset_two_factor(other.user.id, now + 2).await.unwrap(); + let e = seen.one().await; + assert_eq!((e.action, e.actor_user, e.actor_device, e.target_id), (AuditAction::CliTwoFactorReset, None, None, Some(other.user.id.0))); + assert_eq!(details(&e), json(&format!(r#"{{"revoked":{}}}"#, revoked.len()))); + assert!(matches!(meta.reset_two_factor(other.user.id, now + 3).await, Err(MetaError::Conflict(_)))); + assert!(seen.actions().await.is_empty()); + + // An account's confirmation by recovery code, as an admin gives it: the code's use, no device. + let bea = protected(meta, "bea").await; + let mut seen = Seen::new(meta).await; + assert!(meta.confirm_second_factor(bea.user.id, Some(&Proof::Recovery(bea.hashes()[3])), now + 4).await.unwrap()); + let e = seen.one().await; + assert_eq!( + (e.action, e.actor_user, e.actor_device, e.target_id), + (AuditAction::AuthRecoveryCodeUsed, Some(bea.user.id), None, Some(bea.user.id.0)) + ); + // A wrong proof, or the same code again: nothing written. + assert!(matches!(confirm_with(meta, bea.user.id, None, now + 5).await, Err(MetaError::TwoFactorInvalid))); + assert!(matches!( + meta.confirm_second_factor(bea.user.id, Some(&Proof::Recovery(bea.hashes()[3])), now + 6).await, + Err(MetaError::TwoFactorInvalid) + )); + assert!(seen.actions().await.is_empty()); +} + +/// An error in the log fails every two-factor change and changes nothing; a wrong second step is still answered as wrong, since +/// its event is written after the answer is decided and only reported. +async fn an_error_in_the_log_fails_every_two_factor_change(url: &str) { + sqlx::any::install_default_drivers(); + let meta = Meta::open(url).await.unwrap(); + let me = protected(&meta, "ana").await; + let (_, admin) = admin_actor(&meta, "root").await; + let setup = user(&meta, "setup").await; + let setup_device = device(&meta, &setup).await; + let setup_actor = DeviceActor { user: setup.id, device: setup_device, password_hash: "$argon2id$stub".to_owned() }; + meta.two_factor_begin_setup(&setup_actor, b"cipher", SETUP_AT).await.unwrap(); + let plain = user(&meta, "plain").await; + let locker = protected(&meta, "locker").await; + let events_before = all_events(&meta).await.len(); + let now = SETUP_AT + 100; + let generation = gen_of(&meta, "ana").await; + let chal = new_challenge(); + meta.sign_in(me.user.id, ("$argon2id$stub", generation), "pc", &DeviceClient::default(), &token(), &chal, now).await.unwrap(); + let invite = code(INVITE_CHARS); + meta.create_invite("ana", &invite.hash(), now, INVITE_TTL_MS).await.unwrap(); + + break_audit(url).await; + let is_db = |r: Result<(), MetaError>| assert!(matches!(r, Err(MetaError::Db(_))), "{r:?}"); + let hashes = me.hashes(); + is_db(meta.two_factor_confirm(setup.id, setup_device, &*right(FIRST_STEP), &hashes, now).await.map(|_| ())); + is_db(meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 1)), &hashes, now).await); + is_db(meta.two_factor_disable(&me.actor(), &Proof::Recovery(hashes[0]), now).await.map(|_| ())); + is_db(meta.confirm_second_factor(me.user.id, Some(&Proof::Recovery(hashes[1])), now).await.map(|_| ())); + is_db(meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), now).await.map(|_| ())); + is_db(redeem_with_factor(&meta, &invite.hash(), Some(&Proof::Recovery(hashes[2])), now).await.map(|_| ())); + is_db(meta.sign_in(plain.id, ("$argon2id$stub", 0), "pc", &DeviceClient::default(), &token(), &new_challenge(), now).await.map(|_| ())); + is_db(meta.admin_reset_two_factor(&admin, me.user.id, now).await.map(|_| ())); + is_db(meta.reset_two_factor(me.user.id, now).await.map(|_| ())); + // A wrong second step: the answer is the same, whether or not the log works. + let result = meta.complete_second_factor(&chal.id_hash, &totp(&wrong()), &token(), now).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + // The lock is set by the tenth wrong proof although its event can't be written: failing closed there would let a fault in + // the log undo the lock. The answer is the same and the counters and the lock are committed. + for n in 1..=10 { + let result = confirm_with(&meta, locker.user.id, None, now).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "wrong proof {n}: {result:?}"); + } + assert_eq!(meta.two_factor_locked_for(locker.user.id, now).await.unwrap(), Some(15 * MINUTE), "the lock holds"); + repair_audit(url).await; + + // Nothing was changed by any of it: still on, no code used, nobody signed out, the set-up still open. + assert_eq!(all_events(&meta).await.len(), events_before, "no event, no change (the lock's event was lost, only reported)"); + assert!(meta.two_factor_enabled(me.user.id).await.unwrap()); + assert_eq!(meta.two_factor_info(me.user.id, now).await.unwrap().recovery_codes_left, 10); + assert_eq!(live_devices(&meta, &me.user).await, vec![me.first]); + assert!(meta.two_factor_info(setup.id, now).await.unwrap().pending); + assert!(!meta.two_factor_enabled(setup.id).await.unwrap()); + assert_eq!(meta.invite_needs_password(&invite.hash(), now).await.unwrap(), Some(false), "the invite is still there"); + // And the same changes work once the log does. + meta.two_factor_new_recovery_codes(&me.actor(), &totp(&right(FIRST_STEP + 2)), &hashes, now + 1).await.unwrap(); +} + +#[tokio::test] +async fn an_error_in_the_log_fails_every_two_factor_change_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_error_in_the_log_fails_every_two_factor_change(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_error_in_the_log_fails_every_two_factor_change_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_error_in_the_log_fails_every_two_factor_change(&url).await; +} + +// ---- web sessions (migration 0011) ---- + +const WEB_IDLE_MS: i64 = 8 * 60 * MINUTE; +const WEB_ABSOLUTE_MS: i64 = 7 * 24 * 60 * MINUTE; + +fn new_session() -> NewWebSession { + NewWebSession { id_hash: token(), absolute_ms: WEB_ABSOLUTE_MS } +} + +/// An administrator with a password hash the web sign-ins below check against. +async fn web_admin(meta: &Meta, username: &str) -> UserRecord { + meta.create_user(username, ADMIN_HASH, Role::Admin, NOW).await.unwrap() +} + +/// A web sign-in of `user` as the handler would make it after verifying the password. +async fn web_sign_in(meta: &Meta, user: &UserRecord, session: &NewWebSession, now: i64) -> Option { + let generation = gen_of(meta, &user.username).await; + meta.web_sign_in(user.id, (&user.password_hash, generation), &new_challenge(), session, now).await.unwrap() +} + +async fn session_is_valid(meta: &Meta, session: &NewWebSession, now: i64) -> bool { + meta.web_session(&session.id_hash, now, WEB_IDLE_MS).await.unwrap().is_some() +} + +/// Only administrators get a session; it is found by its token hash, ends by idle time, by its absolute lifetime and by +/// logout, and the sign-in is written to the log without a device. +async fn web_sessions_are_made_looked_up_expire_and_log_out(meta: &Meta) { + let root = web_admin(meta, "root").await; + let member = user(meta, "ana").await; + // A member with the right password gets nothing, the same as a wrong generation or a wrong hash. + assert_eq!(web_sign_in(meta, &member, &new_session(), NOW).await, None); + let stale = meta.web_sign_in(root.id, (&root.password_hash, 99), &new_challenge(), &new_session(), NOW).await.unwrap(); + assert_eq!(stale, None, "the password was checked against another generation"); + let wrong_hash = meta.web_sign_in(root.id, ("$argon2id$other", root.auth_gen), &new_challenge(), &new_session(), NOW).await.unwrap(); + assert_eq!(wrong_hash, None); + assert_eq!(meta.web_sessions_open(member.id, NOW, WEB_IDLE_MS).await.unwrap(), 0); + + let session = new_session(); + assert_eq!(web_sign_in(meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let found = meta.web_session(&session.id_hash, NOW + 1, WEB_IDLE_MS).await.unwrap().unwrap(); + assert_eq!((found.user, found.username.as_str(), found.role, found.mfa_verified_ms), (root.id, "root", Role::Admin, None)); + assert_eq!((found.created_ms, found.last_seen_ms, found.absolute_expires_ms), (NOW, NOW, NOW + WEB_ABSOLUTE_MS)); + assert!(meta.web_session(&token(), NOW + 1, WEB_IDLE_MS).await.unwrap().is_none(), "an unknown token"); + let events = all_events(meta).await; + let login = events.iter().find(|e| e.action == AuditAction::AuthLogin).unwrap(); + assert_eq!((login.actor_user, login.actor_device), (Some(root.id), None)); + assert_eq!(details(login), json(r#"{"client_kind":"web"}"#)); + assert!(DeviceClient::new(Some("web"), None, None).is_err(), "devices cannot claim web"); + + // Idle: valid just inside the idle time, over at it, and a use moves it. + assert!(session_is_valid(meta, &session, NOW + WEB_IDLE_MS - 1).await); + assert!(!session_is_valid(meta, &session, NOW + WEB_IDLE_MS).await); + meta.web_session_seen(&session.id_hash, NOW + WEB_IDLE_MS - 1).await.unwrap(); + assert!(session_is_valid(meta, &session, NOW + 2 * WEB_IDLE_MS - 2).await, "a use starts the idle time again"); + // The time never moves back. + meta.web_session_seen(&session.id_hash, NOW + 5).await.unwrap(); + assert_eq!(meta.web_session(&session.id_hash, NOW + WEB_IDLE_MS, WEB_IDLE_MS).await.unwrap().unwrap().last_seen_ms, NOW + WEB_IDLE_MS - 1); + + // Absolute: keeping it busy does not help. + let busy = new_session(); + assert_eq!(web_sign_in(meta, &root, &busy, NOW).await, Some(WebSignIn::Session)); + let mut at = NOW; + while at + WEB_IDLE_MS / 2 < NOW + WEB_ABSOLUTE_MS { + at += WEB_IDLE_MS / 2; + meta.web_session_seen(&busy.id_hash, at).await.unwrap(); + assert!(session_is_valid(meta, &busy, at).await); + } + assert!(!session_is_valid(meta, &busy, NOW + WEB_ABSOLUTE_MS).await, "the absolute lifetime ends it"); + + // Logout ends it at once, and a second logout, or one of an unknown session, is no error. + let out = new_session(); + assert_eq!(web_sign_in(meta, &root, &out, NOW + 10).await, Some(WebSignIn::Session)); + assert!(session_is_valid(meta, &out, NOW + 11).await); + meta.web_logout(&out.id_hash, NOW + 12).await.unwrap(); + meta.web_logout(&out.id_hash, NOW + 13).await.unwrap(); + meta.web_logout(&token(), NOW + 13).await.unwrap(); + assert!(!session_is_valid(meta, &out, NOW + 14).await); + meta.web_session_seen(&out.id_hash, NOW + 15).await.unwrap(); + assert!(!session_is_valid(meta, &out, NOW + 16).await, "a use does not revive a logged-out session"); +} + +/// An administrator with two-factor sign-in on, and a session made with it (the sign-in's second step). `step` is the time +/// step of the code that made the session. +async fn web_admin_with_factor(meta: &Meta, username: &str, step: i64, now: i64) -> (UserRecord, DeviceId, NewWebSession) { + let me = web_admin(meta, username).await; + let first = device(meta, &me).await; + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: me.id, device: first, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", now).await.unwrap(); + meta.two_factor_confirm(me.id, first, &*right(step), &hashes, now + 1).await.unwrap(); + let (chal, session) = (new_challenge(), new_session()); + let generation = gen_of(meta, username).await; + let made = meta.web_sign_in(me.id, (ADMIN_HASH, generation), &chal, &session, now + 2).await.unwrap(); + assert!(matches!(made, Some(WebSignIn::Challenge { .. }))); + meta.web_complete_second_factor(&chal.id_hash, &totp(&right(step + 1)), &session, now + 3).await.unwrap(); + assert!(session_is_valid(meta, &session, now + 4).await); + (me, first, session) +} + +/// Every change that raises `users.auth_gen` ends the account's sessions: a password change and reset, a disable, an enable +/// after it, a removal, and two-factor sign-in turned on, off, or reset by an admin or the command line. +async fn a_web_session_ends_with_every_change_of_the_accounts_generation(meta: &Meta) { + // The acting admin has two-factor sign-in, which resetting another administrator's needs. + let (boss, plain) = admin_actor(meta, "boss").await; + let boss_device = device_of(&plain); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: boss.id, device: boss_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", NOW).await.unwrap(); + meta.two_factor_confirm(boss.id, boss_device, &*right(FIRST_STEP), &hashes, NOW + 1).await.unwrap(); + let actor = AdminActor { two_factor: true, ..plain }; + let mut now = NOW + 10; + + // A password reset by the command line. + let root = web_admin(meta, "root").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &root, &s, now).await, Some(WebSignIn::Session)); + meta.reset_password(root.id, "$argon2id$new", true, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a password reset ends the session"); + now += 10; + + // A password change by a device. + let (ana, ana_actor) = admin_actor(meta, "ana").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &ana, &s, now).await, Some(WebSignIn::Session)); + meta.change_password(ana.id, ADMIN_HASH, "$argon2id$changed", device_of(&ana_actor), now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a password change ends the session"); + now += 10; + + // Disable ends it, and an enable afterwards does not bring it back. + let target = web_admin(meta, "target").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &target, &s, now).await, Some(WebSignIn::Session)); + meta.admin_disable_user(&actor, target.id, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a disabled account's session is over"); + meta.admin_enable_user(&actor, target.id, now + 3).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 4).await, "enabling does not revive it"); + now += 10; + + // A removal. + let doomed = web_admin(meta, "doomed").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &doomed, &s, now).await, Some(WebSignIn::Session)); + meta.admin_remove_user(&actor, doomed.id, now + 1).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 2).await, "a removed account's session is over"); + now += 10; + + // Two-factor sign-in turned on: a session from before it is over (a set-up that is not confirmed changes nothing). + let guarded = web_admin(meta, "guarded").await; + let guarded_device = device(meta, &guarded).await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &guarded, &s, now).await, Some(WebSignIn::Session)); + let setup = DeviceActor { user: guarded.id, device: guarded_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", now + 1).await.unwrap(); + assert!(session_is_valid(meta, &s, now + 2).await, "an open set-up is no change"); + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + meta.two_factor_confirm(guarded.id, guarded_device, &*right(FIRST_STEP), &hashes, now + 3).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 4).await, "turning two-factor sign-in on ends the session"); + now += 10; + + // Turned off by the account itself. + let (own, own_device, s) = web_admin_with_factor(meta, "own", FIRST_STEP, now).await; + let setup = DeviceActor { user: own.id, device: own_device, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_disable(&setup, &totp(&right(FIRST_STEP + 5)), now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "turning two-factor sign-in off ends the session"); + now += 20; + + // Reset by an admin, and by the command line. + let (by_admin, _, s) = web_admin_with_factor(meta, "byadmin", FIRST_STEP, now).await; + meta.admin_reset_two_factor(&actor, by_admin.id, now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "an admin's reset ends the session"); + now += 20; + let (by_cli, _, s) = web_admin_with_factor(meta, "bycli", FIRST_STEP, now).await; + meta.reset_two_factor(by_cli.id, now + 10).await.unwrap(); + assert!(!session_is_valid(meta, &s, now + 11).await, "the command line's reset ends the session"); + + // Another account's session is untouched by all of that. + let bystander = web_admin(meta, "bystander").await; + let s = new_session(); + assert_eq!(web_sign_in(meta, &bystander, &s, now + 30).await, Some(WebSignIn::Session)); + meta.reset_password(root.id, "$argon2id$again", true, now + 31).await.unwrap(); + assert!(session_is_valid(meta, &s, now + 32).await); +} + +/// An admin with two-factor sign-in gets a challenge of kind `web`, which no device sign-in can use and which makes no +/// device, and completes it with a code, once. The second factor's counters and lock are the device sign-in's. +async fn a_web_sign_in_with_two_factor_needs_its_own_challenge(meta: &Meta) { + let me = web_admin(meta, "root").await; + let first = device(meta, &me).await; + let hashes: Vec<[u8; 32]> = (0..RECOVERY_CODES).map(|_| code(INVITE_CHARS).hash()).collect(); + let setup = DeviceActor { user: me.id, device: first, password_hash: ADMIN_HASH.to_owned() }; + meta.two_factor_begin_setup(&setup, b"cipher-of-the-secret", SETUP_AT).await.unwrap(); + meta.two_factor_confirm(me.id, first, &*right(FIRST_STEP), &hashes, SETUP_AT + 1).await.unwrap(); + + let generation = gen_of(meta, "root").await; + let chal = new_challenge(); + let session = new_session(); + let made = meta.web_sign_in(me.id, (ADMIN_HASH, generation), &chal, &session, SETUP_AT + 10).await.unwrap(); + assert_eq!(made, Some(WebSignIn::Challenge { expires_ms: SETUP_AT + 10 + 5 * MINUTE })); + assert!(!session_is_valid(meta, &session, SETUP_AT + 11).await, "no session without the second factor"); + assert_eq!(live_devices(meta, &me).await, vec![first], "and no device"); + // The challenge is seen as a web one only. + assert_eq!(meta.web_challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), Some((me.id, "root".to_owned()))); + assert_eq!(meta.challenge_owner(&chal.id_hash, SETUP_AT + 11).await.unwrap(), None, "a device sign-in does not see it"); + let as_device = meta.complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &token(), SETUP_AT + 12).await; + assert!(matches!(as_device, Err(MetaError::ChallengeGone)), "{as_device:?}"); + assert_eq!(live_devices(meta, &me).await, vec![first], "still no new device, and the code was not used"); + // And the other way round. + let device_chal = new_challenge(); + let made = meta.sign_in(me.id, (ADMIN_HASH, generation), "pc", &DeviceClient::default(), &token(), &device_chal, SETUP_AT + 13).await.unwrap(); + assert!(matches!(made, Some(SignIn::Challenge { .. }))); + let as_web = meta.web_complete_second_factor(&device_chal.id_hash, &totp(&right(FIRST_STEP + 1)), &session, SETUP_AT + 14).await; + assert!(matches!(as_web, Err(MetaError::ChallengeGone)), "{as_web:?}"); + assert!(!session_is_valid(meta, &session, SETUP_AT + 15).await); + + // Wrong codes are counted (four, and the fifth would delete the challenge), then the right one makes the session, once. + for _ in 0..4 { + let result = meta.web_complete_second_factor(&chal.id_hash, &totp(&wrong()), &session, SETUP_AT + 20).await; + assert!(matches!(result, Err(MetaError::TwoFactorInvalid)), "{result:?}"); + } + let done = meta.web_complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 1)), &session, SETUP_AT + 21).await.unwrap(); + assert_eq!(done, me.id); + let found = meta.web_session(&session.id_hash, SETUP_AT + 22, WEB_IDLE_MS).await.unwrap().unwrap(); + assert_eq!(found.mfa_verified_ms, Some(SETUP_AT + 21), "the session says when the second factor was given"); + assert_eq!(live_devices(meta, &me).await, vec![first], "a web sign-in makes no device"); + let login = all_events(meta).await.into_iter().find(|event| event.action == AuditAction::AuthLogin).unwrap(); + assert_eq!(details(&login), json(r#"{"client_kind":"web"}"#)); + let failed = all_events(meta).await.into_iter().find(|event| event.action == AuditAction::AuthSecondFactorFailed).unwrap(); + assert_eq!(details(&failed), json(r#"{"client_kind":"web","step":"totp"}"#)); + let again = meta.web_complete_second_factor(&chal.id_hash, &totp(&right(FIRST_STEP + 2)), &new_session(), SETUP_AT + 23).await; + assert!(matches!(again, Err(MetaError::ChallengeGone)), "once: {again:?}"); + + // The same time step is not accepted again (the counter is the device sign-in's and the admin confirmation's). + let generation = gen_of(meta, "root").await; + let replay = new_challenge(); + meta.web_sign_in(me.id, (ADMIN_HASH, generation), &replay, &new_session(), SETUP_AT + 24).await.unwrap(); + let replayed = meta.web_complete_second_factor(&replay.id_hash, &totp(&right(FIRST_STEP + 1)), &new_session(), SETUP_AT + 25).await; + assert!(matches!(replayed, Err(MetaError::TwoFactorInvalid)), "{replayed:?}"); + // Wrong proofs lock the second factor for every way of using it: a device sign-in's step too. + for n in 0..12 { + let _ = confirm_with(meta, me.id, None, SETUP_AT + 30 + n).await; + } + let locked = meta.web_complete_second_factor(&replay.id_hash, &totp(&right(FIRST_STEP + 9)), &new_session(), SETUP_AT + 50).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); + let locked = meta.complete_second_factor(&device_chal.id_hash, &totp(&right(FIRST_STEP + 9)), &token(), SETUP_AT + 51).await; + assert!(matches!(locked, Err(MetaError::TwoFactorLocked { .. })), "{locked:?}"); +} + +/// Pruning deletes sessions that are over (logged out, expired, idle), never a valid one, and at most the batch asked for. +async fn web_sessions_are_pruned_in_bounded_batches(meta: &Meta) { + let root = web_admin(meta, "root").await; + let late = NOW + WEB_ABSOLUTE_MS - 2 * WEB_IDLE_MS; + let now = NOW + WEB_ABSOLUTE_MS - WEB_IDLE_MS; + let keep = new_session(); + assert_eq!(web_sign_in(meta, &root, &keep, now - MINUTE).await, Some(WebSignIn::Session)); + for n in 0..5 { + assert_eq!(web_sign_in(meta, &root, &new_session(), NOW + n).await, Some(WebSignIn::Session)); + } + let out = new_session(); + assert_eq!(web_sign_in(meta, &root, &out, late).await, Some(WebSignIn::Session)); + meta.web_logout(&out.id_hash, late + 1).await.unwrap(); + + assert_eq!(meta.web_sessions_count_over(now, WEB_IDLE_MS).await.unwrap(), 6, "five idle ones and the logged-out one"); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 4, "a batch is bounded"); + assert_eq!(meta.web_sessions_count_over(now, WEB_IDLE_MS).await.unwrap(), 2); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 2); + assert_eq!(meta.web_sessions_prune(now, WEB_IDLE_MS, 4).await.unwrap(), 0, "nothing is left to delete"); + assert!(session_is_valid(meta, &keep, now).await, "a valid session is never pruned"); + // Past the absolute lifetime the last one goes too. + let end = NOW + 2 * WEB_ABSOLUTE_MS; + assert_eq!(meta.web_sessions_prune(end, WEB_IDLE_MS, 100).await.unwrap(), 1); + assert_eq!(meta.web_sessions_count_over(end, WEB_IDLE_MS).await.unwrap(), 0); +} + +/// An admin change made on a browser session is audited without a device, and a session that was logged out, or whose account +/// changed, can no longer make one. +async fn a_web_actor_changes_accounts_until_its_session_ends(meta: &Meta) { + // The store judges the session by its own clock, which follows the time each call below is made at. + let clock = std::sync::Arc::new(std::sync::atomic::AtomicI64::new(NOW)); + let ticking = clock.clone(); + let meta = &meta.clone().with_clock(std::sync::Arc::new(move || ticking.load(std::sync::atomic::Ordering::SeqCst))); + let at_ms = |t: i64| { + clock.store(t, std::sync::atomic::Ordering::SeqCst); + t + }; + let root = web_admin(meta, "root").await; + let session = new_session(); + assert_eq!(web_sign_in(meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let actor = AdminActor { + user: root.id, + via: AdminVia::Web { id_hash: session.id_hash, idle_ms: WEB_IDLE_MS }, + password_hash: ADMIN_HASH.to_owned(), + two_factor: false, + }; + assert!(!format!("{actor:?}").contains(&format!("{:?}", session.id_hash)), "the session's hash is not in Debug"); + let ana = user(meta, "ana").await; + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 1)).await.unwrap(); + let events = all_events(meta).await; + let changed = events.iter().find(|e| e.action == AuditAction::AdminQuotaChanged).unwrap(); + assert_eq!((changed.actor_user, changed.actor_device), (Some(root.id), None), "no device acted"); + // Another account's session is not this admin's. + let other = web_admin(meta, "other").await; + let theirs = new_session(); + assert_eq!(web_sign_in(meta, &other, &theirs, NOW).await, Some(WebSignIn::Session)); + let borrowed = AdminActor { via: AdminVia::Web { id_hash: theirs.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + assert!(matches!(meta.admin_update_user(&borrowed, ana.id, None, Some(None), at_ms(NOW + 2)).await, Err(MetaError::DeviceRevoked))); + // A session that does not exist. + let unknown = AdminActor { via: AdminVia::Web { id_hash: token(), idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + assert!(matches!(meta.admin_update_user(&unknown, ana.id, None, Some(None), at_ms(NOW + 3)).await, Err(MetaError::DeviceRevoked))); + // Logged out: refused, and nothing changed. + meta.web_logout(&session.id_hash, NOW + 4).await.unwrap(); + assert!(matches!(meta.admin_update_user(&actor, ana.id, None, Some(None), at_ms(NOW + 5)).await, Err(MetaError::DeviceRevoked))); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, Some(MIN_QUOTA_BYTES)); + // A session that expired between the request's start and the change (the handler's check was passed, the transaction runs + // later): refused by the idle time and by the absolute lifetime, and nothing changed. + let slow = new_session(); + assert_eq!(web_sign_in(meta, &root, &slow, NOW + 20).await, Some(WebSignIn::Session)); + let slow_actor = AdminActor { via: AdminVia::Web { id_hash: slow.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + let idle = meta.admin_update_user(&slow_actor, ana.id, None, Some(None), at_ms(NOW + 20 + WEB_IDLE_MS)).await; + assert!(matches!(idle, Err(MetaError::DeviceRevoked)), "idle: {idle:?}"); + meta.admin_update_user(&slow_actor, ana.id, None, Some(None), at_ms(NOW + 20 + WEB_IDLE_MS - 1)).await.unwrap(); + let busy = new_session(); + assert_eq!(web_sign_in(meta, &root, &busy, NOW + 30).await, Some(WebSignIn::Session)); + let busy_actor = AdminActor { via: AdminVia::Web { id_hash: busy.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + let mut at = NOW + 30; + while at + WEB_IDLE_MS / 2 < NOW + 30 + WEB_ABSOLUTE_MS { + at += WEB_IDLE_MS / 2; + meta.web_session_seen(&busy.id_hash, at).await.unwrap(); + } + meta.admin_update_user(&busy_actor, ana.id, None, Some(None), at_ms(at)).await.unwrap(); + let absolute = meta.admin_update_user(&busy_actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 30 + WEB_ABSOLUTE_MS)).await; + assert!(matches!(absolute, Err(MetaError::DeviceRevoked)), "absolute: {absolute:?}"); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, None); + // A session of a changed account: the generation moved on. + let late = new_session(); + assert_eq!(web_sign_in(meta, &root, &late, NOW + 6).await, Some(WebSignIn::Session)); + let late_actor = AdminActor { via: AdminVia::Web { id_hash: late.id_hash, idle_ms: WEB_IDLE_MS }, ..actor.clone() }; + meta.admin_update_user(&late_actor, ana.id, None, Some(None), at_ms(NOW + 7)).await.unwrap(); + meta.reset_password(root.id, "$argon2id$new", true, NOW + 8).await.unwrap(); + let after = meta.admin_update_user(&late_actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), at_ms(NOW + 9)).await; + assert!(matches!(after, Err(MetaError::DeviceRevoked)), "{after:?}"); + assert_eq!(admin_view(meta, ana.id).await.quota_bytes, None); +} + +// ---- the migration from 0010 to 0011 ---- + +/// Drops what migration 0011 added and forgets it ran, which leaves the database as 0010 left it (with its rows). +async fn back_to_0010(url: &str) { + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + drop_history_trash_and_manifest_index(&mut conn).await; + sqlx::query("DROP TABLE web_sessions").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM schema_version WHERE version >= 11").execute(&mut conn).await.unwrap(); +} + +/// A database from the release before the web admin keeps its accounts, devices and audit log, gets an empty session table +/// that works (also after the migration is rolled back and run again), and a database newer than the program is still refused. +async fn migrating_from_0010_adds_the_web_sessions(url: &str) { + sqlx::any::install_default_drivers(); + let (root, old_token, events) = { + let meta = Meta::open(url).await.unwrap(); + let root = web_admin(&meta, "root").await; + let old_token = token(); + meta.create_device(root.id, "old pc", &old_token, NOW).await.unwrap(); + meta.sign_in_device(root.id, (&root.password_hash, root.auth_gen), "pc", &DeviceClient::default(), &token(), NOW).await.unwrap().unwrap(); + let events = all_events(&meta).await.len(); + (root, old_token, events) + }; + back_to_0010(url).await; + assert_eq!(schema_version(url).await, 10); + + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14, "the current schema includes history, trash and the reference index after web sessions"); + assert_eq!(meta.device_by_token(&old_token).await.unwrap().map(|a| a.user), Some(root.id)); + assert_eq!(all_events(&meta).await.len(), events, "the audit rows are kept"); + let session = new_session(); + assert_eq!(web_sign_in(&meta, &root, &session, NOW + 5).await, Some(WebSignIn::Session)); + assert!(session_is_valid(&meta, &session, NOW + 6).await); + + // The rollback: the table goes, the version is forgotten, and opening again makes the table again. + drop(meta); + back_to_0010(url).await; + assert_eq!(schema_version(url).await, 10); + let meta = Meta::open(url).await.unwrap(); + assert_eq!(schema_version(url).await, 14); + assert!(!session_is_valid(&meta, &session, NOW + 6).await, "the table is new and empty"); + drop(meta); + + // A database from a newer Craft Cloud is still refused. + use sqlx::Connection; + let mut conn = sqlx::AnyConnection::connect(url).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES (15)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(url).await, Err(MetaError::Corrupt(_)))); +} + +#[tokio::test] +async fn migrating_from_0010_adds_the_web_sessions_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + migrating_from_0010_adds_the_web_sessions(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn migrating_from_0010_adds_the_web_sessions_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + migrating_from_0010_adds_the_web_sessions(&url).await; +} + +// ---- a change that waits for its locks while its session ends ---- + +/// The time a change judges its session by is read after the change has its locks: one that waited while the session went idle +/// is refused, and changes nothing. +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(url: &str) { + use sqlx::Connection; + use std::sync::Arc; + use std::sync::atomic::{AtomicI64, Ordering}; + sqlx::any::install_default_drivers(); + let now = Arc::new(AtomicI64::new(NOW)); + let ticking = now.clone(); + let meta = Meta::open(url).await.unwrap().with_clock(Arc::new(move || ticking.load(Ordering::SeqCst))); + let root = web_admin(&meta, "root").await; + let ana = user(&meta, "ana").await; + let session = new_session(); + assert_eq!(web_sign_in(&meta, &root, &session, NOW).await, Some(WebSignIn::Session)); + let actor = AdminActor { + user: root.id, + via: AdminVia::Web { id_hash: session.id_hash, idle_ms: WEB_IDLE_MS }, + password_hash: ADMIN_HASH.to_owned(), + two_factor: false, + }; + // Another connection holds the write lock (SQLite) or the admin's row (Postgres) until the test lets go. + let mut holder = sqlx::AnyConnection::connect(url).await.unwrap(); + if url.starts_with("sqlite:") { + sqlx::query("BEGIN IMMEDIATE").execute(&mut holder).await.unwrap(); + } else { + sqlx::query("BEGIN").execute(&mut holder).await.unwrap(); + sqlx::query("UPDATE users SET username = username WHERE username = 'root'").execute(&mut holder).await.unwrap(); + } + let change = { + let (meta, actor, ana) = (meta.clone(), actor.clone(), ana.id); + tokio::spawn(async move { meta.admin_update_user(&actor, ana, None, Some(Some(MIN_QUOTA_BYTES)), NOW + 1).await }) + }; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert!(!change.is_finished(), "the change is waiting for the lock"); + now.store(NOW + WEB_IDLE_MS + 5, Ordering::SeqCst); + sqlx::query("COMMIT").execute(&mut holder).await.unwrap(); + let result = change.await.unwrap(); + assert!(matches!(result, Err(MetaError::DeviceRevoked)), "{result:?}"); + assert_eq!(admin_view(&meta, ana.id).await.quota_bytes, None, "nothing changed"); + // The same change while the session is valid goes through. + now.store(NOW + 2, Ordering::SeqCst); + meta.admin_update_user(&actor, ana.id, None, Some(Some(MIN_QUOTA_BYTES)), NOW + 2).await.unwrap(); +} + +#[tokio::test] +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_session_that_ends_while_a_change_waits_for_its_locks_does_not_authorise_it(&url).await; +} + +/// A file that comes back from the trash gets a new current version with its last content, after the last one and +/// without a save record, so a delete that was sent against the old version (and whose answer a machine never heard) +/// is refused as a version mismatch and can't delete the restored file again; a folder has no version. +async fn a_restored_file_gets_a_new_version_with_its_old_content(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 2).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + delete_file(meta, &u, dev, node, versions[1].id, NOW + 10).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, node, NOW + 20).await.unwrap(); + let new = done.version.unwrap(); + assert!(!versions.iter().any(|v| v.id == new)); + assert_eq!(done.seq, before.seq + 1, "one change of the space"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3); + assert_eq!( + (history[0].id, history[0].parent_version, history[0].size, history[0].save, history[0].seq), + (new, Some(versions[1].id), 4, false, done.seq) + ); + assert!(history.iter().any(|v| v.id == versions[1].id && v.save), "the old version keeps its record"); + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!((f.version, f.manifest, f.mtime_ms, f.save.clone()), (new, versions[1].manifest, versions[1].mtime_ms, None)); + + // The delete a machine sent before the restore, with the old version, is refused and the file stays. + let stale = meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(versions[1].id) }], NOW + 30).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(new) })]); + assert!(!live_name(meta, &u, node).await.2); + + // A folder has no version. + let dir = folder(meta, &u, dev, r, "plain-dir", NOW + 40).await; + delete_folder(meta, &u, dev, dir, NOW + 41).await; + assert_eq!(meta.restore_from_trash(u.space, dev, dir, NOW + 42).await.unwrap().version, None); +} + +/// A restore never takes the tree past its depth limit: a folder that comes back below a live folder that has moved +/// deep meanwhile is refused as too deep, for the folder itself and for a file whose folder would come back, and +/// the refusal changes nothing; a file in a folder at the limit is fine. +async fn a_restore_does_not_exceed_the_depth_limit(meta: &Meta) { + use craft_cloud_proto::limits::MAX_TREE_DEPTH; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let shallow = folder(meta, &u, dev, r, "shallow", NOW).await; + let c = folder(meta, &u, dev, shallow, "c", NOW).await; + let (f, fv) = file_in(meta, &u, dev, c, "f.txt", NOW).await; + let spare = folder(meta, &u, dev, shallow, "spare", NOW).await; + let (g, gv) = file_in(meta, &u, dev, spare, "g.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + delete_folder(meta, &u, dev, c, NOW + 2).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 3).await; + // A chain of folders down to depth MAX_TREE_DEPTH - 1, and the live folder moved to its end: depth MAX_TREE_DEPTH. + let mut ops = Vec::new(); + let mut parent = r; + for i in 0..MAX_TREE_DEPTH - 1 { + let id = NodeId::generate(); + ops.push(Op::Mkdir { node: id, parent, name: name(&format!("d{i}")) }); + parent = id; + } + meta.commit(u.space, dev, &ops, NOW + 4).await.unwrap(); + meta.commit(u.space, dev, &[Op::Move { node: shallow, from_parent: r, from_name: name("shallow"), parent, name: name("shallow") }], NOW + 5) + .await + .unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let trash_before = trash_ids(meta, &u).await; + + // The deleted folder would sit one level too deep, and so would the folder of the file: refused, nothing changes. + for node in [c, f] { + let refused = meta.restore_from_trash(u.space, dev, node, NOW + 6).await; + assert_eq!(refused_restore(refused), vec![(0, ConflictKind::TooDeep)], "restoring {node}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, trash_before); + } + // A file may sit in the deepest folder: it comes back, in `spare`, which is live (it was never deleted). + let done = meta.restore_from_trash(u.space, dev, g, NOW + 7).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (spare, "g.txt")); +} + +// ---- the reference index ---- + +/// A manifest over `chunks` (each one byte), the chunks recorded as `u`'s; the manifest itself is not recorded. +async fn manifest_over(meta: &Meta, u: &UserRecord, chunks: &[ChunkId]) -> Manifest { + let unique: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(u.space, &unique, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), chunks.len() as u64, FileHash::from_bytes([chunks.len() as u8; 32]), refs).unwrap() +} + +fn fresh_chunks(n: usize) -> Vec { + (0..n).map(|_| random_chunk()).collect() +} + +fn edge_list(chunks: &[ChunkId]) -> Vec<(u64, ChunkId)> { + chunks.iter().enumerate().map(|(i, c)| (i as u64, *c)).collect() +} + +/// The index of a manifest is written in the transaction that records it, exactly: in order, repeated chunks at each +/// of their places; recording it again (another space, the same bytes) changes nothing; a manifest the quota refuses +/// leaves no index behind; an empty file's manifest is complete with no edges. +async fn a_new_manifest_has_exact_edges_complete_in_its_own_transaction(meta: &Meta) { + use craft_cloud_meta::{EdgeCheck, IndexState}; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let (a, b, c) = (random_chunk(), random_chunk(), random_chunk()); + let chunks = [a, b, a, c]; + let m = manifest_over(meta, &ana, &chunks).await; + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert_eq!(meta.index_check(&m).await.unwrap(), EdgeCheck::Exact); + + // The same manifest from another space: nothing is written twice. + let edges = meta.index_edge_count().await.unwrap(); + let again = manifest_over(meta, &bob, &chunks).await; + meta.add_manifest(bob.space, &again, 100, NOW).await.unwrap(); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + + // An empty file. + let empty = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 0, FileHash::from_bytes([0; 32]), vec![]).unwrap(); + meta.add_manifest(ana.space, &empty, 10, NOW).await.unwrap(); + assert_eq!(meta.index_state(&empty.id()).await.unwrap(), IndexState::Complete { total: 0 }); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + + // A manifest the stored-bytes cap refuses leaves no index: its transaction rolls back whole. + let capped = meta.clone().with_stored_bytes_cap(Some(50)); + let carol = user(&capped, "carol").await; + let chunks = fresh_chunks(3); + let refused = manifest_over(&capped, &carol, &chunks).await; + assert!(matches!(capped.add_manifest(carol.space, &refused, 1_000, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + assert_eq!(capped.index_state(&refused.id()).await.unwrap(), IndexState::Absent); + assert_eq!(capped.index_edge_count().await.unwrap(), edges); + // A manifest whose chunks the space lacks is refused before anything is written. + let lacking = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 1, FileHash::from_bytes([7; 32]), vec![ChunkRef { id: random_chunk(), len: 1 }]) + .unwrap(); + assert!(matches!(meta.add_manifest(ana.space, &lacking, 100, NOW).await, Err(MetaError::MissingChunks(_)))); + assert_eq!(meta.index_state(&lacking.id()).await.unwrap(), IndexState::Absent); +} + +/// A manifest longer than the budget is indexed in batches of at most that many edges, each its own transaction, and is +/// complete only after the comparison with its manifest. +async fn a_long_manifest_is_indexed_in_batches_of_the_budget(meta: &Meta) { + use craft_cloud_meta::{IndexState, Validation}; + let meta = meta.clone().with_index_batch_rows(5); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + let before = meta.index_edge_count().await.unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 5, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 5, "no more than the budget in the recording transaction"); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + + for (expected_done, expected_rows) in [(10u64, 10u64), (12, 12)] { + let state = meta.index_stage(&m).await.unwrap(); + assert_eq!(state, IndexState::Building { done: expected_done, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, expected_rows, "at most the budget more per stage"); + } + // Every edge is written, and still it is not complete: only the comparison makes it so. + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 12, total: 12 }); + assert_eq!(meta.index_validate(&m).await.unwrap(), Validation::Complete); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + // Once complete, more stages change nothing. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 12); + + // `index_complete` does all of it, and again is a no-op. + let more = fresh_chunks(23); + let long = manifest_over(&meta, &ana, &more).await; + meta.add_manifest(ana.space, &long, 100, NOW).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + assert_eq!(meta.index_state(&long.id()).await.unwrap(), IndexState::Complete { total: 23 }); + assert_eq!(meta.index_edges(&long.id()).await.unwrap(), edge_list(&more)); +} + +/// The index is built from what a manifest says, not from what is there: wrong, swapped, missing and extra edges, a +/// repeated chunk collapsed or changed, and a wrong edge written before the build started are each found by the +/// comparison, the index does not become complete, and a build that repeats the manifest's own list makes it exact. +async fn an_index_is_complete_only_when_every_edge_is_right_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let (a, b, c, d, e, f) = (random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk()); + let spare = random_chunk(); + // `(what, chunks, damage SQL, expected first problem)`: `$1` is a chunk ID and `$2` the manifest. + type Case<'a> = (&'a str, Vec, &'a str, (u64, EdgeProblemKind)); + let cases: Vec = vec![ + ( + "wrong chunk", + vec![a, b, c, d, e, f], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a missing edge", + vec![a, b, c, d, e, f], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 4 AND chunk <> $1", + (4, EdgeProblemKind::MissingEdge), + ), + ( + "an extra edge", + vec![a, b, c, d, e, f], + "INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($2, 6, $1)", + (6, EdgeProblemKind::ExtraEdge), + ), + ( + "a repeated chunk changed", + vec![a, b, a, c], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a repeated chunk collapsed", + vec![a, b, a, c], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 2 AND chunk = $1", + (2, EdgeProblemKind::MissingEdge), + ), + ]; + for (what, chunks, damage, (ordinal, kind)) in cases { + // A distinct manifest per case: its chunks differ in size by the case. + let chunks: Vec = + chunks.iter().map(|c| ChunkId::from_bytes(*blake3::hash(&[c.as_bytes().as_slice(), what.as_bytes()].concat()).as_bytes())).collect(); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + assert!(matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { .. }), "{what}: all edges written, not yet compared"); + // Damage it with every edge written, before the comparison. + let wrong = chunks.first().map(|c| *c.as_bytes()).unwrap_or(*spare.as_bytes()); + let other = if damage.starts_with("UPDATE") { *spare.as_bytes() } else { wrong }; + sqlx::query(sqlx::AssertSqlSafe(damage.to_owned())).bind(other.to_vec()).bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let outcome = meta.index_validate(&m).await.unwrap(); + match outcome { + Validation::Mismatch(problem) => assert_eq!((problem.ordinal, problem.kind), (ordinal, kind), "{what}"), + other => panic!("{what}: expected a mismatch, got {other:?}"), + } + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{what}: not complete"); + // A build repeats the manifest's own list and ends exact. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }, "{what}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{what}"); + } + + // A wrong edge that was there before the build began is not taken for the right one. + let chunks = fresh_chunks(4); + let m = manifest_over(&meta, &ana, &chunks).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "recorded with a wrong edge present"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + + // A complete index that was damaged afterwards is found by the checks that read it, not trusted. + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 3 }); + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 0") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert!(matches!(meta.index_check(&m).await.unwrap(), craft_cloud_meta::EdgeCheck::Mismatch(p) if p.ordinal == 0)); + // The builder leaves a complete row alone (it is the checker's to report), and a mismatch there is an error, not a silent rewrite. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 3 }); +} + +/// A build stopped part way resumes where it was, after the database was closed and opened again, and a manifest +/// recorded before the index existed (no row at all) is found and indexed. +async fn an_index_build_resumes_after_a_restart_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let (ana, m) = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + (ana, m) + }; + // The "restart": a new handle on the same database. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + assert_eq!(meta.index_edge_count().await.unwrap(), 8); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + + // A manifest from before the index: recorded, no row, no edges. + let old_chunks = fresh_chunks(6); + let old = manifest_over(&meta, &ana, &old_chunks).await; + meta.add_manifest(ana.space, &old, 100, NOW).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + assert_eq!(meta.index_state(&old.id()).await.unwrap(), IndexState::Absent); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![old.id()]); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent), (2, 1, 1)); + meta.index_complete(&old).await.unwrap(); + assert_eq!(meta.index_edges(&old.id()).await.unwrap(), edge_list(&old_chunks)); + assert_eq!(meta.index_audit(5).await.unwrap().complete, 2); +} + +/// The migration that adds the index does not pretend: manifests recorded before it have no row and are reported as not +/// indexed until a build indexes them. +async fn the_index_migration_leaves_old_manifests_unindexed_on(url: &str) { + use craft_cloud_meta::IndexState; + let (m, ana) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &fresh_chunks(3)).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + (m, ana) + }; + let _ = ana; + { + let mut conn = raw(url).await; + for statement in ["DROP TABLE manifest_chunks", "DROP TABLE manifest_index", "DELETE FROM schema_version WHERE version >= 14"] { + sqlx::query(statement).execute(&mut conn).await.unwrap(); + } + } + let meta = Meta::open(url).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent, "recorded before the index: not indexed"); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent, audit.building), (1, 0, 1, 0)); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 14); +} + +/// What the database alone can say is wrong, in seeded damage: orphan edges, an index row without its manifest, an edge +/// to a chunk that is not recorded, a root naming a manifest that is not recorded; and the notices: manifests no version +/// or snapshot names (the live-document asset case) and chunks no manifest names. +async fn the_audit_names_what_is_wrong_on(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + + // A healthy file, an asset no version names, and a chunk no manifest names. + let chunks = fresh_chunks(3); + let file_manifest = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &file_manifest, 100, NOW).await.unwrap(); + let node = NodeId::generate(); + let v = version(file_manifest.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("f.txt"), version: v }], NOW).await.unwrap(); + let asset = manifest_over(&meta, &ana, &fresh_chunks(2)).await; + meta.add_manifest(ana.space, &asset, 100, NOW).await.unwrap(); + let stray = random_chunk(); + meta.add_chunk(ana.space, &stray, 10, NOW).await.unwrap(); + let healthy = meta.index_audit(5).await.unwrap(); + assert_eq!((healthy.manifests, healthy.complete, healthy.building, healthy.absent), (2, 2, 0, 0)); + assert_eq!(healthy.unrooted_manifests.count, 1, "the asset is named by no version"); + assert_eq!(healthy.unrooted_manifests.examples, vec![asset.id().to_string()]); + assert_eq!(healthy.unreferenced_chunks.count, 1); + for finding in [ + &healthy.index_without_manifest, + &healthy.edges_without_index, + &healthy.complete_with_wrong_count, + &healthy.unknown_dependencies, + &healthy.roots_without_manifest, + &healthy.roots_not_indexed, + &healthy.space_blobs_without_blob, + ] { + assert_eq!(finding.count, 0, "{finding:?}"); + } + + // Damage: orphan edges, an index row without a manifest, an unknown dependency, a root without its manifest. + let ghost = ManifestId::from_bytes([0xaa; 32]); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 0, $2)") + .bind(ghost.as_bytes().to_vec()) + .bind(stray.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + let ghost_row = ManifestId::from_bytes([0xbb; 32]); + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, 0, 0, 1)") + .bind(ghost_row.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 0 AND id = $1").bind(chunks[1].as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 1 AND id = $1").bind(file_manifest.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let damaged = meta.index_audit(5).await.unwrap(); + assert_eq!(damaged.edges_without_index.examples, vec![ghost.to_string()]); + assert_eq!(damaged.index_without_manifest.count, 2, "the made-up row and the manifest whose record was removed"); + assert!(damaged.index_without_manifest.examples.contains(&ghost_row.to_string())); + assert_eq!(damaged.unknown_dependencies.count, 1); + assert_eq!(damaged.unknown_dependencies.examples, vec![format!("{}/{}", file_manifest.id(), chunks[1])]); + assert_eq!(damaged.roots_without_manifest.examples, vec![file_manifest.id().to_string()], "the version names a manifest that is gone"); + assert_eq!(damaged.space_blobs_without_blob.count, 2, "the memberships of the removed blobs"); + // A complete index whose edge count is not its chunk count. + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal = 0") + .bind(asset.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(meta.index_audit(5).await.unwrap().complete_with_wrong_count.examples, vec![asset.id().to_string()]); +} + +/// Every path that records or uses a manifest leaves the index exactly as the manifests made it: commit, update, +/// restore of a version, restore from the trash, a late save record, a live document's import and compaction. Their +/// roots are always indexed, and none of them writes an edge. +async fn the_index_stays_exact_through_every_write_path(meta: &Meta) { + use craft_cloud_proto::live::node_doc; + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let r = root(meta, &ana).await; + let clean = |label: &'static str, edges: u64| async move { + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.complete, audit.manifests, "{label}: every manifest is complete"); + assert_eq!(audit.roots_not_indexed.count + audit.roots_without_manifest.count + audit.unknown_dependencies.count, 0, "{label}"); + assert_eq!(audit.edges_without_index.count + audit.index_without_manifest.count + audit.complete_with_wrong_count.count, 0, "{label}"); + assert_eq!(meta.index_edge_count().await.unwrap(), edges, "{label}: the path wrote no edge"); + }; + let (c1, c2, c3) = (fresh_chunks(3), fresh_chunks(2), fresh_chunks(4)); + let (m1, m2, m3) = (manifest_over(meta, &ana, &c1).await, manifest_over(meta, &ana, &c2).await, manifest_over(meta, &ana, &c3).await); + for m in [&m1, &m2, &m3] { + meta.add_manifest(ana.space, m, 100, NOW).await.unwrap(); + } + let edges = 3 + 2 + 4; + clean("uploads", edges).await; + + let node = NodeId::generate(); + let v1 = version(m1.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("path.txt"), version: v1.clone() }], NOW).await.unwrap(); + clean("create", edges).await; + let v2 = version(m2.id()); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v1.id, version: v2.clone() }], NOW + 1).await.unwrap(); + clean("update", edges).await; + let restored = meta.restore_version(ana.space, dev, node, v1.id, v2.id, NOW + 2).await.unwrap(); + clean("restore of a version", edges).await; + let record = craft_cloud_proto::SaveRecord { doc: node_doc(ana.space, node), frontiers: vec![(1, 2)] }; + meta.attach_save(ana.space, node, v2.id, &record).await.unwrap(); + clean("a late save record", edges).await; + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(restored.version) }], NOW + 3).await.unwrap(); + clean("delete", edges).await; + meta.restore_from_trash(ana.space, dev, node, NOW + 4).await.unwrap(); + clean("restore from the trash", edges).await; + + // A live document of a file: its import snapshot and a compaction's snapshot are roots too. + let doc_node = file(meta, &ana, "doc.txt").await; + let doc = node_doc(ana.space, doc_node); + let first = snapshot(meta, &ana, 100).await; + let edges = meta.index_edge_count().await.unwrap(); + meta.create_live_document(ana.space, doc, "document:wordcraft", doc_node, &first.manifest, NOW).await.unwrap(); + clean("a live document's import", edges).await; + meta.append_live_update(ana.space, doc, dev, 1, &[1; 10], NOW).await.unwrap(); + let compacted = snapshot(meta, &ana, 40).await; + let edges = meta.index_edge_count().await.unwrap(); + assert!(meta.compact_live_doc(ana.space, doc, 1, &compacted.manifest).await.unwrap()); + clean("a compaction", edges).await; + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.unrooted_manifests.count, 1, "only m3 is named by nothing"); +} + +/// A build runs beside commits and uploads: manifests recorded before the index (no rows) are indexed while new files +/// are committed and new manifests recorded, and the result is exact for all of them. +async fn a_build_runs_beside_commits_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + let mut old = Vec::new(); + for i in 0..6 { + let chunks = fresh_chunks(7 + i); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + old.push((m, chunks)); + } + let builder = { + let (meta, old) = (meta.clone(), old.clone()); + tokio::spawn(async move { + for (m, _) in &old { + meta.index_complete(m).await.unwrap(); + } + }) + }; + let writer = { + let (meta, space) = (meta.clone(), ana.space); + tokio::spawn(async move { + let mut made = Vec::new(); + for i in 0..8 { + let chunks = fresh_chunks(5); + let ids: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(space, &ids, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + let m = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([i; 32]), refs).unwrap(); + meta.add_manifest(space, &m, 100, NOW).await.unwrap(); + // As the server's upload handler does, beside the builder: the part the recording left. + meta.index_complete(&m).await.unwrap(); + let node = NodeId::generate(); + let v = NewVersion { id: VersionId::generate(), manifest: m.id(), mtime_ms: NOW, exec: false, save: None }; + meta.commit(space, dev, &[Op::Create { node, parent: r, name: name(&format!("w{i}.txt")), version: v }], NOW).await.unwrap(); + made.push((m, chunks)); + } + made + }) + }; + builder.await.unwrap(); + let made = writer.await.unwrap(); + for (m, chunks) in old.iter().chain(made.iter()) { + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(chunks)); + } + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete), (14, 14)); + assert_eq!(audit.roots_not_indexed.count, 0); +} + +#[tokio::test] +async fn an_index_is_complete_only_when_every_edge_is_right_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_is_complete_only_when_every_edge_is_right_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_is_complete_only_when_every_edge_is_right_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_is_complete_only_when_every_edge_is_right_on(&url).await; +} + +#[tokio::test] +async fn an_index_build_resumes_after_a_restart_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_build_resumes_after_a_restart_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_build_resumes_after_a_restart_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_build_resumes_after_a_restart_on(&url).await; +} + +#[tokio::test] +async fn the_index_migration_leaves_old_manifests_unindexed_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_index_migration_leaves_old_manifests_unindexed_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_index_migration_leaves_old_manifests_unindexed_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_index_migration_leaves_old_manifests_unindexed_on(&url).await; +} + +#[tokio::test] +async fn the_audit_names_what_is_wrong_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_audit_names_what_is_wrong_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_audit_names_what_is_wrong_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_audit_names_what_is_wrong_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_runs_beside_commits_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_runs_beside_commits_on(&url).await; +} + +/// An edge at a negative ordinal, which pagination starting at -1 once skipped, is found by the comparison: the index +/// of such a manifest is not marked complete, however it got there, and a build throws the edge away. +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let spare = random_chunk(); + for ordinal in [-1i64, -7, i64::MIN] { + let chunks = fresh_chunks(5); + let m = manifest_over(&meta, &ana, &chunks).await; + // The stray edge is there before the manifest is recorded, and stays after all its own edges were written. + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(ordinal) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + match meta.index_check(&m).await.unwrap() { + craft_cloud_meta::EdgeCheck::Mismatch(p) => { + assert_eq!((p.ordinal, p.kind), (ordinal.unsigned_abs(), EdgeProblemKind::NegativeOrdinal), "{ordinal}") + } + other => panic!("{ordinal}: expected a mismatch, got {other:?}"), + } + assert!(matches!(meta.index_validate(&m).await.unwrap(), Validation::Mismatch(p) if p.kind == EdgeProblemKind::NegativeOrdinal), "{ordinal}"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{ordinal}: not complete"); + // The build rebuilds from the manifest, and the stray edge is gone. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 5 }, "{ordinal}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{ordinal}"); + } +} + +/// Throwing away the edges of an index that starts again takes batches of the index budget, each its own transaction, +/// and a run that stops between two of them leaves a marker the next one resumes from; nothing is written or marked +/// complete while edges of the old list are left. +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(12); + let spare = random_chunk(); + let m = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + m + }; + let mut conn = raw(url).await; + // A wrong edge, and one at a negative ordinal; the index was not marked complete. + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, -1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // The reset starts (what a build does on a mismatch) and the process stops right after: only the marker is there. + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // A new handle carries on, one batch of 3 rows per transaction, and writes nothing meanwhile. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + assert_eq!(meta.index_edge_count().await.unwrap(), 13); + let mut left = 13u64; + while left > 0 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }, "{left} edges left"); + let now = meta.index_edge_count().await.unwrap(); + assert_eq!(left - now, left.min(3), "a batch deletes at most the budget"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. })); + left = now; + } + // The last batch (fewer rows than the budget) put the row back to building, so the next stage writes edges. + assert_eq!(meta.index_edge_count().await.unwrap(), 0); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// A build that meets a wrong edge resets the index across several batches by itself, and ends exact. +async fn a_build_resets_a_wrong_index_in_batches_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + let mut conn = raw(url).await; + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 7") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 10 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// An extra edge at the highest possible ordinal, `i64::MAX`, after valid ones is reported as a missing edge at the first +/// ordinal that is not there, and the report is stable. +/// +/// This does not reach the `raw.checked_add(1)` that guards the page cursor in `compare`. An edge at `i64::MAX` is +/// returned as `MissingEdge` as soon as it is seen, because its ordinal is above the next expected one, and no +/// manifest has the 2^63 chunks that would make it the expected one, so the cursor step at `i64::MAX` cannot be +/// reached through the public meta API. The test pins that a high-ordinal edge is rejected without an error or an +/// overflow, not the checked addition. +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(url: &str) { + use craft_cloud_meta::{EdgeCheck, EdgeProblemKind}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(i64::MAX) + .bind(random_chunk().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + let expected = EdgeCheck::Mismatch(craft_cloud_meta::EdgeProblem { ordinal: 3, kind: EdgeProblemKind::MissingEdge }); + assert_eq!(meta.index_check(&m).await.unwrap(), expected); + assert_eq!(meta.index_check(&m).await.unwrap(), expected, "the report is stable"); + let edges = meta.index_edges(&m.id()).await.unwrap(); + assert_eq!(edges.len(), 4); + assert_eq!(edges.last().unwrap().0, i64::MAX as u64, "the stored edge at the maximum ordinal is still there"); +} + +/// A reset of 12 edges at budget 3 takes four full batches and one empty batch before returning to building state 0. +async fn an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let before = meta.index_edge_count().await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + for batch in 1..=4 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap(), before - batch * 3); + } + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 2, "four full batches leave reset state 2 until an empty batch proves completion"); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 0); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }, "the next call writes instead of looping"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// Two independent Postgres handles finish one damaged manifest reset without duplicate or missing ordinals. +async fn concurrent_index_completes_reset_one_manifest_on(url: &str) { + use craft_cloud_meta::IndexState; + let left = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let right = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&left, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&left, &ana, &chunks).await; + left.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + left.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + // Force the overlap: a third connection holds the row lock of the manifest's index row, both builders start and are + // proven to wait on it, and only then is it released, so both reach the reset together. + let mut holder = raw(url).await; + sqlx::query("BEGIN").execute(&mut holder).await.unwrap(); + sqlx::query("SELECT 1 FROM manifest_index WHERE manifest = $1 FOR UPDATE").bind(m.id().as_bytes().to_vec()).execute(&mut holder).await.unwrap(); + + let left_manifest = m.clone(); + let right_manifest = m.clone(); + let left_builder = left.clone(); + let mut first = tokio::spawn(async move { left_builder.index_complete(&left_manifest).await }); + let mut second = tokio::spawn(async move { right.index_complete(&right_manifest).await }); + let mut waiting = 0i64; + for _ in 0..300 { + waiting = sqlx::query_scalar( + "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock' AND query LIKE '%manifest_index%FOR UPDATE%'", + ) + .fetch_one(&mut conn) + .await + .unwrap(); + if waiting >= 2 || first.is_finished() || second.is_finished() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + let overlapped = waiting == 2 && !first.is_finished() && !second.is_finished(); + sqlx::query("ROLLBACK").execute(&mut holder).await.unwrap(); + if !overlapped { + first.abort(); + second.abort(); + } + assert!(overlapped, "both builders must be blocked on the manifest's index row at once (waiting backends: {waiting})"); + let joined = tokio::time::timeout(std::time::Duration::from_secs(30), async { + (&mut first).await.unwrap().unwrap(); + (&mut second).await.unwrap().unwrap(); + }) + .await; + if joined.is_err() { + first.abort(); + second.abort(); + } + assert!(joined.is_ok(), "concurrent index completion must finish within 30 seconds"); + + assert_eq!(left.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(left.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + let (edges, ordinals): (i64, i64) = sqlx::query_as("SELECT COUNT(*), COUNT(DISTINCT ordinal) FROM manifest_chunks WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!((edges, ordinals), (12, 12)); +} + +#[tokio::test] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&url).await; +} + +#[tokio::test] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_reset_of_an_index_goes_in_batches_and_resumes_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_reset_of_an_index_goes_in_batches_and_resumes_on(&url).await; +} + +#[tokio::test] +async fn a_build_resets_a_wrong_index_in_batches_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_resets_a_wrong_index_in_batches_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_resets_a_wrong_index_in_batches_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_resets_a_wrong_index_in_batches_on(&url).await; +} + +#[tokio::test] +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&url).await; +} + +#[tokio::test] +async fn an_exact_multiple_index_reset_finishes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_exact_multiple_index_reset_finishes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_index_completes_reset_one_manifest_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + concurrent_index_completes_reset_one_manifest_on(&url).await; } diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 6ff15c5d..f32e886c 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -5,11 +5,16 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::hash::ChunkId; -use crate::ids::{DeviceId, Epoch, NodeId, SpaceId}; +use crate::ids::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; use crate::ops::{Conflict, Op}; /// What apps see of these bodies lives in `craft-cloud-api` (no postcard), re-exported here. -pub use craft_cloud_api::api::{DeviceInfo, ErrorAnswer, PairingState, PairingStatus, codes}; +pub use craft_cloud_api::api::{ + AUDIT_ACTIONS, AccountInfo, AccountUsage, AdminCode, AdminConfirm, AdminCreateUser, AdminCreatedUser, AdminDevices, AdminManifestIndex, + AdminPasswordReset, AdminServerInfo, AdminStorageAccount, AdminStorageOverview, AdminUpdateUser, AdminUserInfo, AdminUserList, AuditDetails, + AuditEvent, AuditExportEnd, AuditPage, DeviceInfo, ErrorAnswer, PairingState, PairingStatus, RecoveryCodes, RevokedOthers, TwoFactorChallenge, + TwoFactorSetupResponse, TwoFactorStatus, UsageEstimateReason, UsageEstimates, codes, valid_audit_action, +}; /// Protocol versions this build speaks. A client and server must share at least one. pub const PROTOCOL_MIN: u32 = 1; @@ -46,13 +51,21 @@ pub struct SpaceResponse { pub username: String, } -/// `POST /api/v1/auth/login`. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +/// `POST /api/v1/auth/login`. `client_kind`, `platform` and `app_version` are optional and only shown in the device +/// list, never a basis for a permission: `client_kind` is `agent`, `desktop` or `cli`, the others are free text of at +/// most 32 bytes (all without control characters). A body without them works as before. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct LoginRequest { pub username: String, pub password: String, /// Shown in the device list, e.g. the computer's name. pub device_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, } impl std::fmt::Debug for LoginRequest { @@ -61,6 +74,9 @@ impl std::fmt::Debug for LoginRequest { .field("username", &self.username) .field("password", &"") .field("device_name", &self.device_name) + .field("client_kind", &self.client_kind) + .field("platform", &self.platform) + .field("app_version", &self.app_version) .finish() } } @@ -97,14 +113,27 @@ impl std::fmt::Debug for PairingCreated { /// `POST /api/v1/join` (no auth): redeem an invite or an add-a-computer code (normalised or as typed) for a new /// device; answered with a [`LoginResponse`]. `password` is required, and only taken, for an invite to an account -/// that has none yet. Its `Debug` output hides the code and the password. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +/// that has none yet. `client_kind`, `platform` and `app_version` are as in [`LoginRequest`]. `totp_code` or +/// `recovery_code` (one of them) is the second factor of an account that has two-factor sign-in: the server answers +/// `two_factor_required` without using the code up, and the same body is sent again with it. Its `Debug` output hides +/// the code, the password and the second factor. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct JoinRequest { pub code: String, /// Shown in the device list, e.g. the computer's name. pub device_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, } impl std::fmt::Debug for JoinRequest { @@ -113,6 +142,109 @@ impl std::fmt::Debug for JoinRequest { .field("code", &"") .field("device_name", &self.device_name) .field("password", &self.password.as_ref().map(|_| "")) + .field("client_kind", &self.client_kind) + .field("platform", &self.platform) + .field("app_version", &self.app_version) + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /api/v1/account/password` (device auth): changes the password, answered `204`. Every other device of the +/// account is signed out, and its open invite and pairings are deleted. A wrong `current_password` is `403 +/// wrong_password` (never 401), a `new_password` that breaks the rules `400 weak_password`. Its `Debug` output hides +/// both passwords. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangePasswordRequest { + pub current_password: String, + pub new_password: String, +} + +impl std::fmt::Debug for ChangePasswordRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChangePasswordRequest").field("current_password", &"").field("new_password", &"").finish() + } +} + +/// `PATCH /api/v1/devices/{id}` (device auth): renames one of the account's devices that isn't revoked; answered with +/// its [`DeviceInfo`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenameDeviceRequest { + pub name: String, +} + +// ---- two-factor sign-in (`/api/v1/account/two-factor/…`, `/api/v1/auth/login/second-factor`) ---- +// +// The second factor is a TOTP code (`totp_code`, six digits) or a recovery code (`recovery_code`, 16 letters and +// digits, with dashes or without). A wrong one is `403 two_factor_invalid` on a signed-in route and `401` at sign-in and +// join (which are not signed in); too many wrong ones lock the account's second factor for a while (`429 +// two_factor_locked` with `Retry-After`). Every type here that holds a secret hides it in `Debug`. + +/// `POST /account/two-factor/setup`: the account's password starts setting two-factor sign-in up. Its `Debug` output +/// hides the password. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorSetupRequest { + pub password: String, +} + +impl std::fmt::Debug for TwoFactorSetupRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorSetupRequest").field("password", &"").finish() + } +} + +/// `POST /account/two-factor/confirm`: the first code from the authenticator app turns two-factor sign-in on. Its +/// `Debug` output hides the code. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorConfirmRequest { + pub totp_code: String, +} + +impl std::fmt::Debug for TwoFactorConfirmRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorConfirmRequest").field("totp_code", &"").finish() + } +} + +/// `POST /account/two-factor/disable` and `POST /account/two-factor/recovery-codes`: the password and one second +/// factor (`recovery-codes` takes `totp_code` only). Its `Debug` output hides all three. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TwoFactorProofRequest { + pub password: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for TwoFactorProofRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TwoFactorProofRequest") + .field("password", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) + .finish() + } +} + +/// `POST /auth/login/second-factor`: the challenge `two_factor_required` gave, and one second factor. Answered with a +/// [`LoginResponse`]. Its `Debug` output hides all of it. +#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SecondFactorRequest { + pub challenge: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub totp_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recovery_code: Option, +} + +impl std::fmt::Debug for SecondFactorRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecondFactorRequest") + .field("challenge", &"") + .field("totp_code", &self.totp_code.as_ref().map(|_| "")) + .field("recovery_code", &self.recovery_code.as_ref().map(|_| "")) .finish() } } @@ -145,6 +277,166 @@ pub struct SaveAttach { pub save: crate::SaveRecord, } +/// Versions in one `GET /api/v1/nodes/{id}/history` page when `limit` is not given. +pub const HISTORY_DEFAULT_LIMIT: u32 = 50; +/// Most versions in one history page; a larger `limit` is `invalid`. +pub const HISTORY_MAX_LIMIT: u32 = 200; + +/// Where the next history page starts: after this `(seq, version)` in the newest-first order. On the wire it is an +/// opaque string; clients pass back the `next` they were given and nothing else. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HistoryCursor { + pub seq: u64, + pub version: VersionId, +} + +impl std::fmt::Display for HistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.seq, self.version) + } +} + +/// A history cursor that does not read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BadHistoryCursor; + +impl std::fmt::Display for BadHistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("not a history cursor") + } +} + +impl std::error::Error for BadHistoryCursor {} + +impl std::str::FromStr for HistoryCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted, so one cursor has one spelling. + fn from_str(s: &str) -> Result { + let (seq, version) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = HistoryCursor { seq: seq.parse().map_err(|_| BadHistoryCursor)?, version: version.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One version of a file in its history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryVersion { + pub id: VersionId, + pub parent_version: Option, + pub epoch: Epoch, + /// The space's change sequence number of the commit that made this version. + pub seq: u64, + pub size: u64, + pub mtime_ms: i64, + /// The device that committed it. + pub device: DeviceId, + pub created_ms: i64, + /// Whether a save record (a Craft app's live-document save) hangs on this version. + pub save: bool, + /// Thinned out of the history: its row stays, but it can't be restored. Always `false` until thinning exists. + pub pruned: bool, +} + +/// `GET /api/v1/nodes/{id}/history?cursor&limit`: the versions of a file, newest first by `(seq, id)`. `next` is the +/// `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryResponse { + pub versions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/nodes/{id}/restore`: make `version` the file's content again as a new version. `expected_version` +/// must still be the file's current version, else the answer is `conflict`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreRequest { + pub version: VersionId, + pub expected_version: VersionId, +} + +/// A restore that applied: the new version, which follows `parent_version` (the one that was current). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreResponse { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + +/// Items in one `GET /api/v1/trash` page when `limit` is not given. +pub const TRASH_DEFAULT_LIMIT: u32 = 50; +/// Most items in one trash page; a larger `limit` is `invalid`. +pub const TRASH_MAX_LIMIT: u32 = 200; + +/// Where the next trash page starts: after this `(deleted_ms, node)` in the newest-deleted-first order. Opaque on the +/// wire, like [`HistoryCursor`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TrashCursor { + pub deleted_ms: i64, + pub node: NodeId, +} + +impl std::fmt::Display for TrashCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.deleted_ms, self.node) + } +} + +impl std::str::FromStr for TrashCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted. + fn from_str(s: &str) -> Result { + let (deleted_ms, node) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = TrashCursor { deleted_ms: deleted_ms.parse().map_err(|_| BadHistoryCursor)?, node: node.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashItem { + pub id: NodeId, + /// The folder it was in when it was deleted (itself deleted, perhaps). + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + pub deleted_ms: i64, + /// `deleted_ms` plus the server's `retention.trash_days`: when the server may remove the node for good. Nothing is + /// removed yet, so this is a promise of the earliest time, not an event. + pub purge_after_ms: i64, +} + +/// `GET /api/v1/trash?cursor&limit`: the deleted nodes of your space that were not removed from the trash, newest +/// deletion first by `(deleted_ms, id)`. `next` is the `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashResponse { + pub items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/trash/{id}/restore`: the node is back. `parent` and `name` say where, which differs from where it was +/// when a folder of its path was merged into a live folder of the same name, or when `renamed` is set: the name was +/// taken, so the node got a conflict name like the sync engine makes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashRestoreResponse { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + /// For a file, its new current version: a copy of its last one (the same content), so a delete sent against the old + /// version, whose answer a machine never heard, can't delete it again. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub seq: u64, + pub epoch: Epoch, +} + /// A commit that applied. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitResponse { @@ -159,8 +451,24 @@ pub struct QuotaUse { pub limit: u64, } +/// What the server knows of its own room, with `server_full`. Each number is left out when the server has none (a +/// store on S3 has no filesystem to measure; no `capacity` is set). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServerRoom { + /// Bytes still available on the store's filesystem (disk store only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub free_bytes: Option, + /// Total size of the store's filesystem (disk store only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_bytes: Option, + /// What the admin said the store may hold (`capacity`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capacity_bytes: Option, +} + /// Every error answer. `error` is a stable code, one of [`codes`]; `message` is for people. `conflicts` is -/// set for `conflict`, `missing` for `missing_chunks`, `quota` for `quota_exceeded`. Clients ignore codes and JSON +/// set for `conflict`, `missing` for `missing_chunks`, `quota` for `quota_exceeded`, `server` for `server_full`, +/// `two_factor` for `two_factor_required`. Clients ignore codes and JSON /// fields they don't know (the server may add more later), so match on the codes you handle and treat the rest by /// HTTP status. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -173,6 +481,12 @@ pub struct ErrorBody { pub missing: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub quota: Option, + /// Set for `server_full`: what the server knows of its room. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub server: Option, + /// Set for `two_factor_required`: the challenge to answer at `POST /auth/login/second-factor`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub two_factor: Option, } /// Response header carrying the space's epoch on every signed-in request. @@ -184,15 +498,115 @@ mod tests { #[test] fn secrets_stay_out_of_debug_output() { - let req = LoginRequest { username: "ana".into(), password: "hunter2-secret".into(), device_name: "pc".into() }; + let req = LoginRequest { username: "ana".into(), password: "hunter2-secret".into(), device_name: "pc".into(), ..LoginRequest::default() }; assert!(!format!("{req:?}").contains("hunter2-secret")); let res = LoginResponse { device: DeviceId::generate(), token: "tok-secret".into() }; assert!(!format!("{res:?}").contains("tok-secret")); - let join = JoinRequest { code: "K7M4Q9TX".into(), device_name: "pc".into(), password: Some("hunter2-secret".into()) }; + let join = + JoinRequest { code: "K7M4Q9TX".into(), device_name: "pc".into(), password: Some("hunter2-secret".into()), ..JoinRequest::default() }; let shown = format!("{join:?}"); assert!(!shown.contains("K7M4Q9TX") && !shown.contains("hunter2-secret"), "{shown}"); let created = PairingCreated { id: Uuid::nil(), code: "K7M4Q9TX".into(), expires_ms: 1 }; assert!(!format!("{created:?}").contains("K7M4Q9TX")); + let change = ChangePasswordRequest { current_password: "old-secret-pw".into(), new_password: "new-secret-pw".into() }; + let shown = format!("{change:?}"); + assert!(!shown.contains("old-secret-pw") && !shown.contains("new-secret-pw"), "{shown}"); + assert_eq!(shown.matches("").count(), 2); + let confirm = AdminConfirm { + password: Some("admin-secret-pw".into()), + totp_code: Some("918273".into()), + recovery_code: Some("RECOVERYSECRET00".into()), + }; + let shown = format!("{confirm:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273") && !shown.contains("RECOVERYSECRET"), "{shown}"); + let reset = + AdminPasswordReset { password: Some("admin-secret-pw".into()), totp_code: Some("918273".into()), ..AdminPasswordReset::default() }; + let shown = format!("{reset:?}"); + assert!(!shown.contains("admin-secret-pw") && !shown.contains("918273"), "{shown}"); + let create = + AdminCreateUser { username: "ana".into(), role: "member".into(), password: Some("admin-secret-pw".into()), ..AdminCreateUser::default() }; + assert!(!format!("{create:?}").contains("admin-secret-pw")); + let update = AdminUpdateUser { password: Some("admin-secret-pw".into()), ..AdminUpdateUser::default() }; + assert!(!format!("{update:?}").contains("admin-secret-pw")); + let code = AdminCode { code: "ABCDEFGHJKMNPQRS".into(), join_url: Some("cloud.example.org/ABCD-EFGH-JKMN-PQRS".into()), expires_ms: 1 }; + let shown = format!("{code:?} {:?}", AdminCreatedUser { user: sample_user(), invite: code.clone() }); + assert!(!shown.contains("ABCDEFGH") && !shown.contains("cloud.example.org"), "{shown}"); + } + + #[test] + fn two_factor_secrets_stay_out_of_debug_output() { + let setup = TwoFactorSetupResponse { + secret: "JBSWY3DPEHPK3PXP".into(), + otpauth_uri: "otpauth://totp/Craft%20Cloud:ana?secret=JBSWY3DPEHPK3PXP".into(), + expires_ms: 5, + }; + let shown = format!("{setup:?}"); + assert!(!shown.contains("JBSWY3DP") && !shown.contains("otpauth:"), "{shown}"); + let codes = RecoveryCodes { recovery_codes: vec!["ABCD-EFGH-JKMN-PQRS".into(), "WXYZ-0123-4567-89AB".into()] }; + let shown = format!("{codes:?}"); + assert!(!shown.contains("ABCD") && !shown.contains("WXYZ"), "{shown}"); + let second = + SecondFactorRequest { challenge: "challenge-secret".into(), totp_code: Some("123456".into()), recovery_code: Some("RC-SECRET".into()) }; + let shown = format!("{second:?}"); + assert!(!shown.contains("challenge-secret") && !shown.contains("123456") && !shown.contains("RC-SECRET"), "{shown}"); + let proof = TwoFactorProofRequest { password: "pw-secret-1".into(), totp_code: Some("654321".into()), recovery_code: None }; + let shown = format!("{proof:?}"); + assert!(!shown.contains("pw-secret-1") && !shown.contains("654321"), "{shown}"); + assert!(!format!("{:?}", TwoFactorSetupRequest { password: "pw-secret-1".into() }).contains("pw-secret-1")); + assert!(!format!("{:?}", TwoFactorConfirmRequest { totp_code: "654321".into() }).contains("654321")); + let join = JoinRequest { + code: "K7M4Q9TX".into(), + totp_code: Some("111222".into()), + recovery_code: Some("RC-SECRET".into()), + ..JoinRequest::default() + }; + let shown = format!("{join:?}"); + assert!(!shown.contains("111222") && !shown.contains("RC-SECRET"), "{shown}"); + let body = ErrorBody { + error: codes::TWO_FACTOR_REQUIRED.into(), + message: "m".into(), + two_factor: Some(TwoFactorChallenge { challenge: "challenge-secret".into(), expires_ms: 1 }), + ..ErrorBody::default() + }; + assert!(!format!("{body:?}").contains("challenge-secret")); + // The challenge travels in the error body, and an older client that doesn't know the field ignores it. + let json = serde_json::to_value(&body).unwrap(); + assert_eq!(json["two_factor"], serde_json::json!({ "challenge": "challenge-secret", "expires_ms": 1 })); + assert_eq!(serde_json::from_value::(json).unwrap(), body); + } + + fn sample_user() -> AdminUserInfo { + AdminUserInfo { + id: Uuid::nil(), + username: "ana".into(), + role: "member".into(), + created_ms: 1, + disabled_ms: None, + two_factor_enabled: false, + devices_active: 0, + stored_bytes: 0, + quota_bytes: None, + invite_pending: false, + } + } + + #[test] + fn an_update_tells_a_missing_quota_from_a_null_one() { + let missing: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "role": "admin", "password": "p" })).unwrap(); + assert_eq!(missing.quota_bytes, None); + let null: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": null })).unwrap(); + assert_eq!(null.quota_bytes, Some(None)); + let set: AdminUpdateUser = serde_json::from_value(serde_json::json!({ "quota_bytes": 5_000_000 })).unwrap(); + assert_eq!(set.quota_bytes, Some(Some(5_000_000))); + assert!(serde_json::from_value::(serde_json::json!({ "quota_bytes": -1 })).is_err()); + assert_eq!(serde_json::to_value(&set).unwrap(), serde_json::json!({ "quota_bytes": 5_000_000 })); + } + + #[test] + fn an_admin_body_ignores_fields_it_does_not_know() { + let confirm: AdminConfirm = serde_json::from_value(serde_json::json!({ "password": "p", "totp_code": "123456", "later": 1 })).unwrap(); + assert_eq!((confirm.password.as_deref(), confirm.totp_code.as_deref(), confirm.recovery_code), (Some("p"), Some("123456"), None)); + assert_eq!(serde_json::from_value::(serde_json::json!({})).unwrap().password, None); } #[test] @@ -204,11 +618,42 @@ mod tests { assert_eq!(waiting, PairingStatus { state: PairingState::Waiting, device_name: None }); let join: JoinRequest = serde_json::from_value(serde_json::json!({ "code": "c", "device_name": "d" })).unwrap(); assert_eq!(join.password, None); + assert_eq!((join.client_kind, join.platform, join.app_version), (None, None, None)); + let login = LoginRequest { username: "ana".into(), password: "pw".into(), device_name: "pc".into(), ..LoginRequest::default() }; + assert_eq!(serde_json::to_value(&login).unwrap(), serde_json::json!({ "username": "ana", "password": "pw", "device_name": "pc" })); assert_eq!(codes::INVALID_CODE, "invalid_code"); assert_eq!(codes::PASSWORD_REQUIRED, "password_required"); assert_eq!(codes::WEAK_PASSWORD, "weak_password"); } + #[test] + fn audit_bodies_have_a_plain_json_form() { + let details: AuditDetails = serde_json::from_str(r#"{"revoked":2,"a_later_field":"x","quota_bytes":null}"#).unwrap(); + assert_eq!(details.revoked, Some(2)); + assert_eq!(details.quota_bytes, Some(None)); + assert_eq!(serde_json::to_string(&details).unwrap(), r#"{"quota_bytes":null,"revoked":2}"#); + assert_eq!(serde_json::from_str::("{}").unwrap(), AuditDetails::default()); + assert_eq!(serde_json::to_string(&AuditDetails::default()).unwrap(), "{}"); + let end = AuditExportEnd { complete: Some(true), truncated: None, exported: 3 }; + assert_eq!(serde_json::to_string(&end).unwrap(), r#"{"complete":true,"exported":3}"#); + let event = AuditEvent { + id: Uuid::nil(), + at_ms: 5, + actor_user: None, + actor_device: None, + action: "auth.login_failed".into(), + target_kind: String::new(), + target_id: None, + outcome: "failure".into(), + details: AuditDetails { step: Some("password".into()), ..AuditDetails::default() }, + count: 4, + }; + let json = serde_json::to_value(&event).unwrap(); + assert_eq!(json["actor_user"], serde_json::Value::Null); + assert_eq!(json["count"], 4); + assert_eq!(serde_json::from_value::(json).unwrap(), event); + } + #[test] fn error_bodies_omit_empty_extras() { let body = ErrorBody { error: "not_found".into(), message: "no such chunk".into(), ..ErrorBody::default() }; @@ -217,6 +662,36 @@ mod tests { assert_eq!(serde_json::from_value::(json).unwrap(), body); } + #[test] + fn account_usage_round_trips_and_ignores_unknown_fields() { + let response = AccountUsage { + stored_bytes: 17, + quota_bytes: Some(99), + trash_bytes: None, + history_bytes: None, + estimates: UsageEstimates { index_complete: false, reason: Some(UsageEstimateReason::IndexIncomplete) }, + }; + let value = serde_json::to_value(&response).unwrap(); + assert_eq!(serde_json::from_value::(value.clone()).unwrap(), response); + let mut value = value; + value.as_object_mut().unwrap().insert("future_field".into(), serde_json::json!(1)); + assert_eq!(serde_json::from_value::(value).unwrap(), response); + } + + #[test] + fn account_usage_decodes_complete_and_incomplete_estimates() { + let complete: AccountUsage = + serde_json::from_str(r#"{"stored_bytes":11,"quota_bytes":null,"trash_bytes":0,"history_bytes":0,"estimates":{"index_complete":true}}"#) + .unwrap(); + assert_eq!(complete.estimates, UsageEstimates { index_complete: true, reason: None }); + let incomplete: AccountUsage = serde_json::from_str( + r#"{"stored_bytes":11,"quota_bytes":null,"trash_bytes":null,"history_bytes":null,"estimates":{"index_complete":false,"reason":"index_incomplete"}}"#, + ).unwrap(); + assert_eq!(incomplete.trash_bytes, None); + assert_eq!(incomplete.history_bytes, None); + assert_eq!(incomplete.estimates.reason, Some(UsageEstimateReason::IndexIncomplete)); + } + #[test] fn a_quota_answer_says_how_much_is_used() { let body = ErrorBody { @@ -230,6 +705,22 @@ mod tests { assert_eq!(serde_json::from_value::(json).unwrap(), body); } + #[test] + fn a_server_full_answer_says_what_the_server_knows_of_its_room() { + let body = ErrorBody { + error: codes::SERVER_FULL.into(), + message: "m".into(), + server: Some(ServerRoom { free_bytes: Some(5), total_bytes: Some(100), capacity_bytes: None }), + ..ErrorBody::default() + }; + let json = serde_json::to_value(&body).unwrap(); + assert_eq!(json["server"], serde_json::json!({ "free_bytes": 5, "total_bytes": 100 }), "unknown numbers are left out"); + assert_eq!(serde_json::from_value::(json).unwrap(), body); + // An answer without the field (an older server, or another code) reads. + let plain: ErrorBody = serde_json::from_value(serde_json::json!({ "error": codes::SERVER_FULL, "message": "m" })).unwrap(); + assert_eq!(plain.server, None); + } + #[test] fn unknown_error_fields_are_ignored() { let json = serde_json::json!({ "error": "something_new", "message": "m", "retry_in": 3 }); @@ -237,6 +728,42 @@ mod tests { assert_eq!(body.error, "something_new"); } + #[test] + fn history_cursors_have_one_spelling() { + let cursor = HistoryCursor { seq: 7, version: VersionId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in + ["", "7", "7.", ".x", "x.y", "+7.00000000-0000-0000-0000-000000000000", &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] + { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + assert_eq!("-1.00000000-0000-0000-0000-000000000000".parse::(), Err(BadHistoryCursor)); + } + + #[test] + fn trash_cursors_have_one_spelling() { + let cursor = TrashCursor { deleted_ms: 1_760_000_000_000, node: NodeId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in ["", "7", "7.", ".x", "x.y", &format!("+{text}"), &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + } + + #[test] + fn history_and_restore_bodies_have_a_plain_json_form() { + let restore: RestoreRequest = serde_json::from_value(serde_json::json!({ + "version": "01a12b0b-9fca-73db-b82d-c7c33683601e", + "expected_version": "01a12b0b-9fca-73db-b82d-c7c33683601f", + "unknown": 1 + })) + .unwrap(); + assert_ne!(restore.version, restore.expected_version); + let last = HistoryResponse { versions: vec![], next: None }; + assert_eq!(serde_json::to_value(&last).unwrap(), serde_json::json!({ "versions": [] })); + } + #[test] fn error_codes_are_stable() { assert_eq!(codes::MISSING_CHUNKS, "missing_chunks"); diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 4f219f67..6054bbdd 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -12,6 +12,7 @@ async-trait.workspace = true blake3.workspace = true bytes.workspace = true craft-cloud-proto.workspace = true +futures-util.workspace = true object_store.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/store/src/disk.rs b/crates/store/src/disk.rs index cea80de5..9c3436fc 100644 --- a/crates/store/src/disk.rs +++ b/crates/store/src/disk.rs @@ -5,8 +5,10 @@ //! their parents as well, so a crash can't lose the entry that points at a new shard directory. //! A crash leaves the old object or the new one. -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use async_trait::async_trait; use bytes::Bytes; @@ -15,6 +17,8 @@ use crate::{Backend, StoreError}; pub struct DiskBackend { root: PathBuf, + /// How many directories listings have opened, for tests that check a page doesn't walk finished directories again. + dirs_read: Arc, } impl DiskBackend { @@ -22,12 +26,34 @@ impl DiskBackend { pub fn new(root: impl Into) -> Result { let root = root.into(); std::fs::create_dir_all(&root)?; - Ok(Self { root }) + Ok(Self { root, dirs_read: Arc::new(AtomicUsize::new(0)) }) } fn full_path(&self, path: &str) -> PathBuf { self.root.join(path) } + + /// The number of directories listings have read so far. Only meant for tests of the listing's cost. + #[doc(hidden)] + pub fn directories_read(&self) -> usize { + self.dirs_read.load(Ordering::Relaxed) + } +} + +/// The name of a write's temporary file: `.tmp-` and a hyphenated lowercase UUIDv7. +fn temp_name(id: uuid::Uuid) -> String { + format!("{TEMP_PREFIX}{}", id.hyphenated()) +} + +const TEMP_PREFIX: &str = ".tmp-"; + +/// Whether `name` is exactly what [`temp_name`] writes. Any other `.tmp-` name is somebody else's file and +/// is listed as foreign. +fn is_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + uuid::Uuid::try_parse(rest).is_ok_and(|id| id.get_version() == Some(uuid::Version::SortRand) && id.hyphenated().to_string() == rest) } fn write_atomic(target: &Path, data: &[u8]) -> io::Result<()> { @@ -35,7 +61,7 @@ fn write_atomic(target: &Path, data: &[u8]) -> io::Result<()> { // Directories this write will create, deepest first. Their entries in the parents must be flushed too. let created: Vec<&Path> = dir.ancestors().take_while(|p| !p.as_os_str().is_empty() && !p.exists()).collect(); std::fs::create_dir_all(dir)?; - let tmp = dir.join(format!(".tmp-{}", uuid::Uuid::now_v7())); + let tmp = dir.join(temp_name(uuid::Uuid::now_v7())); let result = (|| { let mut file = std::fs::File::create(&tmp)?; file.write_all(data)?; @@ -73,15 +99,38 @@ async fn blocking(f: impl FnOnce() -> io::Result + Send + #[async_trait] impl Backend for DiskBackend { + fn kind(&self) -> &'static str { + "disk" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { let target = self.full_path(path); blocking(move || write_atomic(&target, &data)).await } async fn get_raw(&self, path: &str) -> Result, StoreError> { + self.get_raw_limited(path, usize::MAX).await + } + + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + let target = self.full_path(path); + blocking(move || match std::fs::File::open(&target) { + Ok(file) => { + let mut bytes = Vec::new(); + file.take(u64::try_from(max_bytes.saturating_add(1)).unwrap_or(u64::MAX)).read_to_end(&mut bytes)?; + Ok(Some(Bytes::from(bytes))) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + }) + .await + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { let target = self.full_path(path); - blocking(move || match std::fs::read(&target) { - Ok(bytes) => Ok(Some(Bytes::from(bytes))), + blocking(move || match std::fs::symlink_metadata(&target) { + Ok(meta) if meta.file_type().is_file() => Ok(Some(meta.len())), + Ok(_) => Ok(None), Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), Err(e) => Err(e), }) @@ -97,4 +146,108 @@ impl Backend for DiskBackend { }) .await } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + if Path::new(prefix).is_absolute() || Path::new(prefix).components().any(|part| !matches!(part, Component::Normal(_))) { + return Err(StoreError::Backend("listing prefix must be a relative path".into())); + } + let root = self.root.clone(); + let prefix = prefix.to_owned(); + let after = after.map(str::to_owned); + let dirs_read = Arc::clone(&self.dirs_read); + blocking(move || { + let Some(start) = list_prefix_dir(&root, &prefix)? else { + return Ok(Vec::new()); + }; + let mut out = Vec::new(); + walk_sorted(&start, &prefix, after.as_deref(), limit, &dirs_read, &mut out)?; + Ok(out) + }) + .await + } +} + +fn list_prefix_dir(root: &Path, prefix: &str) -> io::Result> { + let mut current = root.to_path_buf(); + for component in Path::new(prefix).components() { + let Component::Normal(name) = component else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "listing prefix must be a relative path")); + }; + current.push(name); + let metadata = match std::fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Ok(None); + } + } + Ok(Some(current)) +} + +/// Collects up to `limit` file paths below `dir` that sort strictly after `after`, in byte order of the +/// relative path (`/` separators), the order memory and S3 use. `relative` is `dir`'s path relative to the root. +/// Returns `true` once `out` is full. +/// +/// One directory's entries are read and sorted at a time, so memory is bounded by the largest single +/// directory listing (a shard directory holds at most 256 entries) plus `limit` paths. A subtree whose paths +/// all sort at or before `after` is skipped without being opened, and the walk stops as soon as `limit` paths +/// are found, so a page costs the directories on the way to `after` and to the end of the page, not the +/// store. A directory sorts as `name/`, so `a.b` comes before everything in `a/`. +fn walk_sorted(dir: &Path, relative: &str, after: Option<&str>, limit: usize, dirs_read: &AtomicUsize, out: &mut Vec) -> io::Result { + if out.len() >= limit { + return Ok(true); + } + let metadata = match std::fs::symlink_metadata(dir) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Ok(false); + } + let reader = match std::fs::read_dir(dir) { + Ok(reader) => reader, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + dirs_read.fetch_add(1, Ordering::Relaxed); + // (sort key, name, is a directory) + let mut entries: Vec<(String, String, bool)> = Vec::new(); + for entry in reader { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "store path is not valid UTF-8"))?.to_owned(); + let file_type = entry.file_type()?; + if file_type.is_symlink() { + continue; + } + if file_type.is_dir() { + entries.push((format!("{name}/"), name, true)); + } else if file_type.is_file() && !is_temp_name(&name) { + entries.push((name.clone(), name, false)); + } + } + entries.sort_unstable(); + for (_, name, is_dir) in entries { + let path = if relative.is_empty() { name.clone() } else { format!("{relative}/{name}") }; + if is_dir { + let subtree = format!("{path}/"); + // Everything below sorts after `subtree` and starts with it; it is all at or before `after` + // exactly when `after` is greater and is not itself below it. + if after.is_some_and(|cursor| cursor > subtree.as_str() && !cursor.starts_with(&subtree)) { + continue; + } + if walk_sorted(&dir.join(&name), &path, after, limit, dirs_read, out)? { + return Ok(true); + } + } else if after.is_none_or(|cursor| path.as_str() > cursor) { + out.push(path); + if out.len() >= limit { + return Ok(true); + } + } + } + Ok(false) } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 43f37475..b4ff4eb8 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -6,6 +6,15 @@ //! //! Each stored object is one format byte followed by the blob. Format 0 is plain bytes; other values are //! reserved for compression or encryption and are refused today. +//! +//! [`Store::list`] returns bounded, ordered pages scoped to one blob namespace. Foreign object paths remain +//! visible in the page. S3-compatible backends may return incomplete or stale listings, so listing is not a +//! proof that the namespace contains every object. Disk listing reports an I/O error for non-UTF-8 names, +//! which the string path API cannot represent. Disk and memory list in byte order of the relative path and +//! resume from the cursor without visiting what lies before it; a disk page reads one directory at a time. On +//! S3 a page reads the listing only up to its limit, and a key that `object_store` cannot represent (an empty +//! path segment, as in `chunks//a`) ends the listing with an error instead of appearing as foreign; a key +//! with a trailing slash is reported without the slash. Cursors longer than any real path are refused. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod disk; @@ -89,14 +98,106 @@ pub enum StoreError { /// Moves raw bytes. Implementations don't check anything; [`Store`] does. #[async_trait] pub trait Backend: Send + Sync + 'static { + /// What kind of storage this is, for the admin's server overview (`disk`, `s3` or `memory`). A backend that + /// doesn't say is `other`. Never a path, an address or a setting. + fn kind(&self) -> &'static str { + "other" + } /// Writes the object atomically: a reader sees the old object, the new one, or none, never a mix. async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError>; /// `None` when there is no such object. async fn get_raw(&self, path: &str) -> Result, StoreError>; + /// Reads at most `max_bytes` plus one byte. Backends should override this for bounded I/O. + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + Ok(self.get_raw(path).await?.map(|bytes| bytes.slice(..bytes.len().min(max_bytes.saturating_add(1))))) + } + /// Stored object size without reading object contents. + async fn size_raw(&self, _path: &str) -> Result, StoreError> { + Err(StoreError::Backend("backend does not support metadata-only object sizes".into())) + } /// Removing a missing object succeeds. async fn delete_raw(&self, path: &str) -> Result<(), StoreError>; + /// Lists paths below `prefix`, in ascending order, strictly after `after`. + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError>; +} + +/// The longest object path a listing cursor can carry. S3 keys are at most 1024 bytes and Linux paths at most +/// 4096, so no object the backends can hold needs more. +const MAX_CURSOR_PATH_BYTES: usize = 4096; +/// The longest valid token: the `1:c:` header plus two hex digits per path byte. +const MAX_CURSOR_TOKEN_BYTES: usize = 4 + 2 * MAX_CURSOR_PATH_BYTES; + +/// Opaque continuation token returned by [`Store::list`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListCursor(String); + +impl ListCursor { + /// Reconstructs a cursor token saved from a previous page. + pub fn from_token(token: impl Into) -> Self { + Self(token.into()) + } + + /// Returns this cursor's opaque token for storage or transport. + pub fn as_token(&self) -> &str { + &self.0 + } + + fn new(kind: BlobKind, path: &str) -> Result { + if path.len() > MAX_CURSOR_PATH_BYTES { + return Err(StoreError::Backend("object path is too long for a listing cursor".into())); + } + let kind = match kind { + BlobKind::Chunk => "c", + BlobKind::Manifest => "m", + }; + let encoded = path.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect::(); + Ok(Self(format!("1:{kind}:{encoded}"))) + } + + fn decode(&self, kind: BlobKind) -> Result { + let invalid = || StoreError::Backend("invalid listing cursor".into()); + // Checked before anything is split, allocated or decoded: a token is untrusted input. + if self.0.len() > MAX_CURSOR_TOKEN_BYTES { + return Err(invalid()); + } + let mut parts = self.0.splitn(3, ':'); + let version = parts.next().ok_or_else(invalid)?; + let cursor_kind = parts.next().ok_or_else(invalid)?; + let encoded = parts.next().ok_or_else(invalid)?; + let expected = match kind { + BlobKind::Chunk => "c", + BlobKind::Manifest => "m", + }; + if version != "1" || cursor_kind != expected || encoded.is_empty() || encoded.len() % 2 != 0 { + return Err(invalid()); + } + let mut bytes = Vec::with_capacity(encoded.len() / 2); + let mut index = 0; + while index < encoded.len() { + let end = index.checked_add(2).ok_or_else(invalid)?; + let pair = encoded.as_bytes().get(index..end).ok_or_else(invalid)?; + let hex = std::str::from_utf8(pair).map_err(|_| invalid())?; + bytes.push(u8::from_str_radix(hex, 16).map_err(|_| invalid())?); + index = end; + } + let path = String::from_utf8(bytes).map_err(|_| invalid())?; + if !path.starts_with(&format!("{}/", dir(kind))) { + return Err(invalid()); + } + Ok(path) + } } +/// One bounded page. Invalid object names appear in `foreign`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListPage { + pub blobs: Vec, + pub foreign: Vec, + pub next_cursor: Option, +} + +pub const MAX_LIST_PAGE_SIZE: usize = 1000; + /// The blob store the server uses. Cheap to clone. #[derive(Clone)] pub struct Store { @@ -108,6 +209,11 @@ impl Store { Self { backend: Arc::new(backend) } } + /// The backend's kind (`disk`, `s3`, `memory` or `other`). + pub fn kind(&self) -> &'static str { + self.backend.kind() + } + /// Stores `data` under `key` after checking its size and hash. Storing a blob that is already there /// succeeds and changes nothing. pub async fn put(&self, key: BlobKey, data: &[u8]) -> Result<(), StoreError> { @@ -126,13 +232,21 @@ impl Store { /// Reads a blob and checks it. A blob whose bytes no longer match its ID is an error, never served. pub async fn get(&self, key: BlobKey) -> Result { - let object = self.backend.get_raw(&key.path()).await?.ok_or(StoreError::NotFound(key))?; + self.get_sized(key).await.map(|(data, _)| data) + } + + /// Like [`Store::get`], and also says how many bytes the object is stored as (the format byte included), which is + /// what the metadata records as the blob's stored size. + pub async fn get_sized(&self, key: BlobKey) -> Result<(Bytes, u64), StoreError> { + let max_raw = max_len(key.kind).saturating_add(1); + let object = self.backend.get_raw_limited(&key.path(), max_raw).await?.ok_or(StoreError::NotFound(key))?; let Some((&format, _)) = object.split_first() else { return Err(StoreError::Corrupt { key, reason: "empty object" }); }; if format != FORMAT_PLAIN { return Err(StoreError::Corrupt { key, reason: "unknown format byte" }); } + let stored = u64::try_from(object.len()).unwrap_or(u64::MAX); let data = object.slice(1..); if data.len() > max_len(key.kind) { return Err(StoreError::Corrupt { key, reason: "larger than the limit" }); @@ -140,12 +254,75 @@ impl Store { if *blake3::hash(&data).as_bytes() != key.id { return Err(StoreError::Corrupt { key, reason: "hash mismatch" }); } - Ok(data) + Ok((data, stored)) + } + + /// Returns stored object size without materializing the object. + pub async fn stored_size(&self, key: BlobKey) -> Result { + self.backend.size_raw(&key.path()).await?.ok_or(StoreError::NotFound(key)) } pub async fn delete(&self, key: BlobKey) -> Result<(), StoreError> { self.backend.delete_raw(&key.path()).await } + + /// Lists one kind beneath its namespace. S3-compatible listings can be incomplete or stale. + pub async fn list(&self, kind: BlobKind, cursor: Option, limit: usize) -> Result { + if limit == 0 { + return Err(StoreError::Backend("listing limit must be greater than zero".into())); + } + let limit = limit.min(MAX_LIST_PAGE_SIZE); + let after = cursor.as_ref().map(|cursor| cursor.decode(kind)).transpose()?; + if let Some(path) = &after + && !path.starts_with(&format!("{}/", dir(kind))) + { + return Err(StoreError::Backend("invalid listing cursor".into())); + } + let request_limit = limit.checked_add(1).ok_or_else(|| StoreError::Backend("listing limit is too large".into()))?; + let mut paths = self.backend.list_raw(dir(kind), after.as_deref(), request_limit).await?; + paths.sort(); + paths.dedup(); + if paths.iter().any(|path| !path.starts_with(&format!("{}/", dir(kind)))) { + return Err(StoreError::Backend("backend returned object outside listing prefix".into())); + } + let has_more = paths.len() > limit; + paths.truncate(limit); + let next_cursor = if has_more { paths.last().map(|path| ListCursor::new(kind, path)).transpose()? } else { None }; + let mut blobs = Vec::new(); + let mut foreign = Vec::new(); + for path in paths { + match parse_key(kind, &path) { + Some(key) => blobs.push(key), + None => foreign.push(path), + } + } + Ok(ListPage { blobs, foreign, next_cursor }) + } +} + +fn parse_key(kind: BlobKind, path: &str) -> Option { + let mut parts = path.split('/'); + if parts.next()? != dir(kind) { + return None; + } + let first = parts.next()?; + let second = parts.next()?; + let hex = parts.next()?; + if parts.next().is_some() + || first.len() != 2 + || second.len() != 2 + || hex.len() != 64 + || !hex.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + || &hex[..2] != first + || &hex[2..4] != second + { + return None; + } + let mut id = [0u8; 32]; + for (i, byte) in id.iter_mut().enumerate() { + *byte = u8::from_str_radix(hex.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(BlobKey { kind, id }) } #[cfg(test)] @@ -161,4 +338,16 @@ mod tests { assert_eq!(key.path(), format!("chunks/ab/cd/abcd{}", "00".repeat(30))); assert!(BlobKey { kind: BlobKind::Manifest, id }.path().starts_with("manifests/ab/cd/")); } + + #[tokio::test] + async fn disk_read_rejects_huge_object_without_reading_it_all() { + let root = tempfile::tempdir().unwrap(); + let key = BlobKey { kind: BlobKind::Chunk, id: [7; 32] }; + let path = root.path().join(key.path()); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let file = std::fs::File::create(&path).unwrap(); + file.set_len(u64::from(MAX_CHUNK_BYTES) + 1_000_000_000).unwrap(); + let store = Store::new(disk::DiskBackend::new(root.path()).unwrap()); + assert!(matches!(store.get_sized(key).await, Err(StoreError::Corrupt { reason: "larger than the limit", .. }))); + } } diff --git a/crates/store/src/memory.rs b/crates/store/src/memory.rs index 47a007ce..ca41d901 100644 --- a/crates/store/src/memory.rs +++ b/crates/store/src/memory.rs @@ -1,6 +1,6 @@ //! An in-memory backend for tests. -use std::collections::HashMap; +use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use async_trait::async_trait; @@ -11,7 +11,7 @@ use crate::{Backend, StoreError}; /// Clones share the same objects, so a test can keep a handle to the backend it gave a `Store`. #[derive(Default, Clone)] pub struct MemoryBackend { - objects: Arc>>, + objects: Arc>>, } impl MemoryBackend { @@ -19,7 +19,7 @@ impl MemoryBackend { Self::default() } - fn lock(&self) -> Result>, StoreError> { + fn lock(&self) -> Result>, StoreError> { self.objects.lock().map_err(|_| StoreError::Backend("memory store lock poisoned".into())) } @@ -32,6 +32,10 @@ impl MemoryBackend { #[async_trait] impl Backend for MemoryBackend { + fn kind(&self) -> &'static str { + "memory" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { self.lock()?.insert(path.to_owned(), data); Ok(()) @@ -41,8 +45,34 @@ impl Backend for MemoryBackend { Ok(self.lock()?.get(path).cloned()) } + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { + Ok(self.lock()?.get(path).map(|bytes| bytes.slice(..bytes.len().min(max_bytes.saturating_add(1))))) + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + Ok(self.lock()?.get(path).map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX))) + } + async fn delete_raw(&self, path: &str) -> Result<(), StoreError> { self.lock()?.remove(path); Ok(()) } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + let objects = self.lock()?; + let boundary = if prefix.is_empty() { String::new() } else { format!("{prefix}/") }; + // Start at the later of the prefix boundary and `after`, and stop at the first key outside the + // boundary: keys of later namespaces are never visited, and the mutex is held for at most `limit` steps. + let start = match after { + Some(path) if path >= boundary.as_str() => std::ops::Bound::Excluded(path.to_owned()), + _ => std::ops::Bound::Included(boundary.clone()), + }; + Ok(objects + .range((start, std::ops::Bound::Unbounded)) + .map(|(path, _)| path) + .take_while(|path| path.starts_with(&boundary)) + .take(limit) + .cloned() + .collect()) + } } diff --git a/crates/store/src/s3.rs b/crates/store/src/s3.rs index 0557eba2..a9cbb5ba 100644 --- a/crates/store/src/s3.rs +++ b/crates/store/src/s3.rs @@ -3,9 +3,10 @@ use async_trait::async_trait; use bytes::Bytes; +use futures_util::TryStreamExt; use object_store::aws::{AmazonS3, AmazonS3Builder}; use object_store::path::Path; -use object_store::{ObjectStoreExt, PutPayload}; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; use crate::{Backend, StoreError}; @@ -67,16 +68,65 @@ fn backend_error(e: object_store::Error) -> StoreError { StoreError::Backend(e.to_string()) } +/// The path of a listed key relative to the configured prefix, or `None` when the key is outside the +/// listing prefix (`boundary`, which ends in `/`) or outside the backend's root (`root_boundary`). +fn relative_path(full: &str, boundary: &str, root_boundary: &str) -> Option { + if !full.starts_with(boundary) { + return None; + } + full.strip_prefix(root_boundary).map(str::to_owned) +} + +/// `object_store` parses every listed key as a `Path`, and one key it cannot represent (an empty segment, +/// as in `chunks//a`) fails the whole ListObjectsV2 response page it arrived in and ends the stream. Such a +/// key cannot be reported as foreign from here, and the objects of that page are not visible either, so the +/// listing stops with an error that says why. A key with a trailing slash (`chunks/a/`) is not an error: +/// `object_store` strips the slash and the object appears as `chunks/a`. +fn list_error(error: object_store::Error) -> StoreError { + match error { + object_store::Error::InvalidPath { source } => { + StoreError::Backend(format!("S3 listing contains a key that object_store cannot represent, so the listing cannot continue: {source}")) + } + other => backend_error(other), + } +} + #[async_trait] impl Backend for S3Backend { + fn kind(&self) -> &'static str { + "s3" + } + async fn put_raw(&self, path: &str, data: Bytes) -> Result<(), StoreError> { self.store.put(&self.object_path(path), PutPayload::from(data)).await.map_err(backend_error)?; Ok(()) } async fn get_raw(&self, path: &str) -> Result, StoreError> { + self.get_raw_limited(path, usize::MAX).await + } + + async fn get_raw_limited(&self, path: &str, max_bytes: usize) -> Result, StoreError> { match self.store.get(&self.object_path(path)).await { - Ok(result) => Ok(Some(result.bytes().await.map_err(backend_error)?)), + Ok(result) => { + let limit = max_bytes.saturating_add(1); + let mut body = result.into_stream(); + let mut bytes = Vec::with_capacity(limit.min(1024 * 1024)); + while bytes.len() < limit { + let Some(part) = body.try_next().await.map_err(backend_error)? else { break }; + let take = part.len().min(limit.saturating_sub(bytes.len())); + bytes.extend_from_slice(&part[..take]); + } + Ok(Some(Bytes::from(bytes))) + } + Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(backend_error(e)), + } + } + + async fn size_raw(&self, path: &str) -> Result, StoreError> { + match self.store.head(&self.object_path(path)).await { + Ok(meta) => Ok(Some(meta.size)), Err(object_store::Error::NotFound { .. }) => Ok(None), Err(e) => Err(backend_error(e)), } @@ -88,11 +138,90 @@ impl Backend for S3Backend { Err(e) => Err(backend_error(e)), } } + + async fn list_raw(&self, prefix: &str, after: Option<&str>, limit: usize) -> Result, StoreError> { + let object_prefix = self.object_path(prefix); + let normalized_prefix = object_prefix.as_ref().to_owned(); + let boundary = if normalized_prefix.is_empty() { String::new() } else { format!("{normalized_prefix}/") }; + let normalized_root = if self.prefix.is_empty() { String::new() } else { Path::from(self.prefix.as_str()).as_ref().to_owned() }; + let root_boundary = if normalized_root.is_empty() { String::new() } else { format!("{normalized_root}/") }; + let offset = after + .map(|path| { + let full_path = if normalized_root.is_empty() { path.to_owned() } else { format!("{normalized_root}/{path}") }; + Path::parse(full_path).map_err(|error| StoreError::Backend(error.to_string())) + }) + .transpose()?; + let mut stream = if let Some(offset) = &offset { + self.store.list_with_offset(Some(&object_prefix), offset) + } else { + self.store.list(Some(&object_prefix)) + }; + // ListObjectsV2 returns keys in ascending UTF-8 binary order, which is the order `list_raw` promises. + // The stream is read only until `limit` results are collected, then dropped, so a page costs at most + // the S3 requests that cover `limit` keys. A key that is not strictly greater than the one before it + // (a service that lists out of order, such as S3 Express directory buckets) is an error: with an + // unsorted listing a bounded page cannot be correct, and draining the namespace is not an option. + let mut paths: Vec = Vec::new(); + while paths.len() < limit { + let meta = match stream.try_next().await { + Ok(Some(meta)) => meta, + Ok(None) => break, + Err(error) => return Err(list_error(error)), + }; + let Some(relative) = relative_path(meta.location.as_ref(), &boundary, &root_boundary) else { + continue; + }; + if after.is_some_and(|cursor| relative.as_str() <= cursor) { + continue; + } + if paths.last().is_some_and(|previous| relative.as_str() <= previous.as_str()) { + return Err(StoreError::Backend("S3 listing is not in ascending key order".into())); + } + paths.push(relative); + } + Ok(paths) + } } #[cfg(test)] mod debug_tests { - use super::S3Config; + use super::{S3Backend, S3Config, list_error, relative_path}; + use crate::StoreError; + + #[test] + fn relative_path_keeps_the_prefix_boundary_exact() { + assert_eq!(relative_path("root/chunks/ab", "root/chunks/", "root/").as_deref(), Some("chunks/ab")); + assert_eq!(relative_path("root/chunksx/ab", "root/chunks/", "root/"), None); + assert_eq!(relative_path("chunks/ab", "chunks/", "").as_deref(), Some("chunks/ab")); + assert_eq!(relative_path("other/chunks/ab", "root/chunks/", "root/"), None); + } + + #[test] + fn keys_object_store_cannot_parse_give_a_specific_error() { + let source = object_store::path::Path::parse("chunks//a").unwrap_err(); + let message = match list_error(object_store::Error::InvalidPath { source }) { + StoreError::Backend(message) => message, + other => panic!("unexpected {other:?}"), + }; + assert!(message.contains("cannot represent"), "{message}"); + // A trailing slash is normalised away by object_store, before this crate sees the key. + assert_eq!(object_store::path::Path::parse("chunks/a/").unwrap().as_ref(), "chunks/a"); + } + + #[test] + fn object_prefix_uses_object_store_path_normalization() { + let config = S3Config { + bucket: "b".into(), + prefix: "a//b%bar".into(), + region: "us-east-1".into(), + endpoint: None, + access_key_id: "key".into(), + secret_access_key: "secret".into(), + allow_http: false, + }; + let backend = S3Backend::new(&config).unwrap(); + assert_eq!(backend.object_path("chunks").as_ref(), "a/b%25bar/chunks"); + } #[test] fn debug_output_hides_the_secret_key() { diff --git a/crates/store/tests/conformance.rs b/crates/store/tests/conformance.rs index 84150342..ae8ba483 100644 --- a/crates/store/tests/conformance.rs +++ b/crates/store/tests/conformance.rs @@ -10,13 +10,37 @@ use craft_cloud_proto::{ChunkId, ManifestId}; use craft_cloud_store::disk::DiskBackend; use craft_cloud_store::memory::MemoryBackend; use craft_cloud_store::s3::{S3Backend, S3Config}; -use craft_cloud_store::{BlobKey, Store, StoreError}; +use craft_cloud_store::{Backend, BlobKey, ListCursor, MAX_LIST_PAGE_SIZE, Store, StoreError}; fn chunk_key(data: &[u8]) -> BlobKey { BlobKey::chunk(&ChunkId::from_bytes(*blake3::hash(data).as_bytes())) } async fn conformance(store: Store) { + let empty = store.list(craft_cloud_store::BlobKind::Chunk, None, 3).await.unwrap(); + assert!(empty.blobs.is_empty() && empty.foreign.is_empty() && empty.next_cursor.is_none()); + let mut expected = Vec::new(); + for n in 0..8 { + let data = format!("listing conformance {n} {}", uuid::Uuid::now_v7()).into_bytes(); + let key = chunk_key(&data); + store.put(key, &data).await.unwrap(); + expected.push(key); + } + expected.sort_by_key(BlobKey::path); + let mut seen = Vec::new(); + let mut cursor = None; + loop { + let page = store.list(craft_cloud_store::BlobKind::Chunk, cursor, 3).await.unwrap(); + assert!(page.foreign.is_empty()); + assert!(page.blobs.len() <= 3); + seen.extend(page.blobs); + cursor = page.next_cursor.map(|cursor| ListCursor::from_token(cursor.as_token().to_owned())); + if cursor.is_none() { + break; + } + } + assert_eq!(seen, expected); + // Round trip, both kinds. A unique payload per run keeps S3 runs independent. let data = format!("chunk bytes {}", uuid::Uuid::now_v7()).into_bytes(); let key = chunk_key(&data); @@ -69,14 +93,27 @@ async fn s3_backend() { let var = |name: &str| std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set with CRAFT_CLOUD_TEST_S3_ENDPOINT")); let config = S3Config { bucket: var("CRAFT_CLOUD_TEST_S3_BUCKET"), - prefix: format!("test-{}", uuid::Uuid::now_v7()), + prefix: format!("test-{}/a//b%bar", uuid::Uuid::now_v7()), region: "us-east-1".into(), endpoint: Some(endpoint), access_key_id: var("CRAFT_CLOUD_TEST_S3_ACCESS_KEY"), secret_access_key: var("CRAFT_CLOUD_TEST_S3_SECRET_KEY"), allow_http: true, }; - conformance(Store::new(S3Backend::new(&config).unwrap())).await; + let store = Store::new(S3Backend::new(&config).unwrap()); + conformance(store.clone()).await; + let raw = S3Backend::new(&config).unwrap(); + raw.put_raw("chunks/foreign-object", Bytes::new()).await.unwrap(); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); + let sibling_config = S3Config { prefix: format!("{}-sibling", config.prefix), ..config.clone() }; + let sibling = Store::new(S3Backend::new(&sibling_config).unwrap()); + let data = b"sibling prefix object"; + let key = chunk_key(data); + sibling.put(key, data).await.unwrap(); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(!page.blobs.contains(&key)); + assert!(!page.foreign.iter().any(|path| path.starts_with("chunks-sibling/"))); } #[tokio::test] @@ -101,6 +138,126 @@ async fn damaged_objects_are_errors_not_data() { assert!(matches!(store.get(key).await, Err(StoreError::Corrupt { reason: "empty object", .. }))); } +#[tokio::test] +async fn memory_list_is_prefix_scoped_and_paginated() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + let key = BlobKey::chunk(&ChunkId::from_bytes([0x11; 32])); + backend.overwrite_raw(&key.path(), Bytes::new()).unwrap(); + backend.overwrite_raw("chunks//a", Bytes::new()).unwrap(); + backend.overwrite_raw("chunksx/sibling-object", Bytes::new()).unwrap(); + + let first = store.list(craft_cloud_store::BlobKind::Chunk, None, 1).await.unwrap(); + assert!(first.blobs.is_empty()); + assert_eq!(first.foreign, vec!["chunks//a"]); + let second = store.list(craft_cloud_store::BlobKind::Chunk, first.next_cursor, 1).await.unwrap(); + assert_eq!(second.blobs, vec![key]); + assert!(second.foreign.is_empty()); + assert!(second.next_cursor.is_none()); +} + +#[tokio::test] +async fn list_empty_store_returns_empty_first_page() { + let store = Store::new(MemoryBackend::new()); + let page = store.list(craft_cloud_store::BlobKind::Manifest, None, 3).await.unwrap(); + assert!(page.blobs.is_empty() && page.foreign.is_empty() && page.next_cursor.is_none()); +} + +#[tokio::test] +async fn list_rejects_invalid_cursor_and_zero_limit() { + let store = Store::new(MemoryBackend::new()); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, Some(ListCursor::from_token("bad")), 1).await, Err(StoreError::Backend(message)) if message == "invalid listing cursor") + ); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, None, 0).await, Err(StoreError::Backend(message)) if message == "listing limit must be greater than zero") + ); +} + +#[tokio::test] +async fn list_clamps_large_limit() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + for n in 0..=MAX_LIST_PAGE_SIZE { + backend.overwrite_raw(&format!("chunks/foreign-{n:04}"), Bytes::new()).unwrap(); + } + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, MAX_LIST_PAGE_SIZE + 40).await.unwrap(); + assert_eq!(page.foreign.len(), MAX_LIST_PAGE_SIZE); + assert!(page.next_cursor.is_some()); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_skips_temps_and_symlinks_outside_root() { + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("escaped"), b"outside").unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + std::fs::write(root.join("chunks").join(format!(".tmp-{}", uuid::Uuid::now_v7())), b"temp").unwrap(); + std::fs::write(root.join("chunks/foreign-object"), b"foreign").unwrap(); + symlink(outside.path(), root.join("chunks/link")).unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(page.blobs.is_empty()); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); +} + +#[tokio::test] +async fn disk_list_rejects_prefix_outside_root() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(dir.path().join("outside"), b"outside").unwrap(); + let backend = DiskBackend::new(&root).unwrap(); + assert!(matches!(backend.list_raw("../", None, 10).await, Err(StoreError::Backend(_)))); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_does_not_follow_symlink_prefix() { + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(outside.join("sub")).unwrap(); + std::fs::write(outside.join("sub/leak"), b"outside").unwrap(); + symlink(&outside, root.join("link")).unwrap(); + let backend = DiskBackend::new(&root).unwrap(); + assert!(backend.list_raw("link", None, 10).await.unwrap().is_empty()); + assert!(backend.list_raw("link/sub", None, 10).await.unwrap().is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn disk_list_reports_non_utf8_names() { + use std::os::unix::ffi::OsStrExt; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + let name = std::ffi::OsStr::from_bytes(b"bad-\xff"); + std::fs::write(root.join("chunks").join(name), b"foreign").unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + assert!( + matches!(store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await, Err(StoreError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData && error.to_string() == "store path is not valid UTF-8") + ); +} + +#[tokio::test] +async fn disk_list_includes_foreign_and_skips_temporary_files() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + std::fs::write(root.join("chunks").join(format!(".tmp-{}", uuid::Uuid::now_v7())), b"temp").unwrap(); + std::fs::write(root.join("chunks/foreign-object"), b"foreign").unwrap(); + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 10).await.unwrap(); + assert!(page.blobs.is_empty()); + assert_eq!(page.foreign, vec!["chunks/foreign-object"]); +} + #[tokio::test] async fn disk_puts_into_new_shard_directories_survive() { let dir = tempfile::tempdir().unwrap(); @@ -140,3 +297,142 @@ async fn disk_writes_leave_no_temporary_files() { let names: Vec = std::fs::read_dir(shard).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); assert_eq!(names, vec![key.path().rsplit('/').next().unwrap().to_owned()]); } + +/// Walks a whole listing with `limit` per page through the backend and returns every path in order. +async fn list_all_raw(backend: &impl Backend, prefix: &str, limit: usize) -> Vec { + let mut all: Vec = Vec::new(); + loop { + let page = backend.list_raw(prefix, all.last().map(String::as_str), limit).await.unwrap(); + if page.is_empty() { + return all; + } + assert!(page.len() <= limit); + all.extend(page); + } +} + +#[tokio::test] +async fn disk_list_page_does_not_walk_finished_directories_again() { + let dir = tempfile::tempdir().unwrap(); + let backend = DiskBackend::new(dir.path()).unwrap(); + let mut paths = Vec::new(); + for n in 0..60 { + let path = chunk_key(format!("walk cost {n}").as_bytes()).path(); + backend.put_raw(&path, Bytes::from_static(b"\0x")).await.unwrap(); + paths.push(path); + } + paths.sort(); + let shards: std::collections::BTreeSet<&str> = paths.iter().filter_map(|path| path.split('/').nth(1)).collect(); + assert!(shards.len() >= 40, "{} shard directories", shards.len()); + + // Page 1 ends in the middle of the store; page 2 resumes there. + let page_one = backend.list_raw("chunks", None, 30).await.unwrap(); + assert_eq!(page_one, paths[..30]); + let before = backend.directories_read(); + let page_two = backend.list_raw("chunks", page_one.last().map(String::as_str), 2).await.unwrap(); + let read = backend.directories_read() - before; + assert_eq!(page_two, paths[30..32]); + // The root, the directories on the way to the cursor and the ones holding the next two objects. The + // shard directories that sort before the cursor are not opened. + assert!(read <= 8, "page 2 opened {read} directories"); + + // Paging by small steps still yields everything exactly once, in order. + assert_eq!(list_all_raw(&backend, "chunks", 7).await, paths); +} + +#[tokio::test] +async fn disk_and_memory_list_in_the_same_byte_order() { + // `a.b` sorts before `a/x` (`.` is 0x2e, `/` is 0x2f) although the directory name `a` sorts before `a.b`. + let names = ["chunks/a.b", "chunks/a/x", "chunks/a/y/z", "chunks/a-b", "chunks/a0", "chunks/B", "chunks/é"]; + let dir = tempfile::tempdir().unwrap(); + let disk = DiskBackend::new(dir.path()).unwrap(); + let memory = MemoryBackend::new(); + for name in &names { + disk.put_raw(name, Bytes::from_static(b"\0")).await.unwrap(); + memory.put_raw(name, Bytes::from_static(b"\0")).await.unwrap(); + } + let mut expected: Vec = names.iter().map(|name| (*name).to_owned()).collect(); + expected.sort(); + for limit in [1, 2, 3, 100] { + assert_eq!(list_all_raw(&disk, "chunks", limit).await, expected, "disk, limit {limit}"); + assert_eq!(list_all_raw(&memory, "chunks", limit).await, expected, "memory, limit {limit}"); + } +} + +#[tokio::test] +async fn disk_list_skips_only_the_writers_temporary_names() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("root"); + std::fs::create_dir_all(root.join("chunks")).unwrap(); + let writer_name = format!(".tmp-{}", uuid::Uuid::now_v7()); + std::fs::write(root.join("chunks").join(&writer_name), b"in progress").unwrap(); + let foreign_names = [ + ".tmp-not-a-uuid", + ".tmp-", + ".tmp-0190a1b2-c3d4-7e5f-8a6b-7c8d9e0f1a2bx", + ".tmp-0190A1B2-C3D4-7E5F-8A6B-7C8D9E0F1A2B", + ".tmp-6ba7b810-9dad-11d1-80b4-00c04fd430c8", + ]; + for foreign in foreign_names { + std::fs::write(root.join("chunks").join(foreign), b"foreign").unwrap(); + } + let store = Store::new(DiskBackend::new(&root).unwrap()); + let page = store.list(craft_cloud_store::BlobKind::Chunk, None, 20).await.unwrap(); + let mut expected: Vec = foreign_names.iter().map(|name| format!("chunks/{name}")).collect(); + expected.sort(); + assert_eq!(page.foreign, expected); + assert!(!page.foreign.iter().any(|path| path.ends_with(&writer_name))); +} + +#[tokio::test] +async fn memory_list_ignores_other_namespaces() { + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + let chunk = BlobKey::chunk(&ChunkId::from_bytes([0x22; 32])); + let manifest = BlobKey::manifest(&ManifestId::from_bytes([0x33; 32])); + for path in ["a/before", "chunks/zzz-foreign", "chunksx/sibling", "manifests/zzz-foreign", "zzz/after"] { + backend.overwrite_raw(path, Bytes::new()).unwrap(); + } + backend.overwrite_raw(&chunk.path(), Bytes::new()).unwrap(); + backend.overwrite_raw(&manifest.path(), Bytes::new()).unwrap(); + + let first = store.list(craft_cloud_store::BlobKind::Chunk, None, 1).await.unwrap(); + assert_eq!(first.blobs, vec![chunk]); + let second = store.list(craft_cloud_store::BlobKind::Chunk, first.next_cursor, 1).await.unwrap(); + assert_eq!(second.foreign, vec!["chunks/zzz-foreign"]); + assert!(second.next_cursor.is_none()); + let manifests = store.list(craft_cloud_store::BlobKind::Manifest, None, 10).await.unwrap(); + assert_eq!(manifests.blobs, vec![manifest]); + assert_eq!(manifests.foreign, vec!["manifests/zzz-foreign"]); + // A cursor from before the namespace starts at the namespace; one past it finds nothing. + assert_eq!(backend.list_raw("chunks", Some("a/before"), 10).await.unwrap().len(), 2); + assert!(backend.list_raw("chunks", Some("zzz/after"), 10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn list_cursor_is_checked_before_it_is_decoded() { + let invalid = |result: Result| matches!(result, Err(StoreError::Backend(message)) if message == "invalid listing cursor"); + let store = Store::new(MemoryBackend::new()); + let kind = craft_cloud_store::BlobKind::Chunk; + // Otherwise valid cursors (`chunks/` and then filler) that are far longer than any real object path. + let hex = |path: &str| path.bytes().map(|byte| format!("{byte:02x}")).collect::(); + let huge = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4 * 1024 * 1024)); + assert!(invalid(store.list(kind, Some(ListCursor::from_token(huge)), 1).await)); + let just_over = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4096 - "chunks/".len() + 1)); + assert!(invalid(store.list(kind, Some(ListCursor::from_token(just_over)), 1).await)); + let longest = format!("1:c:{}{}", hex("chunks/"), "61".repeat(4096 - "chunks/".len())); + assert!(store.list(kind, Some(ListCursor::from_token(longest)), 1).await.is_ok()); + for garbage in + ["", ":", "1:c:", "1:c:zz", "1:c:6", "1:c:ff", "2:c:6368756e6b732f61", "1:x:6368756e6b732f61", "1:c:6d616e69666573742f61", "not a cursor"] + { + assert!(invalid(store.list(kind, Some(ListCursor::from_token(garbage)), 1).await), "{garbage:?}"); + } + // A real cursor of one kind is refused for the other kind. + let backend = MemoryBackend::new(); + let store = Store::new(backend.clone()); + backend.overwrite_raw("chunks/a", Bytes::new()).unwrap(); + backend.overwrite_raw("chunks/b", Bytes::new()).unwrap(); + let cursor = store.list(kind, None, 1).await.unwrap().next_cursor.unwrap(); + assert!(invalid(store.list(craft_cloud_store::BlobKind::Manifest, Some(cursor.clone()), 1).await)); + assert!(store.list(kind, Some(cursor), 1).await.is_ok()); +} diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index 9a78bb49..fb635346 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -44,6 +44,35 @@ impl Engine { self.state.base.remove(&id); } self.state.sent_deletes.clear(); + // The same for folders, whose deletes carry no version. A folder this machine asked to delete (the answer never + // recorded) that the server reports live is let go like a file whose delete went through (it was brought back + // after the delete, or changed by another machine first), and arrives as it is on the server, not deleted a + // second time. A folder the server no longer has went through, so it is let go too, once it is gone from here + // as well. This is settled deepest first, after the files above: a restored folder is only judged once the + // folders of this machine's own delete below it are settled, or a tombstoned subfolder still tracked in it + // would keep it and the next push would delete it again. A folder the feed did not mention was never touched: + // its delete is sent again. One that still has tracked nodes in it stays. + let live_dirs = std::mem::take(&mut self.state.live_dirs); + let mut asked: Vec<(usize, NodeId)> = + std::mem::take(&mut self.state.sent_dir_deletes).into_iter().map(|id| (self.state.base_path(id).map_or(0, |p| p.depth()), id)).collect(); + asked.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + for (_, id) in asked { + let is_dir = matches!(self.state.base.get(&id).map(|e| &e.kind), Some(BaseKind::Dir { .. })); + if !is_dir || self.state.base.values().any(|e| e.parent == Some(id)) { + continue; + } + let released = if live_dirs.contains(&id) { + true + } else if self.state.remote.contains_key(&id) { + false + } else { + // Deleted on the server; nothing of it is left here when its place is empty. + self.state.base_path(id).is_some_and(|path| matches!(self.stat(&path), Ok(None))) + }; + if released { + self.state.base.remove(&id); + } + } // Changed or new remote nodes, parents first. let mut work: Vec<(usize, NodeId)> = self.state.remote.keys().filter(|id| **id != root).filter_map(|id| Some((self.state.remote_path(*id)?.depth(), *id))).collect(); diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index 55f2335b..029cb4f5 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -609,6 +609,11 @@ impl Engine { if node.deleted { self.state.remote.remove(&node.id); } else { + // A folder this machine asked to delete (the answer never recorded) that the server reports + // changed and live: noted here, judged in `apply_remote` once the files it held are settled. + if self.state.sent_dir_deletes.contains(&node.id) { + self.state.live_dirs.insert(node.id); + } self.state.remote.insert(node.id, RemoteNode::from_state(&node)); } } @@ -672,6 +677,10 @@ impl Engine { } } let (epoch, seq) = first.ok_or_else(|| SyncError::Corrupt("empty listing".into()))?; + // A folder this machine asked to delete (the answer never recorded) that the listing has live is never pushed as + // deleted from that stale request: a listing can't tell a delete that never arrived from one followed by a + // restore, so the folder is let go and arrives as the server has it (the conservative side of a doubt). + self.state.live_dirs.extend(self.state.sent_dir_deletes.iter().filter(|id| remote.contains_key(*id)).copied()); self.state.remote = remote; self.state.epoch = Some(epoch); self.state.seq = seq; diff --git a/crates/sync/src/engine/push.rs b/crates/sync/src/engine/push.rs index 470e36f2..7b5b7357 100644 --- a/crates/sync/src/engine/push.rs +++ b/crates/sync/src/engine/push.rs @@ -649,8 +649,16 @@ impl Engine { _ => None, }) .collect(); - if !deletes.is_empty() { + let dir_deletes: Vec = only_ops + .iter() + .filter_map(|op| match op { + Op::Delete { node, expected: None } => Some(*node), + _ => None, + }) + .collect(); + if !deletes.is_empty() || !dir_deletes.is_empty() { self.state.sent_deletes.extend(deletes.iter().copied()); + self.state.sent_dir_deletes.extend(dir_deletes.iter().copied()); self.checkpoint()?; } let result = self.transport.commit(&only_ops).await; @@ -660,6 +668,9 @@ impl Engine { for (node, _) in &deletes { self.state.sent_deletes.remove(node); } + for node in &dir_deletes { + self.state.sent_dir_deletes.remove(node); + } } match result { Ok(_) => { diff --git a/crates/sync/src/state.rs b/crates/sync/src/state.rs index f88b8cab..323a4daf 100644 --- a/crates/sync/src/state.rs +++ b/crates/sync/src/state.rs @@ -36,6 +36,18 @@ pub struct State { /// the server's changes. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub sent_deletes: BTreeMap, + /// Folders this machine asked the server to delete in a commit whose answer it never recorded. A folder has no + /// version to tell a delete that went through and a later restore from one that never arrived, so the pull judges + /// it by the change feed: a folder the server reports changed and live (brought back from the trash after the + /// delete, or changed by another machine first) is let go, once the files it held are settled, and arrives as it is + /// on the server, instead of being deleted a second time; one the feed doesn't mention was never touched, and its delete is sent again. Cleared + /// once a pass has applied the server's changes. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub sent_dir_deletes: BTreeSet, + /// Of `sent_dir_deletes`, the folders the pull of this pass saw the server report changed and live. Only for the + /// rest of the pass: never saved. + #[serde(skip)] + pub live_dirs: BTreeSet, /// Server deletes held behind a link: tracked nodes missing here while a link or special file that appeared /// since the last finished push carries a name from the node's last local path, so it may be the node, moved with /// its folder (seed 112337's no-pass order; see the push). Each is held for as long as any of its links stands diff --git a/crates/sync/src/transport.rs b/crates/sync/src/transport.rs index f5d63147..b9f55531 100644 --- a/crates/sync/src/transport.rs +++ b/crates/sync/src/transport.rs @@ -19,6 +19,11 @@ pub enum TransportError { /// less than a chunk's worth is ([`Config::quota_floor`](crate::Config::quota_floor)). #[error("the account's storage on the server is full or nearly ({used} of {limit} bytes used); the admin can allow more")] QuotaExceeded { used: u64, limit: u64 }, + /// The server itself is full and refuses every new upload (`server_full`; the admin set `refuse_when_full`). Not the + /// account's quota: no file would fit, whatever its size, so the engine ends the pass and the agent pauses until the + /// admin has freed space or raised the capacity. + #[error("The server is full; the admin needs to free space or raise its capacity")] + ServerFull, /// The change feed can't continue from our sequence number; list the whole tree. #[error("the change feed needs a full listing")] ResyncRequired, diff --git a/crates/sync/tests/common/mod.rs b/crates/sync/tests/common/mod.rs index 68877bbb..a344b22b 100644 --- a/crates/sync/tests/common/mod.rs +++ b/crates/sync/tests/common/mod.rs @@ -80,6 +80,11 @@ pub struct Faults { pub every: AtomicU64, /// Make the next commit apply on the server but report failure, as if its reply was lost. pub lose_commit_reply: AtomicBool, + /// Fail the next commit before it reaches the server. + pub fail_next_commit: AtomicBool, + /// Make the next read of the change feed answer `ResyncRequired`, as if the server's history had been cut short: + /// the machine then lists the tree again. + pub resync_next_changes: AtomicBool, /// Make every commit come back refused, naming an operation the batch doesn't have. pub bad_conflict_index: AtomicBool, /// The machine died (see `MemFs::die_after`): every call fails. @@ -94,6 +99,8 @@ pub struct Faults { /// The next this many chunk uploads are answered as stored but never stored, as by a proxy that answered for /// the server and lost them. pub lose_chunks: AtomicU64, + /// Every chunk upload is refused as by a server that is full itself (`TransportError::ServerFull`). + pub server_full: AtomicBool, calls: AtomicU64, } @@ -158,6 +165,9 @@ impl Transport for Local { async fn changes(&self, since: u64, after: Option) -> Result { self.faults.check()?; + if self.faults.resync_next_changes.swap(false, Ordering::Relaxed) { + return Err(TransportError::ResyncRequired); + } self.meta.changes(self.space, since, after, self.page).await.map_err(meta_err) } @@ -169,6 +179,9 @@ impl Transport for Local { async fn put_chunk(&self, id: &ChunkId, data: Bytes) -> Result<(), TransportError> { self.faults.check()?; self.faults.chunk_uploads.fetch_add(1, Ordering::Relaxed); + if self.faults.server_full.load(Ordering::Relaxed) { + return Err(TransportError::ServerFull); + } // One engine drives a transport at a time, so a load and a store are enough. if let Some(left) = self.faults.lose_chunks.load(Ordering::Relaxed).checked_sub(1) { self.faults.lose_chunks.store(left, Ordering::Relaxed); @@ -207,6 +220,9 @@ impl Transport for Local { async fn commit(&self, ops: &[Op]) -> Result { self.faults.check()?; + if self.faults.fail_next_commit.swap(false, Ordering::Relaxed) { + return Err(TransportError::Unavailable("injected before commit".into())); + } if self.faults.bad_conflict_index.load(Ordering::Relaxed) { return Err(TransportError::Rejected(vec![Conflict { op: ops.len() + 5, kind: ConflictKind::NoSuchNode }])); } diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index a8570f7d..a3f7287b 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3023,6 +3023,25 @@ async fn an_account_with_no_room_left_stops_the_pass() { ); } +#[tokio::test] +async fn a_server_that_is_full_stops_the_pass_with_its_own_message_not_the_accounts() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + write(&mut a, "one.txt", b"one"); + write(&mut a, "two.txt", b"two"); + a.transport().faults.server_full.store(true, Ordering::Relaxed); + let full = a.sync_once().await; + assert!(matches!(&full, Err(craft_cloud_sync::SyncError::Transport(craft_cloud_sync::TransportError::ServerFull))), "{full:?}"); + let shown = full.unwrap_err().to_string(); + assert!(shown.contains("The server is full") && !shown.contains("account"), "{shown}"); + assert_eq!(a.transport().faults.chunk_uploads.load(Ordering::Relaxed), 1, "the pass ended at the first refusal, no file was skipped one by one"); + // Space freed: the next pass uploads both. + a.transport().faults.server_full.store(false, Ordering::Relaxed); + let report = a.sync_once().await.unwrap(); + assert_eq!(report.uploaded.len(), 2, "{report:?}"); +} + #[tokio::test] async fn a_nearly_full_account_counts_as_full() { // 400 bytes of room left: less than one chunk, so the account counts as full and the pass stops, instead of @@ -3153,3 +3172,432 @@ async fn a_live_reset_doc_leaves_the_file_sync_alone() { assert!(!files(&a).iter().any(|(p, _)| p == "gone.txt"), "A's delete was undone"); assert_eq!(read(&b, "a.txt"), b"v2 from the laptop"); } + +// ---- restore from the trash, after a delete whose answer a machine never heard ---- + +/// The node of the file or folder `name` the server holds (deleted or not), in the space of `user`. +async fn node_named(server: &Server, user: &craft_cloud_meta::UserRecord, name: &str) -> craft_cloud_proto::NodeId { + let feed = server.meta.changes(user.space, 0, None, 5000).await.unwrap(); + let found: Vec<_> = feed.nodes.iter().filter(|n| n.name.as_ref().is_some_and(|n| n.as_str() == name)).collect(); + assert_eq!(found.len(), 1, "one node named {name}: {found:?}"); + found[0].id +} + +/// Brings `node` back from the trash, as the web or an admin tool would: not one of the syncing machines. +async fn restore_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + let token = *blake3::hash(craft_cloud_proto::NodeId::generate().as_uuid().as_bytes()).as_bytes(); + let device = server.meta.create_device(user.id, "web", &token, 0).await.unwrap(); + server.meta.restore_from_trash(user.space, device, node, 1_760_000_000_000).await.unwrap(); +} + +/// Takes `node` out of the trash (marks it for removal), as the web or an admin tool would. +async fn remove_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + server.meta.request_purge(user.space, node, 1_760_000_000_000).await.unwrap(); +} + +/// Brings the nodes named in `steps` back from the trash, in order; a name with a `-` in front is taken out of the trash +/// instead (and with it what is below it). Restoring a folder brings back all that is in the trash below it, so a test +/// that wants a folder alone removes the rest first. +async fn trash_steps(server: &Server, user: &craft_cloud_meta::UserRecord, steps: &[&str]) { + for step in steps { + match step.strip_prefix('-') { + Some(name) => remove_from_trash(server, user, node_named(server, user, name).await).await, + None => restore_from_trash(server, user, node_named(server, user, step).await).await, + } + } +} + +/// A machine deleted a file, the server applied it, the answer was lost; the file is brought back from the trash +/// before the machine syncs again. The machine's delete must not be sent again: the file comes back, on both machines. +#[tokio::test] +async fn a_file_restored_after_a_lost_delete_answer_comes_back_and_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "f.txt").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + assert_eq!(files(&b), files(&a)); + // And it stays: nothing deletes it on a later pass, on either machine. + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + let c = machine(&server, &ana, "studio").await; + let mut c = c; + c.sync_once().await.unwrap(); + assert_eq!(files(&c), files(&a)); +} + +/// The same, but the name was taken meanwhile by another file: the restored one comes back under a conflict name and +/// both files reach both machines. +#[tokio::test] +async fn a_file_restored_under_a_conflict_name_after_a_lost_delete_answer_keeps_both_files() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"the first"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + // The desktop sees the delete, and its user makes a new f.txt. + b.sync_once().await.unwrap(); + write(&mut b, "f.txt", b"a new file"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a, &mut b], 6).await; + let on_a = files(&a); + assert_eq!(on_a, files(&b)); + assert_eq!(on_a.len(), 2, "{on_a:?}"); + assert!(on_a.contains(&("f.txt".to_owned(), b"a new file".to_vec())), "{on_a:?}"); + let restored = on_a.iter().find(|(p, _)| p.contains("(conflict, web, ")).expect("the restored file under a conflict name"); + assert_eq!(restored.1, b"the first"); +} + +/// A folder that came back from the trash is merged into a live folder of the same name: the restored file lands in +/// it, and the machine whose delete answer was lost ends up with it, not deleted again. +#[tokio::test] +async fn a_file_restored_into_a_live_folder_after_a_lost_delete_answer_comes_back() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let file = node_named(&server, &ana, "f.txt").await; + // The desktop sees both deletes; its user makes a new folder d with another file. + b.sync_once().await.unwrap(); + b.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut b, "d/other.txt", b"other"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, file).await; + + settle(&mut [&mut a, &mut b], 6).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec()), ("d/other.txt".to_owned(), b"other".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); +} + +/// A folder deleted here, the answer lost, and brought back from the trash: it is not deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} + +/// A folder deleted here whose answer was lost and that nobody brought back is still deleted: the delete is not undone +/// by the doubt about it. +#[tokio::test] +async fn a_folder_deleted_with_a_lost_answer_and_not_restored_stays_deleted() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + settle(&mut [&mut a, &mut b], 6).await; + assert!(a.fs().dirs().is_empty(), "{:?}", a.fs().dirs()); + assert!(b.fs().dirs().is_empty(), "{:?}", b.fs().dirs()); +} + +/// A folder with a file in it, both deleted here, the answer lost, then brought back from the trash in different ways. The +/// folder this machine deleted is never deleted a second time, whichever of the two (or both) comes back. +async fn lost_delete_of_a_folder_with_a_file(restore: &[&str]) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, restore).await; + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +#[tokio::test] +async fn a_non_empty_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let (a, b, server, ana) = lost_delete_of_a_folder_with_a_file(&["-f.txt", "d"]).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has the folder"); +} + +/// A folder restored brings back the file that was deleted with it, which is not deleted a second time either. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_comes_back_with_its_file() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["d"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} + +#[tokio::test] +async fn only_the_file_restored_after_a_lost_delete_answer_brings_its_folder_back_too() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["f.txt"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} + +/// A tree of nested folders with a file at its bottom, all deleted here in one commit whose answer is lost, then +/// brought back from the trash in different ways (by node name). Whatever comes back, nothing is deleted a second time. +async fn lost_delete_of_a_nested_tree( + dirs: &[&str], + file: &str, + restore: &[&str], +) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + for dir in dirs { + a.fs_mut().mkdir(&path(dir)).unwrap(); + } + write(&mut a, file, b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path(file)).unwrap(); + for dir in dirs.iter().rev() { + a.fs_mut().remove_dir(&path(dir)).unwrap(); + } + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, restore).await; + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +/// Both machines and a fresh one have exactly the folders `dirs` and no file, and the server still has them. +async fn assert_only_dirs(m: (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord), dirs: &[&str]) { + let (a, b, server, ana) = m; + let expected: Vec<_> = dirs.iter().map(|d| path(d)).collect(); + assert_eq!(a.fs().dirs(), expected, "the laptop"); + assert_eq!(b.fs().dirs(), expected, "the desktop"); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), expected, "the server still has the folders"); +} + +#[tokio::test] +async fn a_nested_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-sub", "d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_nested_folder_and_its_subfolder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-f.txt", "d"]).await; + assert_only_dirs(m, &["d", "d/sub"]).await; +} + +#[tokio::test] +async fn only_the_deepest_file_restored_after_a_lost_delete_answer_brings_the_whole_chain_back() { + let (a, b, _server, _ana) = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["f.txt"]).await; + let expected = vec![("d/sub/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d"), path("d/sub")]); +} + +#[tokio::test] +async fn a_three_deep_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-a", "d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_three_deep_folder_and_its_middle_folder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-b", "d"]).await; + assert_only_dirs(m, &["d", "d/a"]).await; +} + +#[tokio::test] +async fn only_the_deepest_folder_restored_after_a_lost_delete_answer_brings_its_parents_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-f.txt", "b"]).await; + assert_only_dirs(m, &["d", "d/a", "d/a/b"]).await; +} + +/// The change feed is cut short after the lost delete answer and the restore: the machine lists the tree again, at the +/// same epoch. A restored empty folder is in that listing, and must not be deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_and_a_relist_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} + +/// A folder delete recorded before sending but rejected before reaching the server is treated conservatively after a relist. +#[tokio::test] +async fn an_unsent_folder_delete_and_relist_keeps_the_live_folder() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "keep.txt", b"keep"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + let before = server.meta.space_info(ana.space).await.unwrap().seq; + a.transport().faults.fail_next_commit.store(true, Ordering::Relaxed); + let error = a.sync_once().await.unwrap_err(); + assert!(error.to_string().contains("injected before commit"), "{error}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "the failed request never reached the server"); + + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + let report = a.sync_once().await.unwrap(); + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(files(&a), vec![("keep.txt".to_owned(), b"keep".to_vec())]); + assert_eq!(files(&b), files(&a)); + assert!(report.removed_remotely.is_empty(), "{report:?}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "no delete was sent again"); +} + +/// A same-epoch relist restores the live parent while keeping its deleted descendants deleted. +#[tokio::test] +async fn a_nested_relist_keeps_only_the_restored_parent() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + a.fs_mut().mkdir(&path("d/sub")).unwrap(); + write(&mut a, "d/sub/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/sub/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d/sub")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + trash_steps(&server, &ana, &["-sub", "d"]).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty()); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has only the restored parent"); +} + +/// A tracked child keeps an in-doubt folder in the base when a relist sees it live. +#[tokio::test] +async fn a_tracked_child_keeps_an_in_doubt_folder_from_release() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"original"); + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "e/g.txt", b"other"); + settle(&mut [&mut a, &mut b], 5).await; + let folder = node_named(&server, &ana, "d").await; + let file = node_named(&server, &ana, "f.txt").await; + a.fs_mut().rename(&path("d/f.txt"), &path("e/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.fs_mut().rename(&path("e"), &path("d")).unwrap(); + // Restore a checkpoint with D's delete in doubt. D's tracked file F is still at d/f.txt, inside E's local folder. + let mut state = a.state().clone(); + state.sent_dir_deletes.insert(folder); + let mut a = common::restart(&a, state, |_| {}); + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + a.sync_once().await.unwrap(); + assert!(a.state().base.contains_key(&folder), "the folder is still tracked with its child"); + assert!(a.state().base.contains_key(&file), "the live file is still tracked"); + assert_eq!(files(&a), vec![("d/f.txt".to_owned(), b"original".to_vec()), ("d/g.txt".to_owned(), b"other".to_vec())]); + let listing = server.meta.tree(ana.space, None, 5000).await.unwrap(); + assert!(listing.nodes.iter().any(|node| node.id == folder && !node.deleted), "folder node was deleted"); + assert!(listing.nodes.iter().any(|node| node.id == file && !node.deleted), "tracked file node was deleted"); +} + +/// The local replacement becomes a conflict copy, and the restored file returns at its original path. +#[tokio::test] +async fn a_local_replacement_after_a_lost_delete_reply_keeps_its_content() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + write(&mut a, "f.txt", b"original"); + a.sync_once().await.unwrap(); + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + write(&mut a, "f.txt", b"local replacement"); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a], 6).await; + let on_a = files(&a); + assert!(on_a.contains(&("f.txt".to_owned(), b"original".to_vec())), "{on_a:?}"); + assert!(on_a.iter().any(|(p, bytes)| p.contains("(conflict, laptop, ") && bytes == b"local replacement"), "{on_a:?}"); + let mut b = machine(&server, &ana, "desktop").await; + b.sync_once().await.unwrap(); + assert_eq!(files(&b), files(&a)); +} diff --git a/docs/agent.md b/docs/agent.md index 18e2a394..cfd1cd18 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -18,7 +18,10 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ often, at least every 5 minutes. When the server can't be reached it tries again with growing pauses, up to 5 minutes. If something needs you (the folder or its marker is missing, the device was signed out) it pauses and `status` says why; a missing folder or marker is looked at again every minute, and syncing goes on by - itself once it is back. Ctrl-C lets a running pass finish and stops; a second Ctrl-C stops at once. + itself once it is back. When the server itself is full (its admin turned on `refuse_when_full` and the server's disk + or capacity is used up) the agent pauses with "The server is full; the admin needs to free space or raise its + capacity" (never the account's storage message, which means the account's own quota) and looks again every 10 + minutes; nothing is lost, and syncing goes on once the admin has freed space. Ctrl-C lets a running pass finish and stops; a second Ctrl-C stops at once. - While `run` runs, `sync --approve-deletes` from another terminal hands the approval to it: exactly the deletions `status` listed. If more came since, it lists them instead, approves nothing and exits 2; check the list and run it again. Every list of waiting deletions comes with its set (`set 3f9a…`), also in @@ -59,9 +62,25 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ and `--adopt` as for `login`) signs it in. An admin's invite (`user invite`, in [`server.md`](server.md)) works the same way; for a new account, `join` asks for its password twice on standard input. A code that doesn't work says so, without saying why: make a new one. +- An account with two-factor sign-in (turned on in the app's Security page; see "Two-factor sign-in" in + [`server.md`](server.md)) asks for a second step at `login` and at `join`: after the password, the command prints + `This account has two-factor sign-in. Type the code from your authenticator app (or a recovery code):` and reads one line + from standard input, so a script gives the password line and then the code line. The code is never an argument and + is never stored; a wrong one may be typed again twice (the server itself allows five wrong codes per sign-in). Six digits + are a code from the app; anything else is taken as a recovery code, which works once. Too many wrong codes lock the + account's second step for a while: the command says how long, and the lock does not sign anything out. For a `join` that + needs a code the invite isn't used up until the code is right. A running agent (`run`) does the same through its local + API; the Craft Cloud app asks for the code in its own window. +- Rolling this out: update the server first, then the agents and the app. An agent from before two-factor sign-in can't + complete a sign-in for an account that has it, and it reports that as a wrong password; update the agent, nothing is + broken. The app on a server that doesn't know two-factor sign-in says so on its Security page instead of failing. - Through the running agent: `pause` and `resume` syncing (a pause holds across restarts; recents keep working), `quit` the agent, and `devices` to list the account's machines (`devices remove ` takes one out; to take this machine out, use `logout`). +- The desktop reads storage through the authenticated local API, `GET /v1/usage`. It asks the server for every + request and returns `AccountUsage` unchanged. Signed out, it returns `409 not_signed_in`; when the server cannot be + reached, it returns `503 offline`. The desktop shows trash and old-version values as overlapping estimates and + never adds them to stored bytes. - Recent files, per app, shared by every machine of the account (through the running agent, as an app sees them): @@ -85,3 +104,84 @@ printf '%s\n' 'your password' | ./target/release/craft-cloud-agent login https:/ The first machine to open a file makes it a live document from the file's content (UTF-8 text only); the others join it. Changes go to every machine that has it open as they are made, and edits made offline merge when the machine is back. `doc` never writes the file: what it shows is the live document. + +## The local API's admin calls + +For the Craft Cloud app's admin pages the running agent passes the server's admin API on (`/api/v1/admin/…`, see +[`server.md`](server.md), "Managing accounts over HTTP") under `/v1/admin/…`, with the signed-in device's token. They +have the same checks as every local call (the `Host`, no `Origin`, the token in `Authorization: Bearer`), bodies of at +most 8 KiB, and need the agent to be signed in (`not_signed_in` otherwise). + +| Local call | Server call | Body, answer | +| --- | --- | --- | +| `GET /v1/admin/server` | `GET /admin/server` | the server's counters and kinds; a member is refused `forbidden` | +| `GET /v1/admin/storage` | `GET /admin/storage` | bounded storage overview: totals, quota counts, top ten accounts, manifest-index counts, and the room the server has (optional fields) | +| `GET /v1/admin/users?cursor&limit&query` | `GET /admin/users` | a page of accounts by username; `next` is the cursor of the following page; `query` finds a username containing it | +| `POST /v1/admin/users` | `POST /admin/users` | `{username, role, password, totp_code?, recovery_code?}` → the account and its invite code | +| `PATCH /v1/admin/users/{id}` | `PATCH /admin/users/{id}` | `{role?, quota_bytes?, password, totp_code?, recovery_code?}` → the account (`null` removes the quota) | +| `POST /v1/admin/users/{id}/disable`, `…/enable` | the same | `{password}` → 204 | +| `DELETE /v1/admin/users/{id}` | the same | `{password}` → 202 | +| `POST /v1/admin/users/{id}/invite`, `…/password-reset` | the same | `{password}` (a reset code: also `totp_code?`, `recovery_code?`, `reset_two_factor?`) → `{code, join_url?, expires_ms}` | +| `DELETE /v1/admin/users/{id}/invite` | the same | `{password}` → 204 | +| `GET /v1/admin/users/{id}/devices` | the same | every device of the account | +| `GET /v1/admin/users/{id}/usage` | `GET /admin/users/{id}/usage` | the account's stored bytes, effective quota, and labeled trash/history estimates; unknown or removed account is `not_found` | +| `DELETE /v1/admin/users/{id}/devices/{device_id}` | the same | `{password}` → 204 | +| `POST /v1/admin/users/{id}/reset-two-factor` | the same | `{password, totp_code?, recovery_code?}` → 204: takes the account's two-factor sign-in away and signs its computers out | +| `GET /v1/account` | `GET /account` | the signed-in account: `role`, `two_factor_enabled` (how the app finds out whether to show its admin pages) | +| `GET /v1/account/activity?cursor&before_ms&limit` | the same | the account's own security events, newest first, any account (`user` and `action` are not read) | +| `GET /v1/admin/audit?cursor&before_ms&limit&user&action` | `GET /admin/audit` | the whole log, newest first; `user` an account ID, `action` an event name or a prefix like `auth.` of the closed list; a cursor is passed back as it came | +| `GET /v1/admin/audit/export?before_ms&user&action` | `GET /admin/audit/export` | the log as NDJSON, **streamed** through (chunked): event lines, then a last line `{"complete":true,"exported":n}` or `{"truncated":true,"exported":n}` | + +- **The server decides.** The agent checks no role and makes no decision of its own; it reads each body as the shared + type, writes it to the server again and answers what the server answered, with the server's HTTP status, error code and + message. A wrong password stays `403 wrong_password`, a member's call `403 forbidden`, a disabled admin's + `403 account_disabled`, the last enabled admin's demotion `409 last_admin`, too many tries `429 rate_limited` with the + server's `Retry-After` header. The one status the agent does not pass on is the server's `401`: on a signed-in route it + says this device was revoked, and the local API answers it as every other call does (`409 needs_you`); a local `401` always means a wrong token. +- **The confirmation is not kept.** Every change carries the acting admin's current password in its body, from the app + through the agent to the server. The agent holds it only for the length of the request, never writes or logs it, and + the `Debug` output of every body hides it. The codes the server makes (an invite, a password reset code) are shown by + the app once and kept by neither side. +- **A second factor** joins the confirmation when the admin's account has two-factor sign-in: `totp_code` (a code from the + authenticator app) or `recovery_code`, in the same body as the password, passed on as typed fields and never kept. The server + answers a missing or wrong one `403 two_factor_invalid` and too many `429 two_factor_locked` with the wait (the body's + `retry_after_secs` and the `Retry-After` header); neither is a revoked device, so the agent stays signed in. A password reset + code takes the whole `AdminPasswordReset` (the same fields and `reset_two_factor`, which makes redeeming the code turn the + account's two-factor sign-in off). +- **The audit export is never held whole.** The agent passes the server's stream through chunk by chunk; a client that + reads it (`craft-cloud-app`'s `admin_audit_export`) counts it as whole only with the server's `complete` line and the number + of events it says, and takes a stream that ends without it (a broken connection, a server that failed in the middle) for a + failed export. `truncated` is a good export of the newest events that is not the whole log. Filters and limits are checked + before anything is sent: an `action` outside the closed list is `400 invalid`, as is a `limit` of 0 or over 200 (100 for an + account's own activity). +## Local API: file history + +The local API exposes server history to desktop clients. Paths are relative to the synced folder and resolve through +the agent's saved sync state; unknown or unsynced paths return `404 not_found`, paths outside the root return +`400 invalid`, and folders return `400 invalid`. + +| Method and path | Request | Success | +|---|---|---| +| `GET /v1/files/history?path=&cursor=&limit=` | Query parameters; `cursor` and `limit` optional | `200` with `{versions, next}`; newest first | +| `POST /v1/files/restore` | `{"path":"Notes/todo.txt","version":"…","expected_version":"…"}` | `200` with the new version, its parent, sequence and epoch | + +Restore requires the file's current version to match `expected_version`; otherwise the API answers `409 conflict`. +The agent does not write restored bytes directly. Its normal change feed picks up the new server version and syncs it +to disk. + +## Local API: trash + +Trash items are addressed by their server node ID because deleted nodes are absent from the agent's sync tree. The +agent validates that each ID has the expected UUID form before forwarding it. List responses may include `path`, a +display hint formed from a currently synced parent folder and the item's name; it is `null` when that parent is not +live on this machine. A missing hint does not fail the request. + +| Method and path | Request | Success | +|---|---|---| +| `GET /v1/trash?cursor=&limit=` | Optional paging query parameters | `200` with `{items, next}`, newest deletion first | +| `POST /v1/trash/restore` | `{"id":""}` | `200` with restored node, parent, name, rename flag, version (for files), sequence and epoch | +| `POST /v1/trash/remove` | `{"id":""}` | `204` | + +A successful restore asks the agent for a normal sync pass; it never writes files directly. Removing an item only +marks it out of the trash. It cannot be restored from the trash afterward, but stored space is not freed yet. Server +HTTP status and error code are preserved; a server `401 unauthorized` becomes local `409 needs_you`. diff --git a/docs/design/live.md b/docs/design/live.md index 75e9a4ee..3aa598df 100644 --- a/docs/design/live.md +++ b/docs/design/live.md @@ -24,6 +24,9 @@ per app connection for live sessions. `run` serves it while signed out too. Call one), `GET /v1/pairings/{id}` → `{state: waiting|used|expired, device_name?}`, `DELETE /v1/pairings/{id}` → 204; - `GET /v1/devices` → the account's devices (`current` marks this one); `DELETE /v1/devices/{id}` revokes another one and answers the list after (this one is refused with `cannot_remove_self`: that is `logout`); +- `GET`, `POST`, `PATCH` and `DELETE` `/v1/admin/…`, and `GET /v1/account` and `/v1/account/activity`: the server's admin API + and the account's own audit events passed on for the Craft Cloud app's admin pages, with the server's status and error codes (listed in [`../agent.md`](../agent.md), "The local API's admin + calls"); - `POST /v1/pause` / `POST /v1/resume` → status: while paused no sync pass starts (one under way finishes), and the long-poll, the folder watcher and the live connection hold; the local API and the apps' recents keep working (adds reach the server after resume). Kept in `state.json`, so a paused agent starts paused; `run_state` is diff --git a/docs/design/sync.md b/docs/design/sync.md index 52d60652..9d669dec 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -578,6 +578,225 @@ changes locally, then pushes local changes. These rules came out of the simulati database (not a supported setup) would each count their own, so twice the sign-in attempts and twice the writes. - **Setup**: `craft-cloud-server init` writes the config, creates the data directory and the first admin. Solo use is that one admin account and nothing more. +- **Device sessions** (migration 0007): a device token has no expiry and its format is unchanged (43 characters, stored as a + BLAKE3 hash); what changed is what a device records and how an account ends them. A device records the `client_kind`, + `platform` and `app_version` it said at sign-in (display only; the agent says `agent`, its operating system and its version on every + sign-in and join), and, once revoked, when (`revoked_ms`) and why + (`revoke_reason`). The `revoked` flag is still the check: the three are set together in one transaction and only by + the first revoke, so a second revoke changes neither. A user keeps `password_changed_ms`, and `disabled_ms`, which the + admin API sets and the token check, sign-in, join and pairing read (see "Admin API and roles"). + - **Changing the password** (`POST /account/password`) checks the current password under one of the server-wide + password-check slots (all busy: `429` with `Retry-After`, and the try is given back), then in one transaction sets + the new hash and `password_changed_ms`, revokes every other device (`revoke_reason = password`) and deletes the + account's open invite and all its pairings, so no code made before the change works after it; if the password + changed meanwhile nothing changes (`409 conflict`), and neither does it if the calling device was signed out meanwhile (`401`). Afterwards each revoked device's live connections end at once. The same two + steps run for `user reset-password` (all devices, reason `reset`, unless `--keep-devices`) and `device revoke-all` + (reason `admin`) on the command line, which cannot reach the server's live connections: those end at the next ping + (within about 30 s) and a change-feed long-poll within 60 s. + - **Sign-in and join under a change**: a password sign-in creates its device only if the stored hash is still the one + it checked (the account is locked first), and answers as for a wrong password otherwise, so a device is never + added after a reset or change that signed the others out. Every transaction that revokes devices, changes a + password, redeems an invite or makes a pairing takes the account's row first, which keeps them from deadlocking. + - **Limits**: `password_changes` counts every try, a wrong password included, 5 per account per 15 minutes; the + password and device-name bodies are read up to 4 KiB (the sign-in bodies stay at 1 MiB). +- **Admin API and roles** (migration 0008): an account is an `admin` or a `member`, and every route under + `/api/v1/admin` is behind the `require_admin` layer, which runs after the device check and reads the role the + database gave for that very request (nothing is cached, so a demoted admin is out with the next request). A member + gets `403 forbidden`, a missing or revoked token `401`. The layer is only the first look: every change checks the + acting admin again inside its own transaction. + - **Confirmation.** A device token never expires, so a stolen admin token alone must not be enough to create an + admin, issue a code or sign others out. Every change (creating, changing, disabling, enabling, removing an account, + issuing or revoking an invite, issuing a password reset code, revoking another user's device) carries the acting + admin's current `password` in its JSON body, also the `DELETE` routes (at most 4 KiB). A missing or mistyped body + field is `400 invalid`; a wrong password is `403 wrong_password`, never 401, because the agent reads every 401 on a + signed-in route as a revoked device. Tries are counted per account in the password change's limiter (5 wrong + passwords in 15 minutes, then `429`; a right password is given back), and every check takes one of the + server-wide password-check slots. **An admin whose account has two-factor sign-in** adds `totp_code` (or a + `recovery_code`) to the same body: missing or wrong is `403 two_factor_invalid` (the field is a credential there, not + a form field; a code that is not six digits is `400 invalid`), too many wrong ones `429 two_factor_locked` with + `Retry-After`. The code is checked and used up by the confirmation, on the same counters, limiter and time-step rule + as sign-in's second step (below): **one 30-second step serves one change**, the next change takes the next code or + a recovery code. The password is told first: a wrong one is `403 wrong_password` whatever the code. Resetting another + **administrator's** password or two-factor sign-in takes the acting admin's own two-factor sign-in (`403 forbidden` + without it; the command line on the host is the way then). The change compares the admin's two-factor state with + the one the confirmation saw, under the account's lock, so a confirmation given before they turned it on or off is + `409 conflict`. + - **The acting admin in the transaction.** After the password check (a slow Argon2id run) the admin's role, device or + password can have changed. Every change locks, in this order, (1) the `last_admin` lock if it can take an + administrator away, (2) the user rows of the actor and the target in ascending ID, (3) the rows derived from them, + and then checks that the actor's device is not revoked and the account neither disabled nor marked for removal + (`401`), that the role is still `admin` (`403 forbidden`) and that the password hash is the one verified (`409 + conflict`). Sign-ins, joins and pairings take the user row first too, so none of these can deadlock. + - **The last administrator.** Demoting, disabling or removing an admin who is the last enabled one is `409 + last_admin`. A count cannot be locked with `FOR UPDATE`, and locking each target's row would not stop two admins + from each taking the other away, so every change that can reduce the admins first takes one transaction-wide lock + and counts after it (Postgres `pg_advisory_xact_lock` with a fixed key, SQLite the write transaction, which is + the same thing there). Nobody can disable or remove their own account (`400 invalid`) but may step down when + another admin is left. The command line does not enforce the rule or the confirmation: the operator has the files. + - **Disabled accounts.** Disabling signs every device out (`revoke_reason = disabled`), deletes the open invite or + reset code and the pairings, and sets `disabled_ms`; enabling clears it and brings back no device. A disabled + account's old tokens are `401 unauthorized`; sign-in and join answer `403 account_disabled`, but only after the + password or code was right (a wrong one is `401` or `404 invalid_code` as ever). Issuing is serialised with + disabling: a sign-in, a redeem, a pairing and a new device lock the user row first and look at `disabled_ms` and + `removed_ms` in their own transaction, so a password checked before the disable never turns into a device after + it, and enabling later cannot make such a device usable. **Removing** an account (`DELETE /admin/users/{id}`, `202`) + also sets `removed_ms`: it is disabled, left out of the lists and cannot be enabled, its username stays taken, + and nothing it stored is deleted (freeing it comes with garbage collection). + - **Codes.** `POST /admin/users` makes an account without a password (role `admin` or `member`) and a 7-day invite; + the person chooses the password when they redeem it. An invite for an existing account is only made for one + that still has no password: for an account with a password it would add a computer without asking for it, so + `409 conflict`, and a password reset code is the way in. A **password reset code** (`purpose = reset`, 16 + characters, one hour, one per account like an invite, so each replaces the other) is redeemed at `POST /join`: it + always asks for a new password (`password_required`, `weak_password`), sets it, signs **every** device of the + account out (`revoke_reason = reset`), deletes its pairings, and, as every redeem does, signs the redeeming computer + in. Codes are shown once in the answer (with the whole join link when `public_url` is set) and never logged. + - **Quota.** An admin may set `quota_bytes` for one account (from one chunk, 4 MiB, to `i64::MAX`; `null` removes + it). The limit of a space is its quota, else `stored_bytes_per_user`, else none, and it holds in the look before a + chunk or manifest is stored (`check_room`, also when there is no server-wide cap) and in the count that follows. + A quota under what the account stores deletes nothing and refuses new bytes with `507 quota_exceeded`. + - **Live connections.** The process that revokes a device ends its live connections at once; another server process on + the same Postgres database, and the command line, are noticed at the connection's next ping (about 30 s). + - Logs carry the kind of event and the target's ID at `info`, never a username, password, code or link. +- **Security audit log** (migration 0009): the events that matter for who may do what are rows of `audit_events`, + written by the change they record, in its transaction and before its commit, so a change whose event cannot be written + is not made (fail closed). The names are a closed list in the code: `auth.login`, `auth.login_failed`, + `auth.second_factor_failed`, `auth.recovery_code_used`, `auth.two_factor_locked`, `device.revoked`, + `device.revoked_others`, `account.password_changed`, `account.two_factor_enabled`, `account.two_factor_disabled`, + `account.recovery_codes_regenerated`, `admin.user_created`, `admin.user_disabled`, `admin.user_enabled`, + `admin.user_removed`, `admin.role_changed`, `admin.quota_changed`, `admin.invite_issued`, `admin.invite_revoked`, + `admin.password_reset_issued`, `admin.two_factor_reset`, `admin.device_revoked`, `join.invite_redeemed`, + `join.reset_redeemed`, `join.pairing_redeemed`, `cli.password_reset`, `cli.two_factor_reset`, `cli.device_revoked`, + `cli.devices_revoked_all` and `cli.user_removed`. A name whose change does not exist yet (`user remove`) is listed so + the vocabulary a client filters by does not change; its writer arrives with that change. + - **Rows.** `id` (UUIDv7), `at_ms`, `actor_user` and `actor_device` (`NULL` for the command line, and for a sign-in that + failed: nobody proved who they were), `action`, `target_kind` (`user`, `device` or empty) and `target_id`, `outcome` + (`success`, or `failure` for a failed sign-in), `details`, and for aggregated rows `agg_key` and `agg_count` (the API + calls it `count`). There are no foreign keys: an event keeps the ID of an account that was removed, and the name is + gone because it was never stored. + - **What each operation writes.** A sign-in `auth.login` (actor: the account and its new device); a failed one + `auth.login_failed`. `DELETE /devices/{id}` `device.revoked`; `POST /devices/revoke-others` **one** + `device.revoked_others` with the count (none when no device was left); `POST /account/password` one + `account.password_changed` whose `revoked` counts the devices that went with it. Admin changes write their + `admin.*` event, a `PATCH` with a role and a quota one event for each field that changed, in one transaction; the + devices a disable or removal signs out are counted in its one event. Redeeming an invite, a reset code or a + pairing writes `join.invite_redeemed`, `join.reset_redeemed` (with `revoked`) or `join.pairing_redeemed`. The + command line writes `cli.password_reset`, `cli.device_revoked` and `cli.devices_revoked_all`, with no actor. **A + repeat or a no-op writes nothing**, since nothing changed: disabling a disabled account, signing out a device that is + out, giving a role or quota the account has, revoking an invite that is not there, and so a retried request after a + dropped connection leaves the event of its first run only. Files, uploads and live documents are not audited. + The admin and command line events about a device are about its **account** (`target_kind = user`) and carry the device + in `details`, so the account's own activity shows them. + - **`details` is typed.** Each event has its own few fields, each a closed list of words or a number: `client_kind` + (`agent`, `desktop`, `cli`, `web`) and `platform` (`windows`, `macos`, `linux`, `android`, `ios`, `other`) for a sign-in or + join, `reason` (as in the device list), `step` (`password`, `totp`, `recovery`, `challenge_expired`), `role_from` and + `role_to`, `quota_bytes` (a number, `null` when the quota was removed), `revoked` (a count), `device` (an ID) and `two_factor_removed` (`true`, only on `join.reset_redeemed`). A + device may call its platform any printable text of 32 bytes, an address or a file name included, so anything not in + the list is `other`; `web` means a browser session of the admin pages and cannot be named by a device. Nothing a client typed ever reaches a row. The text is at most 2,048 bytes: if it would be more, + whole fields go, the least important first, and `"truncated": true` is added; bytes are never cut off, which would + leave invalid JSON. **No address, file name, path, device name, name of an account that does not exist, password, + code, challenge, token, key or e-mail address is stored anywhere in a row.** + - **Two-factor sign-in** writes these, all in the transaction of the change unless said otherwise. A sign-in that made a + device writes `auth.login` whichever way it went (no second factor, a code, a recovery code); the challenge `POST + /auth/login` gives writes nothing, since nobody is signed in yet. Turning it on, new recovery codes and turning it off write + `account.two_factor_enabled`, `account.recovery_codes_regenerated` and `account.two_factor_disabled` (the account, from the + device that asked). A recovery code that is used up writes `auth.recovery_code_used` in the transaction that uses it up + (sign-in, join, turning it off, an admin's confirmation, which commits on its own before the change it confirms), by the + account and the device where there is one; never the code. An admin's reset writes `admin.two_factor_reset` (admin, target) + and the command line `cli.two_factor_reset` (no actor, with `revoked`); a reset code made with `reset_two_factor` that is + redeemed writes `join.reset_redeemed` with `two_factor_removed: true`, by the account that redeemed it: the code does not + say which admin made it, and none is made up. The wrong second step of a sign-in or a join is `auth.second_factor_failed` + (`step`: `totp` or `recovery`; `challenge_expired` for a challenge that is unknown, expired, used or out of tries, with no + account when it was not found), aggregated and written after the answer is decided, like `auth.login_failed`. **A wrong proof + on a signed-in route** (turning it off, new recovery codes, an admin's confirmation) **is not written one by one**; the lock + it can cause is: `auth.two_factor_locked`, once when a lock is set or lengthened and never for a refusal while it holds. That + event is written like a failed sign-in's, in a transaction of its own after the counters and the lock are committed, so a + fault in the log can neither undo the lock nor change the answer; it is logged without content and the event is lost. + - **Failed sign-ins** are one row per account, action, client kind and minute, `agg_key` = action + client kind + account + minute, written + after the answer is decided and in a transaction of its own: `INSERT … ON CONFLICT (agg_key) DO UPDATE SET agg_count = + audit_events.agg_count + 1` (the column is qualified, since a bare one is ambiguous against `excluded` on Postgres), + so many tries at one moment make one row that counts them all and keeps the identity, time and details of the first. + Tries with a name no account has share one row per client kind and minute with no account in it, so a guesser can neither put text in + the log nor make it grow without bound. A failure to write this row is logged (no content) and changes neither the + answer nor the limits on tries, which are not stored in the log at all. + - **Reading.** Pages are newest first by `(at_ms, id)` descending. `cursor` is that pair, opaque, and exclusive: the next + page starts strictly behind it, so events with one timestamp are neither repeated nor lost. The export reads 500 rows + per query through a channel of two pages, so a slow client slows the reading instead of filling memory; the end of the + log is fixed when it starts (`at_ms` up to then), at most 100,000 event rows are sent, and the reading stops when the + client goes away. The last line says how it ended; a database error in the middle breaks the connection (and is logged + without content) so that no client can take half an export for a whole one. + - **Append-only, in the code.** `Meta` has no function that changes or deletes a row of `audit_events` except + `audit_prune`, which deletes events older than the retention, and the counter of an aggregated row. A test reads the + sources of `craft-cloud-meta` and allows exactly those two statements. **This deviates from a design with + database triggers and separate roles**: it is a design choice so that the same code runs on both databases (SQLite has no database roles; triggers exist but are not used), and an operator + with access to the database can still change the log. Protection against that is outside this server. + - **Retention.** `[audit] retention_days` (1 to 3650, default 365, the section is optional). Old events are deleted by + `audit prune` (`--dry-run` only counts) and by a write that finds the last complete pruning an hour or more past (kept + in memory, set only when a pruning deleted everything that was due), after its commit and never inside the security + transaction, in batches of 1,000 rows each in a transaction of its own and at most 20 batches a time. A server + that nobody writes to keeps expired events until `audit prune` runs: schedule it (cron, the task scheduler). There + is no periodic task. + - **Personal data.** A row holds account and device IDs and times, which is personal data. `docs/server.md` says how long + it is kept and why; the server stores no address. + - Reading the log is not itself an event. +- **Two-factor sign-in** (migration 0010, after the audit log's 0009): an account may turn on a TOTP second factor (RFC 6238: HMAC-SHA-1, six + digits, 30-second steps, one step either way for a clock that is a little off; the codes authenticator apps make) with + ten recovery codes. With it on, a right password gives **no token**: `POST /auth/login` answers `401 + two_factor_required` with `two_factor: {challenge, expires_ms}`, and `POST /auth/login/second-factor` with the + challenge and a `totp_code` (or `recovery_code`) makes the device. + - **The secret** is 160 random bits, shown once (Base32 and an `otpauth://` URI) and stored only as an AES-256-GCM + ciphertext: a key-name byte, a 12-byte nonce and the ciphertext, with `craft-cloud:totp:v1:` and the account's ID as + associated data, so a ciphertext moved to another account does not open. The key is 32 random bytes in the file the + config names (`[two_factor] key_file`, relative to `config.toml`), which `init` makes and the **server never makes** + (a second server on the same Postgres database, or a restore without the key, would otherwise seal secrets + nobody can open); the AES key is BLAKE3 `derive_key("craft-cloud 2026 totp secret v1", key)` of it. The key's name is + the first byte of the BLAKE3 hash of the key file: one of 256 values, only there to tell the current key from the old one + (the server refuses to start when they are equal). It does not identify a key (two different keys can share a name), and + the GCM authentication tag refuses a wrong key whatever the name says. + **Fail closed**: when the database holds secrets, the server opens **every** one at start and refuses to run if the key + is missing or one does not open, naming the file. With no secrets and no key it runs, and set-up answers `503 + two_factor_unavailable`. **Rotation**: the new key in `key_file`, the old in `key_file_old`, every instance restarted with + both (a running server holds the keys it started with), then `two-factor rekey` rewrites the secrets under the new one + (one account at a time, each read fresh under the account's lock, a final scan, repeatable, refusing when a secret has a + key that is neither), then `key_file_old` goes, after `rekey` reported that nothing is left under another key. Without the key the secrets of a backup are unusable: keep it apart from backups. + - **Recovery codes** are ten of 16 characters in the join codes' alphabet (80 bits, shown in groups of four), stored as + the BLAKE3 hash of the normalised code, no pepper: 80 bits can't be searched even from a leaked database, and the codes + work whatever happens to the key. Each works once; new ones replace all the old. A recovery code stands in for a + code at sign-in, join, turning it off and the admin's confirmation. + - **Set-up** (`POST /account/two-factor/setup`, the password again) keeps a new secret for 10 minutes, bound to the + device that asked and apart from any active secret; another device gets `409 two_factor_pending`; five wrong codes use + it up. `confirm` with the first code turns it on, shows the recovery codes and, in the same transaction, signs every + other device out (`revoke_reason = two_factor`), raises `users.auth_gen` and deletes the account's invite, + pairings and challenges. Two confirmations at once turn it on once. Turning it off (`disable`: password and a code or + recovery code) does the same, and so do `user reset-2fa` and the admin's `reset-two-factor` (which sign **all** devices out). + - **The transaction contract.** Everything that issues a challenge or a device, or uses a proof up, locks the account's + row first (the order every other change of the account uses), checks again under the lock what the handler checked + before (the device still signed in, the password hash still the verified one, `auth_gen`, the account's **current** + two-factor state, the challenge), and only then acts. Sign-in's challenge is made in that transaction: a stale hash + is `401 unauthorized` like a wrong password, an account with it on gets a challenge and no device. The second + step finds the challenge by a plain read to learn the owner, locks the account, checks the challenge again, then uses up + the time step (or recovery code) **and** the challenge and makes the device in one transaction with one commit. + `join` does the same for an invite or reset code. A **time step is accepted once and only a later one after it** + (`last_accepted_step`, updated `WHERE last_accepted_step < step`, shared by sign-in and the admin's confirmations), so + two requests with one code can't both succeed. A challenge is 32 random bytes, kept as a BLAKE3 hash, good 5 minutes, + five tries, once; at most five are open per account (the oldest goes); the address it was made for is only an extra + binding (behind a proxy that hides addresses all clients share one), the secret token is what binds it. A password + change, a reset, turning it off or on and a disable delete the account's challenges. + - **Wrong proofs** are counted and committed in the transaction that refused them, under the account's lock (every check + runs before anything is consumed, so the invite stays, the challenge stays, no device is made): the next request sees the + attempts and the lock, so a challenge takes five wrong proofs and no more, and the tenth sets the lock. Ten wrong ones in a row lock the account's second factor for 15 + minutes; each wrong one after a lock ran out doubles it, up to 24 hours; a right one resets it; `user reset-2fa` lifts + it. While it lasts every path (second step, join, turning off, new recovery codes, the admin's confirmation) answers + `429 two_factor_locked` with `Retry-After`. It is never permanent, so nobody with the bare password can lock an + account out for good. Separate limiter `second_factors`: 10 wrong codes + per account and 30 per address in 5 minutes (a right one is given back), and they count against the sign-in limiter. + - **Join**: an invite or reset code of an account with it on needs `totp_code` or `recovery_code` in the same body; + without it the answer is `401 two_factor_required` (its `challenge` is empty: `/join` has no challenge row, the same + body is sent again with the code) and the code is **not** used up; a wrong factor is `401 two_factor_invalid`, is not + given back to the address's join tries and counts against the account. A reset code made with `reset_two_factor` + turns it off instead of asking. **A pairing needs no second factor** (only a signed-in device can make one, and turning + two-factor sign-in on signs the other devices out), so **any valid device token can add devices**. + - **Status codes**: a wrong code is `401` at sign-in and join (not signed in) and `403` on every signed-in route, where a + 401 would read as a revoked device. The 2FA bodies are at most 4 KiB and a malformed one is `400 invalid`. - **Join codes**: a machine can also get its device token with a code instead of a password. Codes are Crockford base32 (`0-9` and letters without I, L, O, U; on input case is ignored, O reads as 0, I and L as 1, dashes and spaces are dropped), shown in groups of four (`K7M4-Q9TX`), and stored only as their BLAKE3 hash (migration 0004: @@ -630,25 +849,100 @@ changes locally, then pushes local changes. These rules came out of the simulati admins who back up the bucket themselves (bucket versioning or replication); GC must stay paused until that bucket backup is done, or the admin keeps the bucket's own deleted-object retention longer than the GC grace. **Restore** sets a new epoch for every space. -- **Admin CLI**: `init`, `serve`, `user add|reset-password|invite [--revoke]`, `device list|revoke` and `reset-epoch` - exist; `user remove|quota`, `gc`, `verify` (re-hash every chunk), `backup ` and `restore ` come with M4. +- **Web admin and browser sessions** (migration 0011, off unless `[web_admin] enabled = true`): the admin API's operations + also have pages in the browser, under `/web`, **server-rendered HTML in Rust**: no script, no build step, no Node, one + stylesheet served by the server itself (`GET /web/style.css`). The pages are an adapter over the same `craft_cloud_meta` calls + and the same confirmation as the JSON routes, so the rules (last administrator, who may reset whom, quotas) have one home, and the + JSON routes stay as they were. This replaces "no web UI in v1" for **administrators only**: there is no page for a member and + none for files. + - **Setup is explicit.** The pages exist only with `[web_admin] enabled = true`, which the config accepts only with + `behind_proxy = true` and `public_url = "https://host[:port]"` without a path (a TLS proxy in front; `public_url` is the + origin the pages check forms against). Otherwise `/web` is `404` like any unknown route. `idle_minutes` (5 to 1440, default 480) + and `absolute_hours` (1 to 720, default 168) set the session lifetimes; unknown keys are refused. + - **Sessions** (`web_sessions`): the cookie `__Host-craft_cloud_session` holds 32 random bytes (base64url, 43 characters); the + table keeps only their BLAKE3 hash, the account, the account's `auth_gen` at sign-in, `created_ms`, `last_seen_ms`, + `absolute_expires_ms`, `revoked_ms` and `mfa_verified_ms` (when a second factor was given; `NULL` for an account without one). + The cookie is `Secure; HttpOnly; SameSite=Strict; Path=/`, with no `Domain` and no lifetime of its own (the server decides, so + a copied cookie can't outlive its session). A session is valid while it is not logged out, was made less than + `absolute_hours` ago, was used within `idle_minutes` (its last-seen time is moved at most once a minute, by the next + request), the account is neither disabled nor marked for removal, **and the account's `auth_gen` is still the one it + was made under**. Every change that raises `auth_gen` therefore ends the account's sessions with no code of its own: a password + change or reset (device, command line or redeemed code), a disable, an enable, a removal, turning two-factor sign-in on or off, + and a reset of it by an admin or the command line. The role is read from the database for every request, so a demoted admin + gets `403` at once. A change made on a session checks, in its transaction under the account's lock, that the session is still + logged in under the account's current `auth_gen` (the same re-check a device gets) **and still inside its idle and absolute lifetimes at the time + of the change** (the time is read after the change has taken its locks, so a wait for them does not extend a session), so a request that began before the session ended (a slow body, a wait for a lock or the password check) cannot + change anything after it. + - **Sign-in** (`POST /web/login`, then `POST /web/login/second-factor` for an account with two-factor sign-in): the JSON + sign-in's own steps in the same order (name and password shape first, a password-check slot, then the counts, a dummy + hash for a name nobody has), with limiters of their own (`AppState::web_logins`: 30 per client address and 10 per account + in five minutes), so an attack on the pages does not lock the API and the other way round. **Only administrators get a + session**, and every refusal of the password step (unknown name, wrong password, a member, a disabled or removed account) + is the same `401` page, so nothing says which accounts exist or what role they have. The second step uses the second factor's + own code (`totp_code` or `recovery_code`, a time step serving once, five wrong codes deleting the challenge, ten locking the + account for 15 minutes and doubling after that, with the per-account and per-address limits of the device sign-in): the + lock is the account's, so it holds for pages and devices alike. The challenge is a row of `login_challenges` of kind `web`; a + web challenge can never make a device and a device's can never make a session. The sign-in writes `auth.login` with no + device; a failed one the usual `auth.login_failed`. + - **One credential at a time.** The pages know the cookie only; the JSON admin routes know the bearer token only (no JSON + route takes the cookie). A request to either with an `Authorization` header **and** the session cookie is `400 invalid`, + so a client can't be unsure which credential acted. + - **CSRF and origin.** Every form carries `csrf`, a value derived from the session token under a key-derivation context of + its own (so the stored hash can't make it), compared in constant time; missing, wrong or another session's is `403`. Every + request that is not a `GET` or `HEAD`, the sign-in too, must carry an `Origin` equal to the origin of `public_url` or, + without one, a `Referer` on it; an `Origin` that differs (including `null`) is `403` whatever the `Referer`. No `GET` + changes anything (the audit export is a read). The sign-in form has no CSRF token, as no session exists to bind one to: + the origin check is what stops another site from signing a browser in. + - **Headers on every answer under `/web`**, errors and redirects included: `Content-Security-Policy: default-src 'self'; + script-src 'none'; style-src 'self'; img-src 'self'; connect-src 'none'; object-src 'none'; base-uri 'none'; form-action + 'self'; frame-ancestors 'none'`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: no-referrer`, `X-Frame-Options: DENY`, + `Cross-Origin-Opener-Policy` and `Cross-Origin-Resource-Policy: same-origin`, a restrictive `Permissions-Policy`, and + `Cache-Control: no-store` (the stylesheet `no-cache`). No page has a script, an inline style, an event handler or a file from + another host. `Strict-Transport-Security` is the proxy's to send. + - **Statuses** are one contract for the pages: `401` for a missing, expired or unknown session (a `GET` is sent to `/web/login` + with `303` instead; a form post gets the sign-in page), for a wrong username or password at sign-in, and for a sign-in step + (challenge) that is unknown, used up or expired; `403` for a member or demoted admin, a wrong second factor (the page of the + second step again, with the same challenge), a wrong confirmation password or code, a bad CSRF token or origin; `400` for a + malformed form, **including a body that is not a form (no `415`)**; `429` for the limits and the lock, **always with + `Retry-After`** (the end of the window that refused, or the lock's); `413` over 4 KiB; `409`/`404` as the store says. A sign-in + for an account that has no password yet (an open invite) spends the same password check as an unknown name and fails the + same way, in the pages and in `POST /auth/login`. `/web/` has the same headers and origin check as the other paths. Every change asks for the **confirmation in its form**: the admin's password, and `totp_code` or + `recovery_code` when their account has two-factor sign-in (the server asks whatever the page showed), checked by the + JSON routes' function with their counters, so a wrong password counts against the same five-in-fifteen-minutes window. + - **Pages**: overview (counters, kinds), users (search, 50 a page by cursor, create), one user (role and quota, disable, enable, + remove, invite, password reset code, end the open code, reset two-factor, devices with revoke), the audit log (filters + `user`, `action`, 50 a page by cursor, and `GET /web/audit/export`, the JSON export's NDJSON download). Invite and reset + codes are the answer to the `POST` that made them, shown once and kept nowhere; a reload asks the browser to post the form again. + Every value not written by the server's code is HTML-escaped on the way in, including a device's name and platform. + - **Cleanup without a periodic task.** A sign-in deletes up to 100 sessions that are over (logged out, past their lifetime, + idle); `craft-cloud-server web-sessions prune [--dry-run]` deletes all of them (run it from cron, as `audit prune`). Neither + touches a valid session. + - Nothing secret is logged (cookie, CSRF token, challenge, password, codes); events are logged at `info` with kind and target ID. +- **Admin CLI** (the way in without HTTP: it asks for no confirmation and does not enforce `last_admin`, since whoever runs it + has the files): `init` (which also makes the two-factor key file), `serve`, `user add|reset-password [--keep-devices]|invite [--revoke]|reset-2fa`, + `device list|revoke|revoke-all`, `web-sessions prune [--dry-run]`, `two-factor rekey` and `reset-epoch` exist; `user remove|quota`, `gc`, `verify` (re-hash every chunk), `backup ` and `restore ` come with M4. `user invite` prints the join string and link when `public_url` is in the config, else the code and how to finish it; it logs that an invite was made (the username at debug only, never the code). -- No web UI in v1. No telemetry, no update check. +- No web UI for files, and none for members. No telemetry, no update check. API (`/api/v1`, JSON except chunk and manifest bodies; every signed-in response carries the space's epoch in the `craft-epoch` header, and commits/tree/changes carry it in the body too). Errors are -`{error, message, conflicts?, missing?, quota?}` with stable codes (`conflict` 409, `missing_chunks` 409, +`{error, message, conflicts?, missing?, quota?, two_factor?}` with stable codes (`conflict` 409, `missing_chunks` 409, `resync_required` 410, `unauthorized` 401, `rate_limited` 429, `invalid` 400, `not_found` 404, `too_large` 413, `timeout` 408, `quota_exceeded` 507, -`username_taken` 409, `invalid_code` 404, `password_required` 400, `weak_password` 400, `internal` 500; the codes are constants in `craft-cloud-proto::api::codes`). Unknown routes and +`username_taken` 409, `invalid_code` 404, `password_required` 400, `weak_password` 400, `wrong_password` 403, `forbidden` 403 (an +admin route called by a member), `account_disabled` 403 (sign-in and join only), `last_admin` 409, `two_factor_required` 401 (with +`two_factor`), `two_factor_invalid` (401 at sign-in and join, 403 on signed-in routes), `two_factor_unavailable` 503, +`two_factor_pending` 409, `two_factor_locked` 429 (with `Retry-After`), `internal` 500; the codes are constants in `craft-cloud-proto::api::codes`). Unknown routes and methods answer `not_found` / `invalid` (405) the same way: | Call | Purpose | |---|---| | `GET /info` (also unversioned at `/api/info`) | protocol versions, chunker parameter sets, limits (no auth); a client probes `/api/info` first | | `GET /space` | the signed-in user's space: space ID, root node ID, epoch, current seq, username | -| `POST /auth/login` | username + password + device name → device ID + token (shown once); 10 tries per account and 30 per address per 5 minutes; a wrong password and an unknown user look the same | +| `POST /auth/login` | username + password + device name (+ optional `client_kind` `agent`\|`desktop`\|`cli`, `platform` and `app_version`, each at most 32 bytes, shown in the device list and never a basis for a permission; an unknown kind or a control character is `invalid`, checked before the try is counted) → device ID + token (shown once); 10 tries per account and 30 per address per 5 minutes; a wrong password and an unknown user look the same. For an account with two-factor sign-in a right password gives **no token**: `401 two_factor_required` with `two_factor: {challenge, expires_ms}` (the challenge is good 5 minutes, five tries, once) | +| `POST /auth/login/second-factor` | `{challenge, totp_code}` or `{challenge, recovery_code}` (no auth, at most 4 KiB) → device ID + token as a sign-in; a challenge that is unknown, expired, used or no longer fit is `401 unauthorized` (the same for each), a wrong code `401 two_factor_invalid`, a malformed one `400 invalid`, a lock `429 two_factor_locked` with `Retry-After`; `second_factors` limiter | +| `GET /account/usage` | `{stored_bytes, quota_bytes, trash_bytes, history_bytes, estimates: {index_complete, reason?}}`: stored bytes and labeled storage estimates for this device's account | | `GET /tree?after_seq=…&after_node=…&limit=…` | full listing for a first sync or a reconcile, in pages of up to 5,000 (cursor = changed seq + node ID) | | `GET /changes?since=N&after_seq=…&after_node=…&limit=…&wait=30` | changes after N, in pages, long-poll up to 60 s; `resync_required` if N is too old or ahead of the server | | `POST /chunks/missing` | which of these chunk IDs (up to 10,000) this space lacks | @@ -656,10 +950,150 @@ methods answer `not_found` / `invalid` (405) the same way: | `PUT /manifests/{id}` / `GET /manifests/{id}` | one manifest (binary); the server checks the ID, the limits and that every chunk is present | | `POST /commit` | batch of version/mkdir/move/delete operations, one transaction; `409 conflict` listing every conflicting operation (a version whose manifest isn't uploaded is a `missing_manifest` conflict) | | `GET /nodes/{id}/history` / `POST /nodes/{id}/restore` | list versions / make an old one current (M4) | +| `GET /account` | the signed-in account, flat: `{user_id, username, role, created_ms, password_changed_ms, two_factor_enabled}` (`password_changed_ms` is `null` until the password changes through the server) | +| `GET /account/two-factor` | `{enabled, enabled_ms, recovery_codes_left, pending}` | +| `POST /account/two-factor/setup` | `{password}` → `{secret, otpauth_uri, expires_ms}` (secret Base32 without padding, shown once; the set-up lasts 10 minutes, belongs to this device, replaces none that is on). `409 conflict` when it is on, `409 two_factor_pending` when another device has a set-up open, `503 two_factor_unavailable` without a key, `403 wrong_password`; the password's tries are the password change's (5 wrong per account per 15 minutes) and take a check slot | +| `POST /account/two-factor/confirm` | `{totp_code}` → `{recovery_codes: [10]}` shown once. Turns it on, signs every other device out, deletes the invite, pairings and challenges. At most 5 wrong codes per set-up; `403 two_factor_invalid`; `409 conflict` for no set-up, an expired one, or one another device started | +| `POST /account/two-factor/disable` | `{password, totp_code}` or `{password, recovery_code}` → 204. Deletes the secret, codes and challenges and signs every other device out; wrong code `403 two_factor_invalid`, `429 two_factor_locked` | +| `POST /account/two-factor/recovery-codes` | `{password, totp_code}` → `{recovery_codes: [10]}`; the old ones stop working | +| `POST /account/password` | `{current_password, new_password}` → 204. Signs every *other* device out and deletes the account's open invite and pairings, in one transaction; the device that asked stays signed in. A wrong current password is `403 wrong_password` (never 401: see below), a new one that breaks the rules or equals the current one `400 weak_password`; 5 tries per account per 15 minutes (`429 rate_limited`); body at most 4 KiB, passwords at most 1,024 bytes | +| `GET /devices` | your devices: `{id, name, created_ms, last_used_ms, revoked, current, client_kind, platform, app_version, revoked_ms, revoke_reason}`; the last five are new and a client may ignore them (`revoke_reason` is `user`, `others`, `password`, `admin`, `disabled`, `two_factor`, `idle` or `reset`) | +| `PATCH /devices/{id}` | `{name}` (at most 4 KiB, the device-name rule) → 200 with the device. Only your own devices that are not revoked, otherwise `404 not_found` | +| `POST /devices/revoke-others` | signs out every other device of the account (and deletes their pairings) → `{revoked: n}`, 0 when there is none | +| `DELETE /devices/{id}` | signs one of your devices out → 204; its time and reason (`user`) are kept, and a second call changes neither | +| `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | +| `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | +| `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash and whose folder is not in the trash itself (those are listed through the folder), newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | +| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders and, for a folder, all that is in the trash below it, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | +| `DELETE /trash/{id}` | takes a node out of the trash (204), and, for a folder, all that is in the trash below it: only marks them for removal, nothing is freed yet; up to 10,000 items below a folder at once, shallowest first; the rest is then listed and goes by a further call | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | -| `POST /join` | code + device name (+ password for an invite to a new account) → device ID + token, as a sign-in (no auth); `invalid_code`, `password_required`, `weak_password`, `rate_limited` | +| `POST /join` | code + device name (+ password for an invite to a new account **and always for a password reset code**, + the optional client fields of a sign-in) → device ID + token, as a sign-in (no auth); `invalid_code`, `password_required`, `weak_password`, `account_disabled`, `rate_limited`. A reset code signs every other device of the account out. Optional `totp_code` or `recovery_code`: an invite or reset code of an account with two-factor sign-in needs one (`401 two_factor_required` without, code not used up; `401 two_factor_invalid`, not given back to the address's tries), unless the reset code was made with `reset_two_factor`; a pairing code never does | +| `GET /admin/server` | (admin) `{version, users, devices_active, stored_bytes, live_bytes, storage_kind (disk\|s3\|memory), database_kind (sqlite\|postgres)}`: counters and kinds only, no path, address or setting | +| `GET /admin/storage` | (admin) bounded overview: account/storage totals, quota counts, top ten accounts, and manifest-index counts; plus the optional `store_kind`, `capacity_bytes` (from `[storage] capacity`), `store_fs_total_bytes` / `store_fs_free_bytes` (disk store), `server_fs_total_bytes` / `server_fs_free_bytes` and `database_bytes` (SQLite), each left out when unknown | +| `GET /admin/users/{id}/usage` | (admin) `{stored_bytes, quota_bytes, trash_bytes, history_bytes, estimates}` for one account; `quota_bytes` is effective quota; unknown or removed account is `not_found` | +| `GET /admin/users?cursor&limit&query` | (admin) `{users: [{id, username, role, created_ms, disabled_ms, two_factor_enabled, devices_active, stored_bytes, quota_bytes, invite_pending}], next?}` by username; `limit` 1 to 200 (default 50, otherwise `invalid`); `cursor` is opaque (the last username, base64url) and `next` is missing on the last page; `query` finds usernames containing it (case-insensitive, at most 64 bytes of printable characters, `%` and `_` are ordinary characters). Accounts marked for removal are left out | +| `POST /admin/users` | (admin, confirmation) `{username, role, password}` → `{user, invite: {code, join_url?, expires_ms}}`: an account with no password and a 7-day invite; `username_taken` 409 (also for an account marked for removal), `invalid` for a bad name or role | +| `PATCH /admin/users/{id}` | (admin, confirmation) `{role?, quota_bytes?, password}` → the user. Neither field is `invalid`; a missing `quota_bytes` leaves it, `null` removes it; `last_admin` 409 | +| `POST /admin/users/{id}/disable`, `/enable` | (admin, confirmation) → 204. Disable signs every device out and deletes the codes and pairings; not oneself (`invalid`), `last_admin`; enabling an account marked for removal is `conflict` | +| `DELETE /admin/users/{id}` | (admin, confirmation) → 202. Marks the account for removal as above; nothing is deleted | +| `POST /admin/users/{id}/invite`, `DELETE …/invite` | (admin, confirmation) → `{code, join_url?, expires_ms}` / 204: an invite only for an account that has no password (`conflict` otherwise); the delete ends the account's invite or reset code (`not_found` when there is none) | +| `POST /admin/users/{id}/password-reset` | (admin, confirmation) `{reset_two_factor?: bool}` → `{code, join_url?, expires_ms}`: a one-hour reset code, redeemed at `POST /join`; with `reset_two_factor` redeeming it also turns the account's two-factor sign-in off, else an account that has it needs its second factor to redeem. Another admin's needs two-factor sign-in on the acting admin's own (`forbidden`), one's own is `invalid` | +| `POST /admin/users/{id}/reset-two-factor` | (admin, confirmation) → 204: takes the account's two-factor sign-in away, deletes its challenges and signs **every** device out (`revoke_reason = two_factor`); not one's own (`invalid`), none to reset is `409 conflict`, another admin's needs the acting admin's own two-factor sign-in (`forbidden`) | +| `GET /admin/users/{id}/devices`, `DELETE …/devices/{device_id}` | (admin; the delete with confirmation) → `{devices: [device as in GET /devices]}`, revoked ones too / 204 (`revoke_reason = admin`); another user's device or an unknown user is `not_found` | +| `GET /account/activity?cursor&before_ms&limit` | the signed-in account's own security events, newest first: `{events: [event], next?}`; the events it acted in or was the target of, with the device shown only for its own acts, and the details only of events about itself (not of what a former administrator once did to others). `limit` 1 to 100 (default 50, otherwise `invalid`); `cursor` is opaque and exclusive and wins over `before_ms`, which only limits the first page; a cursor this server did not make is `invalid` | +| `GET /admin/audit?cursor&before_ms&limit&user&action` | (admin) the whole log in the same form (`limit` up to 200); `user` is an account ID (events it acted in or was the target of), `action` an event name or a group ending in a dot like `auth.`; anything else is `invalid` | +| `GET /admin/audit/export?user&action&before_ms` | (admin) the same filters (`before_ms` must not be negative) as NDJSON (`application/x-ndjson`, always offered as `craft-cloud-audit.ndjson`), streamed: one event per line, newest first, then one last line, `{"complete":true,"exported":n}`, or `{"truncated":true,"exported":n}` when there were more events than the export holds (100,000). A stream that ends without either line broke off and is not a whole export | + +**Account usage estimates.** `stored_bytes` keeps the existing unit: the stored size of the space's chunks and manifests, each once, including each format byte. `quota_bytes` is the effective quota: the per-account override when set, otherwise the configured `stored_bytes_per_user` cap, otherwise `null`. `trash_bytes` is the stored size of blobs referenced only by versions of deleted nodes still in the space; no live node version or live-document snapshot/import snapshot may reference them. A pending `purge_requested_ms` still counts: it hides the node from the trash list but frees no data yet. `history_bytes` is the stored size of blobs referenced only by non-current versions of live nodes; no current version, saved version (one that carries a save record), trashed node or live-document snapshot/import snapshot may reference them; a chunk counts only if every manifest naming it qualifies, so a chunk shared with a saved version stays out. A saved version of a deleted node still counts in `trash_bytes`, because purging the node removes it. Both include each shared blob once per space. Unrooted manifests are not counted. Both numbers are labeled, overlapping estimates; never add either to the other or to `stored_bytes`. If any manifest belonging to the space has an incomplete reference index, both estimates are `null` and `estimates` is `{index_complete: false, reason: "index_incomplete"}`; the server does not guess. + +**History and restore.** Versions are never removed today, so the history lists every version of a file, and `pruned` +is always `false`: the field is in the contract for the day versions can be thinned (a thinned version will stay +in the list with `pruned: true`, its metadata intact, and can't be restored). A restore is an ordinary `update` made +by the server: it checks, under the same lock as a commit, that the node is a file of the caller's space that is not +deleted and has no live document, that `version` is a version of this node, that its manifest is still in the space, +and that `expected_version` is still the file's current version; then it records a new version with a fresh ID whose +parent is the current version, with the manifest, modification time and executable bit of the old one, at the space's +next sequence number. The epoch does not change (only `reset-epoch` and backup restores do that), the change feed +reports the file like any other update, and a restore needs no quota because the manifest is already the space's. +The new version carries **no save record**: a save record names a state of a live document, and a restore +copies none and invents none. A restore is an ordinary new version, so agents take it in under the normal rules, +including the conflict rules for a machine with unsynced edits of its own. A file that has a live document is +refused, so file versions and live-document state stay separate. Unknown node, foreign node, a version of +another file and a missing manifest are all `not_found`; a deleted file, a folder and a file with a live document are +`invalid`; a stale `expected_version` is `conflict` (409, with the conflict list as for a commit). + +**Trash.** The trash is the set of tombstones with the time of the deletion. `nodes.deleted_ms` is set when a node goes +from live to deleted (not when a deleted node is deleted again) and cleared when it comes back, by an edit or by a restore +from the trash, so the next deletion starts its time afresh; `nodes.purge_requested_ms` is set by `DELETE /trash/{id}` +and cleared when the node comes back. Nodes deleted before the migration that added them are dated at the migration. A +deleted node without a time (damaged data) is dated at the next read of the trash, logged without names, and never +counts as expired. `DELETE /trash/{id}` frees nothing: versions, manifests, chunks and the tombstone stay, the change feed +and the sequence number do not change, and the node's history stays readable; the node only leaves the trash list and can +no longer be restored from it. (An edit that brings it back still wins, as for any deleted file.) Asking twice is fine. +A node is *in the trash* when it is deleted and not marked by `purge_requested_ms`. The list shows a node only if its +parent is not in the trash, so a deleted folder stands for everything below it. (A folder can only be deleted once it is +empty, so what is below it was deleted before it, in the same change or earlier; the rule does not depend on the times.) +A restore from the trash brings back the node asked for and the deleted folders above it, root-most first, keeping their +IDs; for a folder it also brings back every node below it that is in the trash, recursively, whenever it was deleted +(a file deleted on its own earlier inside the folder comes back with it); a node marked for removal, and what hangs +only from it, stays deleted. Each of them is revived like a single restore (new version for a file, `deleted_ms` and +the mark cleared, `changed_seq` set); a node below the folder keeps its name, except that of two deleted nodes of one +name in one folder the one deleted last keeps it and the other gets a conflict name. A removal of a folder marks +everything in the trash below it the same way. Both take at most 10,000 nodes below the folder per call, in the order +depth, then newest deletion first, then ID, so a node never comes back or is marked without its folder; there is no error +for more. What is beyond has its folder live (restore) or marked (removal), or hangs under a folder that does, so it is +listed on its own and a further call takes it, bounded the same way. A restore also makes the per-row work under the write +lock at most that many nodes. It creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: +where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted +folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back +as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is +taken too, and a restore with no free name is `conflict` and changes nothing. A restored *file* gets a **new version**: a copy of its last one (the same manifest, modification time and +executable bit, no save record), after it, in the same change. A machine that deleted the file, whose answer it never +heard (the server applied the delete), still names the old version in its delete; after a restore the server refuses +that delete as a version mismatch, and the engine, which already treats a node that came back at another version as one +whose delete went through, lets the node go and downloads the restored file as a new one. Without this the machine would +send the delete again and delete the restored file. A folder has no version, so a deleted folder that was brought back +looks the same as one that was never deleted; the engine therefore remembers the folder deletes it sent without hearing +the answer (`State::sent_dir_deletes`) and, when the next pull reports such a folder changed and live, lets it go instead +of deleting it again (a folder the feed does not mention was never touched, and its delete is sent again). A restore is +otherwise an ordinary change to live nodes, possibly with a new parent or name. Older agents lack the folder rule: with +one, a folder restored in the short window after a delete whose answer was lost can be deleted a second time (the folder +is empty and can be restored again), so update agents with or soon after the server. Restoring a *version* of a deleted file is still refused: bring the file back +from the trash first. + +**Reference index and `verify`.** The chunks a manifest names live in its object in the store. `manifest_chunks` +(manifest, ordinal, chunk) is a copy of that list in the database, and `manifest_index` records per manifest how many +chunks it names, how many edges are written (ordinals 0 up to that number, never a gap) and whether the list was +compared with the manifest (state 1) or is still being built (state 0). A manifest with no row is not indexed. The index +is derived data and is never the authority; it exists so the database can answer "which chunks does this manifest need" and +"which manifests use this chunk". It is written in the transaction that records a manifest (`add_manifest`), up to a budget +of 5,000 edges per transaction; a longer manifest (up to about a million chunks) is left being built, and the upload +handler finishes it in batches of at most that many edges before it answers, so no write lock is held for a long list. +`craft-cloud-server index build` indexes manifests recorded before the index existed (and any left unfinished), reading +each from the store and checking it against its ID first; it resumes where it stopped and may run beside the server. +A manifest is marked complete only after every edge was compared, ordinal by ordinal, with the manifest it was made from: +equal counts are not enough, and repeated chunks, wrong IDs, swapped order, missing and extra edges are all found. An +index that does not match its manifest while it is still being built is thrown away and written again; a complete index +that is wrong is only reported. Throwing the edges away is itself done in batches of the same size (a marker on the index row says it is under way, so an interrupted run resumes), never in one transaction; an edge at a negative ordinal is a mismatch like any other. Nothing in this slice frees anything: no deletion, no claims, no collection, no retention. + +`craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing: the database and the +storage directory must exist already (a wrong path is an error, not a new empty server with "no problems found"), and a +database whose schema is older than the program is refused, not upgraded (start the server once to upgrade it). A SQLite database is opened read-only (`Meta::open_existing`, `mode=ro`): the database file and the WAL keep their bytes, also when a crashed process left committed data in the WAL, which the command reads; it may leave an empty `-wal` and a `-shm` file (the rebuilt index of the WAL, not data). It lists +problems (which make it exit with an error): a manifest +object that is missing, damaged or not a manifest; a complete index that does not equal its manifest; edges whose manifest +has no index row; an index row without a recorded manifest; an edge or a manifest that names a chunk that is not recorded; +a chunk object that is missing or damaged; a size the database records that is not what the object has (the stored length of a chunk or manifest object, a chunk length the manifest names against the length recorded for that chunk, the file size recorded for a manifest); a version, snapshot or import snapshot that names a manifest that is not +recorded; blob memberships without a blob. It also gives notices (which do not): manifests whose index is not complete, +manifests no version or snapshot names, and chunks no complete manifest names. **The manifests no version or snapshot +names are not garbage.** A live document's binary assets (images and the like, uploaded through the agent's blob +interface as a manifest of their own) are named only by the document's state, which the server stores as opaque bytes, so +the database holds no reference to them. Until that gap is closed (the owner decides how assets are rooted), anything that +frees storage has to treat these manifests, and the chunks they name, as in use. + +**GC inventory and dry run.** `craft-cloud-server gc --dry-run [--json] --config ` lists each store namespace in +pages of at most 1,000 objects and checks each page against `blobs` in one batched query. It reports recorded objects, +orphans (store objects without a `blobs` row), and foreign keys (invalid names for that namespace). Listing errors name +the namespace and page; they are errors, never an empty inventory. Orphans are only candidates after a first-sight grace +period, which cannot be measured without recording state; a future collector would need two runs apart. Their size comes +from a metadata-only stored-size read because `Store::list` does not include sizes. + +Recorded chunks no complete-indexed manifest names are candidates only when all indexes are complete and database checks +report no problems. `Meta::index_audit` reads chunk rows, index states and roots under one read snapshot (Postgres uses +`REPEATABLE READ READ ONLY`); the store listing is a separate time-point diagnostic. These content-addressed objects +are immutable. A future collector must recheck roots before claiming because a reference can be added after this +snapshot. Summary equality across separate audits is not a concurrency guarantee: compensating writes can preserve +counts and examples. Incomplete indexes protect unknown dependencies. Existing reference roots are versions +(including saved versions and trash history), live snapshots and import snapshots, as queried by `Meta::index_audit` in +`crates/meta/src/index.rs`; this command adds no root kinds. Manifests with no version or snapshot root are never +candidates: they are `protected: may be a live document's asset`. Chunk roots that a complete index reports as used stay +protected. A dry run writes no rows, claims, inventory, generation, run record, store object or config. It prints +`nothing was changed`. On SQLite the command follows the rule of `verify`: it opens the existing database, may leave an +empty `-wal` and a `-shm` file (the WAL's rebuilt shared-memory index, not data), and changes neither the database file +nor the WAL's bytes, which its tests compare, with a crashed process's non-empty WAL too. On PostgreSQL the tests compare a +fingerprint of every table and the sequences. Both compare the whole store tree. Grace-period measurement, claims and +deletion belong to a later owner-approved slice. **Client contract** (what the sync agent relies on): @@ -672,7 +1106,13 @@ methods answer `not_found` / `invalid` (405) the same way: `version_exists` conflicts for exactly the retried operations mean the first attempt applied. - A long-poll refused because 16 are already waiting on the space answers `429 rate_limited` with `Retry-After`; wait that long before polling again. -- Unknown JSON fields are ignored in both directions, so later protocol versions can add fields. +- Unknown JSON fields are ignored in both directions, so later protocol versions can add fields. The device list's client + fields and revoke time and reason were added that way: a body without them reads as empty. +- **A signed-in route never answers 401 for anything but the token.** The agent reads every 401 on a signed-in route as + "this device was revoked" and signs out. A wrong password on a route that asks for one (`POST /account/password`) is + therefore `403 wrong_password`; 401 is for a missing, unknown or revoked token (a disabled account's tokens are revoked), and for + the sign-in and join routes. The admin routes follow the same rule: a wrong confirmation password is 403, a member is 403. + A wrong code or recovery code is `403 two_factor_invalid` on a signed-in route and `401` only at sign-in's second step and join. - After an admin restores a backup they run `craft-cloud-server reset-epoch --all`; agents then reconcile. ## Agent and launcher API @@ -691,6 +1131,16 @@ methods answer `not_found` / `invalid` (405) the same way: Calls: `status`, `login`, `logout`, `folders` (get/set synced or cloud-only), `history`, `restore`, `conflicts`, `pending-deletes` (approve or reject a mass-delete pause), `pause`, `resume`. + + Two-factor sign-in (see [`docs/agent.md`](../agent.md)): `login` and `join` answer 401 `two_factor_required` with the + server's challenge when the account has it; `POST /v1/login/second-factor {challenge, totp_code | recovery_code}` finishes + a login, and `join` is sent again with `totp_code` or `recovery_code` (its challenge is empty: the invite isn't consumed + by the first try). `GET /v1/account/two-factor`, `POST .../setup`, `.../confirm`, `.../disable`, `.../recovery-codes`, + `POST /v1/account/password` and `POST /v1/devices/revoke-others` pass through to the server for the app's Security + page. The local answers keep the server's code: `two_factor_invalid` (401 at sign-in, 403 when signed in), + `two_factor_locked` (429 with `Retry-After`), `wrong_password`, `two_factor_pending`, `two_factor_unavailable`; a gone + challenge is 409 `two_factor_expired`. Only a 401 on a signed-in call means this device was signed out; a 403 or 429 + never does. Codes, passwords and secrets are never stored and are redacted in every `Debug`. - **CLI**: `craft-cloud-agent login | status | pin | unpin | history | restore | approve-deletes | pause | resume`. Same operations as the local API, so it is usable and testable before any launcher work. @@ -1014,9 +1464,10 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** - **Names**: property tests that the validator rejects every unsafe form, and that case-fold + NFC uniqueness matches expectations for tricky cases (`ß`/`SS`, Turkish dotted I, combining accents). - **Server**: integration tests against an in-process server, **run once per backend combination** - (SQLite + disk always; Postgres and S3 in CI through service containers, locally when - `CRAFT_CLOUD_TEST_PG` / `CRAFT_CLOUD_TEST_S3_*` are set; CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS` so a - missing backend fails instead of skipping). The S3 test server is SeaweedFS (Apache-2.0), run as a + (SQLite + disk always; Postgres + disk locally when `CRAFT_CLOUD_TEST_PG` is set; Postgres + S3 + when `CRAFT_CLOUD_TEST_PG` / `CRAFT_CLOUD_TEST_S3_*` are set, including CI through service + containers; CI sets `CRAFT_CLOUD_REQUIRE_BACKENDS` so a missing backend fails instead of skipping). + The S3 test server is SeaweedFS (Apache-2.0), run as a container, never linked. Cases: auth, quota, batch commit atomicity, `missing_chunks`, GC never removes a referenced or protected chunk while commits run, thinning, `backup` + `restore` gives the same state with a new epoch, `resync_required`. @@ -1061,7 +1512,7 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** | M3b2b1 | Agent on its own: seed 112337, `run` loop, watcher, long-poll, stability window, backoff, status file | DONE (2026-10-14) | | M3b2b2 | Simulated network (`turmoil`): partitions, delays, dropped connections | DONE (2026-10-15) | | M3c | Fixes from the full repo review | DONE (2026-10-15) | -| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | TODO | +| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | IN PROGRESS: history and restore of file versions, and the trash (list, restore, mark for removal), done (server API only); the rest TODO | | M5 | Selective sync, large-media polish (a few chunks in flight at once), ignore rules | TODO | | M6 | Packaging: binaries, Docker, systemd, built-in TLS, self-hosting guide | TODO | | M7 | Local control API + launcher integration PR | TODO | @@ -1158,7 +1609,7 @@ From the reviews of M3c, less what their fix rounds did: ## Out of scope for Phase 1 -Hosted service; collaboration and sharing; locking; web UI; mobile; virtual files and on-demand download; +Hosted service; collaboration and sharing; locking; a web UI for users' files (the administrators' pages, see "Web admin and browser sessions", are in); mobile; virtual files and on-demand download; end-to-end encryption (not built, but chunk IDs are opaque and stored chunks have a format byte so it can come later); per-chunk compression (same format byte); a packed small-chunk upload endpoint (one request per chunk until measurements say otherwise); compatibility with any other sync service. diff --git a/docs/server.md b/docs/server.md index bf2a6492..675d672f 100644 --- a/docs/server.md +++ b/docs/server.md @@ -25,13 +25,93 @@ For a non-loopback `--listen`, `init` also needs `--behind-proxy` and the proxy' For Postgres or S3, edit `[database]` and `[storage]` in `config.toml`. +### Storage capacity + +The admin's storage overview (`GET /admin/storage`, shown by the Craft Cloud app) tells how much room the server has. +Say how much the blob store may hold with `capacity` in `[storage]`, as a size such as `"1 TiB"` or `"500 GB"`: a whole +number or one with one decimal (`"1.5 TB"`), then `B`, `KB`, `MB`, `GB`, `TB` (powers of 1000) or `KiB`, `MiB`, `GiB`, `TiB` +(powers of 1024). A plain positive integer is a number of bytes, as for `stored_bytes_per_user`. Anything else, and +zero, stops the server from starting with a message that names the setting. + +```toml +[storage] +kind = "disk" +path = "/var/lib/craft-cloud/blobs" +capacity = "1 TiB" +``` + +- `init` makes a disk store and takes the size as `--capacity "1 TiB"` (it asks nothing: its only input is the password + on standard input, so it can run in a script). Without `--capacity` none is written. For an S3 store, which `init` does + not make, add `capacity` to `[storage]` by hand: S3 has no size to ask for, so the capacity you set is the only size the + overview can show for it. A server without `capacity` starts as before and shows no capacity. +- For a disk store the overview also has the total and free bytes of the filesystem that holds the store's directory. + For the server's own data (the SQLite file) it has the total and free bytes of its filesystem and the size of the + database, including its `-wal` and `-shm` files. With Postgres it has none of these: the database server's disk is not + the server's to measure. "Free" is what the server's own user may still use. +- The numbers are read when the overview is asked for, with a filesystem query each and without scanning the stored + data. By default the capacity is information only: the server does not refuse uploads when the store is full or past + the capacity. To make it refuse, see "A full server" below. + +### A full server + +A disk that fills up makes uploads fail halfway, and a store past its capacity has nothing to say about it. Turn on +`refuse_when_full` in `[storage]` and the server instead refuses new uploads in good time, with a clear answer, while +everything else keeps working: + +```toml +[storage] +kind = "disk" +path = "/var/lib/craft-cloud/blobs" +capacity = "1 TiB" # optional on disk, required for S3 +refuse_when_full = true # default false +min_free = "1 GiB" # disk only; the free space to keep, default 1 GiB +``` + +- It is off by default, and an upgrade changes nothing until you set it. With it on, a new chunk or manifest (which is + also how a live document's snapshot goes up) is refused when the store's filesystem would be left with less than + `min_free` (disk only), or when the bytes stored would pass `capacity` (disk or S3, when `capacity` is set). The + answer is `507 server_full`, with the free space, the total and the capacity as far as the server knows them. It is + not `quota_exceeded`, which is one account's own limit. +- Reads, downloads, deletes, the trash, garbage collection, live document updates (they live in the database) and the + admin's calls are never refused, so you can free space while the server is full. Changes to the database itself + (a sign-in, a commit) still work; a SQLite file on a full disk still can fail on its own. +- The stored total and the free space are read at most every 5 seconds, and the bytes the server wrote itself since are + added in between, so the check costs next to nothing; a refusal ends within about 5 seconds of space being freed. + The log says at `warn` when the server starts refusing and when it stops, with the reason and no file names. +- `capacity` is a soft limit: uploads that are in flight at the same moment are each checked against the numbers as they + were, so the store can pass it by up to the bodies in flight (at most 8 per account, 4 MiB each for chunks). The stored + total is the sum of what every account stored, and a chunk two accounts both uploaded counts for both, so refusing can + start before the store's physical size reaches `capacity`. The filesystem reserve, which is measured, has no such gap. +- A chunk or manifest the account already has is accepted again while the server is full: it adds nothing. A request + that declares a body over the size limit is answered `413`, never `server_full`. If the free space can't be read + within 2 seconds (a dead network mount, say), the reserve is not checked until it can be, and the log says so once. With + a large body in flight a client can see its connection closed by the early refusal instead of the `507` itself; an + agent then treats it as a network error and retries, which is safe. +- `min_free` without `refuse_when_full`, `min_free` on S3 and `refuse_when_full` on S3 without `capacity` are refused when + the server starts, since they would do nothing. +- The agent turns `server_full` into a pause: "The server is full; the admin needs to free space or raise its capacity", + and looks again every 10 minutes. The local API's status carries the cause as a code too (`run_state`: `paused` with `reason_code`: `server_full`, or `quota_exceeded` for an account over its quota), which an app can match instead of the words. An agent from before this release does not know the code and treats the HTTP 507 + as a server error: it shows the server's message as an error and tries again with growing pauses, up to 5 minutes. Nothing + is lost either way. A config with `refuse_when_full` or `min_free` is refused by an older server (it rejects unknown + settings), so remove the lines before going back to one. + +`init` also makes `two-factor.key` next to `config.toml` and names it in the config (`[two_factor] key_file`). It is the key +for two-factor sign-in (below). **Back it up, apart from your database backups.** + To cap what one account may store, set `stored_bytes_per_user` (in bytes; there is no cap by default) in `config.toml`. It counts every chunk and manifest an account uploaded once, and nothing frees space until garbage collection arrives, including what a refused file stored before it was refused. A file that would go over it is skipped by the agent, which says so; the rest of the folder keeps syncing. Nearly full counts as full: with less than 4 MiB (one chunk) left, the agent pauses and says the account's storage is full. -A cap under 4 MiB is refused when the server starts, since not even one chunk would fit. Per-user quotas on -file sizes come later, with history. +A cap under 4 MiB is refused when the server starts, since not even one chunk would fit. An admin may override +the cap for an individual account as described below. + +Signed-in devices can read `GET /api/v1/account/usage` for the space's stored bytes, the effective quota, and labeled +estimates of bytes referenced only by trash or old versions. The estimates deduplicate shared blobs per space, exclude +live-document snapshots and unrooted app assets, and are never added to stored bytes. The history estimate treats a +version an app saved (one with a save record) as kept. The effective quota is the account override when set, otherwise +the configured `stored_bytes_per_user` cap; it is `null` when neither applies. An incomplete manifest index makes both +estimates `null`. Live documents have a cap of their own: `live_bytes_per_user` (in bytes; 1 GiB, `1_073_741_824`, by default) is the most one account's live documents (recent-file lists and documents of files) may hold together. It counts @@ -55,9 +135,310 @@ invite replaces the old one, and `user invite --revoke name` ends it. Every late that is signed in (`craft-cloud-agent add-computer`, see [`agent.md`](agent.md)). Codes that don't work are refused alike (wrong, expired or used), and an address that tries ten in a minute is held off for the rest of it. +## Devices, passwords and signing out + +An account's devices (computers and apps) each hold a token that does not expire. Accounts end them like this: + +- A person changes their own password in a Craft Cloud app that offers it (`POST /api/v1/account/password`). That signs + out every other device of the account, deletes its open invite and add-a-computer codes, and keeps the device + that asked. Five tries (wrong passwords count) are allowed per account every 15 minutes. +- `craft-cloud-server user --config cc-data/config.toml reset-password name` sets a new password (read from standard + input), **signs out every device of the account** and deletes its open invite and add-a-computer codes: a token that + was stolen does not survive the reset. It prints how many devices were signed out. Add `--keep-devices` to leave the + devices signed in (the old behaviour); the codes are deleted either way. +- `craft-cloud-server device --config cc-data/config.toml revoke-all name` signs out every device of an account + without touching the password. `device revoke name DEVICE_ID` signs out one; `device list name` shows them. + +The command line runs in its own process, so it cannot reach the server's open connections. A signed-out device's live +connection ends at its next ping (within about 30 seconds), and a change-feed long-poll within 60 seconds (when it +ends, it is answered `401 unauthorized`, never with changes); every other request is refused at once (`401 +unauthorized`). Devices signed out through the API end their live connections +immediately. + +## Two-factor sign-in + +An account can turn on a second factor in an app that offers it: a code from an authenticator app (TOTP: Aegis, 1Password, +Google Authenticator and the like), with ten recovery codes to keep. With it on, a right password does not sign anyone in: +the app also asks for the code, and so does joining a computer with an invite or an admin's password reset code. A +computer joined with an add-a-computer code (a pairing) asks for nothing, since only a computer that is already signed in +can make one: **any signed-in computer can add more**, and turning two-factor sign-in on signs every other one out. + +- **The key.** The authenticator's secret is stored only encrypted (AES-256-GCM, bound to its account), with the key in the + file `[two_factor] key_file` names (32 random bytes; a relative path is relative to `config.toml`). `init` makes it; **the + server never makes it** (a second server on the same database, or a restore without the key, would otherwise seal + secrets nobody can open). **Without this key the secrets in a database backup are unusable**: keep a copy apart from + the backups, and keep it out of everyone's reach. On Unix `init` makes it owner-only; + **on Windows the server sets no file permissions**, so put it somewhere only the account that runs the server can read. + The recovery codes do not depend on the key, so they still work if it is lost; the authenticator codes do not. +- **Failing closed.** When the database holds two-factor secrets, `serve` opens every one of them at start and refuses to + run if the key is missing, unreadable, or does not open all of them; the error names the key file the config expects. + A server without a key and without any secrets runs, and setting it up answers `503 two_factor_unavailable`. Several + servers on one Postgres database must all have the same key. +- **Rotating the key.** A running server keeps the keys it started with, so rotate in this order: (1) make a new key file, + put it in `key_file` and the old one in `key_file_old`; (2) **restart every server instance on the database** with that + config (until then an instance still holds only the old key and goes on sealing new secrets with it); (3) run + `craft-cloud-server two-factor rekey --config cc-data/config.toml` (it rewrites every secret, active and set-up, under the + new key, one account at a time, reads each account's secrets fresh under its lock so a set-up confirmed meanwhile is not + missed, scans again at the end, can be run again, prints only counts, and refuses without changing anything if a secret + was sealed with a key that is neither); (4) remove `key_file_old` and restart **only after `rekey` has reported that every + secret is under the current key** (an error that says secrets are still under another key means it is not done: restart the + instances you missed and run it again). The server runs in the middle of a rotation, with both keys. +- **A locked-out account.** A person who lost their authenticator and their recovery codes: `craft-cloud-server user + --config cc-data/config.toml reset-2fa name` deletes its two-factor secret, recovery codes and sign-in challenges, lifts + any lock, **signs every device out** (it prints how many) and deletes its open invite and add-a-computer codes. They sign + in with their password and can set it up again. Over HTTP an admin does the same with `POST /admin/users/{id}/reset-two-factor` + (below). Both need no key. +- **Wrong codes lock for a while.** Ten wrong codes or recovery codes in a row (at sign-in, joining, turning it off, + making new recovery codes, an admin's confirmation) lock the account's second factor for 15 minutes; each wrong one after + that doubles the time, up to 24 hours; a right one resets it. While it lasts everything answers `429 two_factor_locked` + with `Retry-After`. The lock is never for good, so someone who only has a password can not lock an account out forever. + Wrong codes also count against the sign-in limit (10 per account and 30 per address in 5 minutes, then `429`). +- **Recovery codes** are ten of 16 letters and digits (shown in groups of four), each good once. Only a hash is stored. +- **Older apps.** An agent from before this release shows every `401` at sign-in as "wrong password", so an account with + two-factor sign-in can't sign in with it (it never gets a token). Update the server first, then the Craft Cloud + desktop package with its agent, then stand-alone agents. +- **Logs** never carry a secret, code, recovery code, challenge, password or key. + +## Managing accounts over HTTP + +Accounts have a role, `admin` or `member`, and the routes under `/api/v1/admin` are for admins only (the first admin is +still made by `init`). They are an API for a tool or an admin app; the server's own admin pages are an option, see "The admin +pages in the browser" below. Every change asks +for **a fresh confirmation in its body**, the acting admin's own current password (and, for an admin who has two-factor +sign-in, a `totp_code` or `recovery_code` too), because a device's token never expires and must not be enough on its own: + +```sh +curl -s -X POST https://cloud.example.org/api/v1/admin/users/USER_ID/disable \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' -d '{"password":"your password"}' +``` + +- `GET /admin/server` shows counters (accounts, signed-in devices, stored bytes, live-document bytes) and the kinds of + database and storage, nothing else. `GET /admin/users` lists accounts 50 at a time (`limit` up to 200, `cursor` from + the last answer's `next`, `query` to find a username). +- `GET /admin/storage` returns bounded storage totals, effective quota totals and near/over-quota counts, the ten + largest accounts, manifest-index counts, and the room the server has (see "Storage capacity"). `GET /admin/users/{id}/usage` returns one account's stored bytes, + effective quota, and labeled trash/history estimates; unknown and removed accounts return `404 not_found`. +- `POST /admin/users` makes an account (role `admin` or `member`) with no password and gives an **invite**, shown once: + the person chooses their password when they join with it (a whole join link is included when `public_url` is set). + `POST /admin/users/{id}/invite` makes a new invite, but only for an account that still has no password; for one with a + password, make a **password reset code** (`POST …/password-reset`, good for an hour). Whoever redeems + it in Craft Cloud chooses a new password, **every device of the account is signed out, and the computer that redeems + it is signed in**. If the account has two-factor sign-in, redeeming the code needs its second factor too, unless the + admin asked for `"reset_two_factor": true`, which turns it off with the redeem. Codes are only shown in the answer, + never in the log. +- `PATCH /admin/users/{id}` changes the role and/or `quota_bytes`, a limit on what that account may store that replaces + `stored_bytes_per_user` for it (at least 4 MiB; `null` removes it; effective quota is the override, else configured + cap, else unlimited; it also holds when there is no server-wide cap; one + below what the account stores deletes nothing and refuses new bytes). +- `POST …/disable` signs every device of an account out and refuses its sign-in and joins (`account_disabled`); + `POST …/enable` allows them again but signs nothing back in. `DELETE /admin/users/{id}` marks an account for removal: + it is disabled for good and leaves the list, its name stays taken, and **nothing it stored is deleted yet** (that comes with + garbage collection). You cannot disable or remove yourself, and the last enabled admin cannot be demoted, disabled or + removed (`last_admin`). +- `GET /admin/users/{id}/devices` lists an account's devices and `DELETE …/devices/{device_id}` signs one out. +- `POST …/reset-two-factor` takes an account's two-factor sign-in away (it lost its authenticator and its recovery codes): + its secret, recovery codes and sign-in challenges go, and **every device is signed out**. Not your own account (turn + that off in your account), and nothing to reset is `409 conflict`. +- Resetting another **admin's** password or two-factor sign-in over HTTP needs two-factor sign-in on your own account + (`403 forbidden` without it); the command line on the host is the way if no admin has it. +- **A code serves one change.** The server accepts one 30-second step of an account's authenticator once, and only a later + step after it; sign-in and the admin's confirmations share that. So a second change in the same 30 seconds takes the + next code (wait for it) or an unused recovery code. + +A wrong confirmation password is `403 wrong_password` (never 401); five wrong ones in 15 minutes hold the account off for +the rest of the window. A wrong or missing code for an admin who has two-factor sign-in is `403 two_factor_invalid`. A signed-out device's live connection ends at once in the server process that did it; another +server process on the same database, or the command line, finds out at the connection's next ping (within about 30 seconds). +The command line stays as the way in without HTTP and enforces neither the confirmation nor `last_admin`. + +## The admin pages in the browser + +The server can serve admin pages: users, invites and reset codes, a user's devices, the audit log and its download, in +plain server-rendered HTML (no script, no Node, nothing to build). They are **off by default** and only administrators can +sign in. They do the same things as the admin API with the same confirmation (your password again for every change, and a +code when your account has two-factor sign-in). +The Craft Cloud desktop app also has admin storage views: a server overview with quota and manifest-index summaries, +and per-account usage details. Both use the authenticated admin API; members do not see admin pages. + +```toml +public_url = "https://cloud.example.org" # the address of the pages, https, no path +behind_proxy = true +trusted_proxies = ["10.0.0.2"] + +[web_admin] +enabled = true +idle_minutes = 480 # a session ends when unused for this long (5 to 1440, default 8 hours) +absolute_hours = 168 # and at the latest this long after signing in (1 to 720, default 7 days) +``` + +Open `https://cloud.example.org/web` and sign in as an admin. The server refuses to start with `enabled = true` unless +`behind_proxy = true` and `public_url` is an `https://` address with no path: the pages' cookie is `Secure` (a browser only +sends it over HTTPS) and every form is checked against the origin of `public_url`. + +**Behind the proxy.** + +- The proxy terminates TLS and forwards `/web` and `/api` (and `/join`) to the server's `listen` address. It must pass the + `Origin` and `Cookie` headers through and must not rewrite the `Host` to something that changes the origin the browser + sees: browsers send `Origin: https://cloud.example.org`, and anything else is refused with `403`. Serving the pages under a + path prefix is not supported (`public_url` may not have a path). +- Put the proxy in `trusted_proxies` and let it **overwrite** `X-Forwarded-For` with the client's address; the sign-in limits + count the address that yields. A proxy that passes a client's own `X-Forwarded-For` along lets a visitor pick the address + the limits count. A misconfigured server shows itself at start (the config is refused) or as `403` on every form (the + proxy changed the origin). +- Send `Strict-Transport-Security` from the proxy; the server does not. Do not cache `/web` in the proxy: every answer says + `Cache-Control: no-store`. +- Bind `listen` to the proxy-facing address only. The cookie never leaves the browser over plain HTTP, but the plain HTTP + port must not be reachable by anyone but the proxy. + +**Sessions.** The session is a random value in the cookie `__Host-craft_cloud_session` (`Secure`, `HttpOnly`, +`SameSite=Strict`); the database keeps only its hash. A session ends when you sign out, after `idle_minutes` without use, +after `absolute_hours`, and **whenever the account changes**: a password change or reset, disabling, enabling or removing it, +and turning its two-factor sign-in on or off or resetting it. A demoted admin is out at the next request. Signing in again +makes a new session; there is no "remember me". Because the cookie is `SameSite=Strict`, opening the pages from a link on +another site shows the sign-in page once, and the next click on the pages themselves works. + +**Cleaning up.** Expired sessions are deleted a few at a time when someone signs in. On a server that is signed in to rarely, +or to empty the table: + +```sh +craft-cloud-server web-sessions --config cc-data/config.toml prune # deletes what is over, prints how many +craft-cloud-server web-sessions --config cc-data/config.toml prune --dry-run # only counts +``` + +Run it from cron or the task scheduler (daily is plenty), like `audit prune`. A valid session is never touched. There is no +background task. + +**What the log shows.** A sign-in to the pages is `auth.login` without a device; changes made there are the usual +`admin.*` events with the admin as the actor and no device. Failed sign-ins, second-factor failures and the lock are the +usual `auth.*` events. Nothing secret is logged: not the cookie, the page's forms, a code or a password. + +**Accessibility.** The pages are plain HTML: labelled fields, tables with captions and header cells, a skip link, visible +focus and no script, so they work with a keyboard and a screen reader and scale with the browser's zoom. They have been +checked by reading the markup and the tests, **not yet with a screen reader**: report what does not work. + +## The security audit log + +The server keeps a log of the events that matter for who may do what, in its database: sign-ins and failed sign-ins (a +row per account, client kind and minute, with a count), devices signed out, password changes, an administrator's changes to accounts, +roles, quotas, invites, reset codes and devices, codes redeemed on a new computer, and what the command line does to an +account (`user reset-password`, `user reset-2fa`, `device revoke`, `device revoke-all`). Two-factor sign-in writes its +changes, a recovery code's use, a lock and the wrong second steps of a sign-in or a join (never a code), but not each wrong code +on a signed-in route. Reading it: + +- Every account sees its own events, and what was done to it, with `GET /api/v1/account/activity`. +- Admins read all of it with `GET /api/v1/admin/audit` (filters `user`, an account ID, and `action`, an event name like + `auth.login` or a group like `auth.`; pages by `cursor`) and export it with `GET /api/v1/admin/audit/export`, newline + delimited JSON, newest first. The **last line** says whether the export is whole: `{"complete":true,"exported":n}`, or + `{"truncated":true,"exported":n}` when the log holds more than the 100,000 events an export carries (narrow it with + `user` and `action`). An export that ends without either line broke off: do not treat it as complete. Reading the log is + not logged. +- On the server's host: `craft-cloud-server audit --config cc-data/config.toml list [--user NAME] [--action auth.] [--limit N]` + prints the newest events as text (accounts by ID, not by name). + +**What is in it, and what is not.** A row has a time, the IDs of the account and device that acted and of what it acted +on, an event name, success or failure, and a few typed values (how many devices were signed out, the role before and after, +which kind of client and which platform out of a short list). Client kinds are `agent`, `desktop`, `cli` and `web`; `web` +means a browser session of the admin pages. It has **no IP addresses, no file names, no device names, no +names of accounts that do not exist, no passwords, codes, tokens or other secrets**; what a client types is never copied +into it. Because it holds account IDs and times it is personal data about the people who use the server: tell them +what you keep it for (finding out who changed what and who signed in or tried to), and for how long. An event keeps the ID +of an account that was removed; the name is not stored. + +**How long.** Events are kept 365 days. Set `retention_days` (1 to 3650) in an optional `[audit]` section of `config.toml`: + +```toml +[audit] +retention_days = 365 +``` + +`craft-cloud-server audit --config cc-data/config.toml prune` deletes the events older than that and prints how many +(`--dry-run` only counts). The server also deletes expired events when it writes a new one, at most once an hour and in small +batches that never hold up a sign-in; **a server nobody uses keeps its expired events** until `prune` runs, so run it from cron +or the task scheduler, for example daily. There is no background task. + +The log is append-only in the server's code, not enforced by the database (a design choice: the same code runs on SQLite and Postgres, and SQLite has no database roles; triggers exist but are not used). Someone +who can write to the database can change it; keep the database file and Postgres user private. + +## Upgrading + +This release adds migrations (`0007`, device sessions, `0008`, the admin API: reset codes, per-account quotas, the +removal mark, `0009`, the audit log, `0010`, two-factor sign-in: the secrets, recovery codes and sign-in challenges, +`0011`, the admin pages' browser sessions, `0012`, the history index, `0013`, the trash, and `0014`, the manifest reference +index). The server applies them when it starts. A database it +has migrated cannot be opened by an older server: that one refuses to start ("the database was created by a newer Craft +Cloud"). Rolling back therefore means restoring a backup taken before the upgrade, so take one first. Upgrade the +server first: agents and apps keep working unchanged, **except that an account that turns on two-factor sign-in can't sign in +with an agent from before this release** (see "Two-factor sign-in"). Then update the desktop package with its matching agent, +then agents installed on their own. The `init` of a data directory made before this release did not make a key: add one +with `[two_factor] key_file` in `config.toml` (32 random bytes, owner-only) before anyone sets up two-factor sign-in. A config +that has the new `[audit]` or `[web_admin]` section is refused by an older server (it rejects unknown settings), so remove the section before +going back to one. To turn the admin pages on after the upgrade: take the backup, start the new server (it migrates), then add +`[web_admin]` as described above and restart behind the proxy. Several servers on one Postgres database must all run this +release before any of them is given `[web_admin]`, as for every migration. Likewise a config with `capacity` in `[storage]` +is refused by an older server, so remove the line before going back to one. The new fields of `GET /admin/storage` are +optional: an older app reads a newer server's answer, and a newer app reads an older server's, without them. +## Version history + +The server keeps every version of every file; nothing is thinned or removed yet. A signed-in device can list a file's +versions (`GET /api/v1/nodes/{id}/history`) and make an old one current again (`POST /api/v1/nodes/{id}/restore`), +which adds a new version at the end of the file's history rather than rewinding it, so other computers pick it up +through the normal change feed. This needs no setting and no database migration, and an older server answers +`not_found` for both paths. The agent and the apps do not use them yet. + +## The trash + +A deleted file or folder stays in the trash, and in the server's records, until it is removed for good. A signed-in +device lists the trash (`GET /api/v1/trash`), brings a node back (`POST /api/v1/trash/{id}/restore`, which names it +differently if its name has been taken meanwhile) or takes it out of the trash (`DELETE /api/v1/trash/{id}`). Taking +it out only marks it: **nothing is deleted from the server yet**, no version or chunk is freed, and the node can't be +brought back from the trash afterwards. A folder in the trash is listed alone: everything that is in the trash below it +(deleted and not removed from the trash) comes back with it when you bring the folder back, and goes with it when you +take the folder out, **also a file that was deleted on its own, earlier, inside that folder**. Those items are listed +only through the folder. A folder brings back, or takes out, up to 10,000 items below it at once, the shallowest first; if +there are more, the rest then appears in the trash list on its own (its folder is live again or out of the trash), and +you bring it back or remove it by a further call. The only setting is the number of days a deleted node waits, shown to clients as +`purge_after_ms`; it removes nothing yet: + +```toml +[retention] +trash_days = 30 # 1 to 3650 +``` + +A file brought back gets a new version with its old content, so another computer's delete that was sent just before can't delete it again. Update the agents with or soon after the server: an older agent can delete a *folder* a second time if that folder was brought back right after a delete whose answer it never heard (the folder is empty and can be brought back again). + +The change adds two columns and an index to the metadata (migration 0013). An older server refuses the +upgraded database and rejects the `[retention]` section, so take a backup before upgrading and do not roll back. + +## Checking the storage + +`craft-cloud-server verify --config cc-data/config.toml` checks the database and the blob storage against each other +and prints what it finds; `--json` prints a machine-readable report and `--skip-chunk-reads` skips reading every chunk +object (which hashes it, and takes as long as reading all your data). It changes nothing and exits with an error if +it found a problem. It opens a SQLite database read-only: the database file and its WAL keep their bytes, also after a +crash that left committed data in the WAL, which it reads. Opening may leave an empty `-wal` and a `-shm` file behind +(SQLite's rebuilt index of the WAL, not data). The report also says how many manifests have a complete reference index, which is what a later +cleanup would need before it could free anything. After upgrading, run `craft-cloud-server index build --config +cc-data/config.toml` once: it indexes the manifests stored before the index existed, writing only that index, in small +steps, and it can be stopped and run again. Nothing in the server frees storage yet. + +The change adds two tables to the metadata (migration 0014). An older server refuses the upgraded database, +so take a backup before upgrading and do not roll back. + +`craft-cloud-server gc --dry-run --config cc-data/config.toml` inventories stored chunks and manifests, compares them with +the database and prints a preview; `--json` prints schema version 1. It changes no data, on SQLite or on PostgreSQL. +Like `verify`, it opens a SQLite database that exists already and creates nothing; opening a WAL-mode database may leave +an empty `-wal` file and a `-shm` file behind (SQLite's rebuilt index of the WAL, not data), and the database file and +the WAL's contents stay as they were, also after a crash that left data in the WAL. Store objects with no `blobs` row are orphan candidates, but need two runs apart to establish a future +first-sight grace period. Recorded chunks no complete indexed manifest names are candidates only when every manifest index +is complete and database checks pass. They come from one read-only database snapshot; the store listing is a separate, +time-point diagnostic. A future collector must recheck roots before claiming a chunk. Current versions, saved versions, +trash items, live snapshots and import snapshots protect their chunks. Unrooted manifests are protected because they may be live document assets. Without +`--dry-run`, the command refuses: deleting is not implemented yet. Listing errors fail the command; they do not mean the +store is empty. Nothing is deleted. + ## Restoring a backup -With the server stopped, restore the database and the chunk storage from the backup, then run +With the server stopped, restore the database and the chunk storage from the backup (and the two-factor key file, if +accounts use two-factor sign-in: the server will not start without it), then run `craft-cloud-server reset-epoch --config cc-data/config.toml --all`, and only then start `serve` again. Every account then starts a new epoch, so each synced machine reconciles with the restored state instead of trusting its own change counter. `reset-epoch --config … name` resets one account. (A reset run while the server is up still @@ -71,3 +452,156 @@ every machine then sends up what the emptied log lacks, so nothing any machine h affected. A document whose own history is over 16 MiB stays full until compacting the log is supported. An account may have 64 live documents holding updates and 1,024 in all (one per app today); past that, new ones stay on the machines. Live documents can't be deleted yet. + +## Docker + +`packaging/docker/` holds a hardened container image of the server and a Docker Compose deployment: the server +behind Caddy, which terminates TLS. By default the database (SQLite) and the blobs live on a Docker volume; Postgres +and S3 are overlays. It needs Docker Compose 2.32 or newer on Linux and builds the image locally: there is no image +to pull and nothing is published to a registry. The first build wants about 4 GiB of memory for the Docker engine (it +uses two compiler jobs). + +### Quick start + +```sh +cd packaging/docker +./install.sh --domain cloud.example.org --admin you +``` + +`install.sh` checks Docker and the free disk space, writes `.env` (mode 0600, never committed) with random database +passwords, builds the image, starts the stack and creates the first admin. The admin's password is asked for without +echo, or read from a pipe, and reaches the server through standard input only. On a terminal it asks for the +settings you did not give; `./install.sh --help` lists the options (`--tls acme|internal|custom`, `--email`, +`--db sqlite|postgres`, `--http-port`, `--https-port`, `--no-admin`, `--no-start`). It never overwrites an existing +`.env`. Afterwards `docker compose ps` and `docker compose logs -f server` show the state, and the domain's +`/api/info` answers through Caddy. + +To set it up by hand, copy `.env.example` to `.env`, fill it in, and run `docker compose build` and +`docker compose up -d --wait`. + +### What runs + +| Service | Image | Notes | +|---|---|---| +| `server` | built from `packaging/docker/Dockerfile` | distroless `cc`, uid 65532, plain HTTP on port 8080 inside the stack | +| `caddy` | `caddy:2.11-alpine` | the only service with published ports (80 and 443); TLS and the redirect from HTTP | +| `secrets-init` | `caddy:2.11-alpine` | one-shot, no network: renders `config.toml` and writes the secret files | +| `db`, `db-init` | `postgres:17-alpine` | only with `compose.postgres.yaml` | + +Every base image is pinned by the digest of its multi-arch index, with the tag in a comment next to it. To update +one, change the tag and the digest together (`docker buildx imagetools inspect :` prints the index +digest). + +Every service has a read-only root filesystem, no capabilities (`cap_drop: [ALL]`), `no-new-privileges`, memory and +process limits, and a `noexec` tmpfs for `/tmp`. The exceptions are explained next to each service: Postgres and +`secrets-init` start as root and need a few capabilities to fix ownership, Caddy needs to bind ports 80 and 443, and +`secrets-init` (in the Postgres and S3 overlays) is not read-only because Compose writes its secret files into the +root filesystem. + +Three networks keep the parts apart: `edge` (published ports and outbound access, Caddy only), `frontend` (internal; +Caddy to the server) and `backend` (internal; the server to Postgres, Caddy is not attached). + +### Configuration and secrets + +The server reads only its TOML file. `config.toml.in` is the template and `render-config.sh` fills it. The +`secrets-init` service renders it into the `secrets` volume (mode 0400, owner 65532) and the server mounts it +read-only at `/etc/craft-cloud/config.toml`. The file holds the database URL (with the password, for Postgres) and, +for S3, the keys. The server's `Debug` output hides the URL's credentials, and `docker inspect` of the running +services shows no secret, since none is in an environment or a Compose file. The values are checked against short +lists of allowed characters before they are written, so a value cannot break the TOML or add a setting. + +To change a setting in `.env` (`CC_*`, see `.env.example`), edit the file, run `docker compose up -d --force-recreate secrets-init` +(it must be recreated to see the new values: Compose gives a container its secrets when it is created, and a plain +`up` would start the old container again and write the old values back), then `docker compose restart server`. + +The server believes `X-Forwarded-For` only from Caddy. `trusted_proxies` takes single addresses, no networks, so +Caddy has a fixed address (`CC_CADDY_IP`, `172.29.215.2` in the network `CC_FRONTEND_SUBNET`, `172.29.215.0/24`). If +these defaults clash with a network on your host, change both together, and `CC_FRONTEND_POOL` (where the other +containers get addresses from) with them. + +### TLS modes + +Set `CC_TLS_MODE` in `.env`: + +- `acme` (default): a public certificate from Let's Encrypt. The domain must resolve to the host, and the ports must + be reachable from the internet (with other ports, set `CC_HTTP_PORT`, `CC_HTTPS_PORT` and a matching + `CC_PUBLIC_URL`). +- `internal`: a certificate from Caddy's own CA, for tests. Clients do not trust it. There is no HSTS in this mode. +- `custom`: your own certificate. Put `fullchain.pem` and `private.key` in `packaging/docker/tls/` (or `CC_TLS_DIR`), + readable by root only, and restart Caddy to load a renewed one. + +Caddy sets a request header timeout of 10 seconds, as the checklist above asks. It writes no access log, because URLs hold invite +codes (`/join/`); its error log leaves out the request's URI and headers for the same reason. + +### Postgres + +```sh +./install.sh --domain cloud.example.org --db postgres +``` + +or add `COMPOSE_FILE=compose.yaml:compose.postgres.yaml` to `.env`. This adds a Postgres container and a one-shot +`db-init` that creates the role `craft_cloud`. The server migrates its database when it opens it, so that role owns +the database `craft_cloud` and its schema and may change them. That is less separation than an owner role plus a +runtime role. The role is not a superuser and has no other privileges; `PUBLIC` may not connect to the database or +create in its schema, and the role cannot connect to `postgres` or `template1`. The superuser password exists only +for the Postgres container and `db-init`. Blobs stay on the data volume. + +To rotate the database password, change `CC_SECRET_DB_PASSWORD` in `.env` (characters from `A-Za-z0-9._~-`, at least +32), then run `docker compose up -d --force-recreate secrets-init`, +`docker compose up -d --force-recreate db-init` and `docker compose restart server`. The superuser password cannot be changed that way: Postgres applies it only when +the data directory is created. Change it with `ALTER ROLE postgres` first and then in `.env`. + +The connection between the server and Postgres is not encrypted (the server's `sqlx` has no TLS feature). It never +leaves the internal `backend` network. An external Postgres over TLS is not covered. + +### S3 + +`compose.s3.yaml` replaces the disk storage with an S3 bucket: fill in the `CC_S3_*` and `CC_SECRET_S3_*` lines of +`.env.example` in `.env` and add `compose.s3.yaml` to `COMPOSE_FILE`. The keys end up in `config.toml` like the +database password, not in the server's environment. The server joins the `edge` network to reach the endpoint. The +image holds the CA certificates the server needs for HTTPS endpoints. The overlay has been tried with a local S3 mock +over plain HTTP (`CC_S3_ALLOW_HTTP=true`), not with a real S3 service or an HTTPS endpoint. + +### The first admin and more accounts + +`install.sh` creates the first admin. By hand, with the password on standard input: + +```sh +printf '%s\n' 'a password' | docker compose run --rm --no-deps -T server user --config /etc/craft-cloud/config.toml add --admin you +``` + +More accounts: the same line without `--admin`, or `docker compose run --rm --no-deps server user --config +/etc/craft-cloud/config.toml invite name`, which prints the whole join string because `public_url` is set. The +other commands of this guide (`device`, `reset-epoch`, `live`) run the same way. + +### Data, upgrading and backups + +The data is in the volume `data` (`/var/lib/craft-cloud` in the container): `meta.db` (with SQLite) and `blobs/`. +The image creates the directory for uid 65532 with mode 0700, so a new named volume gets that owner. A bind mount +instead of the volume must be prepared first: `chown 65532:65532 DIR && chmod 0700 DIR`. + +To upgrade, update the source, then run `docker compose build server` and `docker compose up -d --wait`. The server +migrates the database when it opens it and refuses a database newer than itself, so back up before an upgrade and do +not go back to an older image without the backup. + +There is no backup automation yet. Stop the server (`docker compose stop server`), copy the volume (SQLite: +`meta.db`, `meta.db-wal`, `meta.db-shm` and `blobs/` together; Postgres: `pg_dump` from the `db` container plus +`blobs/`), and start it again. Restoring is described above, including `reset-epoch`. + +`docker compose down` keeps the volumes. `docker compose down -v` deletes all data. + +### Troubleshooting + +- `secrets-init` fails with a message such as `PUBLIC_URL has a value that is empty or has characters that are not + allowed`: a setting in `.env` is wrong. The message names the setting and never shows the value. +- The server exits at once with `unable to open database file`: the data directory is not owned by 65532 (see bind + mounts above). +- `server` is not healthy: `docker compose logs server`. The check is `craft-cloud-server healthcheck`, a + `GET /api/info` request to `127.0.0.1:8080` inside the container. +- All devices share one sign-in budget: `trusted_proxies` does not contain Caddy's address. `CC_CADDY_IP` must lie in + `CC_FRONTEND_SUBNET`; run `docker compose up -d --force-recreate secrets-init` and `docker compose up -d` after changing either. + +### Not covered + +Postgres over TLS, TLS in the server itself (Caddy terminates it), Kubernetes or Helm files, a registry image and +release automation, and backup automation (copy the data as above until the server can create backups itself). diff --git a/packaging/docker/.env.example b/packaging/docker/.env.example new file mode 100644 index 00000000..119b3e63 --- /dev/null +++ b/packaging/docker/.env.example @@ -0,0 +1,44 @@ +# Settings of the Docker deployment. install.sh writes a real `.env` from this list (mode 0600, never committed); +# copy this file to `.env` and fill it in by hand if you do not use install.sh. Nothing in this file is a secret. + +# --- required --------------------------------------------------------------- +# The domain computers reach the server at, and the matching public URL (add :port if HTTPS is not on 443). +CC_DOMAIN=cloud.example.org +CC_PUBLIC_URL=https://cloud.example.org + +# --- TLS --------------------------------------------------------------------- +# acme (Let's Encrypt), internal (Caddy's own CA, for tests) or custom (files in CC_TLS_DIR). +CC_TLS_MODE=acme +CC_ACME_EMAIL= +CC_TLS_DIR=./tls +CC_HTTP_PORT=80 +CC_HTTPS_PORT=443 + +# --- image ------------------------------------------------------------------- +CC_IMAGE=craft-cloud:local +CC_VERSION=0.0.0 +CC_REVISION=unknown + +# --- networks ------------------------------------------------------------------ +# Caddy's fixed address on the internal frontend network. The server believes X-Forwarded-For from this address only +# (trusted_proxies takes single addresses, no networks), so it must lie inside CC_FRONTEND_SUBNET. +CC_FRONTEND_SUBNET=172.29.215.0/24 +CC_CADDY_IP=172.29.215.2 +# Where the other containers get their addresses from: inside CC_FRONTEND_SUBNET, and not containing CC_CADDY_IP. +CC_FRONTEND_POOL=172.29.215.128/25 +CC_EDGE_SUBNET=172.29.216.0/24 + +# --- secrets (the Postgres profile; install.sh fills them with random values) ----------------------------------- +# Only characters A-Z a-z 0-9 . _ ~ - and at least 32 of them: the database password becomes part of a database URL. +CC_SECRET_DB_SUPERUSER_PASSWORD= +CC_SECRET_DB_PASSWORD= + +# --- S3 storage (compose.s3.yaml) ---------------------------------------------------------------------------- +CC_S3_BUCKET= +CC_S3_PREFIX= +CC_S3_REGION=us-east-1 +# Leave empty for Amazon S3; set it for MinIO and other S3-compatible services. +CC_S3_ENDPOINT= +CC_S3_ALLOW_HTTP=false +CC_SECRET_S3_ACCESS_KEY_ID= +CC_SECRET_S3_SECRET_ACCESS_KEY= diff --git a/packaging/docker/.gitattributes b/packaging/docker/.gitattributes new file mode 100644 index 00000000..0c6475c3 --- /dev/null +++ b/packaging/docker/.gitattributes @@ -0,0 +1,2 @@ +# Scripts and configs are mounted into Linux containers and must keep LF line endings. +* text eol=lf diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 00000000..b6a3e0b7 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,63 @@ +# Container image of craft-cloud-server: a Debian trixie build stage and a distroless runtime that holds the binary +# and nothing else (no shell, no package manager). It runs as uid 65532. +# +# The build context is the repository root, because Cargo needs the manifests of all workspace members: +# docker build -f packaging/docker/Dockerfile -t craft-cloud:local . +# docker build -f packaging/docker/Dockerfile --build-arg CRAFT_CLOUD_VERSION=0.1.0 \ +# --build-arg CRAFT_CLOUD_REVISION="$(git rev-parse HEAD)" -t craft-cloud:0.1.0 . +# +# Every base image is pinned by the digest of its multi-arch index; the tag in the comment is for humans. Update +# the tag and the digest together (see "Docker" in docs/server.md). + +# rust:1.95-trixie (egui 0.36 needs Rust 1.95; glibc 2.41 like the runtime image below) +ARG RUST_IMAGE=rust:1.95-trixie@sha256:f49565f188ee00bc2a18dd418183f2c5f23ef7d6e691890517ed341a598f67c3 +# debian:trixie-slim (only used to create the data directory with its owner and mode) +ARG DEBIAN_IMAGE=debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4 +# gcr.io/distroless/cc-debian13:nonroot (glibc, libgcc and CA certificates; no shell, no package manager; uid 65532) +ARG RUNTIME_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:08476c244888bfb425bae1ae4dd7c17f093e972aab8426883d97f3c6f280f7c1 + +# ---- build --------------------------------------------------------------------- +FROM ${RUST_IMAGE} AS build +# Docker builds often run in a small VM: keep the compiler's parallelism and memory use low. +ENV CARGO_BUILD_JOBS=2 \ + CARGO_INCREMENTAL=0 \ + CARGO_PROFILE_RELEASE_STRIP=symbols +WORKDIR /src +COPY Cargo.toml Cargo.lock ./ +# Cargo resolves every workspace member, including the apps and xtask, even when only the server package is built. +# Nothing of them but the server enters the image. +COPY .cargo .cargo +COPY crates crates +COPY apps apps +COPY xtask xtask +RUN cargo build --release --locked -p craft-cloud-server \ + && install -D target/release/craft-cloud-server /out/craft-cloud-server + +# ---- files that need a shell to create --------------------------------------------- +FROM ${DEBIAN_IMAGE} AS files +# The data directory belongs to the runtime user from the start, so a fresh named volume inherits the owner and the +# mode 0700. +RUN install -d -m 0700 -o 65532 -g 65532 /rootfs/var/lib/craft-cloud + +# ---- runtime ----------------------------------------------------------------------- +FROM ${RUNTIME_IMAGE} +ARG CRAFT_CLOUD_VERSION=0.0.0 +ARG CRAFT_CLOUD_REVISION=unknown +LABEL org.opencontainers.image.title="craft-cloud-server" \ + org.opencontainers.image.description="Self-hosted sync and storage server for the Craft App Suite" \ + org.opencontainers.image.source="https://github.com/itrycodinganditsbad/craft-cloud" \ + org.opencontainers.image.licenses="MIT OR Apache-2.0" \ + org.opencontainers.image.version="${CRAFT_CLOUD_VERSION}" \ + org.opencontainers.image.revision="${CRAFT_CLOUD_REVISION}" +COPY --from=files --chown=65532:65532 /rootfs/var/lib/craft-cloud /var/lib/craft-cloud +COPY --from=build /out/craft-cloud-server /usr/local/bin/craft-cloud-server +USER 65532:65532 +EXPOSE 8080 +# The SQLite database and the blob storage (when it is on disk) live here: the only writable place besides /tmp. +VOLUME ["/var/lib/craft-cloud"] +# The image has no shell and no curl: the binary asks its own listener (GET /api/info on 127.0.0.1:8080). +HEALTHCHECK --interval=15s --timeout=5s --start-period=30s --retries=3 \ + CMD ["/usr/local/bin/craft-cloud-server", "healthcheck"] +ENTRYPOINT ["/usr/local/bin/craft-cloud-server"] +# The config file is mounted by Compose; see packaging/docker/compose.yaml. +CMD ["serve", "--config", "/etc/craft-cloud/config.toml"] diff --git a/packaging/docker/caddy/Caddyfile b/packaging/docker/caddy/Caddyfile new file mode 100644 index 00000000..3d522b18 --- /dev/null +++ b/packaging/docker/caddy/Caddyfile @@ -0,0 +1,72 @@ +# Reverse proxy of the Docker deployment: TLS in front of craft-cloud-server, which speaks plain HTTP only. +# +# CC_DOMAIN, CC_PUBLIC_URL, CC_TLS_MODE and CC_ACME_EMAIL come from the Compose environment: +# acme a public certificate from Let's Encrypt (the domain must resolve here, ports 80 and 443 be reachable) +# internal a certificate from Caddy's own CA, for local tests (browsers warn) +# custom your own certificate: fullchain.pem and private.key in the directory CC_TLS_DIR + +{ + # Not exposed: the admin API listens on loopback inside this container only. + admin localhost:2019 + # Caddy cannot install its internal CA into a read-only container and would log an error about it. + skip_install_trust + email "{$CC_ACME_EMAIL}" + + # Caddy logs a failed request (an upstream error, for example) with its full URI, which would hold an invite code + # (/join/) and any query. The log keeps neither the URI nor the headers (Authorization, Cookie). + log { + output stderr + format filter { + wrap console + fields { + request>uri delete + request>headers delete + } + } + } + + servers { + timeouts { + # The server cuts off slow request bodies itself, but asks the proxy to bound the request headers. + read_header 10s + } + } +} + +# Plain HTTP: redirect to the configured public URL, which keeps a port other than 443. An explicit site for the HTTP +# address replaces Caddy's automatic redirect, which would drop the port. ACME HTTP challenges are answered before this. +http://{$CC_DOMAIN} { + redir {$CC_PUBLIC_URL}{uri} 308 +} + +{$CC_DOMAIN} { + import tls-{$CC_TLS_MODE}.caddy + + # Only fills in the generic headers where the server sent none (the leading `?`). HSTS comes from the tls file of + # a real certificate; a test certificate must not pin the browser to HTTPS. + header { + ?X-Content-Type-Options "nosniff" + -Server + } + + # No `request_body { max_size ... }` on purpose: uploads are streamed and the server limits each route itself. + reverse_proxy server:8080 { + # Pass response bytes on at once: downloads stream and nothing is held back. + flush_interval -1 + + # Send requests only while the server answers its info endpoint. + health_uri /api/info + health_status 200 + health_interval 10s + health_timeout 3s + + transport http { + dial_timeout 5s + # The time until the server starts to answer; there is no limit on how long a body takes to transfer. + response_header_timeout 60s + } + } + + # No access log on purpose: a log of full URLs would keep invite codes (/join/). The error log above leaves + # out the URI and the headers. If you add an access log, filter them out the same way. +} diff --git a/packaging/docker/caddy/tls-acme.caddy b/packaging/docker/caddy/tls-acme.caddy new file mode 100644 index 00000000..676383aa --- /dev/null +++ b/packaging/docker/caddy/tls-acme.caddy @@ -0,0 +1,3 @@ +# A certificate from Caddy's default public issuers (Let's Encrypt, ZeroSSL). Nothing to configure. +# HSTS only with a real certificate, so not in the internal mode. +header Strict-Transport-Security "max-age=31536000" diff --git a/packaging/docker/caddy/tls-custom.caddy b/packaging/docker/caddy/tls-custom.caddy new file mode 100644 index 00000000..e26ac61d --- /dev/null +++ b/packaging/docker/caddy/tls-custom.caddy @@ -0,0 +1,4 @@ +# Your own certificate: the directory CC_TLS_DIR holds fullchain.pem and private.key and is mounted read-only at +# /certs. Keep the key readable by root only. Caddy reads the files at start; restart it to load a renewed one. +tls /certs/fullchain.pem /certs/private.key +header Strict-Transport-Security "max-age=31536000" diff --git a/packaging/docker/caddy/tls-internal.caddy b/packaging/docker/caddy/tls-internal.caddy new file mode 100644 index 00000000..06e1f32c --- /dev/null +++ b/packaging/docker/caddy/tls-internal.caddy @@ -0,0 +1,2 @@ +# A certificate from Caddy's internal CA. For tests only: browsers and agents do not trust it. No HSTS. +tls internal diff --git a/packaging/docker/compose.postgres.yaml b/packaging/docker/compose.postgres.yaml new file mode 100644 index 00000000..d21108de --- /dev/null +++ b/packaging/docker/compose.postgres.yaml @@ -0,0 +1,99 @@ +# Overlay for Postgres instead of SQLite: +# docker compose -f compose.yaml -f compose.postgres.yaml up -d --wait +# Adds the `db` container (on the internal `backend` network, which Caddy is not attached to) and `db-init`, which +# creates the server's database role. The data volume still holds the blobs. +# +# The connection is not encrypted: the server's sqlx has no TLS feature. That is acceptable on an internal network of +# one host only. An external Postgres over TLS is not covered. + +x-postgres-image: &postgres-image postgres:17-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b + +services: + # Compose puts a file secret into the container's root filesystem, so this one service cannot be read-only here. + secrets-init: + read_only: false + environment: + DB_KIND: postgres + secrets: + - db_superuser_password + - db_password + + db: + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + image: *postgres-image + environment: + POSTGRES_DB: craft_cloud + POSTGRES_USER: postgres + POSTGRES_PASSWORD_FILE: /run/secrets/db_superuser_password + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums" + volumes: + - pgdata:/var/lib/postgresql/data + # Read by the image's entrypoint while it still runs as root. + - type: volume + source: secrets + target: /run/secrets + read_only: true + volume: {subpath: db} + depends_on: + secrets-init: {condition: service_completed_successfully} + tmpfs: + - /tmp + - /run/postgresql + # The entrypoint starts as root to fix the ownership of the data directory and then drops to the postgres user; it + # needs exactly these capabilities. + cap_add: [CHOWN, FOWNER, DAC_OVERRIDE, SETUID, SETGID] + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "postgres", "-d", "craft_cloud"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + networks: [backend] + mem_limit: 1g + pids_limit: 200 + stop_grace_period: 60s + + # Creates the database role of the server, or refreshes its password. Runs on every `up`; safe to run again. + db-init: + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + image: *postgres-image + # The postgres user of the alpine image. + user: "70:70" + entrypoint: ["/bin/sh", "/db-init.sh"] + environment: + PGHOST: db + PGDATABASE: craft_cloud + volumes: + - ./db-init.sh:/db-init.sh:ro + - type: volume + source: secrets + target: /run/secrets + read_only: true + volume: {subpath: db-init} + depends_on: + db: {condition: service_healthy} + # For the private password file db-init.sh writes. + tmpfs: + - /tmp:size=1m,mode=1777,noexec,nosuid,nodev + healthcheck: {disable: true} + networks: [backend] + mem_limit: 128m + pids_limit: 50 + + server: + depends_on: + db-init: {condition: service_completed_successfully} + networks: [backend] + +volumes: + pgdata: {} + +secrets: + db_superuser_password: {environment: CC_SECRET_DB_SUPERUSER_PASSWORD} + db_password: {environment: CC_SECRET_DB_PASSWORD} diff --git a/packaging/docker/compose.s3.yaml b/packaging/docker/compose.s3.yaml new file mode 100644 index 00000000..dc3561b4 --- /dev/null +++ b/packaging/docker/compose.s3.yaml @@ -0,0 +1,29 @@ +# Overlay for S3-compatible storage instead of the local disk (combine it with compose.postgres.yaml if you like): +# docker compose -f compose.yaml -f compose.s3.yaml up -d --wait +# Set CC_S3_* and CC_SECRET_S3_* in .env. The keys go into config.toml (mode 0400, readable by the server only) like +# the database password; they are not in the server's environment. +# +# The server needs a route to the S3 endpoint, so it also joins the `edge` network, the only one with outbound access. +# That network publishes no port of the server. + +services: + # Compose puts a file secret into the container's root filesystem, so this one service cannot be read-only here. + secrets-init: + read_only: false + environment: + STORAGE_KIND: s3 + S3_BUCKET: ${CC_S3_BUCKET:?set CC_S3_BUCKET in .env} + S3_PREFIX: ${CC_S3_PREFIX:-} + S3_REGION: ${CC_S3_REGION:-us-east-1} + S3_ENDPOINT: ${CC_S3_ENDPOINT:-} + S3_ALLOW_HTTP: ${CC_S3_ALLOW_HTTP:-false} + secrets: + - s3_access_key_id + - s3_secret_access_key + + server: + networks: [edge] + +secrets: + s3_access_key_id: {environment: CC_SECRET_S3_ACCESS_KEY_ID} + s3_secret_access_key: {environment: CC_SECRET_S3_SECRET_ACCESS_KEY} diff --git a/packaging/docker/compose.yaml b/packaging/docker/compose.yaml new file mode 100644 index 00000000..9ffa4982 --- /dev/null +++ b/packaging/docker/compose.yaml @@ -0,0 +1,149 @@ +# Docker Compose deployment of Craft Cloud: the server (SQLite and local disk) behind Caddy, which terminates TLS. +# Postgres and S3 are overlays: -f compose.yaml -f compose.postgres.yaml [-f compose.s3.yaml]. See "Docker" in +# docs/server.md, and run ./install.sh for a first setup. +# +# ./install.sh # writes .env, builds the image, starts the stack +# docker compose up -d --wait # after that +# +# Secrets never appear in this file or in the environment of a service that runs. Compose reads them from the +# gitignored .env and gives them to the one-shot `secrets-init` service only, which writes them as files (mode 0400, +# owned by the reading user) into the `secrets` volume. Every other service mounts just its own subdirectory +# read-only. +# +# Volume subpaths (used below) need Docker Compose 2.32 or newer. + +name: craft-cloud + +x-hardening: &hardening + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + +services: + # Renders config.toml and distributes the secrets (see secrets-init.sh). It has no network at all, runs as root for + # about a second and needs only the capabilities to change the owner and mode of the files it writes. + secrets-init: + <<: *hardening + # Any image with busybox will do; the one of Caddy is needed anyway. + image: caddy:2.11-alpine@sha256:e70b164da81ef698b3ad19af799ea6086ce6d9e48ed4079eb9fe26f6fbf14ee5 + user: "0:0" + entrypoint: ["/bin/sh", "/secrets-init.sh"] + network_mode: none + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER] + # Not secrets: what render-config.sh puts into config.toml. + environment: + PUBLIC_URL: ${CC_PUBLIC_URL:?set CC_PUBLIC_URL in .env, or run ./install.sh} + TRUSTED_PROXY: ${CC_CADDY_IP:-172.29.215.2} + DB_KIND: sqlite + STORAGE_KIND: disk + volumes: + - ./secrets-init.sh:/secrets-init.sh:ro + - ./render-config.sh:/render-config.sh:ro + - ./config.toml.in:/config.toml.in:ro + - secrets:/out + healthcheck: {disable: true} + mem_limit: 64m + pids_limit: 50 + + server: + <<: *hardening + restart: unless-stopped + image: ${CC_IMAGE:-craft-cloud:local} + build: + context: ../.. + dockerfile: packaging/docker/Dockerfile + args: + CRAFT_CLOUD_VERSION: ${CC_VERSION:-0.0.0} + CRAFT_CLOUD_REVISION: ${CC_REVISION:-unknown} + user: "65532:65532" + command: ["serve", "--config", "/etc/craft-cloud/config.toml"] + volumes: + # The SQLite database and the blobs. A new named volume inherits the owner (65532) and mode (0700) of the image's + # directory. A bind mount must be given both first: chown 65532:65532 DIR && chmod 0700 DIR. + - data:/var/lib/craft-cloud + - type: volume + source: secrets + target: /etc/craft-cloud + read_only: true + volume: {subpath: server} + tmpfs: + - /tmp:size=16m,mode=1777,noexec,nosuid,nodev + depends_on: + secrets-init: {condition: service_completed_successfully} + # The image has no shell and no curl: the binary asks its own listener. + healthcheck: + test: ["CMD", "/usr/local/bin/craft-cloud-server", "healthcheck", "--addr", "127.0.0.1:8080"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 30s + networks: [frontend] + mem_limit: 1g + pids_limit: 512 + stop_grace_period: 30s + + caddy: + <<: *hardening + restart: unless-stopped + image: caddy:2.11-alpine@sha256:e70b164da81ef698b3ad19af799ea6086ce6d9e48ed4079eb9fe26f6fbf14ee5 + ports: + - "${CC_HTTP_PORT:-80}:80" + - "${CC_HTTPS_PORT:-443}:443" + - "${CC_HTTPS_PORT:-443}:443/udp" + environment: + CC_DOMAIN: ${CC_DOMAIN:?set CC_DOMAIN in .env, or run ./install.sh} + CC_TLS_MODE: ${CC_TLS_MODE:-acme} + CC_PUBLIC_URL: ${CC_PUBLIC_URL:?set CC_PUBLIC_URL in .env, or run ./install.sh} + CC_ACME_EMAIL: ${CC_ACME_EMAIL:-} + volumes: + - ./caddy:/etc/caddy:ro + - caddy_data:/data + - caddy_config:/config + # Only read in the `custom` TLS mode. + - ${CC_TLS_DIR:-./tls}:/certs:ro + tmpfs: + - /tmp + # Runs as root inside the container (the official image does), with no capability except binding ports 80 and 443. + cap_add: [NET_BIND_SERVICE] + depends_on: + server: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "http://127.0.0.1:2019/config/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + edge: {} + # The fixed address the server lists in trusted_proxies. + frontend: + ipv4_address: ${CC_CADDY_IP:-172.29.215.2} + mem_limit: 256m + pids_limit: 200 + +networks: + # Published ports and outbound access (ACME); only Caddy is attached. + edge: + ipam: + config: + - subnet: ${CC_EDGE_SUBNET:-172.29.216.0/24} + # Caddy to the server. `internal`: no route to the outside world. The other containers get their addresses from the + # upper half, so none of them can take Caddy's fixed address (CC_CADDY_IP) first. + frontend: + internal: true + ipam: + config: + - subnet: ${CC_FRONTEND_SUBNET:-172.29.215.0/24} + ip_range: ${CC_FRONTEND_POOL:-172.29.215.128/25} + # Server and database in the Postgres profile. `internal`, and Caddy is not attached. + backend: + internal: true + +volumes: + # The SQLite database and, with disk storage, the blobs: the data to back up. + data: {} + # config.toml and the secret files written by `secrets-init`, one directory per consuming service. + secrets: {} + caddy_data: {} + caddy_config: {} diff --git a/packaging/docker/config.toml.in b/packaging/docker/config.toml.in new file mode 100644 index 00000000..d5b684ea --- /dev/null +++ b/packaging/docker/config.toml.in @@ -0,0 +1,33 @@ +# Template of the server's config.toml in the Docker deployment. render-config.sh fills the placeholders (a name +# between two at signs) and keeps the lines whose marker prefix is active: the database kind, the storage kind, and +# for S3 the optional endpoint and plain-HTTP lines. Every other marker line is dropped. The values are checked +# before they are written. +# The rendered file is written by the secrets-init service into the secrets volume (mode 0400, owner 65532); it can +# hold the database password and S3 keys, so nobody but the server can read it. + +# The server speaks plain HTTP inside the stack. Caddy terminates TLS and has the fixed address in trusted_proxies, +# whose X-Forwarded-For the server believes (it takes single addresses, no networks). +listen = "0.0.0.0:8080" +behind_proxy = true +trusted_proxies = ["@TRUSTED_PROXY@"] +public_url = "@PUBLIC_URL@" + +# Optional limits, in bytes. See docs/server.md. +# stored_bytes_per_user = 2_000_000_000_000 +# live_bytes_per_user = 1_073_741_824 + +[database] +#sqlite#url = "sqlite:/var/lib/craft-cloud/meta.db" +#postgres#url = "postgres://craft_cloud:@DB_PASSWORD@@db:5432/craft_cloud" + +[storage] +#disk#kind = "disk" +#disk#path = "/var/lib/craft-cloud/blobs" +#s3#kind = "s3" +#s3#bucket = "@S3_BUCKET@" +#s3#prefix = "@S3_PREFIX@" +#s3#region = "@S3_REGION@" +#s3#access_key_id = "@S3_ACCESS_KEY_ID@" +#s3#secret_access_key = "@S3_SECRET_ACCESS_KEY@" +#s3endpoint#endpoint = "@S3_ENDPOINT@" +#s3http#allow_http = true diff --git a/packaging/docker/db-init.sh b/packaging/docker/db-init.sh new file mode 100755 index 00000000..49de4d13 --- /dev/null +++ b/packaging/docker/db-init.sh @@ -0,0 +1,52 @@ +#!/bin/sh +case $- in *x*) set +x ;; esac +# One-shot role setup for the Postgres profile (service `db-init`). It runs as the postgres user of the Postgres +# image, reads the passwords from secret files and never prints them. Safe to run again: every run applies the +# password and the role's attributes, which is also how the password is rotated. +# +# The server migrates its database when it opens it, so its role owns the database and the schema `public` and may +# change them. That is less separation than an owner role plus a runtime role would give. The role is not a +# superuser and has no other privileges, and nobody else may connect to its database or to the other databases. +set -eu +umask 077 + +# The superuser password goes into a private password file (mode 0600, on the tmpfs at /tmp, removed on exit), so it +# is neither in a process's environment nor on a command line. The password only holds the characters +# A-Za-z0-9._~-, which need no escaping in this file format. +passfile=$(mktemp /tmp/pgpass.XXXXXX) +trap 'rm -f "$passfile"' EXIT INT TERM +printf '*:*:*:postgres:%s\n' "$(cat /run/secrets/db_superuser_password)" >"$passfile" +PGUSER=postgres +PGPASSFILE=$passfile +export PGUSER PGPASSFILE + +# POSTGRES_PASSWORD_FILE only applies when the data directory is first created, so a changed superuser password in +# .env cannot log in here. +if ! psql --no-psqlrc --quiet --command 'SELECT 1' >/dev/null 2>&1; then + cat >&2 <<'MSG' +db-init: cannot log in as the Postgres superuser "postgres" (or the database is unreachable). +If CC_SECRET_DB_SUPERUSER_PASSWORD in .env was changed after the first start, the database still has the old +password: the image applies it only when the data directory is created. Put the old value back in .env. +MSG + exit 1 +fi + +# psql reads the password itself (backticks), so it is not on a command line. +psql --no-psqlrc --set ON_ERROR_STOP=1 --quiet <<'SQL' +\set db_password `cat /run/secrets/db_password` + +SELECT 'CREATE ROLE craft_cloud LOGIN' + WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'craft_cloud')\gexec + +ALTER ROLE craft_cloud LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'db_password'; + +ALTER DATABASE craft_cloud OWNER TO craft_cloud; +ALTER SCHEMA public OWNER TO craft_cloud; +REVOKE ALL ON DATABASE craft_cloud FROM PUBLIC; +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +-- The role must not be able to connect to the maintenance databases either. +REVOKE CONNECT ON DATABASE postgres FROM PUBLIC; +REVOKE CONNECT ON DATABASE template1 FROM PUBLIC; +SQL + +echo "db-init: the database role craft_cloud is ready" diff --git a/packaging/docker/install.sh b/packaging/docker/install.sh new file mode 100755 index 00000000..98ad74e2 --- /dev/null +++ b/packaging/docker/install.sh @@ -0,0 +1,231 @@ +#!/bin/sh +# Sets up the Docker deployment of Craft Cloud in this directory: checks the requirements, writes `.env` (mode 0600) +# with random database passwords, builds the image, starts the stack and, if asked, creates the first admin. +# +# ./install.sh [--domain cloud.example.org] [--tls acme|internal|custom] [--email you@example.org] +# [--db sqlite|postgres] [--http-port 80] [--https-port 443] [--admin name | --no-admin] [--no-start] +# +# Missing values are asked for when a terminal is attached; without one, --domain is required. The admin's password +# is read from standard input without echo (or from a pipe: printf '%s\n' "$PASSWORD" | ./install.sh ... --admin you) +# and goes to the server over a pipe: never into an argument, the environment or a log. +# +# It never overwrites an existing `.env`. It runs from a clone of the repository or an unpacked release archive; it +# downloads nothing itself (Docker pulls the pinned base images). +set -eu +umask 077 + +cd "$(dirname "$0")" + +die() { + echo "install.sh: $*" >&2 + exit 1 +} + +domain= +tls_mode= +acme_email= +db_kind= +http_port= +https_port= +admin= +no_admin=0 +no_start=0 +interactive=0 +[ -t 0 ] && interactive=1 + +while [ $# -gt 0 ]; do + case $1 in + --domain | --tls | --email | --db | --http-port | --https-port | --admin) + [ $# -ge 2 ] || die "$1 needs a value" + case $1 in + --domain) domain=$2 ;; + --tls) tls_mode=$2 ;; + --email) acme_email=$2 ;; + --db) db_kind=$2 ;; + --http-port) http_port=$2 ;; + --https-port) https_port=$2 ;; + --admin) admin=$2 ;; + esac + shift 2 + ;; + --no-admin) + no_admin=1 + shift + ;; + --no-start) + no_start=1 + shift + ;; + -h | --help) + sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) die "unknown option $1 (see --help)" ;; + esac +done + +# --- requirements --------------------------------------------------------------------------------------------------- +command -v docker >/dev/null 2>&1 || die "docker is not installed" +compose_version=$(docker compose version --short 2>/dev/null) || die "docker compose (version 2) is not available" +# Volume subpaths, which the secret distribution relies on, need Compose 2.32. +major=${compose_version%%.*} +rest=${compose_version#*.} +minor=${rest%%.*} +case "$major$minor" in + *[!0-9]*) die "can't read the Docker Compose version '$compose_version'" ;; +esac +if [ "$major" -lt 2 ] || { [ "$major" -eq 2 ] && [ "$minor" -lt 32 ]; }; then + die "Docker Compose 2.32 or newer is needed (found $compose_version)" +fi +[ ! -e .env ] || die ".env exists already; edit it, or move it away to set up again" + +free_kib=$(df -Pk . | awk 'NR == 2 { print $4 }') +case $free_kib in + "" | *[!0-9]*) ;; + *) [ "$free_kib" -ge 4194304 ] || die "less than 4 GiB of free disk space here; the image build needs room" ;; +esac + +# --- settings ------------------------------------------------------------------------------------------------------- +ask() { + # ask + if [ "$interactive" -eq 1 ]; then + printf '%s [%s]: ' "$2" "$3" + IFS= read -r answer || answer= + eval "$1=\${answer:-\$3}" + else + eval "$1=\$3" + fi +} + +if [ -z "$domain" ]; then + [ "$interactive" -eq 1 ] || die "--domain is required without a terminal" + printf 'Domain the server is reached at (for example cloud.example.org): ' + IFS= read -r domain || domain= +fi +[ -n "$tls_mode" ] || ask tls_mode "TLS (acme, internal, custom)" acme +[ -n "$db_kind" ] || ask db_kind "Database (sqlite, postgres)" sqlite +[ -n "$http_port" ] || ask http_port "HTTP port" 80 +[ -n "$https_port" ] || ask https_port "HTTPS port" 443 +if [ "$tls_mode" = acme ] && [ -z "$acme_email" ] && [ "$interactive" -eq 1 ]; then + printf 'Email for Let'\''s Encrypt notices (optional): ' + IFS= read -r acme_email || acme_email= +fi + +printf '%s' "$domain" | grep -Eq '^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$' || die "the domain has characters that are not allowed" +case $tls_mode in acme | internal | custom) ;; *) die "--tls must be acme, internal or custom" ;; esac +case $db_kind in sqlite | postgres) ;; *) die "--db must be sqlite or postgres" ;; esac +for port in "$http_port" "$https_port"; do + case $port in "" | *[!0-9]* | 0*) die "ports are numbers from 1 to 65535" ;; esac + [ "$port" -le 65535 ] || die "ports are numbers from 1 to 65535" +done +if [ -n "$acme_email" ]; then + printf '%s' "$acme_email" | grep -Eq '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$' || die "the email address has characters that are not allowed" +fi +if [ -n "$admin" ]; then + printf '%s' "$admin" | grep -Eq '^[A-Za-z0-9._-]{1,64}$' || die "usernames are 1-64 letters, digits, '.', '_' or '-'" +fi + +if command -v ss >/dev/null 2>&1; then + for port in "$http_port" "$https_port"; do + if ss -ltn 2>/dev/null | awk '{ print $4 }' | grep -Eq "[:.]$port\$"; then + die "port $port is already in use; pick another with --http-port / --https-port" + fi + done +fi + +public_url="https://$domain" +[ "$https_port" = 443 ] || public_url="$public_url:$https_port" + +version=$(sh ./workspace-version.sh) +revision=$(git rev-parse HEAD 2>/dev/null || true) +[ -n "$revision" ] || revision=unknown + +# --- .env --------------------------------------------------------------------------------------------------------- +# A random password of 40 URL-safe characters (the database password becomes part of a database URL). +random_password() { + value=$(LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null | head -c 40 || true) + [ "${#value}" -eq 40 ] || die "can't read random bytes from /dev/urandom" + printf '%s' "$value" +} + +compose_file=compose.yaml +[ "$db_kind" != postgres ] || compose_file=compose.yaml:compose.postgres.yaml + +{ + echo "# Written by install.sh. Holds the database passwords: keep it private (mode 0600) and out of version control." + echo "COMPOSE_PATH_SEPARATOR=:" + echo "COMPOSE_FILE=$compose_file" + echo "CC_DOMAIN=$domain" + echo "CC_PUBLIC_URL=$public_url" + echo "CC_TLS_MODE=$tls_mode" + echo "CC_ACME_EMAIL=$acme_email" + echo "CC_HTTP_PORT=$http_port" + echo "CC_HTTPS_PORT=$https_port" + echo "CC_IMAGE=craft-cloud:$version" + echo "CC_VERSION=$version" + echo "CC_REVISION=$revision" + if [ "$db_kind" = postgres ]; then + echo "CC_SECRET_DB_SUPERUSER_PASSWORD=$(random_password)" + echo "CC_SECRET_DB_PASSWORD=$(random_password)" + fi +} >.env.new +chmod 600 .env.new +mv .env.new .env +echo "Wrote .env (mode 0600)." + +if [ "$tls_mode" = custom ]; then + mkdir -p tls + chmod 700 tls + echo "Put fullchain.pem and private.key in $(pwd)/tls before the stack starts, then run: docker compose up -d --wait" + no_start=1 +fi + +[ "$no_start" -eq 0 ] || { + echo "Not started. Next: docker compose build && docker compose up -d --wait" + exit 0 +} + +# --- build and start ---------------------------------------------------------------------------------------------- +echo "Building the image craft-cloud:$version (this takes a while the first time)..." +docker compose build server +echo "Starting the stack..." +docker compose up -d --wait + +# --- first admin ---------------------------------------------------------------------------------------------------- +if [ "$no_admin" -eq 0 ] && [ -z "$admin" ] && [ "$interactive" -eq 1 ]; then + printf 'Name of the first admin (empty to skip): ' + IFS= read -r admin || admin= + printf '%s' "$admin" | grep -Eq '^([A-Za-z0-9._-]{1,64})?$' || die "usernames are 1-64 letters, digits, '.', '_' or '-'" +fi + +if [ "$no_admin" -eq 0 ] && [ -n "$admin" ]; then + password= + if [ "$interactive" -eq 1 ]; then + trap 'stty echo 2>/dev/null || true' EXIT INT TERM + stty -echo + printf 'Password for %s: ' "$admin" + IFS= read -r password || password= + printf '\nRepeat the password: ' + IFS= read -r again || again= + stty echo + printf '\n' + [ "$password" = "$again" ] || die "the passwords differ; create the admin later (see docs/server.md)" + unset again + else + IFS= read -r password || password= + fi + [ -n "$password" ] || die "no password given; create the admin later (see docs/server.md)" + # The password travels through a pipe into the container's standard input. A refusal (a password the server's rules + # do not accept) is shown by the server without the password. + if printf '%s\n' "$password" | docker compose run --rm --no-deps -T server user --config /etc/craft-cloud/config.toml add --admin "$admin"; then + unset password + else + unset password + die "the admin was not created; see the message above, then run it by hand (see docs/server.md)" + fi +fi + +echo +echo "Craft Cloud runs at $public_url" +[ "$tls_mode" != internal ] || echo "TLS mode 'internal': the certificate is from Caddy's own CA; clients must be told to trust it (curl -k for a test)." +echo "Next: docker compose ps, and docker compose logs -f server" diff --git a/packaging/docker/render-config.sh b/packaging/docker/render-config.sh new file mode 100755 index 00000000..3231df7c --- /dev/null +++ b/packaging/docker/render-config.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# Renders the server's config.toml from config.toml.in and writes it to standard output. +# +# render-config.sh