From 29befa33fbc23c66e4b5a7245833ae6fd1874882 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 13:26:11 +0100 Subject: [PATCH 01/27] proto: add the history and restore bodies Adds the request and answer types of GET /nodes/{id}/history and POST /nodes/{id}/restore, the opaque history cursor with a single accepted spelling, and the paging limits (default 50, at most 200). --- crates/proto/src/api.rs | 115 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 1 deletion(-) diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 6ff15c5d..129cc97b 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::hash::ChunkId; -use crate::ids::{DeviceId, Epoch, NodeId, SpaceId}; +use crate::ids::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; use crate::ops::{Conflict, Op}; /// What apps see of these bodies lives in `craft-cloud-api` (no postcard), re-exported here. @@ -145,6 +145,93 @@ pub struct SaveAttach { pub save: crate::SaveRecord, } +/// Versions in one `GET /api/v1/nodes/{id}/history` page when `limit` is not given. +pub const HISTORY_DEFAULT_LIMIT: u32 = 50; +/// Most versions in one history page; a larger `limit` is `invalid`. +pub const HISTORY_MAX_LIMIT: u32 = 200; + +/// Where the next history page starts: after this `(seq, version)` in the newest-first order. On the wire it is an +/// opaque string; clients pass back the `next` they were given and nothing else. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HistoryCursor { + pub seq: u64, + pub version: VersionId, +} + +impl std::fmt::Display for HistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.seq, self.version) + } +} + +/// A history cursor that does not read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BadHistoryCursor; + +impl std::fmt::Display for BadHistoryCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("not a history cursor") + } +} + +impl std::error::Error for BadHistoryCursor {} + +impl std::str::FromStr for HistoryCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted, so one cursor has one spelling. + fn from_str(s: &str) -> Result { + let (seq, version) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = HistoryCursor { seq: seq.parse().map_err(|_| BadHistoryCursor)?, version: version.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One version of a file in its history. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryVersion { + pub id: VersionId, + pub parent_version: Option, + pub epoch: Epoch, + /// The space's change sequence number of the commit that made this version. + pub seq: u64, + pub size: u64, + pub mtime_ms: i64, + /// The device that committed it. + pub device: DeviceId, + pub created_ms: i64, + /// Whether a save record (a Craft app's live-document save) hangs on this version. + pub save: bool, + /// Thinned out of the history: its row stays, but it can't be restored. Always `false` until thinning exists. + pub pruned: bool, +} + +/// `GET /api/v1/nodes/{id}/history?cursor&limit`: the versions of a file, newest first by `(seq, id)`. `next` is the +/// `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryResponse { + pub versions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/nodes/{id}/restore`: make `version` the file's content again as a new version. `expected_version` +/// must still be the file's current version, else the answer is `conflict`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreRequest { + pub version: VersionId, + pub expected_version: VersionId, +} + +/// A restore that applied: the new version, which follows `parent_version` (the one that was current). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestoreResponse { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + /// A commit that applied. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitResponse { @@ -237,6 +324,32 @@ mod tests { assert_eq!(body.error, "something_new"); } + #[test] + fn history_cursors_have_one_spelling() { + let cursor = HistoryCursor { seq: 7, version: VersionId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in + ["", "7", "7.", ".x", "x.y", "+7.00000000-0000-0000-0000-000000000000", &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] + { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + assert_eq!("-1.00000000-0000-0000-0000-000000000000".parse::(), Err(BadHistoryCursor)); + } + + #[test] + fn history_and_restore_bodies_have_a_plain_json_form() { + let restore: RestoreRequest = serde_json::from_value(serde_json::json!({ + "version": "01a12b0b-9fca-73db-b82d-c7c33683601e", + "expected_version": "01a12b0b-9fca-73db-b82d-c7c33683601f", + "unknown": 1 + })) + .unwrap(); + assert_ne!(restore.version, restore.expected_version); + let last = HistoryResponse { versions: vec![], next: None }; + assert_eq!(serde_json::to_value(&last).unwrap(), serde_json::json!({ "versions": [] })); + } + #[test] fn error_codes_are_stable() { assert_eq!(codes::MISSING_CHUNKS, "missing_chunks"); From 7c7096c62e91bce0c5f86d54d536228633a51403 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 13:26:11 +0100 Subject: [PATCH 02/27] meta: add version history and restore of an old version history() pages a file's versions newest first by (seq, id) with an exclusive cursor. restore_version() records a new version with the content of an old one as an ordinary update, checking the source, the current version, deletion and live documents under the same space lock as a commit. No migration is needed. --- crates/meta/src/commit.rs | 78 ++++++++++- crates/meta/src/history.rs | 80 +++++++++++ crates/meta/src/lib.rs | 4 +- crates/meta/src/tree.rs | 2 +- crates/meta/tests/behaviour.rs | 241 +++++++++++++++++++++++++++++++++ 5 files changed, 401 insertions(+), 4 deletions(-) create mode 100644 crates/meta/src/history.rs diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index bd8911b9..d74bf856 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -14,11 +14,11 @@ use std::collections::HashMap; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; -use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; +use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, ManifestId, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; use sqlx::Row; use crate::blobs::manifest_file_size; -use crate::codec::{flag, to_i64, uuid_bytes, uuid_from}; +use crate::codec::{flag, hash_from, to_i64, uuid_bytes, uuid_from}; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; @@ -29,6 +29,16 @@ pub struct CommitOutcome { pub epoch: Epoch, } +/// A successful restore of an old version: the version it made, the one that was current, and the space's new +/// sequence number and epoch (unchanged by a restore). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RestoreOutcome { + pub version: VersionId, + pub parent_version: VersionId, + pub seq: u64, + pub epoch: Epoch, +} + struct NodeRow { parent: Option, name: String, @@ -417,6 +427,70 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } + /// Makes `source`, an older version of the file `node`, its content again as a new version: an ordinary + /// `Update` after `expected`, which must still be the current version. The new version has a fresh ID, copies the + /// manifest, modification time and executable bit of `source`, and gets no save record (a save record names a + /// state of a live document, and a restore changes the file only). The epoch stays. Everything is checked under the + /// space's write lock, in the transaction that writes the version. + /// + /// `NotFound` for a node or source version the space does not have (a source of another file included) or whose + /// manifest the space no longer holds. `Invalid` for a directory, a deleted file, or a file that has a live + /// document. `Rejected` (a `VersionMismatch` for operation 0) when `expected` is no longer current. + pub async fn restore_version( + &self, + space: SpaceId, + device: DeviceId, + node: NodeId, + source: VersionId, + expected: VersionId, + now_ms: i64, + ) -> Result { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the restore with every other commit and save of the space. + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let row = ctx.node(node).await?.ok_or(MetaError::NotFound("node"))?; + if row.is_dir { + return Err(MetaError::Invalid("only files have versions to restore")); + } + if row.deleted { + return Err(MetaError::Invalid("the file is deleted: bring it back from the trash first, then restore a version")); + } + let live = sqlx::query("SELECT 1 FROM live_docs WHERE space_id = $1 AND node = $2 LIMIT 1") + .bind(ctx.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await?; + if live.is_some() { + return Err(MetaError::Invalid("this file has a live document: restoring a version applies to plain file versions only")); + } + let found = sqlx::query("SELECT manifest, mtime_ms, exec FROM versions WHERE space_id = $1 AND id = $2 AND node = $3") + .bind(ctx.space()) + .bind(uuid_bytes(source.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *ctx.conn) + .await? + .ok_or(MetaError::NotFound("version"))?; + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&found.try_get::, _>("manifest")?)?), + mtime_ms: found.try_get("mtime_ms")?, + exec: flag(&found, "exec")?, + save: None, + }; + let version = new.id; + match ctx.apply(&Op::Update { node, expected, version: new }).await? { + Ok(()) => {} + // The space holds no manifest of that ID any more: the data is not there to restore. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(MetaError::Rejected(vec![Conflict { op: 0, kind }])), + } + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(RestoreOutcome { version, parent_version: expected, seq, epoch: info.epoch }) + } + /// Adds a save record to `node`'s version `version`, which went up without one (the app's record reached the /// agent only after the upload: it was saved while the agent was away). Appended only: a version that has a /// record keeps it, and the same call again changes nothing. When it is added, the node is in the change feed diff --git a/crates/meta/src/history.rs b/crates/meta/src/history.rs new file mode 100644 index 00000000..060cfacd --- /dev/null +++ b/crates/meta/src/history.rs @@ -0,0 +1,80 @@ +//! A file's version history: the versions of one file, newest first, a page at a time. +//! +//! Versions are never removed, so every row of `versions` for the node is reported. The `pruned` flag of the answer is +//! part of the contract for the day versions can be thinned; until then it is always false. + +use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; +use craft_cloud_proto::{DeviceId, Epoch, NodeId, SpaceId, VersionId}; +use sqlx::Row; + +use crate::codec::{flag, to_i64, to_u64, uuid_bytes, uuid_from}; +use crate::{Meta, MetaError}; + +/// One page of a file's history. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HistoryPage { + pub versions: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, +} + +impl Meta { + /// One page of the versions of the file `node` in `space`, ordered by `(seq, id)` descending. `after` is the + /// previous page's `next` (exclusive). `NotFound` when the space has no such node; `Invalid` for a directory or + /// a `limit` outside 1 to [`HISTORY_MAX_LIMIT`]. A deleted file still has a history. Each call reads one snapshot. + pub async fn history(&self, space: SpaceId, node: NodeId, after: Option, limit: u32) -> Result { + if limit == 0 || limit > HISTORY_MAX_LIMIT { + return Err(MetaError::Invalid("a history page has 1 to 200 versions")); + } + let mut tx = self.read_snapshot().await?; + let found = sqlx::query("SELECT is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *tx) + .await? + .ok_or(MetaError::NotFound("node"))?; + if flag(&found, "is_dir")? { + return Err(MetaError::Invalid("only files have a history")); + } + let (cursor_seq, cursor_version) = match after { + Some(cursor) => (to_i64(cursor.seq)?, uuid_bytes(cursor.version.as_uuid())), + // Above every sequence number and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The cursor is exclusive. + let rows = sqlx::query( + "SELECT id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, \ + CAST(CASE WHEN save IS NULL THEN 0 ELSE 1 END AS BIGINT) AS has_save \ + FROM versions WHERE space_id = $1 AND node = $2 AND (seq < $3 OR (seq = $3 AND id < $4)) \ + ORDER BY seq DESC, id DESC LIMIT $5", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(cursor_seq) + .bind(cursor_version) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut versions = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent_version")?; + versions.push(HistoryVersion { + id: VersionId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent_version: parent.as_deref().map(uuid_from).transpose()?.map(VersionId::from_uuid), + epoch: Epoch::from_uuid(uuid_from(&row.try_get::, _>("epoch")?)?), + seq: to_u64(row.try_get("seq")?)?, + size: to_u64(row.try_get("size")?)?, + mtime_ms: row.try_get("mtime_ms")?, + device: DeviceId::from_uuid(uuid_from(&row.try_get::, _>("device")?)?), + created_ms: row.try_get("created_ms")?, + save: flag(row, "has_save")?, + pruned: false, + }); + } + let next = if more { versions.last().map(|v| HistoryCursor { seq: v.seq, version: v.id }) } else { None }; + Ok(HistoryPage { versions, next }) + } +} diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 1784df29..93777f9e 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -10,6 +10,7 @@ mod blobs; mod codec; mod commit; +mod history; mod join; mod live; mod schema; @@ -24,8 +25,9 @@ use sqlx::any::{AnyConnectOptions, AnyPoolOptions}; use sqlx::{AnyPool, ConnectOptions}; use tokio::sync::{Mutex, OwnedMutexGuard}; -pub use commit::CommitOutcome; +pub use commit::{CommitOutcome, RestoreOutcome}; pub use craft_cloud_proto::BlobKind; +pub use history::HistoryPage; pub use join::{InviteFor, Redeemed}; pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, diff --git a/crates/meta/src/tree.rs b/crates/meta/src/tree.rs index c5b30d05..47f0f9a6 100644 --- a/crates/meta/src/tree.rs +++ b/crates/meta/src/tree.rs @@ -77,7 +77,7 @@ fn node_state(row: &AnyRow) -> Result { impl Meta { /// Starts a read transaction that sees one consistent snapshot. On Postgres the isolation level has to be /// set as the transaction's first statement. SQLite read transactions are already snapshots. - async fn read_snapshot(&self) -> Result, MetaError> { + pub(crate) async fn read_snapshot(&self) -> Result, MetaError> { let mut tx = self.pool.begin().await?; if self.dialect == crate::Dialect::Postgres { sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY").execute(&mut *tx).await?; diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index d875fc30..feafc49a 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -104,6 +104,11 @@ on_every_backend!( ids_are_never_reused, the_change_feed_and_epochs, concurrent_commits_on_one_file_let_exactly_one_win, + history_pages_newest_first_for_own_files_only, + a_restore_is_a_new_version_after_the_head, + a_restore_refuses_what_it_can_not_restore, + concurrent_restores_let_exactly_one_win, + a_restore_and_a_late_save_record_both_apply, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -1809,3 +1814,239 @@ async fn concurrent_opens_migrate_once_on_postgres() { let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); assert_eq!(versions, 6); } + +/// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. +async fn file_with_versions(meta: &Meta, u: &UserRecord, dev: DeviceId, n: usize) -> (NodeId, Vec) { + let r = root(meta, u).await; + let m = manifest(meta, u, 3).await; + let node = NodeId::generate(); + let mut versions = vec![version(m)]; + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name("history.txt"), version: versions[0].clone() }], NOW).await.unwrap(); + for i in 1..n { + let m = manifest(meta, u, 3 + i as u32).await; + let next = NewVersion { mtime_ms: NOW + i as i64, ..version(m) }; + let expected = versions[i - 1].id; + meta.commit(u.space, dev, &[Op::Update { node, expected, version: next.clone() }], NOW + i as i64).await.unwrap(); + versions.push(next); + } + (node, versions) +} + +async fn file_named(meta: &Meta, u: &UserRecord, dev: DeviceId, file_name: &str) -> (NodeId, NewVersion) { + let r = root(meta, u).await; + let node = NodeId::generate(); + let v = version(manifest(meta, u, 5).await); + meta.commit(u.space, dev, &[Op::Create { node, parent: r, name: name(file_name), version: v.clone() }], NOW).await.unwrap(); + (node, v) +} + +fn refused_restore(result: Result) -> Vec<(usize, ConflictKind)> { + match result { + Err(MetaError::Rejected(list)) => list.into_iter().map(|c| (c.op, c.kind)).collect(), + other => panic!("expected a refused restore, got {other:?}"), + } +} + +/// The history comes newest first in pages that follow each other without gaps or repeats, also across versions that +/// share one sequence number (a batch of updates); only files of the caller's own space have one; a deleted file keeps +/// its history; `pruned` is false and `save` tells whether a record hangs on the version. +async fn history_pages_newest_first_for_own_files_only(meta: &Meta) { + use craft_cloud_proto::api::{HISTORY_MAX_LIMIT, HistoryCursor, HistoryVersion}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, mut versions) = file_with_versions(meta, &u, dev, 4).await; + // Three more versions in one commit: they share a sequence number, so the order inside it is by ID. + let (mut ops, mut prev) = (Vec::new(), versions[3].id); + for i in 0..3 { + let next = version(manifest(meta, &u, 20 + i).await); + ops.push(Op::Update { node, expected: prev, version: next.clone() }); + prev = next.id; + versions.push(next); + } + meta.commit(u.space, dev, &ops, NOW + 100).await.unwrap(); + let head = prev; + let epoch = meta.space_info(u.space).await.unwrap().epoch; + + let all = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap(); + assert_eq!(all.next, None); + assert_eq!(all.versions.len(), 7); + let key = |v: &HistoryVersion| (v.seq, *v.id.as_uuid().as_bytes()); + assert!(all.versions.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (seq, id)"); + assert!(all.versions.iter().all(|v| !v.pruned && !v.save && v.device == dev && v.epoch == epoch)); + assert_eq!(all.versions.iter().filter(|v| v.seq == 5).count(), 3, "the batch shares one sequence number"); + let oldest = all.versions.last().unwrap(); + assert_eq!((oldest.id, oldest.parent_version, oldest.seq, oldest.size), (versions[0].id, None, 1, 3)); + let second = &all.versions[all.versions.len() - 2]; + assert_eq!((second.id, second.parent_version, second.mtime_ms, second.size), (versions[1].id, Some(versions[0].id), NOW + 1, 4)); + + for limit in [1u32, 2, 3] { + let (mut seen, mut after, mut pages) = (Vec::new(), None, 0); + loop { + let page = meta.history(u.space, node, after, limit).await.unwrap(); + assert!(page.versions.len() <= limit as usize); + seen.extend(page.versions.iter().cloned()); + pages += 1; + match page.next { + Some(next) => { + let last = page.versions.last().unwrap(); + assert_eq!(next, HistoryCursor { seq: last.seq, version: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.versions, "limit {limit}: every version once, in order"); + assert_eq!(pages, 7usize.div_ceil(limit as usize)); + } + + // The limit is 1 to 200; a cursor past the end gives an empty last page. + assert!(matches!(meta.history(u.space, node, None, 0).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT + 1).await, Err(MetaError::Invalid(_)))); + let past = meta.history(u.space, node, Some(HistoryCursor { seq: 0, version: VersionId::generate() }), 10).await.unwrap(); + assert_eq!((past.versions.len(), past.next), (0, None)); + + // A record hanging on a version shows as `save`. + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + let with_save: Vec<_> = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.into_iter().filter(|v| v.save).collect(); + assert_eq!(with_save.iter().map(|v| v.id).collect::>(), vec![versions[1].id]); + + // Folders have no history, another account's node is not found, and a deleted file keeps its history. + let r = root(meta, &u).await; + assert!(matches!(meta.history(u.space, r, None, 10).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.history(u.space, NodeId::generate(), None, 10).await, Err(MetaError::NotFound(_)))); + let bob = user(meta, "bob").await; + assert!(matches!(meta.history(bob.space, node, None, 10).await, Err(MetaError::NotFound(_)))); + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(head) }], NOW + 200).await.unwrap(); + assert_eq!(meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions.len(), 7); +} + +/// A restore is an ordinary update after the head: a fresh version with the old content (manifest, size, mtime), the +/// head as its parent, the next sequence number, the same epoch and no save record; it shows in the change feed, and +/// the old version stays where it was. +async fn a_restore_is_a_new_version_after_the_head(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let other = meta.create_device(u.id, "phone", &[9u8; 32], NOW).await.unwrap(); + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[0].id, &record).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_version(u.space, other, node, versions[0].id, versions[2].id, NOW + 50).await.unwrap(); + assert_eq!((done.parent_version, done.epoch, done.seq), (versions[2].id, before.epoch, before.seq + 1)); + assert!(!versions.iter().any(|v| v.id == done.version), "a fresh version ID"); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch)); + + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!( + (f.version, f.manifest, f.mtime_ms, f.device, f.save.clone()), + (done.version, versions[0].manifest, versions[0].mtime_ms, other, None) + ); + + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 4); + assert_eq!(history[0].id, done.version); + assert_eq!((history[0].parent_version, history[0].size, history[0].save, history[0].seq), (Some(versions[2].id), 3, false, done.seq)); + assert!(history.iter().any(|v| v.id == versions[0].id && v.save), "the old version and its record stay"); + + // The restored content is the head now: a stale `expected` is refused and changes nothing. + let stale = meta.restore_version(u.space, dev, node, versions[1].id, versions[2].id, NOW + 60).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(done.version) })]); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, after.seq); + // Restoring the content that is already current is allowed: it is one more version. + let again = meta.restore_version(u.space, dev, node, done.version, done.version, NOW + 70).await.unwrap(); + assert_eq!(again.seq, after.seq + 1); +} + +/// What a restore turns down, each leaving the space as it was: a folder, a deleted file, a file with a live document, +/// a node or version that is not there, a version of another file, and anything of another account. +async fn a_restore_refuses_what_it_can_not_restore(meta: &Meta) { + let u = user(meta, "ana").await; + let (space, dev) = (u.space, device(meta, &u).await); + let (file, versions) = file_with_versions(meta, &u, dev, 2).await; + let (second, second_version) = file_named(meta, &u, dev, "second.txt").await; + let r = root(meta, &u).await; + let seq = meta.space_info(space).await.unwrap().seq; + let (old, head) = (versions[0].id, versions[1].id); + let restore = |node: NodeId, source: VersionId, expected: VersionId| { + let meta = meta.clone(); + async move { meta.restore_version(space, dev, node, source, expected, NOW + 9).await } + }; + + assert!(matches!(restore(r, old, head).await, Err(MetaError::Invalid(_))), "a folder"); + assert!(matches!(restore(NodeId::generate(), old, head).await, Err(MetaError::NotFound(_))), "no such node"); + assert!(matches!(restore(file, VersionId::generate(), head).await, Err(MetaError::NotFound(_))), "no such version"); + assert!(matches!(restore(file, second_version.id, head).await, Err(MetaError::NotFound(_))), "a version of another file"); + // The source is checked before the head: a missing source is not a conflict. + assert!(matches!(restore(file, VersionId::generate(), VersionId::generate()).await, Err(MetaError::NotFound(_)))); + assert_eq!(refused_restore(restore(file, old, VersionId::generate()).await).len(), 1, "a head that is not current"); + + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + assert!(matches!(meta.restore_version(bob.space, bob_device, file, old, head, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + + // A file with a live document. + let doc = craft_cloud_proto::live::node_doc(space, second); + meta.create_live_doc(space, doc, "document:kitdemo", Some(second), NOW).await.unwrap(); + assert!(matches!(restore(second, second_version.id, second_version.id).await, Err(MetaError::Invalid(_))), "a live document"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq, "nothing changed so far"); + + // A deleted file. + meta.commit(space, dev, &[Op::Delete { node: file, expected: Some(head) }], NOW + 5).await.unwrap(); + assert!(matches!(restore(file, old, head).await, Err(MetaError::Invalid(_))), "a deleted file"); + assert_eq!(meta.space_info(space).await.unwrap().seq, seq + 1, "only the delete"); + let feed = meta.changes(space, seq, None, MAX_PAGE_NODES).await.unwrap(); + assert!(feed.nodes.len() == 1 && find(&feed.nodes, file).deleted, "still deleted"); +} + +/// Several restores of different old versions against one head: exactly one applies, the rest are refused with a +/// version mismatch, and the space moves one step. +async fn concurrent_restores_let_exactly_one_win(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let head = versions[2].id; + let before = meta.space_info(u.space).await.unwrap().seq; + let mut tasks = Vec::new(); + for i in 0..8 { + let (meta, space, source) = (meta.clone(), u.space, versions[i % 2].id); + tasks.push(tokio::spawn(async move { meta.restore_version(space, dev, node, source, head, NOW + 20).await })); + } + let mut wins = 0; + for task in tasks { + match task.await.unwrap() { + Ok(_) => wins += 1, + Err(MetaError::Rejected(list)) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + Err(other) => panic!("unexpected error {other:?}"), + } + } + assert_eq!(wins, 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before + 1); +} + +/// A restore and a late save record for the head it replaces run together: both apply under the space's one lock, the +/// record stays on the version it was meant for, and the restored version never gets a record. +async fn a_restore_and_a_late_save_record_both_apply(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + let (head, old) = (versions[2].id, versions[0].id); + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let save = tokio::spawn(async move { m1.attach_save(space, node, head, &record).await }); + let restore = tokio::spawn(async move { m2.restore_version(space, dev, node, old, head, NOW + 30).await }); + save.await.unwrap().unwrap(); + let restored = restore.await.unwrap().unwrap(); + let history = meta.history(space, node, None, 50).await.unwrap().versions; + assert_eq!(history.len(), 4); + for v in &history { + assert_eq!(v.save, v.id == head, "only the old head has the record: {}", v.id); + } + assert_eq!(history[0].id, restored.version); + // The create, two updates, the record and the restore. + assert_eq!(meta.space_info(space).await.unwrap().seq, 5); +} From d2a0c0fccf0f170b9d371c662723360495b757c6 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 13:26:11 +0100 Subject: [PATCH 03/27] server: add the history and restore routes GET /api/v1/nodes/{id}/history and POST /api/v1/nodes/{id}/restore, resolved in the caller's own space. A stale expected_version answers 409 conflict. Covers permissions, paging, conflicts, restoring old and missing versions, and parallel restores in the API tests. --- apps/craft-cloud-server/src/api.rs | 51 +++++++- apps/craft-cloud-server/tests/api.rs | 188 +++++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 2 deletions(-) diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index efc5392f..4bbef60f 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -15,8 +15,8 @@ use axum::routing::{delete, get, post, put}; use axum::{Json, Router, middleware}; use craft_cloud_meta::{BlobKind, DeviceAuth, valid_username}; use craft_cloud_proto::api::{ - CommitRequest, CommitResponse, DeviceInfo, InfoResponse, LoginRequest, LoginResponse, MAX_MISSING_IDS, MissingRequest, MissingResponse, - PROTOCOL_MAX, PROTOCOL_MIN, SpaceResponse, codes, + CommitRequest, CommitResponse, DeviceInfo, HISTORY_DEFAULT_LIMIT, HistoryCursor, HistoryResponse, InfoResponse, LoginRequest, LoginResponse, + MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, RestoreRequest, RestoreResponse, SpaceResponse, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_JSON_BYTES, MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, Manifest, ManifestId, NodeId, PageCursor}; @@ -56,6 +56,8 @@ pub fn router(state: Shared) -> Router { .route("/manifests/{id}", put(put_manifest).get(get_manifest).layer(DefaultBodyLimit::max(MAX_MANIFEST_BYTES))) .route("/commit", post(commit).layer(DefaultBodyLimit::max(MAX_BATCH_JSON_BYTES))) .route("/saves", post(attach_save).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/nodes/{id}/history", get(history)) + .route("/nodes/{id}/restore", post(restore).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) .route("/space", get(space)) .route("/tree", get(tree)) .route("/changes", get(changes)) @@ -385,6 +387,51 @@ async fn attach_save( Ok(Json(CommitResponse { seq: outcome.seq, epoch: outcome.epoch })) } +#[derive(Debug, Deserialize)] +struct HistoryQuery { + cursor: Option, + limit: Option, +} + +/// `GET /nodes/{id}/history`: the versions of a file of the caller's own space, newest first. The node is looked +/// up in the space of the device token, so another account's node is `not_found`. +async fn history( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad history cursor"))?; + let page = state.meta.history(auth.space, node, after, q.limit.unwrap_or(HISTORY_DEFAULT_LIMIT)).await?; + Ok(Json(HistoryResponse { versions: page.versions, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /nodes/{id}/restore`: a new version with the content of an old one, as an ordinary update of the file. +async fn restore( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, + request: Request, +) -> Result, ApiError> { + let Path(node) = id?; + let req: RestoreRequest = read_json(&state, request, SMALL_JSON_BYTES).await?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let outcome = tokio::spawn(async move { + let outcome = task_state.meta.restore_version(auth.space, auth.device, node, req.version, req.expected_version, task_state.now_ms()).await?; + task_state.notifier.publish(auth.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "restore task failed"); + ApiError::internal() + })??; + Ok(Json(RestoreResponse { version: outcome.version, parent_version: outcome.parent_version, seq: outcome.seq, epoch: outcome.epoch })) +} + fn cursor(after_seq: Option, after_node: Option) -> Result, ApiError> { match (after_seq, after_node) { (Some(seq), Some(node)) => Ok(Some(PageCursor { seq, node })), diff --git a/apps/craft-cloud-server/tests/api.rs b/apps/craft-cloud-server/tests/api.rs index 83b95140..54207bbf 100644 --- a/apps/craft-cloud-server/tests/api.rs +++ b/apps/craft-cloud-server/tests/api.rs @@ -159,6 +159,9 @@ on_every_backend!( a_client_ahead_of_the_server_must_resync, overlong_usernames_are_refused_before_rate_limiting, the_change_feed_pages, + the_history_of_a_file_pages_newest_first_for_its_owner_only, + restoring_an_old_version_makes_a_new_one, + parallel_restores_against_one_head_let_one_win, every_rejection_is_a_json_error, one_request_cant_list_unbounded_missing_chunks, long_polls_are_capped_per_space, @@ -1430,3 +1433,188 @@ async fn live_subscriptions_are_capped_per_device() { live_quiet(&mut ws).await; assert_eq!(live_subscribe(&mut other, doc, 0).await, vec![ServerFrame::Subscribed { doc, head: 0 }]); } + +// ---- history and restore ---- + +fn update(node: NodeId, expected: VersionId, manifest: ManifestId) -> Op { + Op::Update { node, expected, version: NewVersion { id: VersionId::generate(), manifest, mtime_ms: 7, exec: false, save: None } } +} + +async fn history_page(s: &Server, token: &str, node: NodeId, query: &str) -> reqwest::Response { + s.client.get(format!("{}/nodes/{node}/history{query}", s.base)).bearer_auth(token).send().await.unwrap() +} + +async fn restore_request(s: &Server, token: &str, node: NodeId, body: serde_json::Value) -> reqwest::Response { + s.client.post(format!("{}/nodes/{node}/restore", s.base)).bearer_auth(token).json(&body).send().await.unwrap() +} + +/// A file of `contents.len()` versions, one per entry: its node, the version IDs and manifests, oldest first. +async fn file_with_history(s: &Server, token: &str, contents: &[&[u8]]) -> (NodeId, Vec, Vec) { + let r = root(s, token).await; + let node = NodeId::generate(); + let mut manifests = Vec::new(); + for data in contents { + manifests.push(upload(s, token, data).await); + } + let first = create(node, r, &format!("history-{node}.txt"), manifests[0]); + let Op::Create { version, .. } = &first else { panic!("a create") }; + let mut versions = vec![version.id]; + assert_eq!(commit(s, token, &[first]).await.status(), StatusCode::OK); + for manifest in &manifests[1..] { + let op = update(node, *versions.last().unwrap(), *manifest); + let Op::Update { version, .. } = &op else { panic!("an update") }; + versions.push(version.id); + assert_eq!(commit(s, token, &[op]).await.status(), StatusCode::OK); + } + (node, versions, manifests) +} + +async fn the_history_of_a_file_pages_newest_first_for_its_owner_only(s: &Server) { + use craft_cloud_proto::api::HistoryResponse; + add_user(s, "ana").await; + add_user(s, "bob").await; + let (ana, bob) = (login(s, "ana").await, login(s, "bob").await); + let tag = uuid::Uuid::now_v7(); + let contents: Vec> = (0..5).map(|i| format!("history {tag} {i}").into_bytes()).collect(); + let refs: Vec<&[u8]> = contents.iter().map(Vec::as_slice).collect(); + let (node, versions, _) = file_with_history(s, &ana, &refs).await; + + let all: HistoryResponse = history_page(s, &ana, node, "").await.json().await.unwrap(); + assert_eq!(all.next, None); + assert_eq!(all.versions.iter().map(|v| v.id).collect::>(), versions.iter().rev().copied().collect::>()); + assert!(all.versions.iter().all(|v| !v.pruned && !v.save && v.size == contents[0].len() as u64)); + assert_eq!(all.versions[4].parent_version, None); + assert_eq!(all.versions[0].parent_version, Some(versions[3])); + assert_eq!(all.versions.iter().map(|v| v.seq).collect::>(), vec![5, 4, 3, 2, 1]); + + // Pages of two follow each other through `next`. + let (mut seen, mut query, mut pages) = (Vec::new(), "?limit=2".to_owned(), 0); + loop { + let page: HistoryResponse = history_page(s, &ana, node, &query).await.json().await.unwrap(); + seen.extend(page.versions.iter().map(|v| v.id)); + pages += 1; + match page.next { + Some(next) => query = format!("?limit=2&cursor={next}"), + None => break, + } + } + assert_eq!((pages, seen), (3, all.versions.iter().map(|v| v.id).collect::>())); + + // Bad paging is `invalid`. + for bad in ["?limit=0", "?limit=201", "?limit=-1", "?cursor=nonsense", "?cursor=1", "?cursor=%201.00000000-0000-0000-0000-000000000000"] { + let (status, body) = error(history_page(s, &ana, node, bad).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid"), "{bad}"); + } + assert_eq!(history_page(s, &ana, node, "?limit=200").await.status(), StatusCode::OK); + + // Other accounts, unknown nodes and folders; a missing or wrong token. + let (status, body) = error(history_page(s, &bob, node, "").await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found"), "another account's file"); + let (status, body) = error(history_page(s, &ana, NodeId::generate(), "").await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found")); + let (status, body) = error(history_page(s, &ana, root(s, &ana).await, "").await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid"), "a folder"); + assert_eq!(history_page(s, "not-a-token", node, "").await.status(), StatusCode::UNAUTHORIZED); + assert_eq!(s.client.get(format!("{}/nodes/{node}/history", s.base)).send().await.unwrap().status(), StatusCode::UNAUTHORIZED); + let (status, body) = error(s.client.get(format!("{}/nodes/not-a-uuid/history", s.base)).bearer_auth(&ana).send().await.unwrap()).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid")); + + // A deleted file keeps its history. + let del = Op::Delete { node, expected: Some(*versions.last().unwrap()) }; + assert_eq!(commit(s, &ana, &[del]).await.status(), StatusCode::OK); + let kept: HistoryResponse = history_page(s, &ana, node, "").await.json().await.unwrap(); + assert_eq!(kept.versions.len(), 5); +} + +async fn restoring_an_old_version_makes_a_new_one(s: &Server) { + use craft_cloud_proto::api::{HistoryResponse, RestoreResponse}; + add_user(s, "ana").await; + add_user(s, "bob").await; + let (ana, bob) = (login(s, "ana").await, login(s, "bob").await); + let tag = uuid::Uuid::now_v7(); + let (first, second, third) = (format!("first {tag}").into_bytes(), format!("second {tag}").into_bytes(), format!("third {tag}").into_bytes()); + let (node, versions, _) = file_with_history(s, &ana, &[&first, &second, &third]).await; + let before: SpaceResponse = space_of(s, &ana).await; + + // Another account can neither read nor restore it. + let body = json!({ "version": versions[0], "expected_version": versions[2] }); + let (status, error_body) = error(restore_request(s, &bob, node, body.clone()).await).await; + assert_eq!((status, error_body.error.as_str()), (StatusCode::NOT_FOUND, "not_found")); + assert_eq!(restore_request(s, "not-a-token", node, body.clone()).await.status(), StatusCode::UNAUTHORIZED); + assert_eq!(space_of(s, &ana).await.seq, before.seq, "refused restores change nothing"); + + let res = restore_request(s, &ana, node, body.clone()).await; + assert_eq!(res.status(), StatusCode::OK); + let done: RestoreResponse = res.json().await.unwrap(); + assert_eq!((done.parent_version, done.seq, done.epoch), (versions[2], before.seq + 1, before.epoch)); + assert!(!versions.contains(&done.version)); + + // It is an ordinary change: in the feed, with the old content, and the old versions stay. + let feed: ChangeSet = + s.client.get(format!("{}/changes?since={}", s.base, before.seq)).bearer_auth(&ana).send().await.unwrap().json().await.unwrap(); + let node_state = feed.nodes.iter().find(|n| n.id == node).unwrap(); + let NodeKind::File(state) = &node_state.kind else { panic!("a file") }; + assert_eq!((state.version, state.save.clone(), state.size), (done.version, None, first.len() as u64)); + assert_eq!(download(s, &ana, &state.manifest).await, first); + let history: HistoryResponse = history_page(s, &ana, node, "").await.json().await.unwrap(); + assert_eq!(history.versions.len(), 4); + assert_eq!((history.versions[0].id, history.versions[0].parent_version), (done.version, Some(versions[2]))); + + // The old `expected_version` is stale now: 409 with the conflict list, nothing changes. + let (status, stale) = error(restore_request(s, &ana, node, body).await).await; + assert_eq!((status, stale.error.as_str()), (StatusCode::CONFLICT, "conflict")); + assert_eq!(stale.conflicts.as_ref().map(Vec::len), Some(1)); + assert_eq!(space_of(s, &ana).await.seq, before.seq + 1); + + // A version that is not this file's: 404. Garbage bodies: 400. + let unknown = json!({ "version": VersionId::generate(), "expected_version": done.version }); + let (status, body) = error(restore_request(s, &ana, node, unknown).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found")); + let (other, other_versions, _) = file_with_history(s, &ana, &[format!("other {tag}").as_bytes()]).await; + assert_ne!(other, node); + let foreign = json!({ "version": other_versions[0], "expected_version": done.version }); + let (status, body) = error(restore_request(s, &ana, node, foreign).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found"), "a version of another file"); + for bad in [json!({}), json!({ "version": "x", "expected_version": "y" }), json!({ "version": versions[0] })] { + let (status, body) = error(restore_request(s, &ana, node, bad).await).await; + assert!(status.is_client_error() && body.error == "invalid", "{status} {}", body.error); + } + let folder = root(s, &ana).await; + let (status, body) = error(restore_request(s, &ana, folder, json!({ "version": versions[0], "expected_version": versions[0] })).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid"), "a folder"); + + // A deleted file is not restored from here. + let seq = space_of(s, &ana).await.seq; + assert_eq!(commit(s, &ana, &[Op::Delete { node, expected: Some(done.version) }]).await.status(), StatusCode::OK); + let (status, body) = error(restore_request(s, &ana, node, json!({ "version": versions[0], "expected_version": done.version })).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid"), "a deleted file"); + assert_eq!(space_of(s, &ana).await.seq, seq + 1, "only the delete"); +} + +async fn parallel_restores_against_one_head_let_one_win(s: &Server) { + use craft_cloud_proto::api::RestoreResponse; + add_user(s, "ana").await; + let ana = login(s, "ana").await; + let tag = uuid::Uuid::now_v7(); + let contents: Vec> = (0..3).map(|i| format!("parallel {tag} {i}").into_bytes()).collect(); + let refs: Vec<&[u8]> = contents.iter().map(Vec::as_slice).collect(); + let (node, versions, _) = file_with_history(s, &ana, &refs).await; + let before = space_of(s, &ana).await.seq; + let mut tasks = Vec::new(); + for i in 0..6 { + let (client, url, token) = (s.client.clone(), format!("{}/nodes/{node}/restore", s.base), ana.clone()); + let body = json!({ "version": versions[i % 2], "expected_version": versions[2] }); + tasks.push(tokio::spawn(async move { client.post(url).bearer_auth(token).json(&body).send().await.unwrap() })); + } + let (mut won, mut conflicted) = (Vec::new(), 0); + for task in tasks { + let res = task.await.unwrap(); + match res.status() { + StatusCode::OK => won.push(res.json::().await.unwrap()), + StatusCode::CONFLICT => conflicted += 1, + other => panic!("unexpected status {other}"), + } + } + assert_eq!((won.len(), conflicted), (1, 5)); + assert_eq!(space_of(s, &ana).await.seq, before + 1); +} From 2171728e7242c5282cb8d2ebc1e1037e0a3552cc Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 13:26:12 +0100 Subject: [PATCH 04/27] docs: describe version history and restore Updates the API table and the milestone note in the sync design, and adds a short section to the server guide. --- docs/design/sync.md | 20 ++++++++++++++++++-- docs/server.md | 8 ++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/design/sync.md b/docs/design/sync.md index 52d60652..9eff1762 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -655,12 +655,28 @@ methods answer `not_found` / `invalid` (405) the same way: | `PUT /chunks/{id}` / `GET /chunks/{id}` | one chunk, max 4 MiB, checked by the server; you can only download chunks your space uploaded | | `PUT /manifests/{id}` / `GET /manifests/{id}` | one manifest (binary); the server checks the ID, the limits and that every chunk is present | | `POST /commit` | batch of version/mkdir/move/delete operations, one transaction; `409 conflict` listing every conflicting operation (a version whose manifest isn't uploaded is a `missing_manifest` conflict) | -| `GET /nodes/{id}/history` / `POST /nodes/{id}/restore` | list versions / make an old one current (M4) | +| `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | +| `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | | `POST /join` | code + device name (+ password for an invite to a new account) → device ID + token, as a sign-in (no auth); `invalid_code`, `password_required`, `weak_password`, `rate_limited` | +**History and restore.** Versions are never removed today, so the history lists every version of a file, and `pruned` +is always `false`: the field is in the contract for the day versions can be thinned (a thinned version will stay +in the list with `pruned: true`, its metadata intact, and can't be restored). A restore is an ordinary `update` made +by the server: it checks, under the same lock as a commit, that the node is a file of the caller's space that is not +deleted and has no live document, that `version` is a version of this node, that its manifest is still in the space, +and that `expected_version` is still the file's current version; then it records a new version with a fresh ID whose +parent is the current version, with the manifest, modification time and executable bit of the old one, at the space's +next sequence number. The epoch does not change (only `reset-epoch` and backup restores do that), the change feed +reports the file like any other update, and a restore needs no quota because the manifest is already the space's. +The new version carries **no save record**: a save record names a state of a live document, and a restore replaces +file content only, so the engine keeps preferring a newer local live-document state over the restored file for a +file that has a live document (which is why such a file is refused). Unknown node, foreign node, a version of +another file and a missing manifest are all `not_found`; a deleted file, a folder and a file with a live document are +`invalid`; a stale `expected_version` is `conflict` (409, with the conflict list as for a commit). + **Client contract** (what the sync agent relies on): - Probe `GET /api/info` and refuse to sync when the protocol ranges don't overlap. @@ -1061,7 +1077,7 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** | M3b2b1 | Agent on its own: seed 112337, `run` loop, watcher, long-poll, stability window, backoff, status file | DONE (2026-10-14) | | M3b2b2 | Simulated network (`turmoil`): partitions, delays, dropped connections | DONE (2026-10-15) | | M3c | Fixes from the full repo review | DONE (2026-10-15) | -| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | TODO | +| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | IN PROGRESS: history and restore of file versions done (server API only); the rest TODO | | M5 | Selective sync, large-media polish (a few chunks in flight at once), ignore rules | TODO | | M6 | Packaging: binaries, Docker, systemd, built-in TLS, self-hosting guide | TODO | | M7 | Local control API + launcher integration PR | TODO | diff --git a/docs/server.md b/docs/server.md index bf2a6492..9440d093 100644 --- a/docs/server.md +++ b/docs/server.md @@ -55,6 +55,14 @@ invite replaces the old one, and `user invite --revoke name` ends it. Every late that is signed in (`craft-cloud-agent add-computer`, see [`agent.md`](agent.md)). Codes that don't work are refused alike (wrong, expired or used), and an address that tries ten in a minute is held off for the rest of it. +## Version history + +The server keeps every version of every file; nothing is thinned or removed yet. A signed-in device can list a file's +versions (`GET /api/v1/nodes/{id}/history`) and make an old one current again (`POST /api/v1/nodes/{id}/restore`), +which adds a new version at the end of the file's history rather than rewinding it, so other computers pick it up +through the normal change feed. This needs no setting and no database migration, and an older server answers +`not_found` for both paths. The agent and the apps do not use them yet. + ## Restoring a backup With the server stopped, restore the database and the chunk storage from the backup, then run From ec83bd72552896976ec761ad99e92d0f66a07bec Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 13:35:32 +0100 Subject: [PATCH 05/27] meta, docs: index the history query and correct the restore note A history page now reads versions through an index on (space_id, node, seq, id) in the order of the query, so the page size bounds the work. The migration adds the index on both databases. The restore note in the sync design no longer claims the engine prefers a newer live-document state: a restore is an ordinary new version and files with a live document are refused. --- .../postgres/0008_versions_node_seq.sql | 3 ++ .../sqlite/0008_versions_node_seq.sql | 3 ++ crates/meta/src/history.rs | 6 ++- crates/meta/src/schema.rs | 2 + crates/meta/tests/behaviour.rs | 49 ++++++++++++++++++- docs/design/sync.md | 7 +-- 6 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 crates/meta/migrations/postgres/0008_versions_node_seq.sql create mode 100644 crates/meta/migrations/sqlite/0008_versions_node_seq.sql diff --git a/crates/meta/migrations/postgres/0008_versions_node_seq.sql b/crates/meta/migrations/postgres/0008_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/postgres/0008_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/migrations/sqlite/0008_versions_node_seq.sql b/crates/meta/migrations/sqlite/0008_versions_node_seq.sql new file mode 100644 index 00000000..66f9d9fe --- /dev/null +++ b/crates/meta/migrations/sqlite/0008_versions_node_seq.sql @@ -0,0 +1,3 @@ +-- The history of a file lists its versions newest first by (seq, id): this index lets the database walk them in that +-- order and stop at the page size, instead of reading and sorting every version of the space. +CREATE INDEX versions_node_seq ON versions (space_id, node, seq DESC, id DESC); diff --git a/crates/meta/src/history.rs b/crates/meta/src/history.rs index 060cfacd..2c0dd98b 100644 --- a/crates/meta/src/history.rs +++ b/crates/meta/src/history.rs @@ -41,11 +41,13 @@ impl Meta { // Above every sequence number and every ID. None => (i64::MAX, vec![0xff; 16]), }; - // One row more than asked for tells whether another page follows. The cursor is exclusive. + // One row more than asked for tells whether another page follows. The cursor is exclusive. The row-value + // comparison and the order match the index `versions_node_seq`, so the database seeks to the cursor and stops + // after the page. let rows = sqlx::query( "SELECT id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, \ CAST(CASE WHEN save IS NULL THEN 0 ELSE 1 END AS BIGINT) AS has_save \ - FROM versions WHERE space_id = $1 AND node = $2 AND (seq < $3 OR (seq = $3 AND id < $4)) \ + FROM versions WHERE space_id = $1 AND node = $2 AND (seq, id) < ($3, $4) \ ORDER BY seq DESC, id DESC LIMIT $5", ) .bind(uuid_bytes(space.as_uuid())) diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index e4c5220d..7080c9f2 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -12,6 +12,7 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0004_join.sql"), include_str!("../migrations/sqlite/0005_live_documents.sql"), include_str!("../migrations/sqlite/0006_save_records.sql"), + include_str!("../migrations/sqlite/0008_versions_node_seq.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -20,6 +21,7 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0004_join.sql"), include_str!("../migrations/postgres/0005_live_documents.sql"), include_str!("../migrations/postgres/0006_save_records.sql"), + include_str!("../migrations/postgres/0008_versions_node_seq.sql"), ]; /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index feafc49a..9906458f 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -1787,6 +1787,8 @@ async fn upgrading_counts_what_each_space_already_stores() { sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); // And before save records. + // And before the history index. + sqlx::query("DROP INDEX versions_node_seq").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE versions DROP COLUMN save").execute(&mut conn).await.unwrap(); sqlx::query("DELETE FROM schema_version WHERE version >= 2").execute(&mut conn).await.unwrap(); } @@ -1812,7 +1814,7 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 6); + assert_eq!(versions, 7); } /// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. @@ -2050,3 +2052,48 @@ async fn a_restore_and_a_late_save_record_both_apply(meta: &Meta) { // The create, two updates, the record and the restore. assert_eq!(meta.space_info(space).await.unwrap().seq, 5); } + +/// The database at the newest version when the code knows it: the version table holds one row per migration, and a +/// database written by a newer build is refused rather than half-read. +#[tokio::test] +async fn the_schema_version_is_recorded_and_a_newer_database_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 7); + // Opening again changes nothing. + drop(Meta::open(&url).await.unwrap()); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(rows, 7); + sqlx::query("INSERT INTO schema_version (version) VALUES (8)").execute(&mut conn).await.unwrap(); + drop(conn); + assert!(matches!(Meta::open(&url).await, Err(MetaError::Corrupt(_))), "a newer database is refused"); +} + +/// A history page is read through the index on `(space_id, node, seq, id)`, not by scanning the versions table. +#[tokio::test] +async fn a_history_page_uses_the_node_index_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + let url = format!("sqlite:{}", dir.path().join("meta.db").display()); + drop(Meta::open(&url).await.unwrap()); + use sqlx::Connection; + let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); + let plan = sqlx::query( + "EXPLAIN QUERY PLAN SELECT id FROM versions WHERE space_id = ?1 AND node = ?2 AND (seq, id) < (?3, ?4) ORDER BY seq DESC, id DESC LIMIT ?5", + ) + .bind(vec![0u8; 16]) + .bind(vec![0u8; 16]) + .bind(i64::MAX) + .bind(vec![0xffu8; 16]) + .bind(51i64) + .fetch_all(&mut conn) + .await + .unwrap(); + use sqlx::Row; + let detail: Vec = plan.iter().map(|r| r.get::("detail")).collect(); + assert!(detail.iter().any(|d| d.contains("versions_node_seq")), "{detail:?}"); + assert!(!detail.iter().any(|d| d.contains("TEMP B-TREE")), "no sort: {detail:?}"); +} diff --git a/docs/design/sync.md b/docs/design/sync.md index 9eff1762..c466aa19 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -671,9 +671,10 @@ and that `expected_version` is still the file's current version; then it records parent is the current version, with the manifest, modification time and executable bit of the old one, at the space's next sequence number. The epoch does not change (only `reset-epoch` and backup restores do that), the change feed reports the file like any other update, and a restore needs no quota because the manifest is already the space's. -The new version carries **no save record**: a save record names a state of a live document, and a restore replaces -file content only, so the engine keeps preferring a newer local live-document state over the restored file for a -file that has a live document (which is why such a file is refused). Unknown node, foreign node, a version of +The new version carries **no save record**: a save record names a state of a live document, and a restore +copies none and invents none. A restore is an ordinary new version, so agents take it in under the normal rules, +including the conflict rules for a machine with unsynced edits of its own. A file that has a live document is +refused, so file versions and live-document state stay separate. Unknown node, foreign node, a version of another file and a missing manifest are all `not_found`; a deleted file, a folder and a file with a live document are `invalid`; a stale `expected_version` is `conflict` (409, with the conflict list as for a commit). From 48106b566885747c5bc9c2349919567f375de5f8 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:04:35 +0100 Subject: [PATCH 06/27] proto: add the trash bodies Adds the list, restore and cursor types of GET /trash and POST /trash/{id}/restore, with the same page limits as the history. --- crates/proto/src/api.rs | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 129cc97b..0c077efb 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -232,6 +232,75 @@ pub struct RestoreResponse { pub epoch: Epoch, } +/// Items in one `GET /api/v1/trash` page when `limit` is not given. +pub const TRASH_DEFAULT_LIMIT: u32 = 50; +/// Most items in one trash page; a larger `limit` is `invalid`. +pub const TRASH_MAX_LIMIT: u32 = 200; + +/// Where the next trash page starts: after this `(deleted_ms, node)` in the newest-deleted-first order. Opaque on the +/// wire, like [`HistoryCursor`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TrashCursor { + pub deleted_ms: i64, + pub node: NodeId, +} + +impl std::fmt::Display for TrashCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.deleted_ms, self.node) + } +} + +impl std::str::FromStr for TrashCursor { + type Err = BadHistoryCursor; + + /// Only the exact form [`Display`](std::fmt::Display) writes is accepted. + fn from_str(s: &str) -> Result { + let (deleted_ms, node) = s.split_once('.').ok_or(BadHistoryCursor)?; + let cursor = TrashCursor { deleted_ms: deleted_ms.parse().map_err(|_| BadHistoryCursor)?, node: node.parse().map_err(|_| BadHistoryCursor)? }; + if cursor.to_string() == s { Ok(cursor) } else { Err(BadHistoryCursor) } + } +} + +/// One deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashItem { + pub id: NodeId, + /// The folder it was in when it was deleted (itself deleted, perhaps). + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + pub deleted_ms: i64, + /// `deleted_ms` plus the server's `retention.trash_days`: when the server may remove the node for good. Nothing is + /// removed yet, so this is a promise of the earliest time, not an event. + pub purge_after_ms: i64, +} + +/// `GET /api/v1/trash?cursor&limit`: the deleted nodes of your space that were not removed from the trash, newest +/// deletion first by `(deleted_ms, id)`. `next` is the `cursor` of the following page, absent on the last. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashResponse { + pub items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// `POST /api/v1/trash/{id}/restore`: the node is back. `parent` and `name` say where, which differs from where it was +/// when a folder of its path was merged into a live folder of the same name, or when `renamed` is set: the name was +/// taken, so the node got a conflict name like the sync engine makes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TrashRestoreResponse { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + pub seq: u64, + pub epoch: Epoch, +} + /// A commit that applied. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitResponse { @@ -337,6 +406,16 @@ mod tests { assert_eq!("-1.00000000-0000-0000-0000-000000000000".parse::(), Err(BadHistoryCursor)); } + #[test] + fn trash_cursors_have_one_spelling() { + let cursor = TrashCursor { deleted_ms: 1_760_000_000_000, node: NodeId::generate() }; + assert_eq!(cursor.to_string().parse::(), Ok(cursor)); + let text = cursor.to_string(); + for bad in ["", "7", "7.", ".x", "x.y", &format!("+{text}"), &text.to_uppercase(), &format!("{text}x"), &format!(" {text}")] { + assert_eq!(bad.parse::(), Err(BadHistoryCursor), "{bad:?}"); + } + } + #[test] fn history_and_restore_bodies_have_a_plain_json_form() { let restore: RestoreRequest = serde_json::from_value(serde_json::json!({ From 523ff28d207a3e213f89cbdf61a38432d046326e Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:04:35 +0100 Subject: [PATCH 07/27] meta: add the trash, with deletion times and a restore that works around taken names A migration adds nodes.deleted_ms and nodes.purge_requested_ms and an index for the trash list, and dates old deletions at the migration. The commit path stamps the time when a node goes from live to deleted and clears it when the node comes back. restore_from_trash brings a deleted node and its deleted folders back in one transaction, merging into live folders of the same name and renaming around taken names with the engine's conflict pattern. request_purge only marks a node, so nothing is freed. --- .../meta/migrations/postgres/0009_trash.sql | 8 + crates/meta/migrations/sqlite/0009_trash.sql | 8 + crates/meta/src/commit.rs | 187 ++++++- crates/meta/src/lib.rs | 4 +- crates/meta/src/schema.rs | 2 + crates/meta/src/trash.rs | 140 +++++ crates/meta/tests/behaviour.rs | 506 +++++++++++++++++- 7 files changed, 847 insertions(+), 8 deletions(-) create mode 100644 crates/meta/migrations/postgres/0009_trash.sql create mode 100644 crates/meta/migrations/sqlite/0009_trash.sql create mode 100644 crates/meta/src/trash.rs diff --git a/crates/meta/migrations/postgres/0009_trash.sql b/crates/meta/migrations/postgres/0009_trash.sql new file mode 100644 index 00000000..7fd42245 --- /dev/null +++ b/crates/meta/migrations/postgres/0009_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/migrations/sqlite/0009_trash.sql b/crates/meta/migrations/sqlite/0009_trash.sql new file mode 100644 index 00000000..6da29e88 --- /dev/null +++ b/crates/meta/migrations/sqlite/0009_trash.sql @@ -0,0 +1,8 @@ +-- The trash: when a node was deleted, and when its permanent removal was asked for. `deleted_ms` is set when a node +-- goes from live to deleted and cleared when it comes back. `purge_requested_ms` is set by an explicit removal from the +-- trash and only marks the node: nothing is freed yet. Nodes deleted before this migration start their time in the +-- trash at the migration, so no old deletion looks expired the moment a retention period is introduced. +ALTER TABLE nodes ADD COLUMN deleted_ms BIGINT; +ALTER TABLE nodes ADD COLUMN purge_requested_ms BIGINT; +UPDATE nodes SET deleted_ms = CAST(strftime('%s', 'now') AS BIGINT) * 1000 WHERE deleted = 1; +CREATE INDEX nodes_trash ON nodes (space_id, deleted_ms, id) WHERE deleted = 1; diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index d74bf856..33ac7073 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -13,8 +13,11 @@ use std::collections::HashMap; +use craft_cloud_proto::conflict_name; use craft_cloud_proto::limits::{MAX_BATCH_OPS, MAX_TREE_DEPTH}; -use craft_cloud_proto::{Conflict, ConflictKind, DeviceId, Epoch, ManifestId, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId}; +use craft_cloud_proto::{ + Conflict, ConflictKind, ConflictStamp, DeviceId, Epoch, ManifestId, Name, NewVersion, NodeId, Op, SaveRecord, SpaceId, VersionId, +}; use sqlx::Row; use crate::blobs::manifest_file_size; @@ -39,6 +42,49 @@ pub struct RestoreOutcome { pub epoch: Epoch, } +/// A node that came back from the trash: where it sits now, whether it took a conflict name, and the space's new +/// sequence number and epoch (unchanged). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashRestoreOutcome { + pub node: NodeId, + pub parent: NodeId, + pub name: String, + pub renamed: bool, + pub seq: u64, + pub epoch: Epoch, +} + +/// The most conflict names tried for one restored node. +const MAX_CONFLICT_COUNTER: u32 = 100; + +fn rejected(kind: ConflictKind) -> MetaError { + MetaError::Rejected(vec![Conflict { op: 0, kind }]) +} + +/// `ms` since the Unix epoch as a UTC minute, for a conflict name (the server doesn't know a user's time zone). +fn conflict_stamp(ms: i64) -> ConflictStamp { + let days = ms.div_euclid(86_400_000); + let minutes = ms.rem_euclid(86_400_000) / 60_000; + // Days to a civil date in the proleptic Gregorian calendar, counted from 0000-03-01. + let z = days.saturating_add(719_468); + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let shifted_month = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * shifted_month + 2) / 5 + 1; + let month = if shifted_month < 10 { shifted_month + 3 } else { shifted_month - 9 }; + let year = year_of_era.saturating_add(era.saturating_mul(400)).saturating_add(i64::from(month <= 2)); + ConflictStamp { + year: u16::try_from(year).unwrap_or(0), + month: u8::try_from(month).unwrap_or(1), + day: u8::try_from(day).unwrap_or(1), + hour: u8::try_from(minutes / 60).unwrap_or(0), + minute: u8::try_from(minutes % 60).unwrap_or(0), + utc: true, + } +} + struct NodeRow { parent: Option, name: String, @@ -126,8 +172,12 @@ impl Ctx<'_> { if deleted { self.failed_restores.clear(); } - sqlx::query("UPDATE nodes SET deleted = $1, changed_seq = $2 WHERE space_id = $3 AND id = $4") + // The time in the trash starts at the change from live to deleted and ends when the node comes back: a later + // deletion starts it afresh, and a pending request to remove the node for good dies with the deletion it was for. + let deleted_ms = deleted.then_some(self.now_ms); + sqlx::query("UPDATE nodes SET deleted = $1, deleted_ms = $2, purge_requested_ms = NULL, changed_seq = $3 WHERE space_id = $4 AND id = $5") .bind(i64::from(deleted)) + .bind(deleted_ms) .bind(self.seq) .bind(self.space()) .bind(uuid_bytes(id.as_uuid())) @@ -354,6 +404,101 @@ impl Ctx<'_> { } } } + /// The first of 100 conflict names for `original` in `parent` that no live node has (the first has no counter, + /// then `, 2` to `, 100`), or `None`. + async fn free_conflict_name(&mut self, parent: NodeId, original: &Name, is_dir: bool, device: &str) -> Result, MetaError> { + let stamp = conflict_stamp(self.now_ms); + for counter in 1..=MAX_CONFLICT_COUNTER { + let candidate = conflict_name(original, is_dir, device, stamp, counter).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + if self.live_child(parent, &candidate).await?.is_none() { + return Ok(Some(candidate)); + } + } + Ok(None) + } + + /// Writes where a deleted node comes back and makes it live: one statement, one sequence number. + async fn revive(&mut self, id: NodeId, parent: NodeId, name: &Name) -> Result<(), MetaError> { + sqlx::query( + "UPDATE nodes SET parent = $1, name = $2, fold_key = $3, deleted = 0, deleted_ms = NULL, purge_requested_ms = NULL, changed_seq = $4 \ + WHERE space_id = $5 AND id = $6", + ) + .bind(uuid_bytes(parent.as_uuid())) + .bind(name.as_str().to_owned()) + .bind(name.fold_key().as_str().to_owned()) + .bind(self.seq) + .bind(self.space()) + .bind(uuid_bytes(id.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + /// Brings a node of the trash back, with the deleted folders above it, root-most first and with their IDs. Where + /// a deleted folder's name is taken by a live folder, that folder is used instead (the deleted one stays deleted) + /// and what is below it moves in. Where a name is taken by a file, or the node's own name is taken by anything, the + /// node comes back under a conflict name. Never overwrites, never creates a node. A conflict that can't be solved + /// is an `Err(MetaError::Rejected)`, and the caller drops the transaction. + async fn restore_from_trash(&mut self, node: NodeId) -> Result { + let found = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await?; + let Some(found) = found else { return Err(MetaError::NotFound("node in the trash")) }; + // A node that was removed from the trash is gone from it, whatever the tombstone still holds. + if !flag(&found, "deleted")? || found.try_get::, _>("purge_requested_ms")?.is_some() { + return Err(MetaError::NotFound("node in the trash")); + } + let is_dir = flag(&found, "is_dir")?; + let device = sqlx::query("SELECT name FROM devices WHERE id = $1") + .bind(uuid_bytes(self.device.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .map(|row| row.try_get::("name")) + .transpose()? + .unwrap_or_default(); + let up = self.ancestors(node).await?; + let deleted_run: Vec<&Ancestor> = up.iter().take_while(|a| a.deleted).collect(); + let Some(base) = up.get(deleted_run.len()) else { + // The walk ended inside the deleted run: the depth bound (too deep) or a deleted root (damaged data). + return if up.last().is_some_and(|a| a.parent.is_some()) { + Err(rejected(ConflictKind::TooDeep)) + } else { + Err(MetaError::Corrupt("the root is deleted")) + }; + }; + let mut parent = base.id; + for (index, a) in deleted_run.iter().enumerate().rev() { + let own = Name::new(&a.name).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let taken = self.live_child(parent, &own).await?; + let is_target = index == 0; + if !is_target && let Some(existing) = taken { + // A live folder of that name stands in for the deleted one; a file there means this folder is renamed. + if self.node(existing).await?.is_some_and(|n| n.is_dir) { + parent = existing; + continue; + } + } + let final_name = match taken { + None => own, + Some(existing) => { + let folder = !is_target || is_dir; + match self.free_conflict_name(parent, &own, folder, &device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + } + } + }; + self.revive(a.id, parent, &final_name).await?; + parent = a.id; + } + let placed = self.node(node).await?.ok_or(MetaError::Corrupt("a restored node vanished"))?; + let parent = placed.parent.ok_or(MetaError::Corrupt("a restored node has no parent"))?; + let original = up.first().map(|a| a.name.as_str()).unwrap_or_default(); + Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, seq: 0, epoch: self.epoch }) + } + /// `node` and its ancestors, nearest first; stops after `MAX_TREE_DEPTH + 2` rows, so a walk that did not /// reach the root (last parent is `Some`) is too deep. async fn ancestors(&mut self, node: NodeId) -> Result, MetaError> { @@ -427,6 +572,20 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } + /// Brings the node `node` of the trash back in `space`, as one change of the space (one sequence number, all or + /// nothing): see `Ctx::restore_from_trash` for the rules. `NotFound` for a node that is not in the trash (live, + /// unknown, another space's, or removed from the trash); `Rejected` (`NameTaken`) when no conflict name is free. + pub async fn restore_from_trash(&self, space: SpaceId, device: DeviceId, node: NodeId, now_ms: i64) -> Result { + let mut w = self.begin_write().await?; + let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; + let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; + let outcome = ctx.restore_from_trash(node).await?; + sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; + w.commit().await?; + Ok(TrashRestoreOutcome { seq, ..outcome }) + } + /// Makes `source`, an older version of the file `node`, its content again as a new version: an ordinary /// `Update` after `expected`, which must still be the current version. The new version has a fresh ID, copies the /// manifest, modification time and executable bit of `source`, and gets no save record (a save record names a @@ -455,7 +614,7 @@ impl Meta { return Err(MetaError::Invalid("only files have versions to restore")); } if row.deleted { - return Err(MetaError::Invalid("the file is deleted: bring it back from the trash first, then restore a version")); + return Err(MetaError::Invalid("the file is deleted: bring it back with POST /trash/{id}/restore first, then restore a version")); } let live = sqlx::query("SELECT 1 FROM live_docs WHERE space_id = $1 AND node = $2 LIMIT 1") .bind(ctx.space()) @@ -533,3 +692,25 @@ impl Meta { Ok(CommitOutcome { seq, epoch: info.epoch }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_millisecond_count_reads_as_a_utc_minute() { + let at = |ms: i64| { + let s = conflict_stamp(ms); + (s.year, s.month, s.day, s.hour, s.minute, s.utc) + }; + assert_eq!(at(0), (1970, 1, 1, 0, 0, true)); + assert_eq!(at(1_760_000_000_000), (2025, 10, 9, 8, 53, true)); + // A leap day, and the last minute of a year. + assert_eq!(at(1_709_164_800_000 + 59 * 60_000), (2024, 2, 29, 0, 59, true)); + assert_eq!(at(1_767_225_540_000), (2025, 12, 31, 23, 59, true)); + // Before the epoch and absurd values do not panic. + assert_eq!(at(-1).0, 1969); + let _ = conflict_stamp(i64::MAX); + let _ = conflict_stamp(i64::MIN); + } +} diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 93777f9e..292bc274 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -14,6 +14,7 @@ mod history; mod join; mod live; mod schema; +mod trash; mod tree; mod users; @@ -25,7 +26,7 @@ use sqlx::any::{AnyConnectOptions, AnyPoolOptions}; use sqlx::{AnyPool, ConnectOptions}; use tokio::sync::{Mutex, OwnedMutexGuard}; -pub use commit::{CommitOutcome, RestoreOutcome}; +pub use commit::{CommitOutcome, RestoreOutcome, TrashRestoreOutcome}; pub use craft_cloud_proto::BlobKind; pub use history::HistoryPage; pub use join::{InviteFor, Redeemed}; @@ -33,6 +34,7 @@ pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, MAX_LIVE_DOCUMENTS_PER_SPACE, MAX_LIVE_PAGE, SnapshotRef, }; +pub use trash::{TrashEntry, TrashPage}; pub use tree::{MAX_PAGE_SAVE_BYTES, SpaceInfo}; pub use users::{DeviceAuth, DeviceRecord, Role, UserId, UserRecord, valid_username}; diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index 7080c9f2..df810ed2 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -13,6 +13,7 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0005_live_documents.sql"), include_str!("../migrations/sqlite/0006_save_records.sql"), include_str!("../migrations/sqlite/0008_versions_node_seq.sql"), + include_str!("../migrations/sqlite/0009_trash.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -22,6 +23,7 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0005_live_documents.sql"), include_str!("../migrations/postgres/0006_save_records.sql"), include_str!("../migrations/postgres/0008_versions_node_seq.sql"), + include_str!("../migrations/postgres/0009_trash.sql"), ]; /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one diff --git a/crates/meta/src/trash.rs b/crates/meta/src/trash.rs new file mode 100644 index 00000000..8d7dab83 --- /dev/null +++ b/crates/meta/src/trash.rs @@ -0,0 +1,140 @@ +//! The trash: the deleted nodes of a space, and the request to remove one of them for good. +//! +//! Deleting stays what it always was, a tombstone in the change feed. The trash is that same set of tombstones seen +//! with the time of the deletion. Bringing a node back is [`Meta::restore_from_trash`] in `commit.rs`, because it +//! shares the commit path's rules for names. Removing a node from the trash here only marks it: nothing is freed, +//! so versions, chunks and the tombstone stay as they were. + +use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; +use craft_cloud_proto::{NodeId, SpaceId}; +use sqlx::Row; + +use crate::codec::{flag, to_u64, uuid_bytes, uuid_from}; +use crate::tree::space_info; +use crate::{Dialect, Meta, MetaError}; + +/// A deleted node in the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashEntry { + pub id: NodeId, + pub parent: NodeId, + pub name: String, + pub is_dir: bool, + /// The size of a file's last version. + pub size: Option, + pub deleted_ms: i64, +} + +/// One page of the trash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TrashPage { + pub entries: Vec, + /// The cursor of the next page, if there is one. + pub next: Option, + /// How many deleted nodes without a deletion time were found and stamped with `now` (a damaged database: the + /// migration and the commit path always set it). They are never treated as expired. + pub repaired: u64, +} + +impl Meta { + /// One page of the trash of `space`: its deleted nodes that were not removed from the trash, newest deletion + /// first by `(deleted_ms, id)`, after `after` (exclusive). `Invalid` for a `limit` outside 1 to + /// [`TRASH_MAX_LIMIT`]. A deleted node without a deletion time gets `now_ms` first, so the trash never lists a + /// node as older than it can be shown to be. + pub async fn trash(&self, space: SpaceId, after: Option, limit: u32, now_ms: i64) -> Result { + if limit == 0 || limit > TRASH_MAX_LIMIT { + return Err(MetaError::Invalid("a trash page has 1 to 200 items")); + } + let repaired = self.stamp_unstamped(space, now_ms).await?; + let mut tx = self.read_snapshot().await?; + let (cursor_ms, cursor_node) = match after { + Some(cursor) => (cursor.deleted_ms, uuid_bytes(cursor.node.as_uuid())), + // Above every time and every ID. + None => (i64::MAX, vec![0xff; 16]), + }; + // One row more than asked for tells whether another page follows. The row-value comparison and the order + // match the index `nodes_trash`. + let rows = sqlx::query( + "SELECT n.id, n.parent, n.name, n.is_dir, n.deleted_ms, v.size AS v_size \ + FROM nodes n LEFT JOIN versions v ON v.space_id = n.space_id AND v.id = n.version \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND (n.deleted_ms, n.id) < ($2, $3) \ + ORDER BY n.deleted_ms DESC, n.id DESC LIMIT $4", + ) + .bind(uuid_bytes(space.as_uuid())) + .bind(cursor_ms) + .bind(cursor_node) + .bind(i64::from(limit) + 1) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + let limit = usize::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?; + let more = rows.len() > limit; + let mut entries = Vec::with_capacity(rows.len().min(limit)); + for row in rows.iter().take(limit) { + let parent: Option> = row.try_get("parent")?; + let size: Option = row.try_get("v_size")?; + entries.push(TrashEntry { + id: NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?), + parent: NodeId::from_uuid(uuid_from(&parent.ok_or(MetaError::Corrupt("a deleted node has no parent"))?)?), + name: row.try_get("name")?, + is_dir: flag(row, "is_dir")?, + size: size.map(to_u64).transpose()?, + deleted_ms: row.try_get("deleted_ms")?, + }); + } + let next = if more { entries.last().map(|e| TrashCursor { deleted_ms: e.deleted_ms, node: e.id }) } else { None }; + Ok(TrashPage { entries, next, repaired }) + } + + /// Gives every deleted node of `space` that has no deletion time `now_ms`; returns how many. Looks first without + /// the write lock, since there are none in a sound database. + async fn stamp_unstamped(&self, space: SpaceId, now_ms: i64) -> Result { + let space_key = uuid_bytes(space.as_uuid()); + let any = sqlx::query("SELECT 1 FROM nodes WHERE space_id = $1 AND deleted = 1 AND deleted_ms IS NULL LIMIT 1") + .bind(space_key.clone()) + .fetch_optional(&self.pool) + .await?; + if any.is_none() { + return Ok(0); + } + let mut w = self.begin_write().await?; + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let done = sqlx::query("UPDATE nodes SET deleted_ms = $1 WHERE space_id = $2 AND deleted = 1 AND deleted_ms IS NULL") + .bind(now_ms) + .bind(space_key) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(done.rows_affected()) + } + + /// Takes the deleted node `node` out of the trash: it is no longer listed and can't be restored from the trash. + /// Only marks it (`purge_requested_ms`): no version, chunk or tombstone is touched, so the change feed and the + /// sequence number stay as they are. Asking again changes nothing. `NotFound` when `node` is not a deleted node of + /// `space` in the trash. + pub async fn request_purge(&self, space: SpaceId, node: NodeId, now_ms: i64) -> Result<(), MetaError> { + let mut w = self.begin_write().await?; + // On Postgres this row lock orders the mark with commits and restores of the space. + space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; + let row = sqlx::query("SELECT deleted, purge_requested_ms FROM nodes WHERE space_id = $1 AND id = $2") + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *w.tx) + .await? + .ok_or(MetaError::NotFound("node in the trash"))?; + if !flag(&row, "deleted")? { + return Err(MetaError::NotFound("node in the trash")); + } + if row.try_get::, _>("purge_requested_ms")?.is_some() { + return Ok(()); + } + sqlx::query("UPDATE nodes SET purge_requested_ms = $1 WHERE space_id = $2 AND id = $3") + .bind(now_ms) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *w.tx) + .await?; + w.commit().await?; + Ok(()) + } +} diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index 9906458f..ad7b6f57 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -109,6 +109,15 @@ on_every_backend!( a_restore_refuses_what_it_can_not_restore, concurrent_restores_let_exactly_one_win, a_restore_and_a_late_save_record_both_apply, + deleting_stamps_the_time_once_and_coming_back_clears_it, + the_trash_pages_newest_first_for_its_own_space, + a_restore_brings_the_deleted_folders_back_with_their_ids, + a_restore_merges_into_live_folders_and_renames_around_taken_names, + a_restore_with_no_free_name_changes_nothing, + only_nodes_in_the_trash_can_be_restored_or_removed, + removing_from_the_trash_frees_nothing, + a_trash_restore_and_a_commit_for_the_same_name_agree, + a_trash_restore_and_an_edit_of_the_file_agree, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -1787,6 +1796,10 @@ async fn upgrading_counts_what_each_space_already_stores() { sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); // And before save records. + // And before the trash. + sqlx::query("DROP INDEX nodes_trash").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN deleted_ms").execute(&mut conn).await.unwrap(); + sqlx::query("ALTER TABLE nodes DROP COLUMN purge_requested_ms").execute(&mut conn).await.unwrap(); // And before the history index. sqlx::query("DROP INDEX versions_node_seq").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE versions DROP COLUMN save").execute(&mut conn).await.unwrap(); @@ -1814,7 +1827,7 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 7); + assert_eq!(versions, 8); } /// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. @@ -2063,12 +2076,12 @@ async fn the_schema_version_is_recorded_and_a_newer_database_is_refused() { use sqlx::Connection; let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(latest, 7); + assert_eq!(latest, 8); // Opening again changes nothing. drop(Meta::open(&url).await.unwrap()); let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(rows, 7); - sqlx::query("INSERT INTO schema_version (version) VALUES (8)").execute(&mut conn).await.unwrap(); + assert_eq!(rows, 8); + sqlx::query("INSERT INTO schema_version (version) VALUES (9)").execute(&mut conn).await.unwrap(); drop(conn); assert!(matches!(Meta::open(&url).await, Err(MetaError::Corrupt(_))), "a newer database is refused"); } @@ -2097,3 +2110,488 @@ async fn a_history_page_uses_the_node_index_on_sqlite() { assert!(detail.iter().any(|d| d.contains("versions_node_seq")), "{detail:?}"); assert!(!detail.iter().any(|d| d.contains("TEMP B-TREE")), "no sort: {detail:?}"); } + +// ---- the trash ---- + +/// A folder `name` in `parent`: its node. +async fn folder(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, folder_name: &str, now: i64) -> NodeId { + let node = NodeId::generate(); + meta.commit(u.space, dev, &[Op::Mkdir { node, parent, name: name(folder_name) }], now).await.unwrap(); + node +} + +/// A file `name` in `parent` with one version: its node and version. +async fn file_in(meta: &Meta, u: &UserRecord, dev: DeviceId, parent: NodeId, file_name: &str, now: i64) -> (NodeId, NewVersion) { + let node = NodeId::generate(); + let v = version(manifest(meta, u, 6).await); + meta.commit(u.space, dev, &[Op::Create { node, parent, name: name(file_name), version: v.clone() }], now).await.unwrap(); + (node, v) +} + +async fn delete_file(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, current: VersionId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(current) }], now).await.unwrap(); +} + +async fn delete_folder(meta: &Meta, u: &UserRecord, dev: DeviceId, node: NodeId, now: i64) { + meta.commit(u.space, dev, &[Op::Delete { node, expected: None }], now).await.unwrap(); +} + +async fn trash_ids(meta: &Meta, u: &UserRecord) -> Vec { + meta.trash(u.space, None, 200, NOW).await.unwrap().entries.iter().map(|e| e.id).collect() +} + +/// The name a restore gives a node whose name is taken, in the minute of `NOW` (2025-10-09 08:53 UTC), from a device +/// called `laptop`. +fn conflict_of(original: &str, is_dir: bool, counter: u32) -> String { + let stamp = craft_cloud_proto::ConflictStamp { year: 2025, month: 10, day: 9, hour: 8, minute: 53, utc: true }; + craft_cloud_proto::conflict_name(&name(original), is_dir, "laptop", stamp, counter).unwrap().as_str().to_owned() +} + +async fn live_name(meta: &Meta, u: &UserRecord, node: NodeId) -> (Option, String, bool) { + // The feed from the start, which has deleted nodes too. + let feed = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + let n = find(&feed.nodes, node); + (n.parent, n.name.as_ref().map(|n| n.as_str().to_owned()).unwrap_or_default(), n.deleted) +} + +/// The time in the trash starts when a live node is deleted, not when a deleted one is deleted again; coming back +/// clears it, by an edit or by a restore from the trash, and the next deletion starts afresh. +async fn deleting_stamps_the_time_once_and_coming_back_clears_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (file, v) = file_in(meta, &u, dev, r, "stamp.txt", NOW).await; + assert!(trash_ids(meta, &u).await.is_empty()); + + delete_file(meta, &u, dev, file, v.id, NOW + 1_000).await; + let page = meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms, page.repaired), (1, NOW + 1_000, 0)); + // Deleting a deleted file again is no change: the time stays, and the file is not stamped twice. + delete_file(meta, &u, dev, file, v.id, NOW + 5_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 1_000); + + // An edit brings it back and leaves the trash; a later deletion starts the time again. + let edited = version(manifest(meta, &u, 7).await); + meta.commit(u.space, dev, &[Op::Update { node: file, expected: v.id, version: edited.clone() }], NOW + 6_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "an edit brings the file back"); + delete_file(meta, &u, dev, file, edited.id, NOW + 7_000).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 7_000); + + // A restore from the trash clears it too, and the next deletion starts afresh again. + meta.restore_from_trash(u.space, dev, file, NOW + 8_000).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + delete_file(meta, &u, dev, file, edited.id, NOW + 8_500).await; + assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 8_500); +} + +/// The trash lists a space's deleted files and folders, newest deletion first by `(deleted_ms, id)`, in pages without +/// gaps or repeats (also for nodes deleted in one instant), with sizes of files; it lists nothing of another space and +/// nothing that was removed from the trash. +async fn the_trash_pages_newest_first_for_its_own_space(meta: &Meta) { + use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut made = Vec::new(); + for i in 0..5 { + let (node, v) = file_in(meta, &u, dev, r, &format!("f{i}.txt"), NOW).await; + // The last three go in one instant. + delete_file(meta, &u, dev, node, v.id, NOW + 1_000 + i64::from(i.min(2)) * 10).await; + made.push(node); + } + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + delete_folder(meta, &u, dev, dir, NOW + 5_000).await; + + let all = meta.trash(u.space, None, TRASH_MAX_LIMIT, NOW).await.unwrap(); + assert_eq!((all.entries.len(), all.next), (6, None)); + assert_eq!(all.entries[0].id, dir); + assert!(all.entries[0].is_dir && all.entries[0].size.is_none() && all.entries[0].parent == r); + assert!(all.entries[1..].iter().all(|e| !e.is_dir && e.size == Some(6))); + let key = |e: &craft_cloud_meta::TrashEntry| (e.deleted_ms, *e.id.as_uuid().as_bytes()); + assert!(all.entries.windows(2).all(|w| key(&w[0]) > key(&w[1])), "strictly descending by (deleted_ms, id)"); + for limit in [1u32, 2, 4] { + let (mut seen, mut after) = (Vec::new(), None); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize); + seen.extend(page.entries.iter().cloned()); + match page.next { + Some(next) => { + let last = page.entries.last().unwrap(); + assert_eq!(next, TrashCursor { deleted_ms: last.deleted_ms, node: last.id }); + after = Some(next); + } + None => break, + } + } + assert_eq!(seen, all.entries, "limit {limit}"); + } + assert!(matches!(meta.trash(u.space, None, 0, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(meta.trash(u.space, None, TRASH_MAX_LIMIT + 1, NOW).await, Err(MetaError::Invalid(_)))); + + // Another space sees none of it. A node removed from the trash is not listed any more. + let bob = user(meta, "bob").await; + assert!(meta.trash(bob.space, None, 10, NOW).await.unwrap().entries.is_empty()); + meta.request_purge(u.space, dir, NOW + 6_000).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await.len(), 5); + assert!(!trash_ids(meta, &u).await.contains(&dir)); +} + +/// A file in deleted folders comes back with the folders, every node keeping its ID, as one change of the space. +async fn a_restore_brings_the_deleted_folders_back_with_their_ids(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (f, v) = file_in(meta, &u, dev, b, "f.txt", NOW).await; + let (sibling, sv) = file_in(meta, &u, dev, b, "sibling.txt", NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 1).await; + delete_file(meta, &u, dev, sibling, sv.id, NOW + 2).await; + delete_folder(meta, &u, dev, b, NOW + 3).await; + delete_folder(meta, &u, dev, a, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed), (f, b, "f.txt", false)); + assert_eq!((done.seq, done.epoch), (before.seq + 1, before.epoch)); + let after = meta.space_info(u.space).await.unwrap(); + assert_eq!((after.seq, after.epoch), (before.seq + 1, before.epoch), "one change of the space"); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), false)); + assert_eq!(live_name(meta, &u, b).await, (Some(a), "b".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(b), "f.txt".to_owned(), false)); + assert_eq!(trash_ids(meta, &u).await, vec![sibling], "only the node that was asked for and its folders come back"); + + // The change feed reports exactly the nodes that came back, at the new sequence number, as live. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(a, false), (b, false), (f, false)]; + expected.sort(); + assert_eq!(changed, expected); + let NodeKind::File(state) = &find(&feed.nodes, f).kind else { panic!("a file") }; + assert_eq!(state.version, v.id, "the file's last version is its content again"); +} + +/// A name taken by a file or by anything at the node itself gives a conflict name (never an overwrite); a deleted +/// folder whose name is held by a live folder is merged into it; every name tried is free, and a folder that is renamed +/// is a folder. +async fn a_restore_merges_into_live_folders_and_renames_around_taken_names(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + + // The node's own name is taken: a conflict name, and the live file is untouched. + let (f, fv) = file_in(meta, &u, dev, r, "plan.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (squatter, _) = file_in(meta, &u, dev, r, "PLAN.txt", NOW + 2).await; + let done = meta.restore_from_trash(u.space, dev, f, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("plan.txt", false, 1))); + assert_eq!(live_name(meta, &u, squatter).await, (Some(r), "PLAN.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, f).await, (Some(r), conflict_of("plan.txt", false, 1), false)); + + // A second restore of the same name in the same minute takes the next counter. + let (g, gv) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 3).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 4).await; + let (_h, _) = file_in(meta, &u, dev, r, "plan2.txt", NOW + 5).await; + let (_i, _) = file_in(meta, &u, dev, r, &conflict_of("plan2.txt", false, 1), NOW + 6).await; + let done = meta.restore_from_trash(u.space, dev, g, NOW).await.unwrap(); + assert_eq!(done.name, conflict_of("plan2.txt", false, 2)); + + // A deleted folder held by a live folder of the same name: the live folder takes what is restored. + let a = folder(meta, &u, dev, r, "a", NOW).await; + let b = folder(meta, &u, dev, a, "b", NOW).await; + let (x, xv) = file_in(meta, &u, dev, b, "x.txt", NOW).await; + delete_file(meta, &u, dev, x, xv.id, NOW + 10).await; + delete_folder(meta, &u, dev, b, NOW + 11).await; + delete_folder(meta, &u, dev, a, NOW + 12).await; + let live_a = folder(meta, &u, dev, r, "a", NOW + 13).await; + let done = meta.restore_from_trash(u.space, dev, x, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.as_str()), (false, "x.txt")); + assert_eq!(live_name(meta, &u, b).await, (Some(live_a), "b".to_owned(), false), "b moved into the live folder"); + assert_eq!(live_name(meta, &u, x).await, (Some(b), "x.txt".to_owned(), false)); + assert_eq!(live_name(meta, &u, a).await, (Some(r), "a".to_owned(), true), "the deleted a stays deleted"); + assert_eq!(trash_ids(meta, &u).await, vec![a]); + + // A deleted folder held by a file: the folder comes back under a conflict name, with its content below it. + let c = folder(meta, &u, dev, r, "c", NOW).await; + let (y, yv) = file_in(meta, &u, dev, c, "y.txt", NOW).await; + delete_file(meta, &u, dev, y, yv.id, NOW + 20).await; + delete_folder(meta, &u, dev, c, NOW + 21).await; + let (_c_file, _) = file_in(meta, &u, dev, r, "c", NOW + 22).await; + let done = meta.restore_from_trash(u.space, dev, y, NOW).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (c, "y.txt")); + assert_eq!(live_name(meta, &u, c).await, (Some(r), conflict_of("c", true, 1), false)); + + // A deleted folder restored itself, its name held by a live folder: no merge for the node asked for. + let d = folder(meta, &u, dev, r, "d", NOW).await; + delete_folder(meta, &u, dev, d, NOW + 30).await; + let live_d = folder(meta, &u, dev, r, "d", NOW + 31).await; + let done = meta.restore_from_trash(u.space, dev, d, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("d", true, 1))); + assert_eq!(live_name(meta, &u, live_d).await.1, "d"); +} + +/// When every conflict name is taken too, the restore is refused as a conflict and the space is exactly as it was: no +/// sequence number, no revived node, the node still in the trash. +async fn a_restore_with_no_free_name_changes_nothing(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "busy.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + // The name itself and its 100 conflict names, in one batch. + let m = manifest(meta, &u, 1).await; + let mut names = vec!["busy.txt".to_owned()]; + names.extend((1..=100).map(|n| conflict_of("busy.txt", false, n))); + let ops: Vec = names.iter().map(|n| Op::Create { node: NodeId::generate(), parent: r, name: name(n), version: version(m) }).collect(); + meta.commit(u.space, dev, &ops, NOW + 2).await.unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let tree_before = full_tree(meta, u.space).await.nodes.len(); + + let refused = meta.restore_from_trash(u.space, dev, f, NOW).await; + let list = refused_restore(refused); + assert!(matches!(list[0].1, ConflictKind::NameTaken { .. }), "{list:?}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, vec![f]); + assert_eq!(full_tree(meta, u.space).await.nodes.len(), tree_before); + assert!(meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap().nodes.is_empty()); +} + +/// What is not in the trash can't be restored or removed from it: a live node, an unknown one, another space's, one +/// already removed from the trash. +async fn only_nodes_in_the_trash_can_be_restored_or_removed(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (live, _) = file_in(meta, &u, dev, r, "live.txt", NOW).await; + let (gone, gv) = file_in(meta, &u, dev, r, "gone.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, r, "kept.txt", NOW).await; + delete_file(meta, &u, dev, gone, gv.id, NOW + 1).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1).await; + meta.request_purge(u.space, gone, NOW + 2).await.unwrap(); + let seq = meta.space_info(u.space).await.unwrap().seq; + let bob = user(meta, "bob").await; + let bob_device = device(meta, &bob).await; + + for node in [live, NodeId::generate(), r, gone] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "restore {node}"); + } + for node in [live, NodeId::generate(), r] { + assert!(matches!(meta.request_purge(u.space, node, NOW).await, Err(MetaError::NotFound(_))), "remove {node}"); + } + assert!(matches!(meta.restore_from_trash(bob.space, bob_device, kept, NOW).await, Err(MetaError::NotFound(_))), "another space's node"); + assert!(matches!(meta.request_purge(bob.space, kept, NOW).await, Err(MetaError::NotFound(_)))); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, seq, "nothing changed"); + assert_eq!(trash_ids(meta, &u).await, vec![kept]); +} + +/// Removing from the trash only marks the node: no version, manifest or tombstone changes, the change feed and the +/// sequence number stay, the history is still readable; asking twice changes nothing; an edit that brings the node back +/// ends the mark, and the next deletion lists it again. +async fn removing_from_the_trash_frees_nothing(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 3).await; + delete_file(meta, &u, dev, node, versions[2].id, NOW + 100).await; + let before = meta.space_info(u.space).await.unwrap(); + let feed_before = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + + meta.request_purge(u.space, node, NOW + 200).await.unwrap(); + meta.request_purge(u.space, node, NOW + 300).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty(), "it left the trash"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(), feed_before, "the tombstone is as it was"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3, "every version is still there"); + for v in &versions { + assert!(meta.has_manifest(u.space, &v.manifest).await.unwrap(), "every manifest is still the space's"); + } + + // An edit still brings the file back (an edit beats a deletion), which ends the mark. + let edit = version(manifest(meta, &u, 9).await); + meta.commit(u.space, dev, &[Op::Update { node, expected: versions[2].id, version: edit.clone() }], NOW + 400).await.unwrap(); + assert!(!live_name(meta, &u, node).await.2); + delete_file(meta, &u, dev, node, edit.id, NOW + 500).await; + let page = meta.trash(u.space, None, 10, NOW + 600).await.unwrap(); + assert_eq!((page.entries.len(), page.entries[0].deleted_ms), (1, NOW + 500)); +} + +/// A restore and a commit that wants the same name run together: both are applied in some order and the result is +/// consistent, never two live nodes with one name. +async fn a_trash_restore_and_a_commit_for_the_same_name_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + for round in 0..4 { + let file_name = format!("race{round}.txt"); + let (f, fv) = file_in(meta, &u, dev, r, &file_name, NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let rival = NodeId::generate(); + let rival_op = Op::Create { node: rival, parent: r, name: name(&file_name), version: version(manifest(meta, &u, 2).await) }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let create = tokio::spawn(async move { m2.commit(space, dev, &[rival_op], NOW + 2).await }); + let restored = restore.await.unwrap().unwrap(); + match create.await.unwrap() { + // The commit came first: it holds the name, and the restore went around it. + Ok(_) => { + assert!(restored.renamed, "round {round}"); + assert_eq!(live_name(meta, &u, rival).await.1, file_name); + } + // The restore came first: the commit finds the name taken. + Err(MetaError::Rejected(list)) => { + assert!(matches!(list[0].kind, ConflictKind::NameTaken { .. }), "round {round}"); + assert!(!restored.renamed); + assert_eq!(live_name(meta, &u, f).await.1, file_name); + } + Err(other) => panic!("unexpected error {other:?}"), + } + } + // No two live nodes share a name (the unique index would have refused it; this says what is seen). + let tree = full_tree(meta, u.space).await; + let mut names: Vec = tree.nodes.iter().filter_map(|n| n.name.as_ref().map(|n| n.fold_key().as_str().to_owned())).collect(); + let total = names.len(); + names.sort(); + names.dedup(); + assert_eq!(names.len(), total); +} + +/// A restore and an edit of the same deleted file run together: whichever is first, the file ends live with its +/// versions intact, and the second is either an update on a live file or a "not in the trash". +async fn a_trash_restore_and_an_edit_of_the_file_agree(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (f, fv) = file_in(meta, &u, dev, r, "both.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + let edit = version(manifest(meta, &u, 4).await); + let (m1, m2, space) = (meta.clone(), meta.clone(), u.space); + let edit_op = Op::Update { node: f, expected: fv.id, version: edit.clone() }; + let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); + let update = tokio::spawn(async move { m2.commit(space, dev, &[edit_op], NOW + 2).await }); + let restored = restore.await.unwrap(); + update.await.unwrap().unwrap(); + assert!(matches!(restored, Ok(_) | Err(MetaError::NotFound(_))), "{restored:?}"); + assert_eq!(live_name(meta, &u, f).await, (Some(r), "both.txt".to_owned(), false)); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.history(u.space, f, None, 10).await.unwrap().versions.len(), 2); +} + +// ---- the trash: the migration and a damaged database ---- + +/// A raw connection to the database `url`, for what the library never does (reading and breaking columns). +async fn raw(url: &str) -> sqlx::AnyConnection { + use sqlx::Connection; + sqlx::any::install_default_drivers(); + sqlx::AnyConnection::connect(url).await.unwrap() +} + +/// Puts the database back to the state before the trash migration: no columns, no index, one version fewer. +async fn undo_the_trash_migration(url: &str) { + let mut conn = raw(url).await; + for statement in [ + "DROP INDEX nodes_trash", + "ALTER TABLE nodes DROP COLUMN deleted_ms", + "ALTER TABLE nodes DROP COLUMN purge_requested_ms", + "DELETE FROM schema_version WHERE version >= 8", + ] { + sqlx::query(statement).execute(&mut conn).await.unwrap(); + } +} + +fn wall_clock_ms() -> i64 { + i64::try_from(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()).unwrap() +} + +/// Nodes deleted before the trash existed start their time in the trash at the migration, once: live nodes get no +/// time, and opening the database again does not stamp them again. +async fn the_trash_migration_dates_old_deletions_at_the_migration(url: &str) { + let (ana, file, folder_node) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "old.txt", NOW).await; + let (_kept, _) = file_in(&meta, &ana, dev, r, "kept.txt", NOW).await; + let dir = folder(&meta, &ana, dev, r, "olddir", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + delete_folder(&meta, &ana, dev, dir, NOW + 2).await; + (ana, file, dir) + }; + undo_the_trash_migration(url).await; + let before = wall_clock_ms(); + let meta = Meta::open(url).await.unwrap(); + let after = wall_clock_ms(); + let page = meta.trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(page.repaired, 0, "the migration stamped them, nothing was left to repair"); + let mut ids: Vec<_> = page.entries.iter().map(|e| e.id).collect(); + ids.sort(); + let mut expected = vec![file, folder_node]; + expected.sort(); + assert_eq!(ids, expected, "the live file is not in the trash"); + let stamps: Vec = page.entries.iter().map(|e| e.deleted_ms).collect(); + assert!(stamps.iter().all(|t| (before - 1_000..=after + 1_000).contains(t)), "{stamps:?} not in {before}..{after}"); + assert_eq!(stamps[0], stamps[1], "one baseline for all of them"); + // Another open changes nothing. + drop(meta); + let again = Meta::open(url).await.unwrap().trash(ana.space, None, 10, 0).await.unwrap(); + assert_eq!(again.entries, page.entries); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 8); + let live_with_time: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM nodes WHERE deleted = 0 AND (deleted_ms IS NOT NULL OR purge_requested_ms IS NOT NULL)") + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(live_with_time, 0); +} + +#[tokio::test] +async fn the_trash_migration_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_trash_migration_dates_old_deletions_at_the_migration(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_trash_migration_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_trash_migration_dates_old_deletions_at_the_migration(&url).await; +} + +/// A deleted node without a deletion time (it can't happen through this code) is stamped with the time of the read +/// and reported, never listed as expired; the stamp stays. +async fn a_deleted_node_without_a_time_is_stamped_when_read(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let (file, v) = file_in(&meta, &ana, dev, r, "damaged.txt", NOW).await; + delete_file(&meta, &ana, dev, file, v.id, NOW + 1).await; + let mut conn = raw(url).await; + sqlx::query("UPDATE nodes SET deleted_ms = NULL WHERE deleted = 1").execute(&mut conn).await.unwrap(); + let page = meta.trash(ana.space, None, 10, NOW + 77).await.unwrap(); + assert_eq!((page.repaired, page.entries.len(), page.entries[0].deleted_ms), (1, 1, NOW + 77)); + let later = meta.trash(ana.space, None, 10, NOW + 99).await.unwrap(); + assert_eq!((later.repaired, later.entries[0].deleted_ms), (0, NOW + 77)); +} + +#[tokio::test] +async fn a_damaged_deletion_time_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_deleted_node_without_a_time_is_stamped_when_read(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_damaged_deletion_time_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_deleted_node_without_a_time_is_stamped_when_read(&url).await; +} From aa7f0826bf1f6d0a05f19434b8f041b3b53c9e45 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:04:35 +0100 Subject: [PATCH 08/27] server: add the trash routes and retention.trash_days GET /api/v1/trash, POST /api/v1/trash/{id}/restore and DELETE /api/v1/trash/{id}, resolved in the caller's own space, and the [retention] trash_days setting (default 30, 1 to 3650) that sets purge_after_ms. --- apps/craft-cloud-server/src/api.rs | 83 +++++++++++++- apps/craft-cloud-server/src/config.rs | 52 +++++++++ apps/craft-cloud-server/src/main.rs | 2 +- apps/craft-cloud-server/src/state.rs | 9 ++ apps/craft-cloud-server/tests/api.rs | 159 ++++++++++++++++++++++++++ 5 files changed, 303 insertions(+), 2 deletions(-) diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index 4bbef60f..ac45b4ca 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -16,7 +16,8 @@ use axum::{Json, Router, middleware}; use craft_cloud_meta::{BlobKind, DeviceAuth, valid_username}; use craft_cloud_proto::api::{ CommitRequest, CommitResponse, DeviceInfo, HISTORY_DEFAULT_LIMIT, HistoryCursor, HistoryResponse, InfoResponse, LoginRequest, LoginResponse, - MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, RestoreRequest, RestoreResponse, SpaceResponse, codes, + MAX_MISSING_IDS, MissingRequest, MissingResponse, PROTOCOL_MAX, PROTOCOL_MIN, RestoreRequest, RestoreResponse, SpaceResponse, + TRASH_DEFAULT_LIMIT, TrashCursor, TrashItem, TrashResponse, TrashRestoreResponse, codes, }; use craft_cloud_proto::limits::{MAX_BATCH_JSON_BYTES, MAX_BATCH_OPS, MAX_CHUNK_BYTES, MAX_MANIFEST_BYTES, MAX_PAGE_NODES}; use craft_cloud_proto::{ChangeSet, ChunkId, DeviceId, Manifest, ManifestId, NodeId, PageCursor}; @@ -58,6 +59,9 @@ pub fn router(state: Shared) -> Router { .route("/saves", post(attach_save).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) .route("/nodes/{id}/history", get(history)) .route("/nodes/{id}/restore", post(restore).layer(DefaultBodyLimit::max(SMALL_JSON_BYTES))) + .route("/trash", get(trash)) + .route("/trash/{id}/restore", post(restore_from_trash)) + .route("/trash/{id}", delete(remove_from_trash)) .route("/space", get(space)) .route("/tree", get(tree)) .route("/changes", get(changes)) @@ -432,6 +436,83 @@ async fn restore( Ok(Json(RestoreResponse { version: outcome.version, parent_version: outcome.parent_version, seq: outcome.seq, epoch: outcome.epoch })) } +const MS_PER_DAY: i64 = 86_400_000; + +#[derive(Debug, Deserialize)] +struct TrashQuery { + cursor: Option, + limit: Option, +} + +/// `GET /trash`: the deleted nodes of the caller's own space, newest deletion first. +async fn trash( + State(state): State, + Extension(auth): Extension, + q: Result, QueryRejection>, +) -> Result, ApiError> { + let Query(q) = q?; + let after = q.cursor.as_deref().map(str::parse::).transpose().map_err(|_| ApiError::invalid("bad trash cursor"))?; + let page = state.meta.trash(auth.space, after, q.limit.unwrap_or(TRASH_DEFAULT_LIMIT), state.now_ms()).await?; + if page.repaired > 0 { + tracing::warn!(count = page.repaired, "deleted nodes without a deletion time were stamped with the current time"); + } + let keep = i64::from(state.trash_days).saturating_mul(MS_PER_DAY); + let items = page + .entries + .into_iter() + .map(|e| TrashItem { + id: e.id, + parent: e.parent, + name: e.name, + is_dir: e.is_dir, + size: e.size, + deleted_ms: e.deleted_ms, + purge_after_ms: e.deleted_ms.saturating_add(keep), + }) + .collect(); + Ok(Json(TrashResponse { items, next: page.next.map(|c| c.to_string()) })) +} + +/// `POST /trash/{id}/restore`: a deleted node of the caller's space comes back, in one change. +async fn restore_from_trash( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result, ApiError> { + let Path(node) = id?; + // As a commit: the publish is never skipped for a change that went through. + let task_state = state.clone(); + let outcome = tokio::spawn(async move { + let outcome = task_state.meta.restore_from_trash(auth.space, auth.device, node, task_state.now_ms()).await?; + task_state.notifier.publish(auth.space, outcome.seq); + Ok::<_, ApiError>(outcome) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "trash restore task failed"); + ApiError::internal() + })??; + Ok(Json(TrashRestoreResponse { + node: outcome.node, + parent: outcome.parent, + name: outcome.name, + renamed: outcome.renamed, + seq: outcome.seq, + epoch: outcome.epoch, + })) +} + +/// `DELETE /trash/{id}`: takes a deleted node out of the trash. Only marks it: nothing is freed. +async fn remove_from_trash( + State(state): State, + Extension(auth): Extension, + id: Result, PathRejection>, +) -> Result { + let Path(node) = id?; + state.meta.request_purge(auth.space, node, state.now_ms()).await?; + Ok(StatusCode::NO_CONTENT) +} + fn cursor(after_seq: Option, after_node: Option) -> Result, ApiError> { match (after_seq, after_node) { (Some(seq), Some(node)) => Ok(Some(PageCursor { seq, node })), diff --git a/apps/craft-cloud-server/src/config.rs b/apps/craft-cloud-server/src/config.rs index 725e9a48..5301ac27 100644 --- a/apps/craft-cloud-server/src/config.rs +++ b/apps/craft-cloud-server/src/config.rs @@ -9,6 +9,9 @@ //! # live_bytes_per_user = 1_073_741_824 # cap on what one account's live documents hold, in bytes (1 GiB) //! # public_url = "https://cloud.example.com" # where computers reach the server; `user invite` prints it //! +//! # [retention] +//! # trash_days = 30 # how long a deleted file stays in the trash before the server may remove it (1 to 3650) +//! //! [database] //! url = "sqlite:/var/lib/craft-cloud/meta.db" # or postgres://user:pass@host/db //! @@ -55,12 +58,39 @@ pub enum ConfigError { agent could not tell a full account from a file that is too big. Raise it, or remove it for no cap" )] StoredBytesCapTooSmall(u64, u32), + #[error("retention.trash_days = {0} is not between {MIN_TRASH_DAYS} and {MAX_TRASH_DAYS}")] + TrashDays(u32), #[error("public_url = {0:?} is not a server address like https://cloud.example.com (a host, an optional port and path, no query)")] PublicUrl(String), #[error("storage.{0} is missing (set it in the config or the {1} environment variable)")] MissingS3Credential(&'static str, &'static str), } +/// Fewest and most days a deleted node waits in the trash. +pub const MIN_TRASH_DAYS: u32 = 1; +pub const MAX_TRASH_DAYS: u32 = 3650; +/// Days in the trash when `retention.trash_days` is not set. +pub const DEFAULT_TRASH_DAYS: u32 = 30; + +/// `[retention]`: how long things are kept. Only the trash today; nothing is removed by it yet. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RetentionConfig { + /// Days a deleted node stays in the trash before the server may remove it for good (shown as `purge_after_ms`). + #[serde(default = "default_trash_days")] + pub trash_days: u32, +} + +impl Default for RetentionConfig { + fn default() -> Self { + Self { trash_days: DEFAULT_TRASH_DAYS } + } +} + +fn default_trash_days() -> u32 { + DEFAULT_TRASH_DAYS +} + #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { @@ -90,6 +120,8 @@ pub struct Config { /// trailing slash. Unset: `user invite` prints the bare code and says to put the address in front. #[serde(default)] pub public_url: Option, + #[serde(default)] + pub retention: RetentionConfig, pub database: DatabaseConfig, pub storage: StorageConfig, } @@ -186,6 +218,9 @@ impl Config { { return Err(ConfigError::StoredBytesCapTooSmall(cap, MAX_CHUNK_BYTES)); } + if !(MIN_TRASH_DAYS..=MAX_TRASH_DAYS).contains(&config.retention.trash_days) { + return Err(ConfigError::TrashDays(config.retention.trash_days)); + } let mut config = config; if let Some(url) = &config.public_url { config.public_url = Some(public_url(url)?); @@ -323,6 +358,23 @@ path = "/tmp/blobs" assert!(matches!(Config::parse(&no_proxy), Err(ConfigError::IgnoreForwardedForMisplaced))); } + #[test] + fn the_trash_period_has_a_default_and_bounds() { + assert_eq!(Config::parse(DISK).unwrap().retention, RetentionConfig { trash_days: 30 }); + // A table has to come after the top-level keys, so the section goes at the end. + let at_end = |days: &str| Config::parse(&format!("{DISK}\n[retention]\ntrash_days = {days}\n")); + assert_eq!(at_end("1").unwrap().retention.trash_days, 1); + assert_eq!(at_end("3650").unwrap().retention.trash_days, 3650); + for bad in ["0", "3651", "-1", "\"30\"", "1.5"] { + assert!(at_end(bad).is_err(), "{bad}"); + } + assert!(matches!(at_end("0"), Err(ConfigError::TrashDays(0)))); + assert!(matches!(at_end("3651"), Err(ConfigError::TrashDays(3651)))); + // Nothing but `trash_days` is accepted in the section, and the section may be empty. + assert!(matches!(Config::parse(&format!("{DISK}\n[retention]\ngc_wait_hours = 24\n")), Err(ConfigError::Parse(_)))); + assert_eq!(Config::parse(&format!("{DISK}\n[retention]\n")).unwrap().retention.trash_days, 30); + } + #[test] fn unknown_keys_are_errors() { assert!(matches!(Config::parse(&format!("lisen = 1\n{DISK}")), Err(ConfigError::Parse(_)))); diff --git a/apps/craft-cloud-server/src/main.rs b/apps/craft-cloud-server/src/main.rs index 99384030..7422a5af 100644 --- a/apps/craft-cloud-server/src/main.rs +++ b/apps/craft-cloud-server/src/main.rs @@ -35,7 +35,7 @@ async fn run_server(config_path: &std::path::Path) -> Result<(), String> { tracing::warn!("ignore_forwarded_for is set: every client shares one address for sign-in rate limits"); } let (meta, store) = open(&config).await.map_err(|e| e.to_string())?; - let state = AppState::new(meta, store, config.trusted_proxies.clone()).map_err(|e| e.message)?; + let state = AppState::new(meta, store, config.trusted_proxies.clone()).map_err(|e| e.message)?.with_trash_days(config.retention.trash_days); let listener = tokio::net::TcpListener::bind(config.listen).await.map_err(|e| format!("can't listen on {}: {e}", config.listen))?; tracing::info!(listen = %config.listen, behind_proxy = config.behind_proxy, "Craft Cloud server running"); let shutdown = async { diff --git a/apps/craft-cloud-server/src/state.rs b/apps/craft-cloud-server/src/state.rs index 10d7edc8..5cf81a66 100644 --- a/apps/craft-cloud-server/src/state.rs +++ b/apps/craft-cloud-server/src/state.rs @@ -122,6 +122,8 @@ pub struct AppState { pub(crate) touches: Mutex>, /// Verified against when the username doesn't exist, so both cases cost the same. pub dummy_hash: String, + /// Days a deleted node stays in the trash before the server may remove it (`retention.trash_days`). + pub trash_days: u32, /// The time of day for everything a request records (a device's last use, a stored blob, a commit). clock: Clock, } @@ -143,6 +145,7 @@ impl AppState { limits, touches: Mutex::new(HashMap::new()), dummy_hash: hash_password("craft-cloud dummy password")?, + trash_days: 30, clock: Arc::new(now_ms), }) } @@ -156,6 +159,12 @@ impl AppState { self } + /// The state with another trash period in days (`retention.trash_days`; the default is 30). + pub fn with_trash_days(mut self, days: u32) -> Self { + self.trash_days = days; + self + } + /// The state with another clock: a simulation's, so that what requests record and when a device's last use is /// written again follow simulated time, never the real one. A test seam (the `test-support` feature). #[cfg(feature = "test-support")] diff --git a/apps/craft-cloud-server/tests/api.rs b/apps/craft-cloud-server/tests/api.rs index 54207bbf..f111ca4c 100644 --- a/apps/craft-cloud-server/tests/api.rs +++ b/apps/craft-cloud-server/tests/api.rs @@ -162,6 +162,7 @@ on_every_backend!( the_history_of_a_file_pages_newest_first_for_its_owner_only, restoring_an_old_version_makes_a_new_one, parallel_restores_against_one_head_let_one_win, + the_trash_lists_restores_and_removes_over_http, every_rejection_is_a_json_error, one_request_cant_list_unbounded_missing_chunks, long_polls_are_capped_per_space, @@ -1618,3 +1619,161 @@ async fn parallel_restores_against_one_head_let_one_win(s: &Server) { assert_eq!((won.len(), conflicted), (1, 5)); assert_eq!(space_of(s, &ana).await.seq, before + 1); } + +// ---- the trash ---- + +const DAY_MS: i64 = 86_400_000; + +async fn trash_page(s: &Server, token: &str, query: &str) -> reqwest::Response { + s.client.get(format!("{}/trash{query}", s.base)).bearer_auth(token).send().await.unwrap() +} + +async fn trash_restore(s: &Server, token: &str, node: NodeId) -> reqwest::Response { + s.client.post(format!("{}/trash/{node}/restore", s.base)).bearer_auth(token).send().await.unwrap() +} + +async fn trash_remove(s: &Server, token: &str, node: NodeId) -> reqwest::Response { + s.client.delete(format!("{}/trash/{node}", s.base)).bearer_auth(token).send().await.unwrap() +} + +/// Uploads `data` and makes it the file `node` in `parent`: its first version. +async fn new_file(s: &Server, token: &str, parent: NodeId, file_name: &str, data: &[u8]) -> (NodeId, VersionId) { + let node = NodeId::generate(); + let manifest = upload(s, token, data).await; + let op = create(node, parent, file_name, manifest); + let Op::Create { version, .. } = &op else { panic!("a create") }; + let id = version.id; + assert_eq!(commit(s, token, &[op]).await.status(), StatusCode::OK); + (node, id) +} + +async fn delete_node(s: &Server, token: &str, node: NodeId, version: VersionId) { + assert_eq!(commit(s, token, &[Op::Delete { node, expected: Some(version) }]).await.status(), StatusCode::OK); +} + +async fn the_trash_lists_restores_and_removes_over_http(s: &Server) { + use craft_cloud_proto::api::{HistoryResponse, TrashResponse, TrashRestoreResponse}; + add_user(s, "ana").await; + add_user(s, "bob").await; + let (ana, bob) = (login(s, "ana").await, login(s, "bob").await); + let r = root(s, &ana).await; + let tag = uuid::Uuid::now_v7(); + + let empty: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + assert_eq!((empty.items.len(), empty.next), (0, None)); + let mut files = Vec::new(); + for i in 0..3 { + let data = format!("trash {tag} {i}").into_bytes(); + let (node, version) = new_file(s, &ana, r, &format!("t{i}.txt"), &data).await; + delete_node(s, &ana, node, version).await; + files.push((node, version, data)); + } + + // Newest deletion first, with the time the server will wait (30 days by default). + let all: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + assert_eq!(all.items.len(), 3); + assert!(all.items.iter().all(|i| i.parent == r && !i.is_dir && i.purge_after_ms == i.deleted_ms + 30 * DAY_MS && i.deleted_ms > 0)); + assert!(all.items.windows(2).all(|w| (w[0].deleted_ms, w[0].id.to_string()) > (w[1].deleted_ms, w[1].id.to_string())), "newest first"); + assert_eq!(all.items.iter().find(|i| i.id == files[1].0).unwrap().size, Some(files[1].2.len() as u64)); + let (mut seen, mut query) = (Vec::new(), "?limit=2".to_owned()); + loop { + let page: TrashResponse = trash_page(s, &ana, &query).await.json().await.unwrap(); + seen.extend(page.items.iter().map(|i| i.id)); + match page.next { + Some(next) => query = format!("?limit=2&cursor={next}"), + None => break, + } + } + assert_eq!(seen, all.items.iter().map(|i| i.id).collect::>()); + for bad in ["?limit=0", "?limit=201", "?cursor=nonsense", "?cursor=%201.00000000-0000-0000-0000-000000000000"] { + let (status, body) = error(trash_page(s, &ana, bad).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid"), "{bad}"); + } + + // Nobody else sees, restores or removes it; no token, no trash. + let theirs: TrashResponse = trash_page(s, &bob, "").await.json().await.unwrap(); + assert!(theirs.items.is_empty()); + for res in [trash_restore(s, &bob, files[0].0).await, trash_remove(s, &bob, files[0].0).await] { + let (status, body) = error(res).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found")); + } + assert_eq!(trash_page(s, "not-a-token", "").await.status(), StatusCode::UNAUTHORIZED); + assert_eq!(s.client.post(format!("{}/trash/{}/restore", s.base, files[0].0)).send().await.unwrap().status(), StatusCode::UNAUTHORIZED); + assert_eq!(s.client.delete(format!("{}/trash/{}", s.base, files[0].0)).send().await.unwrap().status(), StatusCode::UNAUTHORIZED); + + // A restore is one change: the file is live again with its content. + let before = space_of(s, &ana).await; + let res = trash_restore(s, &ana, files[0].0).await; + assert_eq!(res.status(), StatusCode::OK); + let done: TrashRestoreResponse = res.json().await.unwrap(); + assert_eq!( + (done.node, done.parent, done.name.as_str(), done.renamed, done.seq, done.epoch), + (files[0].0, r, "t0.txt", false, before.seq + 1, before.epoch) + ); + let feed: ChangeSet = + s.client.get(format!("{}/changes?since={}", s.base, before.seq)).bearer_auth(&ana).send().await.unwrap().json().await.unwrap(); + let state = feed.nodes.iter().find(|n| n.id == files[0].0).unwrap(); + assert!(!state.deleted); + let NodeKind::File(file) = &state.kind else { panic!("a file") }; + assert_eq!(download(s, &ana, &file.manifest).await, files[0].2); + let after: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + assert_eq!(after.items.len(), 2); + // Not in the trash any more, and not there at all. + for node in [files[0].0, NodeId::generate(), r] { + let (status, body) = error(trash_restore(s, &ana, node).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found"), "{node}"); + let (status, body) = error(trash_remove(s, &ana, node).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found"), "{node}"); + } + let (status, body) = error(s.client.post(format!("{}/trash/not-a-uuid/restore", s.base)).bearer_auth(&ana).send().await.unwrap()).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid")); + + // The name was taken meanwhile: the restore goes around it and says so, and the new file is untouched. + let (taken, _) = new_file(s, &ana, r, "t1.txt", format!("squatter {tag}").as_bytes()).await; + let done: TrashRestoreResponse = trash_restore(s, &ana, files[1].0).await.json().await.unwrap(); + assert!(done.renamed && done.name.starts_with("t1 (conflict, laptop, ") && done.name.ends_with(").txt"), "{}", done.name); + let tree: ChangeSet = s.client.get(format!("{}/tree", s.base)).bearer_auth(&ana).send().await.unwrap().json().await.unwrap(); + let names: Vec<_> = + tree.nodes.iter().filter(|n| n.id == taken || n.id == files[1].0).map(|n| n.name.as_ref().unwrap().as_str().to_owned()).collect(); + assert!(names.contains(&"t1.txt".to_owned()) && names.contains(&done.name), "{names:?}"); + + // Removing from the trash only marks it: the history and the content are still there, nothing changes in the feed. + let seq = space_of(s, &ana).await.seq; + assert_eq!(trash_remove(s, &ana, files[2].0).await.status(), StatusCode::NO_CONTENT); + assert_eq!(trash_remove(s, &ana, files[2].0).await.status(), StatusCode::NO_CONTENT, "asking again is the same"); + assert_eq!(space_of(s, &ana).await.seq, seq); + let left: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + assert!(left.items.is_empty()); + let history: HistoryResponse = history_page(s, &ana, files[2].0, "").await.json().await.unwrap(); + assert_eq!(history.versions.len(), 1); + assert_eq!(download(s, &ana, &manifest_for(&files[2].2).id()).await, files[2].2); + let (status, body) = error(trash_restore(s, &ana, files[2].0).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found")); + // The restore of a version refuses a deleted file, and points to the trash. + let (status, body) = error(restore_request(s, &ana, files[2].0, json!({ "version": files[2].1, "expected_version": files[2].1 })).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::BAD_REQUEST, "invalid")); + assert!(body.message.contains("/trash/"), "{}", body.message); +} + +/// A server with another `retention.trash_days` (SQLite and disk). +async fn sqlite_disk_with_trash_days(days: u32) -> Server { + let db = tempfile::tempdir().unwrap(); + let blobs = tempfile::tempdir().unwrap(); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap(); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]).unwrap().with_trash_days(days); + start_state(state, meta, vec![db, blobs], None).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_purge_time_follows_the_configured_trash_days() { + use craft_cloud_proto::api::TrashResponse; + let s = sqlite_disk_with_trash_days(7).await; + add_user(&s, "ana").await; + let ana = login(&s, "ana").await; + let r = root(&s, &ana).await; + let (node, version) = new_file(&s, &ana, r, "week.txt", b"seven days").await; + delete_node(&s, &ana, node, version).await; + let page: TrashResponse = trash_page(&s, &ana, "").await.json().await.unwrap(); + assert_eq!(page.items[0].purge_after_ms - page.items[0].deleted_ms, 7 * DAY_MS); +} From 4023e5ca7836e383a7cf13c953b96f81ba4fc68a Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:04:36 +0100 Subject: [PATCH 09/27] docs: describe the trash Adds the trash routes and rules to the sync design and a section with the new setting and the rollout note to the server guide. --- docs/design/sync.md | 22 +++++++++++++++++++++- docs/server.md | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/design/sync.md b/docs/design/sync.md index c466aa19..563f46e3 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -657,6 +657,9 @@ methods answer `not_found` / `invalid` (405) the same way: | `POST /commit` | batch of version/mkdir/move/delete operations, one transaction; `409 conflict` listing every conflicting operation (a version whose manifest isn't uploaded is a `missing_manifest` conflict) | | `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | | `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | +| `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash, newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | +| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders, as one change: `{node, parent, name, renamed, seq, epoch}`; a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | +| `DELETE /trash/{id}` | takes a node out of the trash (204): only marks it for removal, nothing is freed yet | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | @@ -678,6 +681,23 @@ refused, so file versions and live-document state stay separate. Unknown node, f another file and a missing manifest are all `not_found`; a deleted file, a folder and a file with a live document are `invalid`; a stale `expected_version` is `conflict` (409, with the conflict list as for a commit). +**Trash.** The trash is the set of tombstones with the time of the deletion. `nodes.deleted_ms` is set when a node goes +from live to deleted (not when a deleted node is deleted again) and cleared when it comes back, by an edit or by a restore +from the trash, so the next deletion starts its time afresh; `nodes.purge_requested_ms` is set by `DELETE /trash/{id}` +and cleared when the node comes back. Nodes deleted before the migration that added them are dated at the migration. A +deleted node without a time (damaged data) is dated at the next read of the trash, logged without names, and never +counts as expired. `DELETE /trash/{id}` frees nothing: versions, manifests, chunks and the tombstone stay, the change feed +and the sequence number do not change, and the node's history stays readable; the node only leaves the trash list and can +no longer be restored from it. (An edit that brings it back still wins, as for any deleted file.) Asking twice is fine. +A restore from the trash brings back the node asked for and the deleted folders above it, root-most first, keeping their +IDs; it creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: +where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted +folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back +as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is +taken too, and a restore with no free name is `conflict` and changes nothing. The sync engine sees the restored nodes as ordinary changes to live +nodes, possibly with a new parent or name. Restoring a *version* of a deleted file is still refused: bring the file back +from the trash first. + **Client contract** (what the sync agent relies on): - Probe `GET /api/info` and refuse to sync when the protocol ranges don't overlap. @@ -1078,7 +1098,7 @@ M3b2 is split in two. **M3b2a** hardens the engine and the real disk; **M3b2b** | M3b2b1 | Agent on its own: seed 112337, `run` loop, watcher, long-poll, stability window, backoff, status file | DONE (2026-10-14) | | M3b2b2 | Simulated network (`turmoil`): partitions, delays, dropped connections | DONE (2026-10-15) | | M3c | Fixes from the full repo review | DONE (2026-10-15) | -| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | IN PROGRESS: history and restore of file versions done (server API only); the rest TODO | +| M4 | History, restore, trash, thinning, GC (including the chunks of uploads the stored-bytes cap refused), logical quotas, admin CLI (`user remove\|quota`, `gc`, `verify`), backup/restore | IN PROGRESS: history and restore of file versions, and the trash (list, restore, mark for removal), done (server API only); the rest TODO | | M5 | Selective sync, large-media polish (a few chunks in flight at once), ignore rules | TODO | | M6 | Packaging: binaries, Docker, systemd, built-in TLS, self-hosting guide | TODO | | M7 | Local control API + launcher integration PR | TODO | diff --git a/docs/server.md b/docs/server.md index 9440d093..050493f5 100644 --- a/docs/server.md +++ b/docs/server.md @@ -63,6 +63,23 @@ which adds a new version at the end of the file's history rather than rewinding through the normal change feed. This needs no setting and no database migration, and an older server answers `not_found` for both paths. The agent and the apps do not use them yet. +## The trash + +A deleted file or folder stays in the trash, and in the server's records, until it is removed for good. A signed-in +device lists the trash (`GET /api/v1/trash`), brings a node back (`POST /api/v1/trash/{id}/restore`, which names it +differently if its name has been taken meanwhile) or takes it out of the trash (`DELETE /api/v1/trash/{id}`). Taking +it out only marks it: **nothing is deleted from the server yet**, no version or chunk is freed, and the node can't be +brought back from the trash afterwards. The only setting is the number of days a deleted node waits, shown to clients as +`purge_after_ms`; it removes nothing yet: + +```toml +[retention] +trash_days = 30 # 1 to 3650 +``` + +The change adds two columns and an index to the metadata (migration 0009 in the proposal). An older server refuses the +upgraded database and rejects the `[retention]` section, so take a backup before upgrading and do not roll back. + ## Restoring a backup With the server stopped, restore the database and the chunk storage from the backup, then run From 135688a44d0be45714ab1356533144062bd09ec1 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:36:53 +0100 Subject: [PATCH 10/27] meta, proto: bring a file back as a new version and keep restored folders within the depth limit A file restored from the trash now gets a new current version with the content of its last one (no save record). A machine that sent a delete against the old version and never heard the answer can no longer delete the restored file again: the server refuses that delete as a version mismatch, and the engine already lets go of a node that came back at another version. The restore answer reports the new version. A folder that comes back below a folder that has since moved deep is refused as too deep, and nothing changes. --- crates/meta/src/commit.rs | 51 ++++++++++++++- crates/meta/tests/behaviour.rs | 109 ++++++++++++++++++++++++++++++--- crates/proto/src/api.rs | 4 ++ 3 files changed, 156 insertions(+), 8 deletions(-) diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index 33ac7073..7419cd62 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -50,6 +50,8 @@ pub struct TrashRestoreOutcome { pub parent: NodeId, pub name: String, pub renamed: bool, + /// The file's new version (a copy of its last one); none for a folder. + pub version: Option, pub seq: u64, pub epoch: Epoch, } @@ -490,13 +492,60 @@ impl Ctx<'_> { } } }; + // A folder comes back where it did not stand before (below a live folder that has moved, or merged into + // another): it must not take the tree past its depth limit. A file may sit in the deepest folder. + if (!is_target || is_dir) + && let Err(kind) = self.check_placement(parent, None).await? + { + return Err(rejected(kind)); + } self.revive(a.id, parent, &final_name).await?; parent = a.id; } + // A file comes back as a new version with the content of its last one. A machine that sent the delete and + // never heard the answer still names the old version in it, and so can't delete the restored file again: the + // server refuses that delete as a version mismatch, and the machine's engine already treats a node that came + // back at another version as one whose delete went through. + let version = if is_dir { None } else { Some(self.renew_version(node).await?) }; let placed = self.node(node).await?.ok_or(MetaError::Corrupt("a restored node vanished"))?; let parent = placed.parent.ok_or(MetaError::Corrupt("a restored node has no parent"))?; let original = up.first().map(|a| a.name.as_str()).unwrap_or_default(); - Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, seq: 0, epoch: self.epoch }) + Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, version, seq: 0, epoch: self.epoch }) + } + + /// Gives the file `node` a new current version with the manifest, modification time and executable bit of its last + /// one (no save record), after it. Returns the new version's ID. + async fn renew_version(&mut self, node: NodeId) -> Result { + let row = sqlx::query( + "SELECT n.version AS last, v.manifest, v.mtime_ms, v.exec FROM nodes n \ + JOIN versions v ON v.space_id = n.space_id AND v.id = n.version WHERE n.space_id = $1 AND n.id = $2", + ) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .fetch_optional(&mut *self.conn) + .await? + .ok_or(MetaError::Corrupt("a file has no current version"))?; + let last = VersionId::from_uuid(uuid_from(&row.try_get::, _>("last")?)?); + let new = NewVersion { + id: VersionId::generate(), + manifest: ManifestId::from_bytes(hash_from(&row.try_get::, _>("manifest")?)?), + mtime_ms: row.try_get("mtime_ms")?, + exec: flag(&row, "exec")?, + save: None, + }; + match self.insert_version(node, Some(last), &new).await? { + Ok(()) => {} + // The space no longer holds the manifest: the content is not there to bring back. + Err(ConflictKind::MissingManifest { .. }) => return Err(MetaError::NotFound("manifest")), + Err(kind) => return Err(rejected(kind)), + } + sqlx::query("UPDATE nodes SET version = $1 WHERE space_id = $2 AND id = $3") + .bind(uuid_bytes(new.id.as_uuid())) + .bind(self.space()) + .bind(uuid_bytes(node.as_uuid())) + .execute(&mut *self.conn) + .await?; + Ok(new.id) } /// `node` and its ancestors, nearest first; stops after `MAX_TREE_DEPTH + 2` rows, so a walk that did not diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index ad7b6f57..385f0e9f 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -118,6 +118,8 @@ on_every_backend!( removing_from_the_trash_frees_nothing, a_trash_restore_and_a_commit_for_the_same_name_agree, a_trash_restore_and_an_edit_of_the_file_agree, + a_restored_file_gets_a_new_version_with_its_old_content, + a_restore_does_not_exceed_the_depth_limit, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -2178,9 +2180,10 @@ async fn deleting_stamps_the_time_once_and_coming_back_clears_it(meta: &Meta) { assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 7_000); // A restore from the trash clears it too, and the next deletion starts afresh again. - meta.restore_from_trash(u.space, dev, file, NOW + 8_000).await.unwrap(); + let restored = meta.restore_from_trash(u.space, dev, file, NOW + 8_000).await.unwrap(); assert!(trash_ids(meta, &u).await.is_empty()); - delete_file(meta, &u, dev, file, edited.id, NOW + 8_500).await; + // The restore made a new current version: the next delete names it. + delete_file(meta, &u, dev, file, restored.version.unwrap(), NOW + 8_500).await; assert_eq!(meta.trash(u.space, None, 10, NOW + 9_000).await.unwrap().entries[0].deleted_ms, NOW + 8_500); } @@ -2270,7 +2273,9 @@ async fn a_restore_brings_the_deleted_folders_back_with_their_ids(meta: &Meta) { expected.sort(); assert_eq!(changed, expected); let NodeKind::File(state) = &find(&feed.nodes, f).kind else { panic!("a file") }; - assert_eq!(state.version, v.id, "the file's last version is its content again"); + assert_eq!(state.manifest, v.manifest, "the file's content is what it was"); + assert_eq!(Some(state.version), done.version, "as a new version after the last one"); + assert_ne!(state.version, v.id); } /// A name taken by a file or by anything at the node itself gives a conflict name (never an overwrite); a deleted @@ -2458,8 +2463,9 @@ async fn a_trash_restore_and_a_commit_for_the_same_name_agree(meta: &Meta) { assert_eq!(names.len(), total); } -/// A restore and an edit of the same deleted file run together: whichever is first, the file ends live with its -/// versions intact, and the second is either an update on a live file or a "not in the trash". +/// A restore and an edit of the same deleted file run together: whichever is first, the file ends live with its versions +/// intact. If the edit is first it brings the file back and the restore finds nothing in the trash; if the restore is +/// first it makes a new current version, so the edit, made on the old one, is refused as a stale update. async fn a_trash_restore_and_an_edit_of_the_file_agree(meta: &Meta) { let u = user(meta, "ana").await; let dev = device(meta, &u).await; @@ -2472,8 +2478,14 @@ async fn a_trash_restore_and_an_edit_of_the_file_agree(meta: &Meta) { let restore = tokio::spawn(async move { m1.restore_from_trash(space, dev, f, NOW).await }); let update = tokio::spawn(async move { m2.commit(space, dev, &[edit_op], NOW + 2).await }); let restored = restore.await.unwrap(); - update.await.unwrap().unwrap(); - assert!(matches!(restored, Ok(_) | Err(MetaError::NotFound(_))), "{restored:?}"); + let updated = update.await.unwrap(); + match (&restored, &updated) { + // The edit came first and brought the file back: the restore finds nothing in the trash. + (Err(MetaError::NotFound(_)), Ok(_)) => {} + // The restore came first and made a new current version: the edit, made on the old one, is a stale update. + (Ok(_), Err(MetaError::Rejected(list))) => assert!(matches!(list[0].kind, ConflictKind::VersionMismatch { .. })), + other => panic!("one of them must win: {other:?}"), + } assert_eq!(live_name(meta, &u, f).await, (Some(r), "both.txt".to_owned(), false)); assert!(trash_ids(meta, &u).await.is_empty()); assert_eq!(meta.history(u.space, f, None, 10).await.unwrap().versions.len(), 2); @@ -2595,3 +2607,86 @@ async fn a_damaged_deletion_time_on_postgres() { }; a_deleted_node_without_a_time_is_stamped_when_read(&url).await; } + +/// A file that comes back from the trash gets a new current version with its last content, after the last one and +/// without a save record, so a delete that was sent against the old version (and whose answer a machine never heard) +/// is refused as a version mismatch and can't delete the restored file again; a folder has no version. +async fn a_restored_file_gets_a_new_version_with_its_old_content(meta: &Meta) { + use craft_cloud_proto::api::HISTORY_MAX_LIMIT; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let (node, versions) = file_with_versions(meta, &u, dev, 2).await; + let record = craft_cloud_proto::SaveRecord { doc: craft_cloud_proto::live::node_doc(u.space, node), frontiers: vec![(1, 5)] }; + meta.attach_save(u.space, node, versions[1].id, &record).await.unwrap(); + delete_file(meta, &u, dev, node, versions[1].id, NOW + 10).await; + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, node, NOW + 20).await.unwrap(); + let new = done.version.unwrap(); + assert!(!versions.iter().any(|v| v.id == new)); + assert_eq!(done.seq, before.seq + 1, "one change of the space"); + let history = meta.history(u.space, node, None, HISTORY_MAX_LIMIT).await.unwrap().versions; + assert_eq!(history.len(), 3); + assert_eq!( + (history[0].id, history[0].parent_version, history[0].size, history[0].save, history[0].seq), + (new, Some(versions[1].id), 4, false, done.seq) + ); + assert!(history.iter().any(|v| v.id == versions[1].id && v.save), "the old version keeps its record"); + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let NodeKind::File(f) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!((f.version, f.manifest, f.mtime_ms, f.save.clone()), (new, versions[1].manifest, versions[1].mtime_ms, None)); + + // The delete a machine sent before the restore, with the old version, is refused and the file stays. + let stale = meta.commit(u.space, dev, &[Op::Delete { node, expected: Some(versions[1].id) }], NOW + 30).await; + assert_eq!(refused_restore(stale), vec![(0, ConflictKind::VersionMismatch { current: Some(new) })]); + assert!(!live_name(meta, &u, node).await.2); + + // A folder has no version. + let dir = folder(meta, &u, dev, r, "plain-dir", NOW + 40).await; + delete_folder(meta, &u, dev, dir, NOW + 41).await; + assert_eq!(meta.restore_from_trash(u.space, dev, dir, NOW + 42).await.unwrap().version, None); +} + +/// A restore never takes the tree past its depth limit: a folder that comes back below a live folder that has moved +/// deep meanwhile is refused as too deep, for the folder itself and for a file whose folder would come back, and +/// the refusal changes nothing; a file in a folder at the limit is fine. +async fn a_restore_does_not_exceed_the_depth_limit(meta: &Meta) { + use craft_cloud_proto::limits::MAX_TREE_DEPTH; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let shallow = folder(meta, &u, dev, r, "shallow", NOW).await; + let c = folder(meta, &u, dev, shallow, "c", NOW).await; + let (f, fv) = file_in(meta, &u, dev, c, "f.txt", NOW).await; + let spare = folder(meta, &u, dev, shallow, "spare", NOW).await; + let (g, gv) = file_in(meta, &u, dev, spare, "g.txt", NOW).await; + delete_file(meta, &u, dev, f, fv.id, NOW + 1).await; + delete_folder(meta, &u, dev, c, NOW + 2).await; + delete_file(meta, &u, dev, g, gv.id, NOW + 3).await; + // A chain of folders down to depth MAX_TREE_DEPTH - 1, and the live folder moved to its end: depth MAX_TREE_DEPTH. + let mut ops = Vec::new(); + let mut parent = r; + for i in 0..MAX_TREE_DEPTH - 1 { + let id = NodeId::generate(); + ops.push(Op::Mkdir { node: id, parent, name: name(&format!("d{i}")) }); + parent = id; + } + meta.commit(u.space, dev, &ops, NOW + 4).await.unwrap(); + meta.commit(u.space, dev, &[Op::Move { node: shallow, from_parent: r, from_name: name("shallow"), parent, name: name("shallow") }], NOW + 5) + .await + .unwrap(); + let before = meta.space_info(u.space).await.unwrap(); + let trash_before = trash_ids(meta, &u).await; + + // The deleted folder would sit one level too deep, and so would the folder of the file: refused, nothing changes. + for node in [c, f] { + let refused = meta.restore_from_trash(u.space, dev, node, NOW + 6).await; + assert_eq!(refused_restore(refused), vec![(0, ConflictKind::TooDeep)], "restoring {node}"); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + assert_eq!(trash_ids(meta, &u).await, trash_before); + } + // A file may sit in the deepest folder: it comes back, in `spare`, which is live (it was never deleted). + let done = meta.restore_from_trash(u.space, dev, g, NOW + 7).await.unwrap(); + assert_eq!((done.parent, done.name.as_str()), (spare, "g.txt")); +} diff --git a/crates/proto/src/api.rs b/crates/proto/src/api.rs index 0c077efb..65af1ed3 100644 --- a/crates/proto/src/api.rs +++ b/crates/proto/src/api.rs @@ -297,6 +297,10 @@ pub struct TrashRestoreResponse { pub parent: NodeId, pub name: String, pub renamed: bool, + /// For a file, its new current version: a copy of its last one (the same content), so a delete sent against the old + /// version, whose answer a machine never heard, can't delete it again. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, pub seq: u64, pub epoch: Epoch, } From 015f2de1212275d3698bee9b992a4c9e8f2a09c2 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:36:53 +0100 Subject: [PATCH 11/27] sync: do not delete a folder twice after a delete whose answer was lost A folder has no version to tell a delete that went through and a later restore from one that never arrived. The engine now remembers the folder deletes it sent without hearing the answer, and when the next pull reports such a folder changed and live, lets it go instead of deleting it again. A folder the feed does not mention was never touched and its delete is sent again. Scenario tests cover a file and a folder restored after a lost answer, at its own path, under a conflict name, and merged into a live folder. --- crates/sync/src/engine/apply.rs | 2 + crates/sync/src/engine/mod.rs | 11 +++ crates/sync/src/engine/push.rs | 13 ++- crates/sync/src/state.rs | 8 ++ crates/sync/tests/engine.rs | 146 ++++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 1 deletion(-) diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index 9a78bb49..83eea3e0 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -44,6 +44,8 @@ impl Engine { self.state.base.remove(&id); } self.state.sent_deletes.clear(); + // Folders whose delete was in doubt were judged by the pull, which saw whether the server changed them. + self.state.sent_dir_deletes.clear(); // Changed or new remote nodes, parents first. let mut work: Vec<(usize, NodeId)> = self.state.remote.keys().filter(|id| **id != root).filter_map(|id| Some((self.state.remote_path(*id)?.depth(), *id))).collect(); diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index 55f2335b..396892e5 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -609,6 +609,17 @@ impl Engine { if node.deleted { self.state.remote.remove(&node.id); } else { + // A folder this machine asked to delete (the answer never recorded) that the server reports + // changed and live: its delete went through and it was brought back since (or another machine + // changed it first). It is let go, to arrive as it is on the server, not deleted a second + // time. One the feed doesn't mention was never touched: the delete is sent again. A folder + // that still has tracked nodes in it stays as it is. + if self.state.sent_dir_deletes.remove(&node.id) + && matches!(self.state.base.get(&node.id).map(|e| &e.kind), Some(BaseKind::Dir { .. })) + && !self.state.base.values().any(|e| e.parent == Some(node.id)) + { + self.state.base.remove(&node.id); + } self.state.remote.insert(node.id, RemoteNode::from_state(&node)); } } diff --git a/crates/sync/src/engine/push.rs b/crates/sync/src/engine/push.rs index 470e36f2..7b5b7357 100644 --- a/crates/sync/src/engine/push.rs +++ b/crates/sync/src/engine/push.rs @@ -649,8 +649,16 @@ impl Engine { _ => None, }) .collect(); - if !deletes.is_empty() { + let dir_deletes: Vec = only_ops + .iter() + .filter_map(|op| match op { + Op::Delete { node, expected: None } => Some(*node), + _ => None, + }) + .collect(); + if !deletes.is_empty() || !dir_deletes.is_empty() { self.state.sent_deletes.extend(deletes.iter().copied()); + self.state.sent_dir_deletes.extend(dir_deletes.iter().copied()); self.checkpoint()?; } let result = self.transport.commit(&only_ops).await; @@ -660,6 +668,9 @@ impl Engine { for (node, _) in &deletes { self.state.sent_deletes.remove(node); } + for node in &dir_deletes { + self.state.sent_dir_deletes.remove(node); + } } match result { Ok(_) => { diff --git a/crates/sync/src/state.rs b/crates/sync/src/state.rs index f88b8cab..0e8c3a9f 100644 --- a/crates/sync/src/state.rs +++ b/crates/sync/src/state.rs @@ -36,6 +36,14 @@ pub struct State { /// the server's changes. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub sent_deletes: BTreeMap, + /// Folders this machine asked the server to delete in a commit whose answer it never recorded. A folder has no + /// version to tell a delete that went through and a later restore from one that never arrived, so the pull judges + /// it by the change feed: a folder the server reports changed and live (brought back from the trash after the + /// delete, or changed by another machine first) is let go and arrives as it is on the server, instead of being + /// deleted a second time; one the feed doesn't mention was never touched, and its delete is sent again. Cleared + /// once a pass has applied the server's changes. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub sent_dir_deletes: BTreeSet, /// Server deletes held behind a link: tracked nodes missing here while a link or special file that appeared /// since the last finished push carries a name from the node's last local path, so it may be the node, moved with /// its folder (seed 112337's no-pass order; see the push). Each is held for as long as any of its links stands diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index a8570f7d..9a4470da 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3153,3 +3153,149 @@ async fn a_live_reset_doc_leaves_the_file_sync_alone() { assert!(!files(&a).iter().any(|(p, _)| p == "gone.txt"), "A's delete was undone"); assert_eq!(read(&b, "a.txt"), b"v2 from the laptop"); } + +// ---- restore from the trash, after a delete whose answer a machine never heard ---- + +/// The node of the file or folder `name` the server holds (deleted or not), in the space of `user`. +async fn node_named(server: &Server, user: &craft_cloud_meta::UserRecord, name: &str) -> craft_cloud_proto::NodeId { + let feed = server.meta.changes(user.space, 0, None, 5000).await.unwrap(); + let found: Vec<_> = feed.nodes.iter().filter(|n| n.name.as_ref().is_some_and(|n| n.as_str() == name)).collect(); + assert_eq!(found.len(), 1, "one node named {name}: {found:?}"); + found[0].id +} + +/// Brings `node` back from the trash, as the web or an admin tool would: not one of the syncing machines. +async fn restore_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + let token = *blake3::hash(craft_cloud_proto::NodeId::generate().as_uuid().as_bytes()).as_bytes(); + let device = server.meta.create_device(user.id, "web", &token, 0).await.unwrap(); + server.meta.restore_from_trash(user.space, device, node, 1_760_000_000_000).await.unwrap(); +} + +/// A machine deleted a file, the server applied it, the answer was lost; the file is brought back from the trash +/// before the machine syncs again. The machine's delete must not be sent again: the file comes back, on both machines. +#[tokio::test] +async fn a_file_restored_after_a_lost_delete_answer_comes_back_and_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "f.txt").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + assert_eq!(files(&b), files(&a)); + // And it stays: nothing deletes it on a later pass, on either machine. + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(files(&a), vec![("f.txt".to_owned(), b"keep me".to_vec())]); + let c = machine(&server, &ana, "studio").await; + let mut c = c; + c.sync_once().await.unwrap(); + assert_eq!(files(&c), files(&a)); +} + +/// The same, but the name was taken meanwhile by another file: the restored one comes back under a conflict name and +/// both files reach both machines. +#[tokio::test] +async fn a_file_restored_under_a_conflict_name_after_a_lost_delete_answer_keeps_both_files() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + write(&mut a, "f.txt", b"the first"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + // The desktop sees the delete, and its user makes a new f.txt. + b.sync_once().await.unwrap(); + write(&mut b, "f.txt", b"a new file"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a, &mut b], 6).await; + let on_a = files(&a); + assert_eq!(on_a, files(&b)); + assert_eq!(on_a.len(), 2, "{on_a:?}"); + assert!(on_a.contains(&("f.txt".to_owned(), b"a new file".to_vec())), "{on_a:?}"); + let restored = on_a.iter().find(|(p, _)| p.contains("(conflict, web, ")).expect("the restored file under a conflict name"); + assert_eq!(restored.1, b"the first"); +} + +/// A folder that came back from the trash is merged into a live folder of the same name: the restored file lands in +/// it, and the machine whose delete answer was lost ends up with it, not deleted again. +#[tokio::test] +async fn a_file_restored_into_a_live_folder_after_a_lost_delete_answer_comes_back() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let file = node_named(&server, &ana, "f.txt").await; + // The desktop sees both deletes; its user makes a new folder d with another file. + b.sync_once().await.unwrap(); + b.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut b, "d/other.txt", b"other"); + b.sync_once().await.unwrap(); + restore_from_trash(&server, &ana, file).await; + + settle(&mut [&mut a, &mut b], 6).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec()), ("d/other.txt".to_owned(), b"other".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); +} + +/// A folder deleted here, the answer lost, and brought back from the trash: it is not deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} + +/// A folder deleted here whose answer was lost and that nobody brought back is still deleted: the delete is not undone +/// by the doubt about it. +#[tokio::test] +async fn a_folder_deleted_with_a_lost_answer_and_not_restored_stays_deleted() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + settle(&mut [&mut a, &mut b], 6).await; + assert!(a.fs().dirs().is_empty(), "{:?}", a.fs().dirs()); + assert!(b.fs().dirs().is_empty(), "{:?}", b.fs().dirs()); +} From 81224f0f6d8f1cffd2d75ba785c87016b921b0d2 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 14:36:53 +0100 Subject: [PATCH 12/27] docs, server: correct the agent compatibility of the trash restore A restored file has a new version, and agents need the folder rule for the rare case of a folder restored after a delete whose answer was lost. Reports the new version in the restore answer. --- apps/craft-cloud-server/src/api.rs | 1 + docs/design/sync.md | 16 +++++++++++++--- docs/server.md | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index ac45b4ca..c485b4cd 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -497,6 +497,7 @@ async fn restore_from_trash( parent: outcome.parent, name: outcome.name, renamed: outcome.renamed, + version: outcome.version, seq: outcome.seq, epoch: outcome.epoch, })) diff --git a/docs/design/sync.md b/docs/design/sync.md index 563f46e3..02cf2de7 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -658,7 +658,7 @@ methods answer `not_found` / `invalid` (405) the same way: | `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | | `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | | `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash, newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | -| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders, as one change: `{node, parent, name, renamed, seq, epoch}`; a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | +| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | | `DELETE /trash/{id}` | takes a node out of the trash (204): only marks it for removal, nothing is freed yet | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | @@ -694,8 +694,18 @@ IDs; it creates no node and overwrites none. Under one write lock and in one tra where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is -taken too, and a restore with no free name is `conflict` and changes nothing. The sync engine sees the restored nodes as ordinary changes to live -nodes, possibly with a new parent or name. Restoring a *version* of a deleted file is still refused: bring the file back +taken too, and a restore with no free name is `conflict` and changes nothing. A restored *file* gets a **new version**: a copy of its last one (the same manifest, modification time and +executable bit, no save record), after it, in the same change. A machine that deleted the file, whose answer it never +heard (the server applied the delete), still names the old version in its delete; after a restore the server refuses +that delete as a version mismatch, and the engine, which already treats a node that came back at another version as one +whose delete went through, lets the node go and downloads the restored file as a new one. Without this the machine would +send the delete again and delete the restored file. A folder has no version, so a deleted folder that was brought back +looks the same as one that was never deleted; the engine therefore remembers the folder deletes it sent without hearing +the answer (`State::sent_dir_deletes`) and, when the next pull reports such a folder changed and live, lets it go instead +of deleting it again (a folder the feed does not mention was never touched, and its delete is sent again). A restore is +otherwise an ordinary change to live nodes, possibly with a new parent or name. Older agents lack the folder rule: with +one, a folder restored in the short window after a delete whose answer was lost can be deleted a second time (the folder +is empty and can be restored again), so update agents with or soon after the server. Restoring a *version* of a deleted file is still refused: bring the file back from the trash first. **Client contract** (what the sync agent relies on): diff --git a/docs/server.md b/docs/server.md index 050493f5..3eb26827 100644 --- a/docs/server.md +++ b/docs/server.md @@ -77,6 +77,8 @@ brought back from the trash afterwards. The only setting is the number of days a trash_days = 30 # 1 to 3650 ``` +A file brought back gets a new version with its old content, so another computer's delete that was sent just before can't delete it again. Update the agents with or soon after the server: an older agent can delete a *folder* a second time if that folder was brought back right after a delete whose answer it never heard (the folder is empty and can be brought back again). + The change adds two columns and an index to the metadata (migration 0009 in the proposal). An older server refuses the upgraded database and rejects the `[retention]` section, so take a backup before upgrading and do not roll back. From b7e6cb1a7a2ff06009479caf1c3c41ad9aaaae46 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 15:50:47 +0100 Subject: [PATCH 13/27] sync: judge a folder whose delete was in doubt after the files it held are settled A folder deleted together with its file, whose answer was lost, and then restored alone, still had the file tracked when the pull judged it, so it was kept and deleted again once the file was let go. The pull now only notes which folders the feed reports live, and the decision is made in apply_remote after the files whose deletes went through are let go. Scenarios cover the non-empty folder restored alone, the folder and its file both restored, and only the file restored. --- crates/sync/src/engine/apply.rs | 15 ++++++++-- crates/sync/src/engine/mod.rs | 12 ++------ crates/sync/src/state.rs | 8 ++++-- crates/sync/tests/engine.rs | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 13 deletions(-) diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index 83eea3e0..18a2a33a 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -44,8 +44,19 @@ impl Engine { self.state.base.remove(&id); } self.state.sent_deletes.clear(); - // Folders whose delete was in doubt were judged by the pull, which saw whether the server changed them. - self.state.sent_dir_deletes.clear(); + // The same for folders, whose deletes carry no version. A folder this machine asked to delete (the answer never + // recorded) that the pull saw the server report changed and live was brought back after the delete went through + // (or changed by another machine first): it is let go like a file whose delete went through, and arrives as it + // is on the server, not deleted a second time. This comes after the files above, so a folder whose tracked + // files were just let go no longer holds anything. A folder the feed did not mention was never touched: its + // delete is sent again. One that still has tracked nodes in it stays. + let live_dirs = std::mem::take(&mut self.state.live_dirs); + for id in std::mem::take(&mut self.state.sent_dir_deletes) { + let is_dir = matches!(self.state.base.get(&id).map(|e| &e.kind), Some(BaseKind::Dir { .. })); + if live_dirs.contains(&id) && is_dir && !self.state.base.values().any(|e| e.parent == Some(id)) { + self.state.base.remove(&id); + } + } // Changed or new remote nodes, parents first. let mut work: Vec<(usize, NodeId)> = self.state.remote.keys().filter(|id| **id != root).filter_map(|id| Some((self.state.remote_path(*id)?.depth(), *id))).collect(); diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index 396892e5..6d0cb158 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -610,15 +610,9 @@ impl Engine { self.state.remote.remove(&node.id); } else { // A folder this machine asked to delete (the answer never recorded) that the server reports - // changed and live: its delete went through and it was brought back since (or another machine - // changed it first). It is let go, to arrive as it is on the server, not deleted a second - // time. One the feed doesn't mention was never touched: the delete is sent again. A folder - // that still has tracked nodes in it stays as it is. - if self.state.sent_dir_deletes.remove(&node.id) - && matches!(self.state.base.get(&node.id).map(|e| &e.kind), Some(BaseKind::Dir { .. })) - && !self.state.base.values().any(|e| e.parent == Some(node.id)) - { - self.state.base.remove(&node.id); + // changed and live: noted here, judged in `apply_remote` once the files it held are settled. + if self.state.sent_dir_deletes.contains(&node.id) { + self.state.live_dirs.insert(node.id); } self.state.remote.insert(node.id, RemoteNode::from_state(&node)); } diff --git a/crates/sync/src/state.rs b/crates/sync/src/state.rs index 0e8c3a9f..323a4daf 100644 --- a/crates/sync/src/state.rs +++ b/crates/sync/src/state.rs @@ -39,11 +39,15 @@ pub struct State { /// Folders this machine asked the server to delete in a commit whose answer it never recorded. A folder has no /// version to tell a delete that went through and a later restore from one that never arrived, so the pull judges /// it by the change feed: a folder the server reports changed and live (brought back from the trash after the - /// delete, or changed by another machine first) is let go and arrives as it is on the server, instead of being - /// deleted a second time; one the feed doesn't mention was never touched, and its delete is sent again. Cleared + /// delete, or changed by another machine first) is let go, once the files it held are settled, and arrives as it is + /// on the server, instead of being deleted a second time; one the feed doesn't mention was never touched, and its delete is sent again. Cleared /// once a pass has applied the server's changes. #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] pub sent_dir_deletes: BTreeSet, + /// Of `sent_dir_deletes`, the folders the pull of this pass saw the server report changed and live. Only for the + /// rest of the pass: never saved. + #[serde(skip)] + pub live_dirs: BTreeSet, /// Server deletes held behind a link: tracked nodes missing here while a link or special file that appeared /// since the last finished push carries a name from the node's last local path, so it may be the node, moved with /// its folder (seed 112337's no-pass order; see the push). Each is held for as long as any of its links stands diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index 9a4470da..c73bd29d 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3299,3 +3299,54 @@ async fn a_folder_deleted_with_a_lost_answer_and_not_restored_stays_deleted() { assert!(a.fs().dirs().is_empty(), "{:?}", a.fs().dirs()); assert!(b.fs().dirs().is_empty(), "{:?}", b.fs().dirs()); } + +/// A folder with a file in it, both deleted here, the answer lost, then brought back from the trash in different ways. The +/// folder this machine deleted is never deleted a second time, whichever of the two (or both) comes back. +async fn lost_delete_of_a_folder_with_a_file(restore: &[&str]) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + for name in restore { + restore_from_trash(&server, &ana, node_named(&server, &ana, name).await).await; + } + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +#[tokio::test] +async fn a_non_empty_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let (a, b, server, ana) = lost_delete_of_a_folder_with_a_file(&["d"]).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has the folder"); +} + +#[tokio::test] +async fn a_folder_and_its_file_both_restored_after_a_lost_delete_answer_come_back() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["d", "f.txt"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} + +#[tokio::test] +async fn only_the_file_restored_after_a_lost_delete_answer_brings_its_folder_back_too() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["f.txt"]).await; + let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d")]); +} From 28d7aa73e3b82eecdb8cf1542e1a4edc8f44a00d Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 17:05:04 +0100 Subject: [PATCH 14/27] sync: settle a lost folder delete deepest first and on a relist A folder this machine deleted with a lost answer and that came back from the trash was judged before the folders below it were settled. A tombstoned subfolder still tracked in it kept it, and the next push deleted the restored folder again. The folders of this machine's own delete are now settled deepest first: one the server lists as live, or no longer has and that is gone here too, is let go, and only then are the parents judged. A relist at the same epoch (the change feed answered resync required) did not note which of those folders the server lists as live, so a restored empty folder stayed tracked and was deleted again. The relist now notes them, so a folder the server has is never pushed as deleted from a stale request. Regressions cover d/sub/f and d/a/b/f with only the top folder, a middle folder, or only the deepest node restored, and a relist after the restore. --- crates/sync/src/engine/apply.rs | 30 ++++++--- crates/sync/src/engine/mod.rs | 4 ++ crates/sync/tests/common/mod.rs | 6 ++ crates/sync/tests/engine.rs | 107 ++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 7 deletions(-) diff --git a/crates/sync/src/engine/apply.rs b/crates/sync/src/engine/apply.rs index 18a2a33a..fb635346 100644 --- a/crates/sync/src/engine/apply.rs +++ b/crates/sync/src/engine/apply.rs @@ -45,15 +45,31 @@ impl Engine { } self.state.sent_deletes.clear(); // The same for folders, whose deletes carry no version. A folder this machine asked to delete (the answer never - // recorded) that the pull saw the server report changed and live was brought back after the delete went through - // (or changed by another machine first): it is let go like a file whose delete went through, and arrives as it - // is on the server, not deleted a second time. This comes after the files above, so a folder whose tracked - // files were just let go no longer holds anything. A folder the feed did not mention was never touched: its - // delete is sent again. One that still has tracked nodes in it stays. + // recorded) that the server reports live is let go like a file whose delete went through (it was brought back + // after the delete, or changed by another machine first), and arrives as it is on the server, not deleted a + // second time. A folder the server no longer has went through, so it is let go too, once it is gone from here + // as well. This is settled deepest first, after the files above: a restored folder is only judged once the + // folders of this machine's own delete below it are settled, or a tombstoned subfolder still tracked in it + // would keep it and the next push would delete it again. A folder the feed did not mention was never touched: + // its delete is sent again. One that still has tracked nodes in it stays. let live_dirs = std::mem::take(&mut self.state.live_dirs); - for id in std::mem::take(&mut self.state.sent_dir_deletes) { + let mut asked: Vec<(usize, NodeId)> = + std::mem::take(&mut self.state.sent_dir_deletes).into_iter().map(|id| (self.state.base_path(id).map_or(0, |p| p.depth()), id)).collect(); + asked.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + for (_, id) in asked { let is_dir = matches!(self.state.base.get(&id).map(|e| &e.kind), Some(BaseKind::Dir { .. })); - if live_dirs.contains(&id) && is_dir && !self.state.base.values().any(|e| e.parent == Some(id)) { + if !is_dir || self.state.base.values().any(|e| e.parent == Some(id)) { + continue; + } + let released = if live_dirs.contains(&id) { + true + } else if self.state.remote.contains_key(&id) { + false + } else { + // Deleted on the server; nothing of it is left here when its place is empty. + self.state.base_path(id).is_some_and(|path| matches!(self.stat(&path), Ok(None))) + }; + if released { self.state.base.remove(&id); } } diff --git a/crates/sync/src/engine/mod.rs b/crates/sync/src/engine/mod.rs index 6d0cb158..029cb4f5 100644 --- a/crates/sync/src/engine/mod.rs +++ b/crates/sync/src/engine/mod.rs @@ -677,6 +677,10 @@ impl Engine { } } let (epoch, seq) = first.ok_or_else(|| SyncError::Corrupt("empty listing".into()))?; + // A folder this machine asked to delete (the answer never recorded) that the listing has live is never pushed as + // deleted from that stale request: a listing can't tell a delete that never arrived from one followed by a + // restore, so the folder is let go and arrives as the server has it (the conservative side of a doubt). + self.state.live_dirs.extend(self.state.sent_dir_deletes.iter().filter(|id| remote.contains_key(*id)).copied()); self.state.remote = remote; self.state.epoch = Some(epoch); self.state.seq = seq; diff --git a/crates/sync/tests/common/mod.rs b/crates/sync/tests/common/mod.rs index 68877bbb..17589c1c 100644 --- a/crates/sync/tests/common/mod.rs +++ b/crates/sync/tests/common/mod.rs @@ -80,6 +80,9 @@ pub struct Faults { pub every: AtomicU64, /// Make the next commit apply on the server but report failure, as if its reply was lost. pub lose_commit_reply: AtomicBool, + /// Make the next read of the change feed answer `ResyncRequired`, as if the server's history had been cut short: + /// the machine then lists the tree again. + pub resync_next_changes: AtomicBool, /// Make every commit come back refused, naming an operation the batch doesn't have. pub bad_conflict_index: AtomicBool, /// The machine died (see `MemFs::die_after`): every call fails. @@ -158,6 +161,9 @@ impl Transport for Local { async fn changes(&self, since: u64, after: Option) -> Result { self.faults.check()?; + if self.faults.resync_next_changes.swap(false, Ordering::Relaxed) { + return Err(TransportError::ResyncRequired); + } self.meta.changes(self.space, since, after, self.page).await.map_err(meta_err) } diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index c73bd29d..b33b7c4a 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3350,3 +3350,110 @@ async fn only_the_file_restored_after_a_lost_delete_answer_brings_its_folder_bac assert_eq!(files(&b), expected); assert_eq!(a.fs().dirs(), vec![path("d")]); } + +/// A tree of nested folders with a file at its bottom, all deleted here in one commit whose answer is lost, then +/// brought back from the trash in different ways (by node name). Whatever comes back, nothing is deleted a second time. +async fn lost_delete_of_a_nested_tree( + dirs: &[&str], + file: &str, + restore: &[&str], +) -> (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord) { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + for dir in dirs { + a.fs_mut().mkdir(&path(dir)).unwrap(); + } + write(&mut a, file, b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path(file)).unwrap(); + for dir in dirs.iter().rev() { + a.fs_mut().remove_dir(&path(dir)).unwrap(); + } + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + for name in restore { + restore_from_trash(&server, &ana, node_named(&server, &ana, name).await).await; + } + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + (a, b, server, ana) +} + +/// Both machines and a fresh one have exactly the folders `dirs` and no file, and the server still has them. +async fn assert_only_dirs(m: (common::Machine, common::Machine, Server, craft_cloud_meta::UserRecord), dirs: &[&str]) { + let (a, b, server, ana) = m; + let expected: Vec<_> = dirs.iter().map(|d| path(d)).collect(); + assert_eq!(a.fs().dirs(), expected, "the laptop"); + assert_eq!(b.fs().dirs(), expected, "the desktop"); + assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), expected, "the server still has the folders"); +} + +#[tokio::test] +async fn a_nested_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_nested_folder_and_its_subfolder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["d", "sub"]).await; + assert_only_dirs(m, &["d", "d/sub"]).await; +} + +#[tokio::test] +async fn only_the_deepest_file_restored_after_a_lost_delete_answer_brings_the_whole_chain_back() { + let (a, b, _server, _ana) = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["f.txt"]).await; + let expected = vec![("d/sub/f.txt".to_owned(), b"keep me".to_vec())]; + assert_eq!(files(&a), expected); + assert_eq!(files(&b), expected); + assert_eq!(a.fs().dirs(), vec![path("d"), path("d/sub")]); +} + +#[tokio::test] +async fn a_three_deep_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["d"]).await; + assert_only_dirs(m, &["d"]).await; +} + +#[tokio::test] +async fn a_three_deep_folder_and_its_middle_folder_restored_after_a_lost_delete_answer_come_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["d", "a"]).await; + assert_only_dirs(m, &["d", "d/a"]).await; +} + +#[tokio::test] +async fn only_the_deepest_folder_restored_after_a_lost_delete_answer_brings_its_parents_back() { + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["b"]).await; + assert_only_dirs(m, &["d", "d/a", "d/a/b"]).await; +} + +/// The change feed is cut short after the lost delete answer and the restore: the machine lists the tree again, at the +/// same epoch. A restored empty folder is in that listing, and must not be deleted a second time. +#[tokio::test] +async fn a_folder_restored_after_a_lost_delete_answer_and_a_relist_is_not_deleted_again() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "e").await).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + settle(&mut [&mut a, &mut b], 3).await; + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(b.fs().dirs(), vec![path("e")]); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); +} From 2aad296a7a554c7f259249c7e4dfcce2c4021936 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 19:39:49 +0100 Subject: [PATCH 15/27] sync: pin folder rules after lost deletes --- crates/sync/tests/common/mod.rs | 5 ++ crates/sync/tests/engine.rs | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/crates/sync/tests/common/mod.rs b/crates/sync/tests/common/mod.rs index 17589c1c..dbd1d706 100644 --- a/crates/sync/tests/common/mod.rs +++ b/crates/sync/tests/common/mod.rs @@ -80,6 +80,8 @@ pub struct Faults { pub every: AtomicU64, /// Make the next commit apply on the server but report failure, as if its reply was lost. pub lose_commit_reply: AtomicBool, + /// Fail the next commit before it reaches the server. + pub fail_next_commit: AtomicBool, /// Make the next read of the change feed answer `ResyncRequired`, as if the server's history had been cut short: /// the machine then lists the tree again. pub resync_next_changes: AtomicBool, @@ -213,6 +215,9 @@ impl Transport for Local { async fn commit(&self, ops: &[Op]) -> Result { self.faults.check()?; + if self.faults.fail_next_commit.swap(false, Ordering::Relaxed) { + return Err(TransportError::Unavailable("injected before commit".into())); + } if self.faults.bad_conflict_index.load(Ordering::Relaxed) { return Err(TransportError::Rejected(vec![Conflict { op: ops.len() + 5, kind: ConflictKind::NoSuchNode }])); } diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index b33b7c4a..945227c1 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3457,3 +3457,114 @@ async fn a_folder_restored_after_a_lost_delete_answer_and_a_relist_is_not_delete c.sync_once().await.unwrap(); assert_eq!(c.fs().dirs(), vec![path("e")], "the server still has the folder"); } + +/// A folder delete recorded before sending but rejected before reaching the server is treated conservatively after a relist. +#[tokio::test] +async fn an_unsent_folder_delete_and_relist_keeps_the_live_folder() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "keep.txt", b"keep"); + settle(&mut [&mut a, &mut b], 5).await; + + a.fs_mut().remove_dir(&path("e")).unwrap(); + let before = server.meta.space_info(ana.space).await.unwrap().seq; + a.transport().faults.fail_next_commit.store(true, Ordering::Relaxed); + let error = a.sync_once().await.unwrap_err(); + assert!(error.to_string().contains("injected before commit"), "{error}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "the failed request never reached the server"); + + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + let report = a.sync_once().await.unwrap(); + assert_eq!(a.fs().dirs(), vec![path("e")]); + assert_eq!(files(&a), vec![("keep.txt".to_owned(), b"keep".to_vec())]); + assert_eq!(files(&b), files(&a)); + assert!(report.removed_remotely.is_empty(), "{report:?}"); + assert_eq!(server.meta.space_info(ana.space).await.unwrap().seq, before, "no delete was sent again"); +} + +/// A same-epoch relist restores the live parent while keeping its deleted descendants deleted. +#[tokio::test] +async fn a_nested_relist_keeps_only_the_restored_parent() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + a.fs_mut().mkdir(&path("d/sub")).unwrap(); + write(&mut a, "d/sub/f.txt", b"keep me"); + settle(&mut [&mut a, &mut b], 5).await; + a.fs_mut().remove_file(&path("d/sub/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d/sub")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + restore_from_trash(&server, &ana, node_named(&server, &ana, "d").await).await; + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + settle(&mut [&mut a, &mut b], 6).await; + assert_eq!(a.fs().dirs(), vec![path("d")]); + assert_eq!(b.fs().dirs(), vec![path("d")]); + assert!(files(&a).is_empty() && files(&b).is_empty()); + let mut c = machine(&server, &ana, "studio").await; + c.sync_once().await.unwrap(); + assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has only the restored parent"); +} + +/// A tracked child keeps an in-doubt folder in the base when a relist sees it live. +#[tokio::test] +async fn a_tracked_child_keeps_an_in_doubt_folder_from_release() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + let mut b = machine(&server, &ana, "desktop").await; + a.fs_mut().mkdir(&path("d")).unwrap(); + write(&mut a, "d/f.txt", b"original"); + a.fs_mut().mkdir(&path("e")).unwrap(); + write(&mut a, "e/g.txt", b"other"); + settle(&mut [&mut a, &mut b], 5).await; + let folder = node_named(&server, &ana, "d").await; + let file = node_named(&server, &ana, "f.txt").await; + a.fs_mut().rename(&path("d/f.txt"), &path("e/f.txt")).unwrap(); + a.fs_mut().remove_dir(&path("d")).unwrap(); + a.fs_mut().rename(&path("e"), &path("d")).unwrap(); + // Restore a checkpoint with D's delete in doubt. D's tracked file F is still at d/f.txt, inside E's local folder. + let mut state = a.state().clone(); + state.sent_dir_deletes.insert(folder); + let mut a = common::restart(&a, state, |_| {}); + a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); + + a.sync_once().await.unwrap(); + assert!(a.state().base.contains_key(&folder), "the folder is still tracked with its child"); + assert!(a.state().base.contains_key(&file), "the live file is still tracked"); + assert_eq!(files(&a), vec![("d/f.txt".to_owned(), b"original".to_vec()), ("d/g.txt".to_owned(), b"other".to_vec())]); + let listing = server.meta.tree(ana.space, None, 5000).await.unwrap(); + assert!(listing.nodes.iter().any(|node| node.id == folder && !node.deleted), "folder node was deleted"); + assert!(listing.nodes.iter().any(|node| node.id == file && !node.deleted), "tracked file node was deleted"); +} + +/// The local replacement becomes a conflict copy, and the restored file returns at its original path. +#[tokio::test] +async fn a_local_replacement_after_a_lost_delete_reply_keeps_its_content() { + let server = Server::new().await; + let ana = server.user("ana").await; + let mut a = machine(&server, &ana, "laptop").await; + write(&mut a, "f.txt", b"original"); + a.sync_once().await.unwrap(); + a.fs_mut().remove_file(&path("f.txt")).unwrap(); + a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); + assert!(a.sync_once().await.is_err()); + let old = node_named(&server, &ana, "f.txt").await; + write(&mut a, "f.txt", b"local replacement"); + restore_from_trash(&server, &ana, old).await; + + settle(&mut [&mut a], 6).await; + let on_a = files(&a); + assert!(on_a.contains(&("f.txt".to_owned(), b"original".to_vec())), "{on_a:?}"); + assert!(on_a.iter().any(|(p, bytes)| p.contains("(conflict, laptop, ") && bytes == b"local replacement"), "{on_a:?}"); + let mut b = machine(&server, &ana, "desktop").await; + b.sync_once().await.unwrap(); + assert_eq!(files(&b), files(&a)); +} From f2f51b1c978fdfb39cc3a2532797497f88991600 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 15:07:42 +0100 Subject: [PATCH 16/27] meta: add the reference index of what each manifest names A migration adds manifest_chunks (manifest, ordinal, chunk) and manifest_index (chunk count, edges written, state). The index is written in the transaction that records a manifest, up to a budget of edges per transaction, and a longer manifest is finished in batches by index_complete, which also resumes an interrupted build and indexes manifests recorded before the index existed. An index is marked complete only after every edge was compared with its manifest, so wrong, swapped, missing, extra and collapsed edges are found. index_audit reports what the database alone can tell: orphan edges, index rows without a manifest, unknown dependencies, roots that name unrecorded manifests, and manifests that nothing names. Nothing is deleted except the derived edges of an index that does not match its manifest while it is still being built. --- .../postgres/0010_manifest_index.sql | 18 + .../migrations/sqlite/0010_manifest_index.sql | 18 + crates/meta/src/blobs.rs | 9 +- crates/meta/src/index.rs | 537 +++++++++++++++++ crates/meta/src/lib.rs | 5 + crates/meta/src/schema.rs | 2 + crates/meta/tests/behaviour.rs | 543 +++++++++++++++++- 7 files changed, 1124 insertions(+), 8 deletions(-) create mode 100644 crates/meta/migrations/postgres/0010_manifest_index.sql create mode 100644 crates/meta/migrations/sqlite/0010_manifest_index.sql create mode 100644 crates/meta/src/index.rs diff --git a/crates/meta/migrations/postgres/0010_manifest_index.sql b/crates/meta/migrations/postgres/0010_manifest_index.sql new file mode 100644 index 00000000..340a97ac --- /dev/null +++ b/crates/meta/migrations/postgres/0010_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BYTEA NOT NULL, + ordinal BIGINT NOT NULL, + chunk BYTEA NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BYTEA NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/migrations/sqlite/0010_manifest_index.sql b/crates/meta/migrations/sqlite/0010_manifest_index.sql new file mode 100644 index 00000000..cd491700 --- /dev/null +++ b/crates/meta/migrations/sqlite/0010_manifest_index.sql @@ -0,0 +1,18 @@ +-- The reference index: which chunks each manifest names, in order. It is derived data: the manifest object in the +-- store is the authority, and a row of the index is written from a manifest that was read and checked against its ID. +-- `manifest_index` holds, per manifest, how many chunks it names, how many edges are written (the ordinals 0 up to +-- that number, never a gap) and whether the whole list was checked against the manifest: state 0 is being built, state +-- 1 is complete. A manifest without a row is not indexed yet. Nothing in this index is used to free anything. +CREATE TABLE manifest_chunks ( + manifest BLOB NOT NULL, + ordinal BIGINT NOT NULL, + chunk BLOB NOT NULL, + PRIMARY KEY (manifest, ordinal) +); +CREATE INDEX manifest_chunks_by_chunk ON manifest_chunks (chunk, manifest); +CREATE TABLE manifest_index ( + manifest BLOB NOT NULL PRIMARY KEY, + chunk_count BIGINT NOT NULL, + indexed_ordinals BIGINT NOT NULL, + state BIGINT NOT NULL +); diff --git a/crates/meta/src/blobs.rs b/crates/meta/src/blobs.rs index 45fbd781..a03a146a 100644 --- a/crates/meta/src/blobs.rs +++ b/crates/meta/src/blobs.rs @@ -11,10 +11,10 @@ use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId, SpaceId}; use sqlx::Row; use crate::codec::{hash_from, to_i64, to_u64, uuid_bytes}; -use crate::{Meta, MetaError}; +use crate::{Dialect, Meta, MetaError}; /// How a blob's kind is stored in the `kind` columns. -trait KindCode { +pub(crate) trait KindCode { fn code(self) -> i16; } @@ -41,7 +41,7 @@ async fn space_has(tx: &mut sqlx::AnyConnection, space: SpaceId, kind: BlobKind, const BATCH: usize = 500; /// `$first, $first+1, …` for `count` placeholders. -fn placeholders(first: usize, count: usize) -> String { +pub(crate) fn placeholders(first: usize, count: usize) -> String { (first..first + count).map(|n| format!("${n}")).collect::>().join(", ") } @@ -250,6 +250,9 @@ impl Meta { let id = manifest.id(); let blob = Blob { kind: BlobKind::Manifest, id: id.as_bytes(), stored_size: to_i64(stored_size)?, file_size: Some(to_i64(manifest.size())?) }; record(&mut w.tx, space, blob, now_ms, self.stored_cap).await?; + // The index of the manifest's chunks is kept in the same transaction (see `index`): an accepted manifest is never + // without an index row. + crate::index::register(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; w.commit().await } diff --git a/crates/meta/src/index.rs b/crates/meta/src/index.rs new file mode 100644 index 00000000..973a0fe7 --- /dev/null +++ b/crates/meta/src/index.rs @@ -0,0 +1,537 @@ +//! The reference index: which chunks each manifest names, kept in the database next to the manifests themselves. +//! +//! A manifest is a file's content: a list of chunks. The list lives in the manifest's object in the store, which is +//! the authority; this index is a copy of it that the database can query ("which chunks does this manifest need", +//! "which manifests use this chunk"). Because it is a copy it is only trusted when it is *complete*: every manifest has +//! a row in `manifest_index`, and its state is 1 only after the whole list of edges was compared, ordinal by ordinal, +//! with the manifest it was made from. A row count is never enough. +//! +//! The index is kept in the transaction that records a manifest (`Meta::add_manifest`): up to a budget of rows per +//! transaction. A longer manifest is recorded as being built, and the builder ([`Meta::index_complete`]) writes the +//! rest in further transactions of at most that many rows, so no one holds the write lock for a million rows. The same +//! builder indexes manifests that existed before the index did; it resumes where an interrupted run stopped. +//! +//! Nothing here frees anything. The index is read by the checker (`verify`) and by whatever later decides what is +//! garbage; until it says a manifest is complete, that manifest's chunks are unknown, and unknown means kept. + +use std::collections::HashSet; + +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use sqlx::Row; + +use crate::blobs::{KindCode, placeholders}; +use crate::codec::{hash_from, to_i64, to_u64}; +use crate::{Dialect, Meta, MetaError}; + +/// Edges one transaction writes by default (and how many rows a manifest may have to be indexed whole in the +/// transaction that records it). +pub const INDEX_BATCH_ROWS: usize = 5_000; + +/// Edges per `INSERT` statement. +const ROWS_PER_STATEMENT: usize = 500; + +/// How often a builder starts over for one manifest whose edges do not match it before giving up. +const REBUILDS: usize = 2; + +const BUILDING: i64 = 0; +const COMPLETE: i64 = 1; + +/// How far a manifest's index has come. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IndexState { + /// No row: the manifest was recorded before the index existed and nothing has indexed it yet. + Absent, + /// Some edges are written (`done` of `total`), or all of them but not yet compared with the manifest. + Building { done: u64, total: u64 }, + /// Every edge was compared with the manifest. + Complete { total: u64 }, +} + +/// What a comparison of a manifest's stored edges with the manifest found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeCheck { + /// The edges are exactly the manifest's chunks, in order, nothing more. + Exact, + /// The first place they differ. + Mismatch(EdgeProblem), +} + +/// Where and how stored edges differ from their manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EdgeProblem { + pub ordinal: u64, + pub kind: EdgeProblemKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EdgeProblemKind { + /// The edge at the ordinal names another chunk than the manifest does there. + WrongChunk, + /// The manifest has a chunk at the ordinal that has no edge. + MissingEdge, + /// There is an edge at an ordinal the manifest has no chunk for. + ExtraEdge, +} + +impl EdgeProblemKind { + pub fn as_str(self) -> &'static str { + match self { + EdgeProblemKind::WrongChunk => "wrong_chunk", + EdgeProblemKind::MissingEdge => "missing_edge", + EdgeProblemKind::ExtraEdge => "extra_edge", + } + } +} + +/// The result of checking a manifest's edges and, when they are exact, marking its index complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Validation { + Complete, + Mismatch(EdgeProblem), + /// Another builder changed the index meanwhile; look again. + Raced, +} + +/// What the database alone says about the index (the checker adds what only the store can tell). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IndexAudit { + /// Manifests recorded (`blobs` of kind manifest), and how many of them are in each state of the index. + pub manifests: u64, + pub complete: u64, + pub building: u64, + pub absent: u64, + /// Rows of `manifest_index` whose manifest is not recorded. + pub index_without_manifest: Finding, + /// Manifests that have edges but no `manifest_index` row. + pub edges_without_index: Finding, + /// Complete manifests whose number of edges is not the number of chunks they were indexed with. + pub complete_with_wrong_count: Finding, + /// Edges that name a chunk that is not recorded: (manifest, chunk). + pub unknown_dependencies: Finding, + /// Versions and snapshots that name a manifest that is not recorded. + pub roots_without_manifest: Finding, + /// Manifests named by a version or snapshot whose index is not complete. + pub roots_not_indexed: Finding, + /// Recorded manifests that no version, snapshot or import snapshot names. These may be live-document assets, whose + /// references sit inside the documents (the server cannot read them); they are reported, never judged. + pub unrooted_manifests: Finding, + /// Recorded chunks no complete manifest names (only meaningful once every manifest is complete). + pub unreferenced_chunks: Finding, + /// Space memberships of a blob that is not recorded. + pub space_blobs_without_blob: Finding, +} + +/// A count and a few examples (hex IDs, `manifest` or `manifest/chunk`). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Finding { + pub count: u64, + pub examples: Vec, +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A row of `manifest_index`. +struct IndexRow { + chunk_count: i64, + indexed: i64, + state: i64, +} + +async fn read_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, for_update: bool) -> Result, MetaError> { + let sql = if for_update { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1 FOR UPDATE" + } else { + "SELECT chunk_count, indexed_ordinals, state FROM manifest_index WHERE manifest = $1" + }; + let row = sqlx::query(sql).bind(id.as_bytes().to_vec()).fetch_optional(&mut *conn).await?; + row.map(|r| Ok(IndexRow { chunk_count: r.try_get("chunk_count")?, indexed: r.try_get("indexed_ordinals")?, state: r.try_get("state")? })) + .transpose() +} + +/// The row of a manifest, made (as being built, nothing written) if there is none. On Postgres the row stays locked until the +/// transaction ends, so two builders of one manifest take turns. +async fn ensure_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64, for_update: bool) -> Result { + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, $2, 0, 0) ON CONFLICT DO NOTHING") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + read_state(conn, id, for_update).await?.ok_or(MetaError::Corrupt("an index row vanished")) +} + +/// Throws away the edges of a manifest and starts its index again (the edges are derived: the manifest is the authority). +async fn reset(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result<(), MetaError> { + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(id.as_bytes().to_vec()).execute(&mut *conn).await?; + sqlx::query("UPDATE manifest_index SET chunk_count = $1, indexed_ordinals = 0, state = 0 WHERE manifest = $2") + .bind(total) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Writes the edges for the ordinals `from..to` of `manifest`, 500 to a statement. Edges already there are left alone +/// (the comparison catches one that is wrong). +async fn write_edges(conn: &mut sqlx::AnyConnection, manifest: &Manifest, from: usize, to: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let chunks = manifest.chunks().get(from..to).ok_or(MetaError::Corrupt("an index range outside its manifest"))?; + for (part_index, part) in chunks.chunks(ROWS_PER_STATEMENT).enumerate() { + let rows = (0..part.len()).map(|i| format!("({})", placeholders(i * 3 + 1, 3))).collect::>().join(", "); + let mut query = + sqlx::query(sqlx::AssertSqlSafe(format!("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES {rows} ON CONFLICT DO NOTHING"))); + for (i, chunk) in part.iter().enumerate() { + let ordinal = from + .checked_add(part_index.saturating_mul(ROWS_PER_STATEMENT)) + .and_then(|n| n.checked_add(i)) + .ok_or(MetaError::Corrupt("ordinal overflow"))?; + query = query.bind(id.as_bytes().to_vec()).bind(to_i64(ordinal as u64)?).bind(chunk.id.as_bytes().to_vec()); + } + query.execute(&mut *conn).await?; + } + Ok(()) +} + +/// Compares the stored edges of `manifest`, a page of `page` rows at a time, with its chunks. +async fn compare(conn: &mut sqlx::AnyConnection, manifest: &Manifest, page: usize) -> Result { + let id = manifest.id(); + let chunks = manifest.chunks(); + let limit = i64::try_from(page.max(1)).map_err(|_| MetaError::Invalid("page too large"))?; + let (mut next, mut after): (usize, i64) = (0, -1); + loop { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 AND ordinal > $2 ORDER BY ordinal LIMIT $3") + .bind(id.as_bytes().to_vec()) + .bind(after) + .bind(limit) + .fetch_all(&mut *conn) + .await?; + for row in &rows { + let ordinal = to_u64(row.try_get("ordinal")?)?; + let chunk = hash_from(&row.try_get::, _>("chunk")?)?; + let expected = next as u64; + if ordinal > expected { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: expected, kind: EdgeProblemKind::MissingEdge })); + } + match chunks.get(next) { + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + Some(c) if *c.id.as_bytes() != chunk => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::WrongChunk })), + Some(_) => {} + } + next += 1; + after = to_i64(ordinal)?; + } + if rows.len() < page.max(1) { + break; + } + } + if next < chunks.len() { + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: next as u64, kind: EdgeProblemKind::MissingEdge })); + } + Ok(EdgeCheck::Exact) +} + +/// Marks a manifest's index complete if it is still as it was when its edges were compared (`indexed_ordinals` and +/// `chunk_count` both `total`, state still 0). Returns whether it did. +async fn mark_complete(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result { + let done = sqlx::query("UPDATE manifest_index SET state = 1 WHERE manifest = $1 AND state = 0 AND chunk_count = $2 AND indexed_ordinals = $2") + .bind(id.as_bytes().to_vec()) + .bind(total) + .execute(&mut *conn) + .await?; + Ok(done.rows_affected() == 1) +} + +/// Writes the next batch of edges of `manifest` (at most `budget`) and moves the frontier. Returns the state after it. +pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result { + let id = manifest.id(); + let total = manifest.chunks().len(); + let total_i = to_i64(total as u64)?; + let mut row = ensure_state(conn, &id, total_i, pg).await?; + if row.chunk_count != total_i || row.indexed < 0 || row.indexed > total_i { + // The row does not belong to this manifest's list: the manifest is the authority, so it starts again. + reset(conn, &id, total_i).await?; + row = IndexRow { chunk_count: total_i, indexed: 0, state: BUILDING }; + } + if row.state == COMPLETE { + return Ok(IndexState::Complete { total: total as u64 }); + } + let start = usize::try_from(row.indexed).map_err(|_| MetaError::Corrupt("a negative index frontier"))?; + let end = start.saturating_add(budget.max(1)).min(total); + if end > start { + write_edges(conn, manifest, start, end).await?; + sqlx::query("UPDATE manifest_index SET indexed_ordinals = $1 WHERE manifest = $2 AND indexed_ordinals < $1") + .bind(to_i64(end as u64)?) + .bind(id.as_bytes().to_vec()) + .execute(&mut *conn) + .await?; + } + Ok(IndexState::Building { done: end as u64, total: total as u64 }) +} + +/// Called in the transaction that records a manifest: makes its index row and writes up to a budget of edges, and when +/// that is all of them compares them with the manifest and marks the index complete. A longer manifest is left being +/// built for [`Meta::index_complete`]. A row that already exists is not touched beyond that. +pub(crate) async fn register(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(), MetaError> { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + if let IndexState::Building { done, total: n } = stage(conn, pg, manifest, budget).await? + && done >= n + && compare(conn, manifest, budget).await? == EdgeCheck::Exact + { + mark_complete(conn, &id, total).await?; + } + Ok(()) +} + +impl Meta { + /// The store with another budget of edges per index transaction (the default is [`INDEX_BATCH_ROWS`]). For tests. + pub fn with_index_batch_rows(mut self, rows: usize) -> Self { + self.index_rows = rows.max(1); + self + } + + /// Writes the next batch of `manifest`'s edges, at most the index budget, in one transaction, and says how far the + /// index has come. Call it again until it says all edges are written, then [`Meta::index_validate`]; or call + /// [`Meta::index_complete`], which does both. Each call is safe to repeat and to run beside commits and other builders. + pub async fn index_stage(&self, manifest: &Manifest) -> Result { + let mut w = self.begin_write().await?; + let state = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + Ok(state) + } + + /// Compares every edge of `manifest` with it, in a read of one snapshot, and if they are exact marks its index + /// complete (provided every edge was written and nothing changed meanwhile). Nothing else marks an index complete. + pub async fn index_validate(&self, manifest: &Manifest) -> Result { + let id = manifest.id(); + let total = to_i64(manifest.chunks().len() as u64)?; + let check = { + let mut tx = self.read_snapshot().await?; + let state = read_state(&mut tx, &id, false).await?; + let ready = state.is_some_and(|s| s.chunk_count == total && s.indexed == total); + let check = if ready { Some(compare(&mut tx, manifest, self.index_rows).await?) } else { None }; + tx.commit().await?; + check + }; + match check { + None => Ok(Validation::Raced), + Some(EdgeCheck::Mismatch(problem)) => Ok(Validation::Mismatch(problem)), + Some(EdgeCheck::Exact) => { + let mut w = self.begin_write().await?; + let marked = mark_complete(&mut w.tx, &id, total).await?; + w.commit().await?; + // Already complete (another builder was first) is as good. + if marked || matches!(self.index_state(&id).await?, IndexState::Complete { .. }) { + Ok(Validation::Complete) + } else { + Ok(Validation::Raced) + } + } + } + } + + /// Indexes `manifest` to the end: edges in batches of at most the index budget, each its own transaction, then the + /// comparison that marks it complete. Resumes where an earlier run stopped. Edges that do not match the manifest + /// are thrown away and written again (the manifest is the authority), at most twice; then it is `Corrupt`. + pub async fn index_complete(&self, manifest: &Manifest) -> Result<(), MetaError> { + let id = manifest.id(); + let total = manifest.chunks().len() as u64; + let mut rebuilds = 0; + for _ in 0..(REBUILDS + 1) * 4 { + loop { + match self.index_stage(manifest).await? { + IndexState::Complete { .. } => return Ok(()), + IndexState::Building { done, .. } if done >= total => break, + IndexState::Building { .. } | IndexState::Absent => {} + } + } + match self.index_validate(manifest).await? { + Validation::Complete => return Ok(()), + Validation::Raced => {} + Validation::Mismatch(_) => { + if rebuilds >= REBUILDS { + return Err(MetaError::Corrupt("a manifest's index does not match it after being rebuilt")); + } + rebuilds += 1; + let mut w = self.begin_write().await?; + // Only a row that is not complete is rebuilt here: a complete one that is wrong is the checker's to report. + let row = read_state(&mut w.tx, &id, self.dialect == Dialect::Postgres).await?; + if row.is_some_and(|r| r.state != COMPLETE) { + reset(&mut w.tx, &id, to_i64(total)?).await?; + } + w.commit().await?; + } + } + } + Err(MetaError::Corrupt("a manifest's index could not be completed")) + } + + /// How far `manifest`'s index has come. + pub async fn index_state(&self, manifest: &ManifestId) -> Result { + let mut conn = self.pool.acquire().await?; + Ok(match read_state(&mut conn, manifest, false).await? { + None => IndexState::Absent, + Some(r) if r.state == COMPLETE => IndexState::Complete { total: to_u64(r.chunk_count)? }, + Some(r) => IndexState::Building { done: to_u64(r.indexed)?, total: to_u64(r.chunk_count)? }, + }) + } + + /// Compares the edges stored for `manifest` with it, without changing anything. + pub async fn index_check(&self, manifest: &Manifest) -> Result { + let mut tx = self.read_snapshot().await?; + let check = compare(&mut tx, manifest, self.index_rows).await?; + tx.commit().await?; + Ok(check) + } + + /// The stored edges of a manifest, in order: `(ordinal, chunk)`. For tests and the checker. + pub async fn index_edges(&self, manifest: &ManifestId) -> Result, MetaError> { + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal") + .bind(manifest.as_bytes().to_vec()) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok((to_u64(r.try_get("ordinal")?)?, ChunkId::from_bytes(hash_from(&r.try_get::, _>("chunk")?)?)))).collect() + } + + /// How many edges the index holds in all. + pub async fn index_edge_count(&self) -> Result { + let n: i64 = sqlx::query("SELECT COUNT(*) AS n FROM manifest_chunks").fetch_one(&self.pool).await?.try_get("n")?; + to_u64(n) + } + + /// Up to `limit` recorded manifests after `after` (in ID order) whose index is not complete: the work of a builder. + pub async fn manifests_to_index(&self, after: Option, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |m| m.as_bytes().to_vec()); + let rows = sqlx::query( + "SELECT b.id FROM blobs b LEFT JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = $1 AND b.id > $2 AND (i.manifest IS NULL OR i.state <> 1) \ + ORDER BY b.id LIMIT $3", + ) + .bind(BlobKind::Manifest.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| Ok(ManifestId::from_bytes(hash_from(&r.try_get::, _>("id")?)?))).collect() + } + + /// Up to `limit` recorded blobs of `kind` after `after`, in ID order. + pub async fn blob_ids_after(&self, kind: BlobKind, after: Option<[u8; 32]>, limit: usize) -> Result, MetaError> { + let after = after.map_or_else(|| vec![0u8; 32], |id| id.to_vec()); + let rows = sqlx::query("SELECT id FROM blobs WHERE kind = $1 AND id > $2 ORDER BY id LIMIT $3") + .bind(kind.code()) + .bind(after) + .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) + .fetch_all(&self.pool) + .await?; + rows.iter().map(|r| hash_from(&r.try_get::, _>("id")?)).collect() + } + + /// Which of `ids` are recorded chunks. + pub async fn recorded_chunks(&self, ids: &[ChunkId]) -> Result, MetaError> { + let mut found = HashSet::new(); + for part in ids.chunks(ROWS_PER_STATEMENT) { + let sql = format!("SELECT id FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, part.len())); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(BlobKind::Chunk.code()); + for id in part { + query = query.bind(id.as_bytes().to_vec()); + } + for row in query.fetch_all(&self.pool).await? { + found.insert(ChunkId::from_bytes(hash_from(&row.try_get::, _>("id")?)?)); + } + } + Ok(found) + } + + /// What the database says about the index, its roots and the blobs it names, in a read of one snapshot, with up to + /// `examples` examples of each problem. Changes nothing. + pub async fn index_audit(&self, examples: usize) -> Result { + let mut tx = self.read_snapshot().await?; + let mut audit = IndexAudit::default(); + audit.manifests = scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs WHERE kind = 1").await?; + audit.complete = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state = 1").await?; + audit.building = + scalar(&mut tx, "SELECT COUNT(*) AS n FROM blobs b JOIN manifest_index i ON i.manifest = b.id WHERE b.kind = 1 AND i.state <> 1").await?; + audit.absent = audit.manifests.saturating_sub(audit.complete).saturating_sub(audit.building); + // Every manifest a version, a live document's snapshot or its import snapshot names. + const ROOTS: &str = "(SELECT manifest AS a FROM versions UNION SELECT snapshot AS a FROM live_docs WHERE snapshot IS NOT NULL UNION SELECT import_snapshot AS a FROM live_docs WHERE import_snapshot IS NOT NULL) r"; + audit.index_without_manifest = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = i.manifest)", + ) + .await?; + audit.edges_without_index = finding( + &mut tx, + examples, + &["mc.manifest"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = mc.manifest)", + ) + .await?; + audit.complete_with_wrong_count = finding( + &mut tx, + examples, + &["i.manifest"], + "manifest_index i WHERE i.state = 1 AND i.chunk_count <> (SELECT COUNT(*) FROM manifest_chunks mc WHERE mc.manifest = i.manifest)", + ) + .await?; + audit.unknown_dependencies = finding( + &mut tx, + examples, + &["mc.manifest", "mc.chunk"], + "manifest_chunks mc WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 0 AND b.id = mc.chunk)", + ) + .await?; + audit.roots_without_manifest = + finding(&mut tx, examples, &["r.a"], &format!("{ROOTS} WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a)")) + .await?; + audit.roots_not_indexed = finding( + &mut tx, + examples, + &["r.a"], + &format!( + "{ROOTS} WHERE EXISTS (SELECT 1 FROM blobs b WHERE b.kind = 1 AND b.id = r.a) AND NOT EXISTS (SELECT 1 FROM manifest_index i WHERE i.manifest = r.a AND i.state = 1)" + ), + ) + .await?; + audit.unrooted_manifests = + finding(&mut tx, examples, &["b.id"], &format!("blobs b WHERE b.kind = 1 AND NOT EXISTS (SELECT 1 FROM {ROOTS} WHERE r.a = b.id)")) + .await?; + audit.unreferenced_chunks = + finding(&mut tx, examples, &["b.id"], "blobs b WHERE b.kind = 0 AND NOT EXISTS (SELECT 1 FROM manifest_chunks mc WHERE mc.chunk = b.id)") + .await?; + audit.space_blobs_without_blob = + finding(&mut tx, examples, &["sb.id"], "space_blobs sb WHERE NOT EXISTS (SELECT 1 FROM blobs b WHERE b.kind = sb.kind AND b.id = sb.id)") + .await?; + tx.commit().await?; + Ok(audit) + } +} + +/// A count query's number. +async fn scalar(conn: &mut sqlx::AnyConnection, sql: &'static str) -> Result { + to_u64(sqlx::query(sql).fetch_one(&mut *conn).await?.try_get("n")?) +} + +/// How many distinct `columns` the rows of `from_where` have, and the first few (hex, joined by `/`). Only constant +/// text goes into the SQL. +async fn finding(conn: &mut sqlx::AnyConnection, examples: usize, columns: &[&str], from_where: &str) -> Result { + let select = columns.iter().enumerate().map(|(i, c)| format!("{c} AS c{i}")).collect::>().join(", "); + let counted = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT COUNT(*) AS n FROM (SELECT DISTINCT {select} FROM {from_where}) q"))) + .fetch_one(&mut *conn) + .await?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!("SELECT DISTINCT {select} FROM {from_where} ORDER BY 1 LIMIT {examples}"))) + .fetch_all(&mut *conn) + .await?; + let mut shown = Vec::with_capacity(rows.len()); + for row in &rows { + let mut parts = Vec::with_capacity(columns.len()); + for i in 0..columns.len() { + parts.push(hex(&row.try_get::, _>(format!("c{i}").as_str())?)); + } + shown.push(parts.join("/")); + } + Ok(Finding { count: to_u64(counted.try_get("n")?)?, examples: shown }) +} diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 292bc274..c0ceeed6 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -11,6 +11,7 @@ mod blobs; mod codec; mod commit; mod history; +mod index; mod join; mod live; mod schema; @@ -29,6 +30,7 @@ use tokio::sync::{Mutex, OwnedMutexGuard}; pub use commit::{CommitOutcome, RestoreOutcome, TrashRestoreOutcome}; pub use craft_cloud_proto::BlobKind; pub use history::HistoryPage; +pub use index::{EdgeCheck, EdgeProblem, EdgeProblemKind, Finding, INDEX_BATCH_ROWS, IndexAudit, IndexState, Validation}; pub use join::{InviteFor, Redeemed}; pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, @@ -117,6 +119,8 @@ pub struct Meta { live_held_documents_cap: u64, /// The most bytes all of one space's live documents may hold ([`Meta::with_live_bytes_cap`]). live_bytes_cap: u64, + /// Edges of the reference index one transaction writes ([`Meta::with_index_batch_rows`]). + index_rows: usize, } impl Meta { @@ -173,6 +177,7 @@ impl Meta { live_documents_cap: MAX_LIVE_DOCUMENTS_PER_SPACE, live_held_documents_cap: MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, live_bytes_cap: DEFAULT_LIVE_BYTES_PER_SPACE, + index_rows: INDEX_BATCH_ROWS, }) } diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index df810ed2..7b6be4e1 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -14,6 +14,7 @@ const SQLITE: &[&str] = &[ include_str!("../migrations/sqlite/0006_save_records.sql"), include_str!("../migrations/sqlite/0008_versions_node_seq.sql"), include_str!("../migrations/sqlite/0009_trash.sql"), + include_str!("../migrations/sqlite/0010_manifest_index.sql"), ]; const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0001_init.sql"), @@ -24,6 +25,7 @@ const POSTGRES: &[&str] = &[ include_str!("../migrations/postgres/0006_save_records.sql"), include_str!("../migrations/postgres/0008_versions_node_seq.sql"), include_str!("../migrations/postgres/0009_trash.sql"), + include_str!("../migrations/postgres/0010_manifest_index.sql"), ]; /// Arbitrary fixed key for the Postgres advisory lock that serialises migrations between servers sharing one diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index 385f0e9f..22d223b5 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -120,6 +120,9 @@ on_every_backend!( a_trash_restore_and_an_edit_of_the_file_agree, a_restored_file_gets_a_new_version_with_its_old_content, a_restore_does_not_exceed_the_depth_limit, + a_new_manifest_has_exact_edges_complete_in_its_own_transaction, + a_long_manifest_is_indexed_in_batches_of_the_budget, + the_index_stays_exact_through_every_write_path, batch_size_is_limited, folders_nest_at_most_max_tree_depth, creating_or_moving_into_a_deleted_folder_brings_it_back, @@ -1798,6 +1801,9 @@ async fn upgrading_counts_what_each_space_already_stores() { sqlx::query("DROP TABLE pairings").execute(&mut conn).await.unwrap(); sqlx::query("DROP TABLE invites").execute(&mut conn).await.unwrap(); // And before save records. + // And before the reference index. + sqlx::query("DROP TABLE manifest_chunks").execute(&mut conn).await.unwrap(); + sqlx::query("DROP TABLE manifest_index").execute(&mut conn).await.unwrap(); // And before the trash. sqlx::query("DROP INDEX nodes_trash").execute(&mut conn).await.unwrap(); sqlx::query("ALTER TABLE nodes DROP COLUMN deleted_ms").execute(&mut conn).await.unwrap(); @@ -1829,7 +1835,7 @@ async fn concurrent_opens_migrate_once_on_postgres() { use sqlx::Connection; let mut conn = sqlx::PgConnection::connect(&url).await.unwrap(); let versions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(versions, 8); + assert_eq!(versions, 9); } /// A file with `n` versions, one commit each (the first by create): its node and the versions, oldest first. @@ -2078,12 +2084,12 @@ async fn the_schema_version_is_recorded_and_a_newer_database_is_refused() { use sqlx::Connection; let mut conn = sqlx::SqliteConnection::connect(&url).await.unwrap(); let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(latest, 8); + assert_eq!(latest, 9); // Opening again changes nothing. drop(Meta::open(&url).await.unwrap()); let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(rows, 8); - sqlx::query("INSERT INTO schema_version (version) VALUES (9)").execute(&mut conn).await.unwrap(); + assert_eq!(rows, 9); + sqlx::query("INSERT INTO schema_version (version) VALUES (10)").execute(&mut conn).await.unwrap(); drop(conn); assert!(matches!(Meta::open(&url).await, Err(MetaError::Corrupt(_))), "a newer database is refused"); } @@ -2504,6 +2510,8 @@ async fn raw(url: &str) -> sqlx::AnyConnection { async fn undo_the_trash_migration(url: &str) { let mut conn = raw(url).await; for statement in [ + "DROP TABLE manifest_chunks", + "DROP TABLE manifest_index", "DROP INDEX nodes_trash", "ALTER TABLE nodes DROP COLUMN deleted_ms", "ALTER TABLE nodes DROP COLUMN purge_requested_ms", @@ -2552,7 +2560,7 @@ async fn the_trash_migration_dates_old_deletions_at_the_migration(url: &str) { assert_eq!(again.entries, page.entries); let mut conn = raw(url).await; let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); - assert_eq!(latest, 8); + assert_eq!(latest, 9); let live_with_time: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM nodes WHERE deleted = 0 AND (deleted_ms IS NOT NULL OR purge_requested_ms IS NOT NULL)") .fetch_one(&mut conn) @@ -2690,3 +2698,528 @@ async fn a_restore_does_not_exceed_the_depth_limit(meta: &Meta) { let done = meta.restore_from_trash(u.space, dev, g, NOW + 7).await.unwrap(); assert_eq!((done.parent, done.name.as_str()), (spare, "g.txt")); } + +// ---- the reference index ---- + +/// A manifest over `chunks` (each one byte), the chunks recorded as `u`'s; the manifest itself is not recorded. +async fn manifest_over(meta: &Meta, u: &UserRecord, chunks: &[ChunkId]) -> Manifest { + let unique: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(u.space, &unique, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), chunks.len() as u64, FileHash::from_bytes([chunks.len() as u8; 32]), refs).unwrap() +} + +fn fresh_chunks(n: usize) -> Vec { + (0..n).map(|_| random_chunk()).collect() +} + +fn edge_list(chunks: &[ChunkId]) -> Vec<(u64, ChunkId)> { + chunks.iter().enumerate().map(|(i, c)| (i as u64, *c)).collect() +} + +/// The index of a manifest is written in the transaction that records it, exactly: in order, repeated chunks at each +/// of their places; recording it again (another space, the same bytes) changes nothing; a manifest the quota refuses +/// leaves no index behind; an empty file's manifest is complete with no edges. +async fn a_new_manifest_has_exact_edges_complete_in_its_own_transaction(meta: &Meta) { + use craft_cloud_meta::{EdgeCheck, IndexState}; + let ana = user(meta, "ana").await; + let bob = user(meta, "bob").await; + let (a, b, c) = (random_chunk(), random_chunk(), random_chunk()); + let chunks = [a, b, a, c]; + let m = manifest_over(meta, &ana, &chunks).await; + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert_eq!(meta.index_check(&m).await.unwrap(), EdgeCheck::Exact); + + // The same manifest from another space: nothing is written twice. + let edges = meta.index_edge_count().await.unwrap(); + let again = manifest_over(meta, &bob, &chunks).await; + meta.add_manifest(bob.space, &again, 100, NOW).await.unwrap(); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 4 }); + + // An empty file. + let empty = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 0, FileHash::from_bytes([0; 32]), vec![]).unwrap(); + meta.add_manifest(ana.space, &empty, 10, NOW).await.unwrap(); + assert_eq!(meta.index_state(&empty.id()).await.unwrap(), IndexState::Complete { total: 0 }); + assert_eq!(meta.index_edge_count().await.unwrap(), edges); + + // A manifest the stored-bytes cap refuses leaves no index: its transaction rolls back whole. + let capped = meta.clone().with_stored_bytes_cap(Some(50)); + let carol = user(&capped, "carol").await; + let chunks = fresh_chunks(3); + let refused = manifest_over(&capped, &carol, &chunks).await; + assert!(matches!(capped.add_manifest(carol.space, &refused, 1_000, NOW).await, Err(MetaError::QuotaExceeded { .. }))); + assert_eq!(capped.index_state(&refused.id()).await.unwrap(), IndexState::Absent); + assert_eq!(capped.index_edge_count().await.unwrap(), edges); + // A manifest whose chunks the space lacks is refused before anything is written. + let lacking = + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 1, FileHash::from_bytes([7; 32]), vec![ChunkRef { id: random_chunk(), len: 1 }]) + .unwrap(); + assert!(matches!(meta.add_manifest(ana.space, &lacking, 100, NOW).await, Err(MetaError::MissingChunks(_)))); + assert_eq!(meta.index_state(&lacking.id()).await.unwrap(), IndexState::Absent); +} + +/// A manifest longer than the budget is indexed in batches of at most that many edges, each its own transaction, and is +/// complete only after the comparison with its manifest. +async fn a_long_manifest_is_indexed_in_batches_of_the_budget(meta: &Meta) { + use craft_cloud_meta::{IndexState, Validation}; + let meta = meta.clone().with_index_batch_rows(5); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + let before = meta.index_edge_count().await.unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 5, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 5, "no more than the budget in the recording transaction"); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + + for (expected_done, expected_rows) in [(10u64, 10u64), (12, 12)] { + let state = meta.index_stage(&m).await.unwrap(); + assert_eq!(state, IndexState::Building { done: expected_done, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, expected_rows, "at most the budget more per stage"); + } + // Every edge is written, and still it is not complete: only the comparison makes it so. + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 12, total: 12 }); + assert_eq!(meta.index_validate(&m).await.unwrap(), Validation::Complete); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + // Once complete, more stages change nothing. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap() - before, 12); + + // `index_complete` does all of it, and again is a no-op. + let more = fresh_chunks(23); + let long = manifest_over(&meta, &ana, &more).await; + meta.add_manifest(ana.space, &long, 100, NOW).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + meta.index_complete(&long).await.unwrap(); + assert_eq!(meta.index_state(&long.id()).await.unwrap(), IndexState::Complete { total: 23 }); + assert_eq!(meta.index_edges(&long.id()).await.unwrap(), edge_list(&more)); +} + +/// The index is built from what a manifest says, not from what is there: wrong, swapped, missing and extra edges, a +/// repeated chunk collapsed or changed, and a wrong edge written before the build started are each found by the +/// comparison, the index does not become complete, and a build that repeats the manifest's own list makes it exact. +async fn an_index_is_complete_only_when_every_edge_is_right_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let (a, b, c, d, e, f) = (random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk(), random_chunk()); + let spare = random_chunk(); + // `(what, chunks, damage SQL, expected first problem)`: `$1` is a chunk ID and `$2` the manifest. + type Case<'a> = (&'a str, Vec, &'a str, (u64, EdgeProblemKind)); + let cases: Vec = vec![ + ( + "wrong chunk", + vec![a, b, c, d, e, f], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a missing edge", + vec![a, b, c, d, e, f], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 4 AND chunk <> $1", + (4, EdgeProblemKind::MissingEdge), + ), + ( + "an extra edge", + vec![a, b, c, d, e, f], + "INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($2, 6, $1)", + (6, EdgeProblemKind::ExtraEdge), + ), + ( + "a repeated chunk changed", + vec![a, b, a, c], + "UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 2", + (2, EdgeProblemKind::WrongChunk), + ), + ( + "a repeated chunk collapsed", + vec![a, b, a, c], + "DELETE FROM manifest_chunks WHERE manifest = $2 AND ordinal = 2 AND chunk = $1", + (2, EdgeProblemKind::MissingEdge), + ), + ]; + for (what, chunks, damage, (ordinal, kind)) in cases { + // A distinct manifest per case: its chunks differ in size by the case. + let chunks: Vec = + chunks.iter().map(|c| ChunkId::from_bytes(*blake3::hash(&[c.as_bytes().as_slice(), what.as_bytes()].concat()).as_bytes())).collect(); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + assert!(matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { .. }), "{what}: all edges written, not yet compared"); + // Damage it with every edge written, before the comparison. + let wrong = chunks.first().map(|c| *c.as_bytes()).unwrap_or(*spare.as_bytes()); + let other = if damage.starts_with("UPDATE") { *spare.as_bytes() } else { wrong }; + sqlx::query(sqlx::AssertSqlSafe(damage.to_owned())).bind(other.to_vec()).bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let outcome = meta.index_validate(&m).await.unwrap(); + match outcome { + Validation::Mismatch(problem) => assert_eq!((problem.ordinal, problem.kind), (ordinal, kind), "{what}"), + other => panic!("{what}: expected a mismatch, got {other:?}"), + } + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{what}: not complete"); + // A build repeats the manifest's own list and ends exact. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }, "{what}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{what}"); + } + + // A wrong edge that was there before the build began is not taken for the right one. + let chunks = fresh_chunks(4); + let m = manifest_over(&meta, &ana, &chunks).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "recorded with a wrong edge present"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + + // A complete index that was damaged afterwards is found by the checks that read it, not trusted. + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 3 }); + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 0") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert!(matches!(meta.index_check(&m).await.unwrap(), craft_cloud_meta::EdgeCheck::Mismatch(p) if p.ordinal == 0)); + // The builder leaves a complete row alone (it is the checker's to report), and a mismatch there is an error, not a silent rewrite. + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Complete { total: 3 }); +} + +/// A build stopped part way resumes where it was, after the database was closed and opened again, and a manifest +/// recorded before the index existed (no row at all) is found and indexed. +async fn an_index_build_resumes_after_a_restart_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let (ana, m) = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + (ana, m) + }; + // The "restart": a new handle on the same database. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(4); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Building { done: 8, total: 10 }); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![m.id()]); + assert_eq!(meta.index_edge_count().await.unwrap(), 8); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + assert!(meta.manifests_to_index(None, 10).await.unwrap().is_empty()); + + // A manifest from before the index: recorded, no row, no edges. + let old_chunks = fresh_chunks(6); + let old = manifest_over(&meta, &ana, &old_chunks).await; + meta.add_manifest(ana.space, &old, 100, NOW).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(old.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + assert_eq!(meta.index_state(&old.id()).await.unwrap(), IndexState::Absent); + assert_eq!(meta.manifests_to_index(None, 10).await.unwrap(), vec![old.id()]); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent), (2, 1, 1)); + meta.index_complete(&old).await.unwrap(); + assert_eq!(meta.index_edges(&old.id()).await.unwrap(), edge_list(&old_chunks)); + assert_eq!(meta.index_audit(5).await.unwrap().complete, 2); +} + +/// The migration that adds the index does not pretend: manifests recorded before it have no row and are reported as not +/// indexed until a build indexes them. +async fn the_index_migration_leaves_old_manifests_unindexed_on(url: &str) { + use craft_cloud_meta::IndexState; + let (m, ana) = { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &fresh_chunks(3)).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + (m, ana) + }; + let _ = ana; + { + let mut conn = raw(url).await; + for statement in ["DROP TABLE manifest_chunks", "DROP TABLE manifest_index", "DELETE FROM schema_version WHERE version >= 9"] { + sqlx::query(statement).execute(&mut conn).await.unwrap(); + } + } + let meta = Meta::open(url).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Absent, "recorded before the index: not indexed"); + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete, audit.absent, audit.building), (1, 0, 1, 0)); + let mut conn = raw(url).await; + let latest: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + assert_eq!(latest, 9); +} + +/// What the database alone can say is wrong, in seeded damage: orphan edges, an index row without its manifest, an edge +/// to a chunk that is not recorded, a root naming a manifest that is not recorded; and the notices: manifests no version +/// or snapshot names (the live-document asset case) and chunks no manifest names. +async fn the_audit_names_what_is_wrong_on(url: &str) { + let meta = Meta::open(url).await.unwrap(); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + + // A healthy file, an asset no version names, and a chunk no manifest names. + let chunks = fresh_chunks(3); + let file_manifest = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &file_manifest, 100, NOW).await.unwrap(); + let node = NodeId::generate(); + let v = version(file_manifest.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("f.txt"), version: v }], NOW).await.unwrap(); + let asset = manifest_over(&meta, &ana, &fresh_chunks(2)).await; + meta.add_manifest(ana.space, &asset, 100, NOW).await.unwrap(); + let stray = random_chunk(); + meta.add_chunk(ana.space, &stray, 10, NOW).await.unwrap(); + let healthy = meta.index_audit(5).await.unwrap(); + assert_eq!((healthy.manifests, healthy.complete, healthy.building, healthy.absent), (2, 2, 0, 0)); + assert_eq!(healthy.unrooted_manifests.count, 1, "the asset is named by no version"); + assert_eq!(healthy.unrooted_manifests.examples, vec![asset.id().to_string()]); + assert_eq!(healthy.unreferenced_chunks.count, 1); + for finding in [ + &healthy.index_without_manifest, + &healthy.edges_without_index, + &healthy.complete_with_wrong_count, + &healthy.unknown_dependencies, + &healthy.roots_without_manifest, + &healthy.roots_not_indexed, + &healthy.space_blobs_without_blob, + ] { + assert_eq!(finding.count, 0, "{finding:?}"); + } + + // Damage: orphan edges, an index row without a manifest, an unknown dependency, a root without its manifest. + let ghost = ManifestId::from_bytes([0xaa; 32]); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 0, $2)") + .bind(ghost.as_bytes().to_vec()) + .bind(stray.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + let ghost_row = ManifestId::from_bytes([0xbb; 32]); + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, 0, 0, 1)") + .bind(ghost_row.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 0 AND id = $1").bind(chunks[1].as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM blobs WHERE kind = 1 AND id = $1").bind(file_manifest.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let damaged = meta.index_audit(5).await.unwrap(); + assert_eq!(damaged.edges_without_index.examples, vec![ghost.to_string()]); + assert_eq!(damaged.index_without_manifest.count, 2, "the made-up row and the manifest whose record was removed"); + assert!(damaged.index_without_manifest.examples.contains(&ghost_row.to_string())); + assert_eq!(damaged.unknown_dependencies.count, 1); + assert_eq!(damaged.unknown_dependencies.examples, vec![format!("{}/{}", file_manifest.id(), chunks[1])]); + assert_eq!(damaged.roots_without_manifest.examples, vec![file_manifest.id().to_string()], "the version names a manifest that is gone"); + assert_eq!(damaged.space_blobs_without_blob.count, 2, "the memberships of the removed blobs"); + // A complete index whose edge count is not its chunk count. + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal = 0") + .bind(asset.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(meta.index_audit(5).await.unwrap().complete_with_wrong_count.examples, vec![asset.id().to_string()]); +} + +/// Every path that records or uses a manifest leaves the index exactly as the manifests made it: commit, update, +/// restore of a version, restore from the trash, a late save record, a live document's import and compaction. Their +/// roots are always indexed, and none of them writes an edge. +async fn the_index_stays_exact_through_every_write_path(meta: &Meta) { + use craft_cloud_proto::live::node_doc; + let ana = user(meta, "ana").await; + let dev = device(meta, &ana).await; + let r = root(meta, &ana).await; + let clean = |label: &'static str, edges: u64| async move { + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.complete, audit.manifests, "{label}: every manifest is complete"); + assert_eq!(audit.roots_not_indexed.count + audit.roots_without_manifest.count + audit.unknown_dependencies.count, 0, "{label}"); + assert_eq!(audit.edges_without_index.count + audit.index_without_manifest.count + audit.complete_with_wrong_count.count, 0, "{label}"); + assert_eq!(meta.index_edge_count().await.unwrap(), edges, "{label}: the path wrote no edge"); + }; + let (c1, c2, c3) = (fresh_chunks(3), fresh_chunks(2), fresh_chunks(4)); + let (m1, m2, m3) = (manifest_over(meta, &ana, &c1).await, manifest_over(meta, &ana, &c2).await, manifest_over(meta, &ana, &c3).await); + for m in [&m1, &m2, &m3] { + meta.add_manifest(ana.space, m, 100, NOW).await.unwrap(); + } + let edges = 3 + 2 + 4; + clean("uploads", edges).await; + + let node = NodeId::generate(); + let v1 = version(m1.id()); + meta.commit(ana.space, dev, &[Op::Create { node, parent: r, name: name("path.txt"), version: v1.clone() }], NOW).await.unwrap(); + clean("create", edges).await; + let v2 = version(m2.id()); + meta.commit(ana.space, dev, &[Op::Update { node, expected: v1.id, version: v2.clone() }], NOW + 1).await.unwrap(); + clean("update", edges).await; + let restored = meta.restore_version(ana.space, dev, node, v1.id, v2.id, NOW + 2).await.unwrap(); + clean("restore of a version", edges).await; + let record = craft_cloud_proto::SaveRecord { doc: node_doc(ana.space, node), frontiers: vec![(1, 2)] }; + meta.attach_save(ana.space, node, v2.id, &record).await.unwrap(); + clean("a late save record", edges).await; + meta.commit(ana.space, dev, &[Op::Delete { node, expected: Some(restored.version) }], NOW + 3).await.unwrap(); + clean("delete", edges).await; + meta.restore_from_trash(ana.space, dev, node, NOW + 4).await.unwrap(); + clean("restore from the trash", edges).await; + + // A live document of a file: its import snapshot and a compaction's snapshot are roots too. + let doc_node = file(meta, &ana, "doc.txt").await; + let doc = node_doc(ana.space, doc_node); + let first = snapshot(meta, &ana, 100).await; + let edges = meta.index_edge_count().await.unwrap(); + meta.create_live_document(ana.space, doc, "document:wordcraft", doc_node, &first.manifest, NOW).await.unwrap(); + clean("a live document's import", edges).await; + meta.append_live_update(ana.space, doc, dev, 1, &[1; 10], NOW).await.unwrap(); + let compacted = snapshot(meta, &ana, 40).await; + let edges = meta.index_edge_count().await.unwrap(); + assert!(meta.compact_live_doc(ana.space, doc, 1, &compacted.manifest).await.unwrap()); + clean("a compaction", edges).await; + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!(audit.unrooted_manifests.count, 1, "only m3 is named by nothing"); +} + +/// A build runs beside commits and uploads: manifests recorded before the index (no rows) are indexed while new files +/// are committed and new manifests recorded, and the result is exact for all of them. +async fn a_build_runs_beside_commits_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let dev = device(&meta, &ana).await; + let r = root(&meta, &ana).await; + let mut conn = raw(url).await; + let mut old = Vec::new(); + for i in 0..6 { + let chunks = fresh_chunks(7 + i); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index WHERE manifest = $1").bind(m.id().as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + old.push((m, chunks)); + } + let builder = { + let (meta, old) = (meta.clone(), old.clone()); + tokio::spawn(async move { + for (m, _) in &old { + meta.index_complete(m).await.unwrap(); + } + }) + }; + let writer = { + let (meta, space) = (meta.clone(), ana.space); + tokio::spawn(async move { + let mut made = Vec::new(); + for i in 0..8 { + let chunks = fresh_chunks(5); + let ids: Vec<(ChunkId, u64)> = chunks.iter().map(|c| (*c, 2)).collect(); + meta.add_chunks(space, &ids, NOW).await.unwrap(); + let refs: Vec = chunks.iter().map(|id| ChunkRef { id: *id, len: 1 }).collect(); + let m = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), 5, FileHash::from_bytes([i; 32]), refs).unwrap(); + meta.add_manifest(space, &m, 100, NOW).await.unwrap(); + // As the server's upload handler does, beside the builder: the part the recording left. + meta.index_complete(&m).await.unwrap(); + let node = NodeId::generate(); + let v = NewVersion { id: VersionId::generate(), manifest: m.id(), mtime_ms: NOW, exec: false, save: None }; + meta.commit(space, dev, &[Op::Create { node, parent: r, name: name(&format!("w{i}.txt")), version: v }], NOW).await.unwrap(); + made.push((m, chunks)); + } + made + }) + }; + builder.await.unwrap(); + let made = writer.await.unwrap(); + for (m, chunks) in old.iter().chain(made.iter()) { + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: chunks.len() as u64 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(chunks)); + } + let audit = meta.index_audit(5).await.unwrap(); + assert_eq!((audit.manifests, audit.complete), (14, 14)); + assert_eq!(audit.roots_not_indexed.count, 0); +} + +#[tokio::test] +async fn an_index_is_complete_only_when_every_edge_is_right_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_is_complete_only_when_every_edge_is_right_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_is_complete_only_when_every_edge_is_right_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_is_complete_only_when_every_edge_is_right_on(&url).await; +} + +#[tokio::test] +async fn an_index_build_resumes_after_a_restart_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_index_build_resumes_after_a_restart_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_index_build_resumes_after_a_restart_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_index_build_resumes_after_a_restart_on(&url).await; +} + +#[tokio::test] +async fn the_index_migration_leaves_old_manifests_unindexed_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_index_migration_leaves_old_manifests_unindexed_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_index_migration_leaves_old_manifests_unindexed_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_index_migration_leaves_old_manifests_unindexed_on(&url).await; +} + +#[tokio::test] +async fn the_audit_names_what_is_wrong_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + the_audit_names_what_is_wrong_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_audit_names_what_is_wrong_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + the_audit_names_what_is_wrong_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_runs_beside_commits_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_runs_beside_commits_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_runs_beside_commits_on(&url).await; +} From d0ff336ac0ae30fbdbca7a673100864b6a322891 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 15:07:42 +0100 Subject: [PATCH 17/27] server: add verify and index build, and finish a manifest's index before the upload is acknowledged verify checks the database and the store against each other and reports problems and notices as text or JSON, changing nothing. index build indexes manifests recorded before the index existed, in bounded batches. PUT /manifests completes the manifest's index before answering. --- apps/craft-cloud-server/src/api.rs | 3 + apps/craft-cloud-server/src/cli.rs | 68 ++++++ apps/craft-cloud-server/src/lib.rs | 1 + apps/craft-cloud-server/src/verify.rs | 319 ++++++++++++++++++++++++++ apps/craft-cloud-server/tests/api.rs | 68 ++++++ apps/craft-cloud-server/tests/cli.rs | 220 ++++++++++++++++++ 6 files changed, 679 insertions(+) create mode 100644 apps/craft-cloud-server/src/verify.rs diff --git a/apps/craft-cloud-server/src/api.rs b/apps/craft-cloud-server/src/api.rs index c485b4cd..5d938e71 100644 --- a/apps/craft-cloud-server/src/api.rs +++ b/apps/craft-cloud-server/src/api.rs @@ -334,6 +334,9 @@ async fn put_manifest( state.meta.check_room(auth.space, BlobKind::Manifest, id.as_bytes(), body.len() as u64 + 1).await?; state.store.put(BlobKey::manifest(&id), &body).await?; state.meta.add_manifest(auth.space, &manifest, body.len() as u64 + 1, state.now_ms()).await?; + // The index of its chunks is complete before the upload is acknowledged: a manifest of up to the index budget was indexed + // in the transaction above, a longer one is finished here in batches (and by `index build` if this is cut short). + state.meta.index_complete(&manifest).await?; Ok(StatusCode::NO_CONTENT) } diff --git a/apps/craft-cloud-server/src/cli.rs b/apps/craft-cloud-server/src/cli.rs index a9b4908e..f497f7f0 100644 --- a/apps/craft-cloud-server/src/cli.rs +++ b/apps/craft-cloud-server/src/cli.rs @@ -77,6 +77,28 @@ pub enum Command { #[arg(long)] all: bool, }, + /// Check the database and the blob store against each other and report: the reference index of every manifest, the + /// manifests and chunks the database names against the objects in the store, and what names what. Changes nothing; + /// exits with an error when it finds a problem. Reads every chunk object (`--skip-chunk-reads` leaves that out). + Verify { + /// Path to config.toml. + #[arg(long)] + config: Option, + /// Print the report as JSON. + #[arg(long)] + json: bool, + /// Check chunks against the database only; do not read their objects from the store. + #[arg(long)] + skip_chunk_reads: bool, + }, + /// The reference index of what each manifest names. + Index { + /// Path to config.toml. May come before or after the action. + #[arg(long, global = true)] + config: Option, + #[command(subcommand)] + action: IndexAction, + }, /// Manage live documents. Live { /// Path to config.toml. May come before or after the action. @@ -87,6 +109,18 @@ pub enum Command { }, } +#[derive(Debug, Subcommand)] +pub enum IndexAction { + /// Complete the index of every manifest whose index is not complete: reads each manifest from the store, checks it + /// against its ID, and writes its edges in batches. Writes only that derived index. Safe to stop and run again, and to + /// run beside the server. + Build { + /// Edges written per database transaction (default 5000). + #[arg(long)] + batch_rows: Option, + }, +} + #[derive(Debug, Subcommand)] pub enum LiveAction { /// Empty a user's recents document of an app (one that is full: `quota_exceeded`), or, given a document's ID @@ -132,6 +166,9 @@ pub enum CliError { Store(#[from] craft_cloud_store::StoreError), #[error("{0}")] Usage(String), + /// A check found something wrong (its report was written first). + #[error("{0}")] + Inconsistent(String), #[error("I/O error: {0}")] Io(#[from] std::io::Error), } @@ -241,6 +278,37 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ } Ok(()) } + Command::Verify { config, json, skip_chunk_reads } => { + let (meta, store) = open(&Config::load(&config_path(config)?)?).await?; + let report = crate::verify::verify(&meta, &store, crate::verify::VerifyOptions { read_chunks: !skip_chunk_reads }).await?; + if json { + writeln!(out, "{}", report.to_json())?; + } else { + write!(out, "{}", report.to_text())?; + } + match report.problem_count() { + 0 => Ok(()), + n => Err(CliError::Inconsistent(format!("verify found {n} problem(s)"))), + } + } + Command::Index { config, action: IndexAction::Build { batch_rows } } => { + let (meta, store) = open(&Config::load(&config_path(config)?)?).await?; + let meta = match batch_rows { + Some(0) => return Err(CliError::Usage("--batch-rows must be at least 1".into())), + Some(rows) => meta.with_index_batch_rows(rows), + None => meta, + }; + let report = crate::verify::build_index(&meta, &store).await?; + writeln!(out, "indexed {} manifest(s)", report.indexed)?; + for (id, why) in &report.unreadable { + writeln!(out, "could not index manifest {id}: {why}")?; + } + if report.unreadable.is_empty() { + Ok(()) + } else { + Err(CliError::Inconsistent(format!("{} manifest(s) could not be indexed", report.unreadable.len()))) + } + } Command::Live { config, action: LiveAction::ResetDoc { username, app } } => { let (meta, _) = open(&Config::load(&config_path(config)?)?).await?; let user = user_id(&meta, &username).await?; diff --git a/apps/craft-cloud-server/src/lib.rs b/apps/craft-cloud-server/src/lib.rs index 97ffd436..ffe9e3c7 100644 --- a/apps/craft-cloud-server/src/lib.rs +++ b/apps/craft-cloud-server/src/lib.rs @@ -12,6 +12,7 @@ pub mod live; pub mod notify; pub mod ratelimit; pub mod state; +pub mod verify; use std::net::SocketAddr; use std::sync::Arc; diff --git a/apps/craft-cloud-server/src/verify.rs b/apps/craft-cloud-server/src/verify.rs new file mode 100644 index 00000000..62dec5c5 --- /dev/null +++ b/apps/craft-cloud-server/src/verify.rs @@ -0,0 +1,319 @@ +//! The invariant checker behind `craft-cloud-server verify`, and the index builder behind `craft-cloud-server index +//! build`. +//! +//! `verify` reads and reports; it changes nothing (opening the database brings its schema up to date, as every command +//! does, and that is all). It checks what a later garbage collector would have to trust: that every manifest's list of +//! chunks in the database is complete and equal to the manifest object in the store, that every chunk a manifest names +//! is recorded and its object is there and intact, and that nothing in the database names a manifest or a chunk that does +//! not exist. Anything it cannot establish is reported, not guessed: a manifest it cannot read or whose index is not +//! complete has unknown dependencies, and unknown means kept. +//! +//! Two kinds of finding. *Problems* are inconsistencies and make `verify` fail. *Notices* are facts a reader needs but +//! that are not wrong by themselves: manifests whose index is not complete yet (`index build` completes it), manifests +//! no version or snapshot names (a live document's assets are named only inside the document, where the server cannot +//! read them, so these are not garbage and must not be treated as garbage), and chunks no complete manifest names. + +use std::collections::BTreeMap; + +use craft_cloud_meta::{EdgeCheck, Finding, IndexAudit, IndexState, Meta, MetaError}; +use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; +use craft_cloud_store::{BlobKey, Store, StoreError}; +use serde_json::{Value, json}; + +/// Problems listed in full in a report; further ones are only counted. +const MAX_LISTED_PROBLEMS: usize = 1000; +/// Examples of each audit finding. +const EXAMPLES: usize = 20; +/// IDs one metadata query covers. +const PAGE: usize = 200; + +/// What `verify` should look at. +#[derive(Debug, Clone, Copy)] +pub struct VerifyOptions { + /// Read every chunk object from the store, which checks its hash. Without it only manifests are read and chunks are + /// checked against the database. + pub read_chunks: bool, +} + +impl Default for VerifyOptions { + fn default() -> Self { + Self { read_chunks: true } + } +} + +/// One inconsistency. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Problem { + pub kind: &'static str, + /// The IDs involved in hex (`manifest` or `manifest/chunk`). + pub subject: String, + pub detail: Option, +} + +/// The result of a `verify` run. +#[derive(Debug, Clone)] +pub struct Report { + pub audit: IndexAudit, + /// The first [`MAX_LISTED_PROBLEMS`] problems. + pub problems: Vec, + /// Every problem, by kind. + pub counts: BTreeMap<&'static str, u64>, + pub manifests_checked: u64, + pub chunks_checked: u64, + pub chunk_objects_read: bool, +} + +impl Report { + /// How many problems there are in all. + pub fn problem_count(&self) -> u64 { + self.counts.values().sum() + } + + /// Whether the index covers every recorded manifest, completely. + pub fn index_complete(&self) -> bool { + self.audit.manifests == self.audit.complete + } + + fn add(&mut self, kind: &'static str, subject: String, detail: Option) { + *self.counts.entry(kind).or_insert(0) += 1; + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject, detail }); + } + } + + /// An audit finding that is a problem: counted in full, its examples listed. + fn add_finding(&mut self, kind: &'static str, finding: &Finding) { + if finding.count == 0 { + return; + } + *self.counts.entry(kind).or_insert(0) += finding.count; + for example in &finding.examples { + if self.problems.len() < MAX_LISTED_PROBLEMS { + self.problems.push(Problem { kind, subject: example.clone(), detail: None }); + } + } + } + + /// The report as JSON. + pub fn to_json(&self) -> Value { + let notice = |f: &Finding| json!({ "count": f.count, "examples": f.examples }); + json!({ + "ok": self.problem_count() == 0, + "index": { + "manifests": self.audit.manifests, + "complete": self.audit.complete, + "building": self.audit.building, + "absent": self.audit.absent, + "all_complete": self.index_complete(), + }, + "problems": { + "total": self.problem_count(), + "by_kind": self.counts, + "listed": self.problems.iter().map(|p| json!({ "kind": p.kind, "id": p.subject, "detail": p.detail })).collect::>(), + "truncated": self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX), + }, + "notices": { + "manifests_without_complete_index": self.audit.absent.saturating_add(self.audit.building), + "roots_not_indexed": notice(&self.audit.roots_not_indexed), + "unrooted_manifests": notice(&self.audit.unrooted_manifests), + "unreferenced_chunks": notice(&self.audit.unreferenced_chunks), + }, + "checked": { + "manifests": self.manifests_checked, + "chunks": self.chunks_checked, + "chunk_objects_read": self.chunk_objects_read, + }, + }) + } + + /// The report for people. + pub fn to_text(&self) -> String { + let a = &self.audit; + let mut out = format!( + "manifests: {} recorded, index complete for {}, being built for {}, not indexed for {}\nchecked {} manifest(s) and {} chunk(s){}\n", + a.manifests, + a.complete, + a.building, + a.absent, + self.manifests_checked, + self.chunks_checked, + if self.chunk_objects_read { " (chunk objects read and hashed)" } else { " (chunk objects not read)" }, + ); + if self.problem_count() == 0 { + out.push_str("no problems found\n"); + } else { + out.push_str(&format!("{} problem(s) found:\n", self.problem_count())); + for (kind, count) in &self.counts { + out.push_str(&format!(" {kind}: {count}\n")); + } + for p in &self.problems { + match &p.detail { + Some(detail) => out.push_str(&format!(" {} {} ({detail})\n", p.kind, p.subject)), + None => out.push_str(&format!(" {} {}\n", p.kind, p.subject)), + } + } + if self.problem_count() > u64::try_from(self.problems.len()).unwrap_or(u64::MAX) { + out.push_str(" (more problems than are listed; use --json for counts by kind)\n"); + } + } + if !self.index_complete() { + out.push_str("notice: the index is not complete; `craft-cloud-server index build` completes it. Manifests without a complete index have unknown dependencies.\n"); + } + for (what, finding) in [ + ( + "manifests named by no version or snapshot (live-document assets are named only inside their documents, so these are not garbage)", + &a.unrooted_manifests, + ), + ("recorded chunks no complete manifest names", &a.unreferenced_chunks), + ] { + if finding.count > 0 { + out.push_str(&format!("notice: {} {what}\n", finding.count)); + } + } + out + } +} + +/// Why a manifest or chunk object could not be read. +fn store_problem(error: &StoreError) -> (&'static str, Option) { + match error { + StoreError::NotFound(_) => ("object_missing", None), + StoreError::HashMismatch(_) | StoreError::Corrupt { .. } | StoreError::TooLarge { .. } => ("object_corrupt", Some(error.to_string())), + StoreError::Io(_) | StoreError::Backend(_) => ("store_error", Some(error.to_string())), + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Reads a manifest object and decodes it; `Err` says what is wrong with it. +pub async fn read_manifest(store: &Store, id: &ManifestId) -> Result)> { + let bytes = store.get(BlobKey::manifest(id)).await.map_err(|e| { + let (kind, detail) = store_problem(&e); + ( + if kind == "object_missing" { + "manifest_object_missing" + } else if kind == "object_corrupt" { + "manifest_object_corrupt" + } else { + "store_error" + }, + detail, + ) + })?; + let manifest = Manifest::decode(&bytes).map_err(|e| ("manifest_not_decodable", Some(e.to_string())))?; + if manifest.id() != *id { + return Err(("manifest_id_mismatch", None)); + } + Ok(manifest) +} + +/// Checks the database and the store against each other and reports. Changes nothing. +pub async fn verify(meta: &Meta, store: &Store, options: VerifyOptions) -> Result { + let audit = meta.index_audit(EXAMPLES).await?; + let mut report = Report { + audit: audit.clone(), + problems: Vec::new(), + counts: BTreeMap::new(), + manifests_checked: 0, + chunks_checked: 0, + chunk_objects_read: options.read_chunks, + }; + report.add_finding("index_without_manifest", &audit.index_without_manifest); + report.add_finding("orphan_edges", &audit.edges_without_index); + report.add_finding("index_wrong_count", &audit.complete_with_wrong_count); + report.add_finding("unknown_dependency", &audit.unknown_dependencies); + report.add_finding("root_manifest_unknown", &audit.roots_without_manifest); + report.add_finding("space_blob_without_blob", &audit.space_blobs_without_blob); + + // Every recorded manifest: its object, its index, the chunks it names. + let mut after: Option<[u8; 32]> = None; + loop { + let ids = meta.blob_ids_after(BlobKind::Manifest, after, PAGE).await?; + let Some(last) = ids.last().copied() else { break }; + after = Some(last); + for raw in ids { + report.manifests_checked += 1; + let id = ManifestId::from_bytes(raw); + let manifest = match read_manifest(store, &id).await { + Ok(manifest) => manifest, + Err((kind, detail)) => { + report.add(kind, hex(&raw), detail); + continue; + } + }; + if let IndexState::Complete { .. } = meta.index_state(&id).await? + && let EdgeCheck::Mismatch(problem) = meta.index_check(&manifest).await? + { + report.add("index_mismatch", hex(&raw), Some(format!("ordinal {}: {}", problem.ordinal, problem.kind.as_str()))); + } + let ids: Vec = manifest.chunks().iter().map(|c| c.id).collect(); + for part in ids.chunks(500) { + let recorded = meta.recorded_chunks(part).await?; + for chunk in part { + if !recorded.contains(chunk) { + report.add("chunk_not_recorded", format!("{}/{}", hex(&raw), hex(chunk.as_bytes())), None); + } + } + } + } + } + + // Every recorded chunk's object (it also finds chunks no manifest names, which a manifest walk would not). + let mut after: Option<[u8; 32]> = None; + loop { + let ids = meta.blob_ids_after(BlobKind::Chunk, after, PAGE).await?; + let Some(last) = ids.last().copied() else { break }; + after = Some(last); + for raw in ids { + report.chunks_checked += 1; + if !options.read_chunks { + continue; + } + if let Err(e) = store.get(BlobKey::chunk(&ChunkId::from_bytes(raw))).await { + let (kind, detail) = store_problem(&e); + let kind = match kind { + "object_missing" => "chunk_object_missing", + "object_corrupt" => "chunk_object_corrupt", + other => other, + }; + report.add(kind, hex(&raw), detail); + } + } + } + Ok(report) +} + +/// What an `index build` run did. +#[derive(Debug, Clone, Default)] +pub struct BuildReport { + /// Manifests whose index is complete after the run that were not before. + pub indexed: u64, + /// Manifests that could not be indexed: `(manifest, why)`. Their index stays incomplete. + pub unreadable: Vec<(String, &'static str)>, +} + +/// Completes the index of every recorded manifest whose index is not complete, in batches of at most the index budget +/// per transaction, so it can run beside the server and be stopped and run again. A manifest is read from the store and +/// checked against its ID first; one that cannot be read is reported and left unindexed (its chunks stay unknown). Only +/// the derived index is written. +pub async fn build_index(meta: &Meta, store: &Store) -> Result { + let mut report = BuildReport::default(); + let mut after: Option = None; + loop { + let ids = meta.manifests_to_index(after, PAGE).await?; + let Some(last) = ids.last().copied() else { break }; + after = Some(last); + for id in ids { + match read_manifest(store, &id).await { + Ok(manifest) => { + meta.index_complete(&manifest).await?; + report.indexed += 1; + } + Err((kind, _)) => report.unreadable.push((hex(id.as_bytes()), kind)), + } + } + } + Ok(report) +} diff --git a/apps/craft-cloud-server/tests/api.rs b/apps/craft-cloud-server/tests/api.rs index f111ca4c..73911483 100644 --- a/apps/craft-cloud-server/tests/api.rs +++ b/apps/craft-cloud-server/tests/api.rs @@ -1777,3 +1777,71 @@ async fn the_purge_time_follows_the_configured_trash_days() { let page: TrashResponse = trash_page(&s, &ana, "").await.json().await.unwrap(); assert_eq!(page.items[0].purge_after_ms - page.items[0].deleted_ms, 7 * DAY_MS); } + +// ---- the reference index ---- + +/// A server whose metadata writes only a few index edges per transaction (SQLite and disk). +async fn sqlite_disk_with_index_budget(rows: usize) -> Server { + let db = tempfile::tempdir().unwrap(); + let blobs = tempfile::tempdir().unwrap(); + let meta = Meta::open(&format!("sqlite:{}", db.path().join("meta.db").display())).await.unwrap().with_index_batch_rows(rows); + let store = Store::new(DiskBackend::new(blobs.path()).unwrap()); + let state = AppState::new(meta.clone(), store, vec![]).unwrap(); + start_state(state, meta, vec![db, blobs], None).await +} + +/// A manifest of `n` chunks (the first one twice), their data distinct; uploaded chunks first. +async fn upload_many(s: &Server, token: &str, n: usize, tag: &str) -> (Manifest, Vec) { + let datas: Vec> = (0..n).map(|i| format!("index {tag} {i} {}", uuid::Uuid::now_v7()).into_bytes()).collect(); + let mut order: Vec = (0..n).collect(); + order.push(0); + for data in &datas { + assert_eq!(put_chunk(s, token, data).await, StatusCode::NO_CONTENT); + } + let refs: Vec = order.iter().map(|i| ChunkRef { id: chunk_id(&datas[*i]), len: u32::try_from(datas[*i].len()).unwrap() }).collect(); + let size = refs.iter().map(|r| u64::from(r.len)).sum(); + let manifest = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), size, FileHash::from_bytes([9; 32]), refs.clone()).unwrap(); + (manifest, refs.iter().map(|r| r.id).collect()) +} + +/// A manifest accepted is indexed, exactly and completely, before the upload is acknowledged: a short one in the +/// transaction that records it, a longer one (past the server's budget per transaction) in the batches the handler +/// finishes. Uploading it again, or as another account, changes nothing. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_uploaded_manifest_is_indexed_before_the_upload_is_acknowledged() { + use craft_cloud_meta::IndexState; + let s = sqlite_disk_with_index_budget(3).await; + add_user(&s, "ana").await; + add_user(&s, "bob").await; + let (ana, bob) = (login(&s, "ana").await, login(&s, "bob").await); + for (n, label) in [(2usize, "short"), (9, "long")] { + let (manifest, chunks) = upload_many(&s, &ana, n, label).await; + assert_eq!(put_manifest(&s, &ana, &manifest).await.status(), StatusCode::NO_CONTENT, "{label}"); + assert_eq!(s.meta.index_state(&manifest.id()).await.unwrap(), IndexState::Complete { total: n as u64 + 1 }, "{label}"); + let edges = s.meta.index_edges(&manifest.id()).await.unwrap(); + assert_eq!(edges.iter().map(|(_, c)| *c).collect::>(), chunks, "{label}: in order, the first chunk at both its places"); + // Again, and as another account (which has to upload the chunks too). + let before = s.meta.index_edge_count().await.unwrap(); + assert_eq!(put_manifest(&s, &ana, &manifest).await.status(), StatusCode::NO_CONTENT); + for chunk in manifest.chunks() { + let data = s.client.get(format!("{}/chunks/{}", s.base, chunk.id)).bearer_auth(&ana).send().await.unwrap().bytes().await.unwrap(); + assert_eq!( + s.client.put(format!("{}/chunks/{}", s.base, chunk.id)).bearer_auth(&bob).body(data.to_vec()).send().await.unwrap().status(), + StatusCode::NO_CONTENT + ); + } + assert_eq!(put_manifest(&s, &bob, &manifest).await.status(), StatusCode::NO_CONTENT); + assert_eq!(s.meta.index_edge_count().await.unwrap(), before, "{label}: nothing written twice"); + } + // A manifest naming a chunk the account lacks is refused, and leaves no index behind. + let (manifest, _) = upload_many(&s, &ana, 2, "refused").await; + let lacking = { + let mut refs = manifest.chunks().to_vec(); + refs.push(ChunkRef { id: chunk_id(b"never uploaded"), len: 14 }); + let size = refs.iter().map(|r| u64::from(r.len)).sum(); + Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), size, FileHash::from_bytes([8; 32]), refs).unwrap() + }; + let (status, body) = error(put_manifest(&s, &ana, &lacking).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::CONFLICT, "missing_chunks")); + assert_eq!(s.meta.index_state(&lacking.id()).await.unwrap(), IndexState::Absent); +} diff --git a/apps/craft-cloud-server/tests/cli.rs b/apps/craft-cloud-server/tests/cli.rs index 4a5332bf..bfc882d4 100644 --- a/apps/craft-cloud-server/tests/cli.rs +++ b/apps/craft-cloud-server/tests/cli.rs @@ -294,3 +294,223 @@ fn the_binary_serves_what_init_created() { assert!(body.starts_with("HTTP/1.1 200"), "{body}"); assert!(body.contains("\"chunker_params\":[\"fastcdc-v1\"]"), "{body}"); } + +// ---- verify and the reference index ---- + +/// A data directory with a user and the pieces `verify` looks at: one file's manifest of three chunks (one repeated), +/// recorded and stored as an upload does, and a commit that names it. Returns what the tests damage. +struct Seeded { + dir: std::path::PathBuf, + config: String, + db_url: String, + chunks: Vec, + manifest: craft_cloud_proto::ManifestId, + meta: craft_cloud_meta::Meta, + _keep: tempfile::TempDir, +} + +fn chunk_bytes(n: u8) -> Vec { + format!("verify chunk {n} {}", uuid::Uuid::now_v7()).into_bytes() +} + +async fn seeded() -> Seeded { + use craft_cloud_proto::{ChunkRef, ChunkerParamsId, FileHash, Manifest, Name, NewVersion, NodeId, Op, VersionId}; + use craft_cloud_store::BlobKey; + let keep = tempfile::tempdir().unwrap(); + let dir = keep.path().join("data"); + cli(&["init", "--dir", dir.to_str().unwrap(), "--admin", "ana"], "admin password 1\n").await.unwrap(); + let config_path = dir.join("config.toml"); + let config = Config::load(&config_path).unwrap(); + let (meta, store) = craft_cloud_server::cli::open(&config).await.unwrap(); + let ana = meta.user_by_name("ana").await.unwrap().unwrap(); + let datas = [chunk_bytes(1), chunk_bytes(2)]; + let mut chunks = Vec::new(); + for data in &datas { + let id = craft_cloud_proto::ChunkId::from_bytes(*blake3::hash(data).as_bytes()); + store.put(BlobKey::chunk(&id), data).await.unwrap(); + meta.add_chunk(ana.space, &id, data.len() as u64 + 1, 0).await.unwrap(); + chunks.push(id); + } + // The first chunk twice: [a, b, a]. + let order = [chunks[0], chunks[1], chunks[0]]; + let lens = [datas[0].len() as u32, datas[1].len() as u32, datas[0].len() as u32]; + let refs: Vec = order.iter().zip(lens).map(|(id, len)| ChunkRef { id: *id, len }).collect(); + let size = lens.iter().map(|l| u64::from(*l)).sum(); + let manifest = Manifest::new(ChunkerParamsId::new("fastcdc-v1").unwrap(), size, FileHash::from_bytes([3; 32]), refs).unwrap(); + store.put(BlobKey::manifest(&manifest.id()), &manifest.encode()).await.unwrap(); + meta.add_manifest(ana.space, &manifest, manifest.encode().len() as u64 + 1, 0).await.unwrap(); + let info = meta.space_info(ana.space).await.unwrap(); + let device = meta.create_device(ana.id, "laptop", &[5u8; 32], 0).await.unwrap(); + let v = NewVersion { id: VersionId::generate(), manifest: manifest.id(), mtime_ms: 1, exec: false, save: None }; + meta.commit(ana.space, device, &[Op::Create { node: NodeId::generate(), parent: info.root, name: Name::new("f.txt").unwrap(), version: v }], 0) + .await + .unwrap(); + Seeded { + db_url: config.database.url.clone(), + config: config_path.to_str().unwrap().to_owned(), + dir, + chunks, + manifest: manifest.id(), + meta, + _keep: keep, + } +} + +async fn raw_sqlite(url: &str) -> sqlx::SqliteConnection { + use sqlx::Connection; + sqlx::SqliteConnection::connect(url).await.unwrap() +} + +fn object_path(dir: &std::path::Path, kind: &str, id: &[u8; 32]) -> std::path::PathBuf { + let hex: String = id.iter().map(|b| format!("{b:02x}")).collect(); + dir.join("blobs").join(kind).join(&hex[0..2]).join(&hex[2..4]).join(&hex) +} + +async fn verify_json(c: &str, extra: &[&str]) -> (serde_json::Value, bool) { + let mut args = vec!["verify", "--config", c, "--json"]; + args.extend_from_slice(extra); + match cli(&args, "").await { + Ok(out) => (serde_json::from_str(&out).unwrap(), true), + Err(CliError::Inconsistent(_)) => { + // The report is written before the error: run it again through `run` for its text. + let parsed = Cli::try_parse_from(std::iter::once("craft-cloud-server").chain(args.iter().copied())).unwrap(); + let mut out = Vec::new(); + let _ = run(parsed.command, &mut Cursor::new(Vec::new()), &mut out).await; + (serde_json::from_slice(&out).unwrap(), false) + } + Err(other) => panic!("{other}"), + } +} + +#[tokio::test] +async fn verify_passes_on_a_sound_server_and_changes_nothing() { + let s = seeded().await; + let before = s.meta.index_edge_count().await.unwrap(); + let out = cli(&["verify", "--config", &s.config], "").await.unwrap(); + assert!(out.contains("no problems found"), "{out}"); + assert!(out.contains("checked 1 manifest(s) and 2 chunk(s) (chunk objects read and hashed)"), "{out}"); + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(ok); + assert_eq!(json["ok"], true); + assert_eq!( + (json["index"]["manifests"].as_u64(), json["index"]["complete"].as_u64(), json["index"]["all_complete"].as_bool()), + (Some(1), Some(1), Some(true)) + ); + assert_eq!(json["problems"]["total"], 0); + assert_eq!(s.meta.index_edge_count().await.unwrap(), before); + assert_eq!(s.meta.index_edges(&s.manifest).await.unwrap().len(), 3); +} + +#[tokio::test] +async fn verify_reports_a_missing_chunk_object_and_a_missing_manifest_object() { + let s = seeded().await; + std::fs::remove_file(object_path(&s.dir, "chunks", s.chunks[1].as_bytes())).unwrap(); + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(!ok); + assert_eq!(json["problems"]["by_kind"]["chunk_object_missing"], 1, "{json}"); + assert_eq!(json["problems"]["listed"][0]["id"], s.chunks[1].to_string()); + // Without reading chunk objects it is not seen, and the text says so. + let (json, ok) = verify_json(&s.config, &["--skip-chunk-reads"]).await; + assert!(ok, "{json}"); + assert_eq!(json["checked"]["chunk_objects_read"], false); + assert!(cli(&["verify", "--config", &s.config, "--skip-chunk-reads"], "").await.unwrap().contains("chunk objects not read")); + // A damaged chunk object is reported as damaged, not missing. + std::fs::write(object_path(&s.dir, "chunks", s.chunks[0].as_bytes()), b"\0not the chunk").unwrap(); + let (json, _) = verify_json(&s.config, &[]).await; + assert_eq!(json["problems"]["by_kind"]["chunk_object_corrupt"], 1, "{json}"); + // The manifest object is gone: reported (what it names cannot be checked without it). + std::fs::remove_file(object_path(&s.dir, "manifests", s.manifest.as_bytes())).unwrap(); + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(!ok); + assert_eq!(json["problems"]["by_kind"]["manifest_object_missing"], 1, "{json}"); + // Verify never repairs: the same report again. + let (again, _) = verify_json(&s.config, &[]).await; + assert_eq!(again["problems"]["by_kind"], json["problems"]["by_kind"]); +} + +#[tokio::test] +async fn verify_reports_index_damage_and_unknown_dependencies() { + let s = seeded().await; + let mut conn = raw_sqlite(&s.db_url).await; + // A wrong edge in a complete index, an orphan edge, an index row with no manifest, and a chunk that is not recorded. + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 1") + .bind(s.chunks[0].as_bytes().to_vec()) + .bind(s.manifest.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, 0, $2)") + .bind(vec![0xaa; 32]) + .bind(s.chunks[0].as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("INSERT INTO manifest_index (manifest, chunk_count, indexed_ordinals, state) VALUES ($1, 0, 0, 1)") + .bind(vec![0xbb; 32]) + .execute(&mut conn) + .await + .unwrap(); + let (json, ok) = verify_json(&s.config, &["--skip-chunk-reads"]).await; + assert!(!ok); + let by = &json["problems"]["by_kind"]; + assert_eq!( + (by["index_mismatch"].as_u64(), by["orphan_edges"].as_u64(), by["index_without_manifest"].as_u64()), + (Some(1), Some(1), Some(1)), + "{json}" + ); + let mismatch = json["problems"]["listed"].as_array().unwrap().iter().find(|p| p["kind"] == "index_mismatch").unwrap(); + assert_eq!(mismatch["detail"], "ordinal 1: wrong_chunk"); + // A chunk the database does not record: both the manifest walk and the audit say so. + sqlx::query("DELETE FROM blobs WHERE kind = 0 AND id = $1").bind(s.chunks[0].as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let (json, _) = verify_json(&s.config, &["--skip-chunk-reads"]).await; + assert!(json["problems"]["by_kind"]["chunk_not_recorded"].as_u64().unwrap() >= 1, "{json}"); + assert!(json["problems"]["by_kind"]["unknown_dependency"].as_u64().unwrap() >= 1, "{json}"); + // A version that names a manifest that is not recorded. + sqlx::query("DELETE FROM blobs WHERE kind = 1 AND id = $1").bind(s.manifest.as_bytes().to_vec()).execute(&mut conn).await.unwrap(); + let (json, _) = verify_json(&s.config, &["--skip-chunk-reads"]).await; + assert_eq!(json["problems"]["by_kind"]["root_manifest_unknown"], 1, "{json}"); + // Nothing was repaired or removed by looking. + assert_eq!(sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM manifest_chunks").fetch_one(&mut conn).await.unwrap(), 4); +} + +#[tokio::test] +async fn an_index_built_after_the_fact_is_exact_and_verify_notices_until_then() { + let s = seeded().await; + let mut conn = raw_sqlite(&s.db_url).await; + // The index as an upgrade finds it: manifests recorded, nothing indexed. + sqlx::query("DELETE FROM manifest_chunks").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index").execute(&mut conn).await.unwrap(); + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(ok, "an incomplete index is a notice, not a problem: {json}"); + assert_eq!((json["index"]["absent"].as_u64(), json["index"]["all_complete"].as_bool()), (Some(1), Some(false))); + assert_eq!(json["notices"]["manifests_without_complete_index"], 1); + assert!(cli(&["verify", "--config", &s.config], "").await.unwrap().contains("index build")); + + // A build in batches of two edges, then it is complete, exact, and verify says so. + let out = cli(&["index", "build", "--config", &s.config, "--batch-rows", "2"], "").await.unwrap(); + assert!(out.contains("indexed 1 manifest(s)"), "{out}"); + assert_eq!( + s.meta.index_edges(&s.manifest).await.unwrap().iter().map(|(_, c)| *c).collect::>(), + vec![s.chunks[0], s.chunks[1], s.chunks[0]] + ); + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(ok); + assert_eq!(json["index"]["all_complete"], true); + // Again changes nothing. + assert!(cli(&["index", "build", "--config", &s.config], "").await.unwrap().contains("indexed 0 manifest(s)")); + assert!(matches!(cli(&["index", "build", "--config", &s.config, "--batch-rows", "0"], "").await, Err(CliError::Usage(_)))); + + // A manifest whose object cannot be read is reported and stays unindexed: nothing is guessed from the database. + sqlx::query("DELETE FROM manifest_chunks").execute(&mut conn).await.unwrap(); + sqlx::query("DELETE FROM manifest_index").execute(&mut conn).await.unwrap(); + std::fs::remove_file(object_path(&s.dir, "manifests", s.manifest.as_bytes())).unwrap(); + let built = cli(&["index", "build", "--config", &s.config], "").await; + assert!(matches!(built, Err(CliError::Inconsistent(_))), "{built:?}"); + assert_eq!(s.meta.index_state(&s.manifest).await.unwrap(), craft_cloud_meta::IndexState::Absent); + assert!(s.meta.index_edges(&s.manifest).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn verify_needs_a_config() { + assert!(matches!(cli(&["verify"], "").await, Err(CliError::Usage(m)) if m.contains("--config"))); +} From b70886ddf59a01e89190ee733ddbfe66a54b57e9 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 15:07:42 +0100 Subject: [PATCH 18/27] docs: describe the reference index and verify --- docs/design/sync.md | 27 +++++++++++++++++++++++++++ docs/server.md | 13 +++++++++++++ 2 files changed, 40 insertions(+) diff --git a/docs/design/sync.md b/docs/design/sync.md index 02cf2de7..cc278802 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -708,6 +708,33 @@ one, a folder restored in the short window after a delete whose answer was lost is empty and can be restored again), so update agents with or soon after the server. Restoring a *version* of a deleted file is still refused: bring the file back from the trash first. +**Reference index and `verify`.** The chunks a manifest names live in its object in the store. `manifest_chunks` +(manifest, ordinal, chunk) is a copy of that list in the database, and `manifest_index` records per manifest how many +chunks it names, how many edges are written (ordinals 0 up to that number, never a gap) and whether the list was +compared with the manifest (state 1) or is still being built (state 0). A manifest with no row is not indexed. The index +is derived data and is never the authority; it exists so the database can answer "which chunks does this manifest need" and +"which manifests use this chunk". It is written in the transaction that records a manifest (`add_manifest`), up to a budget +of 5,000 edges per transaction; a longer manifest (up to about a million chunks) is left being built, and the upload +handler finishes it in batches of at most that many edges before it answers, so no write lock is held for a long list. +`craft-cloud-server index build` indexes manifests recorded before the index existed (and any left unfinished), reading +each from the store and checking it against its ID first; it resumes where it stopped and may run beside the server. +A manifest is marked complete only after every edge was compared, ordinal by ordinal, with the manifest it was made from: +equal counts are not enough, and repeated chunks, wrong IDs, swapped order, missing and extra edges are all found. An +index that does not match its manifest while it is still being built is thrown away and written again; a complete index +that is wrong is only reported. Nothing in this slice frees anything: no deletion, no claims, no collection, no retention. + +`craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing (opening the database +brings its schema up to date, as every command does). It lists problems (which make it exit with an error): a manifest +object that is missing, damaged or not a manifest; a complete index that does not equal its manifest; edges whose manifest +has no index row; an index row without a recorded manifest; an edge or a manifest that names a chunk that is not recorded; +a chunk object that is missing or damaged; a version, snapshot or import snapshot that names a manifest that is not +recorded; blob memberships without a blob. It also gives notices (which do not): manifests whose index is not complete, +manifests no version or snapshot names, and chunks no complete manifest names. **The manifests no version or snapshot +names are not garbage.** A live document's binary assets (images and the like, uploaded through the agent's blob +interface as a manifest of their own) are named only by the document's state, which the server stores as opaque bytes, so +the database holds no reference to them. Until that gap is closed (the owner decides how assets are rooted), anything that +frees storage has to treat these manifests, and the chunks they name, as in use. + **Client contract** (what the sync agent relies on): - Probe `GET /api/info` and refuse to sync when the protocol ranges don't overlap. diff --git a/docs/server.md b/docs/server.md index 3eb26827..2c353537 100644 --- a/docs/server.md +++ b/docs/server.md @@ -82,6 +82,19 @@ A file brought back gets a new version with its old content, so another computer The change adds two columns and an index to the metadata (migration 0009 in the proposal). An older server refuses the upgraded database and rejects the `[retention]` section, so take a backup before upgrading and do not roll back. +## Checking the storage + +`craft-cloud-server verify --config cc-data/config.toml` checks the database and the blob storage against each other +and prints what it finds; `--json` prints a machine-readable report and `--skip-chunk-reads` skips reading every chunk +object (which hashes it, and takes as long as reading all your data). It changes nothing and exits with an error if +it found a problem. The report also says how many manifests have a complete reference index, which is what a later +cleanup would need before it could free anything. After upgrading, run `craft-cloud-server index build --config +cc-data/config.toml` once: it indexes the manifests stored before the index existed, writing only that index, in small +steps, and it can be stopped and run again. Nothing in the server frees storage yet. + +The change adds two tables to the metadata (migration 0010 in the proposal). An older server refuses the upgraded database, +so take a backup before upgrading and do not roll back. + ## Restoring a backup With the server stopped, restore the database and the chunk storage from the backup, then run From 54ff35c4b85247c978de647cd30d21f62144f730 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 17:17:20 +0100 Subject: [PATCH 19/27] meta: find negative ordinals and reset an index in bounded batches The comparison of a manifest's edges started its pages after ordinal -1, so an edge at a negative ordinal was never seen and an index with such an edge could be marked complete. The comparison now starts at the lowest number there is and reports an edge at a negative ordinal as a mismatch of its own kind, so completion is never published over it. Starting an index again deleted all of a manifest's edges in one transaction, up to about a million rows, which broke the batch budget and blocked other writers. The reset now marks the index row as resetting (state 2) and deletes at most the index budget per transaction, lowest ordinal first; nothing is written or marked complete until the last edge is gone, and a run that stops between two batches is resumed by the next stage. The old behaviour of a complete row that is wrong being left to the checker is unchanged. Regressions cover negative ordinals (including the lowest possible), a reset across several batches that resumes on a new handle after a stop, and a build that resets a wrong index by itself. --- crates/meta/src/index.rs | 106 +++++++++++++++++----- crates/meta/tests/behaviour.rs | 156 +++++++++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 24 deletions(-) diff --git a/crates/meta/src/index.rs b/crates/meta/src/index.rs index 973a0fe7..115b201a 100644 --- a/crates/meta/src/index.rs +++ b/crates/meta/src/index.rs @@ -11,6 +11,11 @@ //! rest in further transactions of at most that many rows, so no one holds the write lock for a million rows. The same //! builder indexes manifests that existed before the index did; it resumes where an interrupted run stopped. //! +//! When the edges of a manifest do not match it, they are thrown away and written again, in batches of the same size +//! (state 2, "resetting", marks it): an index of a million edges is never deleted in one transaction, and a run that stops +//! between two batches is resumed by the next stage. Rows are compared from the lowest ordinal there is, so an edge at a +//! negative ordinal is found too. +//! //! Nothing here frees anything. The index is read by the checker (`verify`) and by whatever later decides what is //! garbage; until it says a manifest is complete, that manifest's chunks are unknown, and unknown means kept. @@ -35,6 +40,10 @@ const REBUILDS: usize = 2; const BUILDING: i64 = 0; const COMPLETE: i64 = 1; +/// The edges are being thrown away to start the index again, a batch of rows to a transaction. Nothing is written +/// and nothing is marked complete until the last edge is gone and the state is back to [`BUILDING`]. A run that stops +/// halfway leaves this marker, and the next one carries on from the rows that are left. +const RESETTING: i64 = 2; /// How far a manifest's index has come. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -71,6 +80,8 @@ pub enum EdgeProblemKind { MissingEdge, /// There is an edge at an ordinal the manifest has no chunk for. ExtraEdge, + /// There is an edge at a negative ordinal, which no manifest has. The ordinal of the problem is its absolute value. + NegativeOrdinal, } impl EdgeProblemKind { @@ -79,6 +90,7 @@ impl EdgeProblemKind { EdgeProblemKind::WrongChunk => "wrong_chunk", EdgeProblemKind::MissingEdge => "missing_edge", EdgeProblemKind::ExtraEdge => "extra_edge", + EdgeProblemKind::NegativeOrdinal => "negative_ordinal", } } } @@ -161,17 +173,43 @@ async fn ensure_state(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i6 read_state(conn, id, for_update).await?.ok_or(MetaError::Corrupt("an index row vanished")) } -/// Throws away the edges of a manifest and starts its index again (the edges are derived: the manifest is the authority). -async fn reset(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result<(), MetaError> { - sqlx::query("DELETE FROM manifest_chunks WHERE manifest = $1").bind(id.as_bytes().to_vec()).execute(&mut *conn).await?; - sqlx::query("UPDATE manifest_index SET chunk_count = $1, indexed_ordinals = 0, state = 0 WHERE manifest = $2") +/// Marks a manifest's index as starting again: the row takes the manifest's chunk count, writes nothing, and is not +/// complete (the edges are derived: the manifest is the authority). The edges themselves go in [`reset_step`]. +async fn begin_reset(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i64) -> Result<(), MetaError> { + sqlx::query("UPDATE manifest_index SET chunk_count = $1, indexed_ordinals = 0, state = $2 WHERE manifest = $3") .bind(total) + .bind(RESETTING) .bind(id.as_bytes().to_vec()) .execute(&mut *conn) .await?; Ok(()) } +/// Throws away at most `budget` of the edges of a manifest whose index starts again, the lowest ordinals first (a +/// negative one included), so no transaction holds the write lock for the up to a million rows a manifest can have. +/// Returns whether none are left: then the row is building again. Fewer rows deleted than the budget means none are. +async fn reset_step(conn: &mut sqlx::AnyConnection, id: &ManifestId, budget: usize) -> Result { + let limit = i64::try_from(budget.max(1)).map_err(|_| MetaError::Invalid("batch too large"))?; + let deleted = sqlx::query( + "DELETE FROM manifest_chunks WHERE manifest = $1 AND ordinal IN (SELECT ordinal FROM manifest_chunks WHERE manifest = $1 ORDER BY ordinal LIMIT $2)", + ) + .bind(id.as_bytes().to_vec()) + .bind(limit) + .execute(&mut *conn) + .await? + .rows_affected(); + let finished = i64::try_from(deleted).is_ok_and(|n| n < limit); + if finished { + sqlx::query("UPDATE manifest_index SET state = $1 WHERE manifest = $2 AND state = $3") + .bind(BUILDING) + .bind(id.as_bytes().to_vec()) + .bind(RESETTING) + .execute(&mut *conn) + .await?; + } + Ok(finished) +} + /// Writes the edges for the ordinals `from..to` of `manifest`, 500 to a statement. Edges already there are left alone /// (the comparison catches one that is wrong). async fn write_edges(conn: &mut sqlx::AnyConnection, manifest: &Manifest, from: usize, to: usize) -> Result<(), MetaError> { @@ -198,16 +236,22 @@ async fn compare(conn: &mut sqlx::AnyConnection, manifest: &Manifest, page: usiz let id = manifest.id(); let chunks = manifest.chunks(); let limit = i64::try_from(page.max(1)).map_err(|_| MetaError::Invalid("page too large"))?; - let (mut next, mut after): (usize, i64) = (0, -1); + // Pagination starts at the lowest number there is, so an edge at a negative ordinal is not skipped. + let (mut next, mut from): (usize, i64) = (0, i64::MIN); loop { - let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 AND ordinal > $2 ORDER BY ordinal LIMIT $3") + let rows = sqlx::query("SELECT ordinal, chunk FROM manifest_chunks WHERE manifest = $1 AND ordinal >= $2 ORDER BY ordinal LIMIT $3") .bind(id.as_bytes().to_vec()) - .bind(after) + .bind(from) .bind(limit) .fetch_all(&mut *conn) .await?; for row in &rows { - let ordinal = to_u64(row.try_get("ordinal")?)?; + let raw: i64 = row.try_get("ordinal")?; + if raw < 0 { + // Rows come lowest first, so a negative ordinal is the first thing seen: no manifest has such an edge. + return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal: raw.unsigned_abs(), kind: EdgeProblemKind::NegativeOrdinal })); + } + let ordinal = to_u64(raw)?; let chunk = hash_from(&row.try_get::, _>("chunk")?)?; let expected = next as u64; if ordinal > expected { @@ -219,7 +263,10 @@ async fn compare(conn: &mut sqlx::AnyConnection, manifest: &Manifest, page: usiz Some(_) => {} } next += 1; - after = to_i64(ordinal)?; + from = match raw.checked_add(1) { + Some(n) => n, + None => return Ok(EdgeCheck::Mismatch(EdgeProblem { ordinal, kind: EdgeProblemKind::ExtraEdge })), + }; } if rows.len() < page.max(1) { break; @@ -242,19 +289,24 @@ async fn mark_complete(conn: &mut sqlx::AnyConnection, id: &ManifestId, total: i Ok(done.rows_affected() == 1) } -/// Writes the next batch of edges of `manifest` (at most `budget`) and moves the frontier. Returns the state after it. -pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result { +/// Writes the next batch of edges of `manifest` (at most `budget`) and moves the frontier. Returns the state after it, +/// and whether the index is still being thrown away to start again (then this step only deleted edges, at most +/// `budget` of them, and `done` says nothing about edges being written). +pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(IndexState, bool), MetaError> { let id = manifest.id(); let total = manifest.chunks().len(); let total_i = to_i64(total as u64)?; - let mut row = ensure_state(conn, &id, total_i, pg).await?; - if row.chunk_count != total_i || row.indexed < 0 || row.indexed > total_i { - // The row does not belong to this manifest's list: the manifest is the authority, so it starts again. - reset(conn, &id, total_i).await?; - row = IndexRow { chunk_count: total_i, indexed: 0, state: BUILDING }; + let row = ensure_state(conn, &id, total_i, pg).await?; + // A row that does not belong to this manifest's list starts again: the manifest is the authority. + let foreign = row.chunk_count != total_i || row.indexed < 0 || row.indexed > total_i; + if foreign { + begin_reset(conn, &id, total_i).await?; + } else if row.state == COMPLETE { + return Ok((IndexState::Complete { total: total as u64 }, false)); } - if row.state == COMPLETE { - return Ok(IndexState::Complete { total: total as u64 }); + if foreign || row.state == RESETTING { + let finished = reset_step(conn, &id, budget).await?; + return Ok((IndexState::Building { done: 0, total: total as u64 }, !finished)); } let start = usize::try_from(row.indexed).map_err(|_| MetaError::Corrupt("a negative index frontier"))?; let end = start.saturating_add(budget.max(1)).min(total); @@ -266,7 +318,7 @@ pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &M .execute(&mut *conn) .await?; } - Ok(IndexState::Building { done: end as u64, total: total as u64 }) + Ok((IndexState::Building { done: end as u64, total: total as u64 }, false)) } /// Called in the transaction that records a manifest: makes its index row and writes up to a budget of edges, and when @@ -275,7 +327,7 @@ pub(crate) async fn stage(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &M pub(crate) async fn register(conn: &mut sqlx::AnyConnection, pg: bool, manifest: &Manifest, budget: usize) -> Result<(), MetaError> { let id = manifest.id(); let total = to_i64(manifest.chunks().len() as u64)?; - if let IndexState::Building { done, total: n } = stage(conn, pg, manifest, budget).await? + if let (IndexState::Building { done, total: n }, false) = stage(conn, pg, manifest, budget).await? && done >= n && compare(conn, manifest, budget).await? == EdgeCheck::Exact { @@ -296,7 +348,7 @@ impl Meta { /// [`Meta::index_complete`], which does both. Each call is safe to repeat and to run beside commits and other builders. pub async fn index_stage(&self, manifest: &Manifest) -> Result { let mut w = self.begin_write().await?; - let state = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + let (state, _) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; w.commit().await?; Ok(state) } @@ -309,7 +361,7 @@ impl Meta { let check = { let mut tx = self.read_snapshot().await?; let state = read_state(&mut tx, &id, false).await?; - let ready = state.is_some_and(|s| s.chunk_count == total && s.indexed == total); + let ready = state.is_some_and(|s| s.chunk_count == total && s.indexed == total && s.state != RESETTING); let check = if ready { Some(compare(&mut tx, manifest, self.index_rows).await?) } else { None }; tx.commit().await?; check @@ -340,8 +392,13 @@ impl Meta { let mut rebuilds = 0; for _ in 0..(REBUILDS + 1) * 4 { loop { - match self.index_stage(manifest).await? { + let mut w = self.begin_write().await?; + let (state, resetting) = stage(&mut w.tx, self.dialect == Dialect::Postgres, manifest, self.index_rows).await?; + w.commit().await?; + match state { IndexState::Complete { .. } => return Ok(()), + // Still throwing edges away: nothing is written yet, whatever the frontier says. + IndexState::Building { .. } if resetting => {} IndexState::Building { done, .. } if done >= total => break, IndexState::Building { .. } | IndexState::Absent => {} } @@ -357,8 +414,9 @@ impl Meta { let mut w = self.begin_write().await?; // Only a row that is not complete is rebuilt here: a complete one that is wrong is the checker's to report. let row = read_state(&mut w.tx, &id, self.dialect == Dialect::Postgres).await?; + // The edges go in batches of the next stages, each its own transaction, and a stop between them resumes. if row.is_some_and(|r| r.state != COMPLETE) { - reset(&mut w.tx, &id, to_i64(total)?).await?; + begin_reset(&mut w.tx, &id, to_i64(total)?).await?; } w.commit().await?; } diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index 22d223b5..ff9ff316 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -3223,3 +3223,159 @@ async fn a_build_runs_beside_commits_on_postgres() { }; a_build_runs_beside_commits_on(&url).await; } + +/// An edge at a negative ordinal, which pagination starting at -1 once skipped, is found by the comparison: the index +/// of such a manifest is not marked complete, however it got there, and a build throws the edge away. +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(url: &str) { + use craft_cloud_meta::{EdgeProblemKind, IndexState, Validation}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let mut conn = raw(url).await; + let spare = random_chunk(); + for ordinal in [-1i64, -7, i64::MIN] { + let chunks = fresh_chunks(5); + let m = manifest_over(&meta, &ana, &chunks).await; + // The stray edge is there before the manifest is recorded, and stays after all its own edges were written. + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(ordinal) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + match meta.index_check(&m).await.unwrap() { + craft_cloud_meta::EdgeCheck::Mismatch(p) => { + assert_eq!((p.ordinal, p.kind), (ordinal.unsigned_abs(), EdgeProblemKind::NegativeOrdinal), "{ordinal}") + } + other => panic!("{ordinal}: expected a mismatch, got {other:?}"), + } + assert!(matches!(meta.index_validate(&m).await.unwrap(), Validation::Mismatch(p) if p.kind == EdgeProblemKind::NegativeOrdinal), "{ordinal}"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. }), "{ordinal}: not complete"); + // The build rebuilds from the manifest, and the stray edge is gone. + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 5 }, "{ordinal}"); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks), "{ordinal}"); + } +} + +/// Throwing away the edges of an index that starts again takes batches of the index budget, each its own transaction, +/// and a run that stops between two of them leaves a marker the next one resumes from; nothing is written or marked +/// complete while edges of the old list are left. +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(12); + let spare = random_chunk(); + let m = { + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + m + }; + let mut conn = raw(url).await; + // A wrong edge, and one at a negative ordinal; the index was not marked complete. + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(spare.as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, -1, $2)") + .bind(m.id().as_bytes().to_vec()) + .bind(spare.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // The reset starts (what a build does on a mismatch) and the process stops right after: only the marker is there. + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // A new handle carries on, one batch of 3 rows per transaction, and writes nothing meanwhile. + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + assert_eq!(meta.index_edge_count().await.unwrap(), 13); + let mut left = 13u64; + while left > 0 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }, "{left} edges left"); + let now = meta.index_edge_count().await.unwrap(); + assert_eq!(left - now, left.min(3), "a batch deletes at most the budget"); + assert!(!matches!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { .. })); + left = now; + } + // The last batch (fewer rows than the budget) put the row back to building, so the next stage writes edges. + assert_eq!(meta.index_edge_count().await.unwrap(), 0); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// A build that meets a wrong edge resets the index across several batches by itself, and ends exact. +async fn a_build_resets_a_wrong_index_in_batches_on(url: &str) { + use craft_cloud_meta::IndexState; + let chunks = fresh_chunks(10); + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + while !matches!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done, total } if done == total) {} + let mut conn = raw(url).await; + sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 7") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 10 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +#[tokio::test] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on(&url).await; +} + +#[tokio::test] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_reset_of_an_index_goes_in_batches_and_resumes_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_reset_of_an_index_goes_in_batches_and_resumes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_reset_of_an_index_goes_in_batches_and_resumes_on(&url).await; +} + +#[tokio::test] +async fn a_build_resets_a_wrong_index_in_batches_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + a_build_resets_a_wrong_index_in_batches_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_resets_a_wrong_index_in_batches_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + a_build_resets_a_wrong_index_in_batches_on(&url).await; +} From d7c68987f3d9a0ffc9c0be76fd4dd281830fdfd1 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 17:17:20 +0100 Subject: [PATCH 20/27] server, meta, store: verify only an existing server and check the sizes it reads `verify` opened SQLite with create enabled and made missing storage directories, so a wrong path gave a new empty server and "no problems found". It now opens the database without creating it (`Meta::open_existing`: a missing file, a file with no Craft Cloud data, or a schema older than the program is an error, and nothing is created or upgraded) and requires the storage directory to exist. `index build` still uses the creating open. A successful read of a chunk or manifest object discarded its bytes, so seeded size corruption passed. Reading now reports the stored length (format byte included) and verify checks it against `blobs.stored_size` for chunks and manifests, the file size recorded for a manifest against the manifest, and the chunk lengths a manifest names against the length recorded for each chunk (one less than the stored size). Regressions cover a wrong database path, a wrong storage path, and seeded wrong sizes with and without reading chunk objects. --- apps/craft-cloud-server/src/cli.rs | 19 ++++- apps/craft-cloud-server/src/error.rs | 2 +- apps/craft-cloud-server/src/verify.rs | 101 +++++++++++++++++++------- apps/craft-cloud-server/tests/cli.rs | 90 +++++++++++++++++++++++ crates/meta/src/index.rs | 38 +++++++--- crates/meta/src/lib.rs | 29 +++++++- crates/meta/src/schema.rs | 31 ++++++++ crates/store/src/lib.rs | 9 ++- docs/design/sync.md | 10 ++- 9 files changed, 283 insertions(+), 46 deletions(-) diff --git a/apps/craft-cloud-server/src/cli.rs b/apps/craft-cloud-server/src/cli.rs index f497f7f0..ce867d20 100644 --- a/apps/craft-cloud-server/src/cli.rs +++ b/apps/craft-cloud-server/src/cli.rs @@ -185,6 +185,23 @@ pub async fn open(config: &Config) -> Result<(Meta, Store), CliError> { Ok((meta, store)) } +/// Like [`open`], for a command that only reads: the database and the storage must exist already, and nothing is +/// created or upgraded. A config that points at the wrong place is an error, not a new empty server that has no problems. +pub async fn open_existing(config: &Config) -> Result<(Meta, Store), CliError> { + let meta = Meta::open_existing(&config.database.url).await?; + let store = match (&config.storage, config.storage.s3()?) { + (_, Some(s3)) => Store::new(S3Backend::new(&s3)?), + (StorageConfig::Disk { path }, None) => { + if !path.is_dir() { + return Err(CliError::Usage(format!("the storage directory {} does not exist (is the config's path right?)", path.display()))); + } + Store::new(DiskBackend::new(path)?) + } + (StorageConfig::S3 { .. }, None) => return Err(CliError::Usage("S3 storage without settings".into())), + }; + Ok((meta, store)) +} + /// `--config` is global, which clap can't also make required, so it is checked here. fn config_path(config: Option) -> Result { config.ok_or_else(|| CliError::Usage("--config is required".into())) @@ -279,7 +296,7 @@ pub async fn run(command: Command, input: &mut impl BufRead, out: &mut impl Writ Ok(()) } Command::Verify { config, json, skip_chunk_reads } => { - let (meta, store) = open(&Config::load(&config_path(config)?)?).await?; + let (meta, store) = open_existing(&Config::load(&config_path(config)?)?).await?; let report = crate::verify::verify(&meta, &store, crate::verify::VerifyOptions { read_chunks: !skip_chunk_reads }).await?; if json { writeln!(out, "{}", report.to_json())?; diff --git a/apps/craft-cloud-server/src/error.rs b/apps/craft-cloud-server/src/error.rs index 73d5a18f..998260f2 100644 --- a/apps/craft-cloud-server/src/error.rs +++ b/apps/craft-cloud-server/src/error.rs @@ -100,7 +100,7 @@ impl From for ApiError { MetaError::DocFull { .. } | MetaError::LiveFull { .. } | MetaError::TooManyDocs { .. } | MetaError::TooManyDocRows { .. } => { Self::new(StatusCode::INSUFFICIENT_STORAGE, codes::QUOTA_EXCEEDED, e.to_string()) } - MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::UnsupportedUrl => { + MetaError::Db(_) | MetaError::Corrupt(_) | MetaError::Missing(_) | MetaError::UnsupportedUrl => { tracing::error!(error = %e, "metadata error"); Self::internal() } diff --git a/apps/craft-cloud-server/src/verify.rs b/apps/craft-cloud-server/src/verify.rs index 62dec5c5..a40edcb7 100644 --- a/apps/craft-cloud-server/src/verify.rs +++ b/apps/craft-cloud-server/src/verify.rs @@ -1,8 +1,9 @@ //! The invariant checker behind `craft-cloud-server verify`, and the index builder behind `craft-cloud-server index //! build`. //! -//! `verify` reads and reports; it changes nothing (opening the database brings its schema up to date, as every command -//! does, and that is all). It checks what a later garbage collector would have to trust: that every manifest's list of +//! `verify` reads and reports; it changes nothing: the database and the storage must exist already (a wrong path is an +//! error, not a new empty server with no problems), and a database whose schema is older is refused, not upgraded. It +//! checks what a later garbage collector would have to trust: that every manifest's list of //! chunks in the database is complete and equal to the manifest object in the store, that every chunk a manifest names //! is recorded and its object is there and intact, and that nothing in the database names a manifest or a chunk that does //! not exist. Anything it cannot establish is reported, not guessed: a manifest it cannot read or whose index is not @@ -189,7 +190,12 @@ fn hex(bytes: &[u8]) -> String { /// Reads a manifest object and decodes it; `Err` says what is wrong with it. pub async fn read_manifest(store: &Store, id: &ManifestId) -> Result)> { - let bytes = store.get(BlobKey::manifest(id)).await.map_err(|e| { + read_manifest_sized(store, id).await.map(|(manifest, _)| manifest) +} + +/// Like [`read_manifest`], with the bytes the object is stored as (the format byte included). +async fn read_manifest_sized(store: &Store, id: &ManifestId) -> Result<(Manifest, u64), (&'static str, Option)> { + let (bytes, stored) = store.get_sized(BlobKey::manifest(id)).await.map_err(|e| { let (kind, detail) = store_problem(&e); ( if kind == "object_missing" { @@ -206,7 +212,7 @@ pub async fn read_manifest(store: &Store, id: &ManifestId) -> Result Resul // Every recorded manifest: its object, its index, the chunks it names. let mut after: Option<[u8; 32]> = None; loop { - let ids = meta.blob_ids_after(BlobKind::Manifest, after, PAGE).await?; - let Some(last) = ids.last().copied() else { break }; + let records = meta.blob_records_after(BlobKind::Manifest, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; after = Some(last); - for raw in ids { + for record in records { + let raw = record.id; report.manifests_checked += 1; let id = ManifestId::from_bytes(raw); - let manifest = match read_manifest(store, &id).await { - Ok(manifest) => manifest, + let (manifest, stored) = match read_manifest_sized(store, &id).await { + Ok(read) => read, Err((kind, detail)) => { report.add(kind, hex(&raw), detail); continue; } }; + // What the database records of the manifest is what the object is: the bytes it is stored as, and the size + // of the file it describes. + if record.stored_size != stored { + report.add( + "manifest_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + if record.file_size != Some(manifest.size()) { + let recorded = record.file_size.map_or_else(|| "none".to_owned(), |n| n.to_string()); + report.add( + "manifest_file_size_mismatch", + hex(&raw), + Some(format!("the database records a file of {recorded} bytes, the manifest says {}", manifest.size())), + ); + } if let IndexState::Complete { .. } = meta.index_state(&id).await? && let EdgeCheck::Mismatch(problem) = meta.index_check(&manifest).await? { report.add("index_mismatch", hex(&raw), Some(format!("ordinal {}: {}", problem.ordinal, problem.kind.as_str()))); } - let ids: Vec = manifest.chunks().iter().map(|c| c.id).collect(); - for part in ids.chunks(500) { - let recorded = meta.recorded_chunks(part).await?; - for chunk in part { - if !recorded.contains(chunk) { + for refs in manifest.chunks().chunks(500) { + let part: Vec = refs.iter().map(|c| c.id).collect(); + let recorded = meta.recorded_chunk_sizes(&part).await?; + for chunk in &part { + if !recorded.contains_key(chunk) { report.add("chunk_not_recorded", format!("{}/{}", hex(&raw), hex(chunk.as_bytes())), None); } } + // The manifest says how long each chunk is; the database records the bytes it is stored as, one more + // than that (the format byte). With the chunk objects read below, this ties the length the manifest + // names to the bytes in the store. + for chunk in refs { + if let Some(stored) = recorded.get(&chunk.id) + && *stored != u64::from(chunk.len).saturating_add(1) + { + report.add( + "chunk_length_mismatch", + format!("{}/{}", hex(&raw), hex(chunk.id.as_bytes())), + Some(format!("the manifest says {} bytes, the database records {stored} stored", chunk.len)), + ); + } + } } } } @@ -263,22 +301,35 @@ pub async fn verify(meta: &Meta, store: &Store, options: VerifyOptions) -> Resul // Every recorded chunk's object (it also finds chunks no manifest names, which a manifest walk would not). let mut after: Option<[u8; 32]> = None; loop { - let ids = meta.blob_ids_after(BlobKind::Chunk, after, PAGE).await?; - let Some(last) = ids.last().copied() else { break }; + let records = meta.blob_records_after(BlobKind::Chunk, after, PAGE).await?; + let Some(last) = records.last().map(|r| r.id) else { break }; after = Some(last); - for raw in ids { + for record in records { + let raw = record.id; report.chunks_checked += 1; if !options.read_chunks { continue; } - if let Err(e) = store.get(BlobKey::chunk(&ChunkId::from_bytes(raw))).await { - let (kind, detail) = store_problem(&e); - let kind = match kind { - "object_missing" => "chunk_object_missing", - "object_corrupt" => "chunk_object_corrupt", - other => other, - }; - report.add(kind, hex(&raw), detail); + match store.get_sized(BlobKey::chunk(&ChunkId::from_bytes(raw))).await { + // Read, and its hash is right: its length is what the database records. + Ok((_, stored)) => { + if stored != record.stored_size { + report.add( + "chunk_stored_size_mismatch", + hex(&raw), + Some(format!("the database records {} bytes, the object is {stored}", record.stored_size)), + ); + } + } + Err(e) => { + let (kind, detail) = store_problem(&e); + let kind = match kind { + "object_missing" => "chunk_object_missing", + "object_corrupt" => "chunk_object_corrupt", + other => other, + }; + report.add(kind, hex(&raw), detail); + } } } } diff --git a/apps/craft-cloud-server/tests/cli.rs b/apps/craft-cloud-server/tests/cli.rs index bfc882d4..49279e8a 100644 --- a/apps/craft-cloud-server/tests/cli.rs +++ b/apps/craft-cloud-server/tests/cli.rs @@ -514,3 +514,93 @@ async fn an_index_built_after_the_fact_is_exact_and_verify_notices_until_then() async fn verify_needs_a_config() { assert!(matches!(cli(&["verify"], "").await, Err(CliError::Usage(m)) if m.contains("--config"))); } + +/// The config of `s` with its database url and its storage path replaced where given. +fn config_with(s: &Seeded, database: Option<&std::path::Path>, storage: Option<&std::path::Path>) -> String { + let text = std::fs::read_to_string(&s.config).unwrap(); + let slashed = |p: &std::path::Path| p.to_string_lossy().replace('\\', "/"); + let lines: Vec = text + .lines() + .map(|line| match (line.trim_start(), database, storage) { + (l, Some(db), _) if l.starts_with("url =") => format!("url = \"sqlite:{}\"", slashed(db)), + (l, _, Some(path)) if l.starts_with("path =") => format!("path = \"{}\"", slashed(path)), + _ => line.to_owned(), + }) + .collect(); + let target = s.dir.join("wrong.toml"); + std::fs::write(&target, lines.join("\n")).unwrap(); + target.to_str().unwrap().to_owned() +} + +#[tokio::test] +async fn verify_of_a_wrong_database_path_is_an_error_and_creates_nothing() { + let s = seeded().await; + let nowhere = s.dir.join("nowhere"); + let missing_db = nowhere.join("meta.db"); + let config = config_with(&s, Some(&missing_db), None); + let err = cli(&["verify", "--config", &config], "").await.expect_err("a database that is not there must not verify"); + assert!(!matches!(err, CliError::Inconsistent(_)), "an error, not a report: {err}"); + assert!(!missing_db.exists() && !nowhere.exists(), "nothing is created"); + // The same in a folder that exists: no new empty database appears in it. + let existing_dir_db = s.dir.join("other.db"); + let config = config_with(&s, Some(&existing_dir_db), None); + assert!(cli(&["verify", "--config", &config, "--json"], "").await.is_err()); + assert!(!existing_dir_db.exists(), "no empty database was created"); + // A file that is not a Craft Cloud database is refused too. + let empty = s.dir.join("empty.db"); + std::fs::write(&empty, b"").unwrap(); + let config = config_with(&s, Some(&empty), None); + assert!(cli(&["verify", "--config", &config], "").await.is_err()); + assert_eq!(std::fs::metadata(&empty).unwrap().len(), 0, "and left as it was"); +} + +#[tokio::test] +async fn verify_of_a_wrong_storage_path_is_an_error_and_creates_nothing() { + let s = seeded().await; + let missing = s.dir.join("no-blobs-here"); + let config = config_with(&s, None, Some(&missing)); + let err = cli(&["verify", "--config", &config], "").await.expect_err("storage that is not there must not verify"); + assert!(matches!(err, CliError::Usage(_)), "{err}"); + assert!(!missing.exists(), "the storage directory is not created"); +} + +#[tokio::test] +async fn verify_reports_sizes_the_database_records_wrongly() { + let s = seeded().await; + let mut conn = raw_sqlite(&s.db_url).await; + let (a, b) = (s.chunks[0], s.chunks[1]); + // A chunk recorded as longer than the manifest says (the manifest lists it twice), and a chunk whose stored length + // differs from the object's. + sqlx::query("UPDATE blobs SET stored_size = stored_size + 7 WHERE kind = 0 AND id = $1") + .bind(a.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("UPDATE blobs SET stored_size = stored_size - 1 WHERE kind = 0 AND id = $1") + .bind(b.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // The manifest's stored size and its file's size. + sqlx::query("UPDATE blobs SET stored_size = stored_size + 3, file_size = file_size + 1 WHERE kind = 1 AND id = $1") + .bind(s.manifest.as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + // Without reading the chunk objects the database's own sizes are checked against the manifests. + let (json, ok) = verify_json(&s.config, &["--skip-chunk-reads"]).await; + assert!(!ok, "{json}"); + let by = &json["problems"]["by_kind"]; + assert_eq!(by["chunk_length_mismatch"], 3, "{json}"); + assert_eq!(by["manifest_stored_size_mismatch"], 1, "{json}"); + assert_eq!(by["manifest_file_size_mismatch"], 1, "{json}"); + assert!(by["chunk_stored_size_mismatch"].is_null(), "the objects were not read: {json}"); + // Reading them also compares the stored lengths with the objects. + let (json, ok) = verify_json(&s.config, &[]).await; + assert!(!ok); + assert_eq!(json["problems"]["by_kind"]["chunk_stored_size_mismatch"], 2, "{json}"); + assert_eq!(json["problems"]["by_kind"]["chunk_length_mismatch"], 3, "{json}"); + // Looking repairs nothing. + let (again, _) = verify_json(&s.config, &[]).await; + assert_eq!(again["problems"]["by_kind"], json["problems"]["by_kind"]); +} diff --git a/crates/meta/src/index.rs b/crates/meta/src/index.rs index 115b201a..b3f77363 100644 --- a/crates/meta/src/index.rs +++ b/crates/meta/src/index.rs @@ -19,7 +19,7 @@ //! Nothing here frees anything. The index is read by the checker (`verify`) and by whatever later decides what is //! garbage; until it says a manifest is complete, that manifest's chunks are unknown, and unknown means kept. -use std::collections::HashSet; +use std::collections::HashMap; use craft_cloud_proto::{BlobKind, ChunkId, Manifest, ManifestId}; use sqlx::Row; @@ -45,6 +45,15 @@ const COMPLETE: i64 = 1; /// halfway leaves this marker, and the next one carries on from the rows that are left. const RESETTING: i64 = 2; +/// What the database records of a blob: its ID, the bytes it is stored as (the format byte included) and, for a +/// manifest, the size of its file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobRecord { + pub id: [u8; 32], + pub stored_size: u64, + pub file_size: Option, +} + /// How far a manifest's index has come. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IndexState { @@ -473,29 +482,38 @@ impl Meta { rows.iter().map(|r| Ok(ManifestId::from_bytes(hash_from(&r.try_get::, _>("id")?)?))).collect() } - /// Up to `limit` recorded blobs of `kind` after `after`, in ID order. - pub async fn blob_ids_after(&self, kind: BlobKind, after: Option<[u8; 32]>, limit: usize) -> Result, MetaError> { + /// Up to `limit` recorded blobs of `kind` after `after`, in ID order, with what the database says of their sizes. + pub async fn blob_records_after(&self, kind: BlobKind, after: Option<[u8; 32]>, limit: usize) -> Result, MetaError> { let after = after.map_or_else(|| vec![0u8; 32], |id| id.to_vec()); - let rows = sqlx::query("SELECT id FROM blobs WHERE kind = $1 AND id > $2 ORDER BY id LIMIT $3") + let rows = sqlx::query("SELECT id, stored_size, file_size FROM blobs WHERE kind = $1 AND id > $2 ORDER BY id LIMIT $3") .bind(kind.code()) .bind(after) .bind(i64::try_from(limit).map_err(|_| MetaError::Invalid("limit too large"))?) .fetch_all(&self.pool) .await?; - rows.iter().map(|r| hash_from(&r.try_get::, _>("id")?)).collect() + rows.iter() + .map(|r| { + let file_size: Option = r.try_get("file_size")?; + Ok(BlobRecord { + id: hash_from(&r.try_get::, _>("id")?)?, + stored_size: to_u64(r.try_get("stored_size")?)?, + file_size: file_size.map(to_u64).transpose()?, + }) + }) + .collect() } - /// Which of `ids` are recorded chunks. - pub async fn recorded_chunks(&self, ids: &[ChunkId]) -> Result, MetaError> { - let mut found = HashSet::new(); + /// Of `ids`, the recorded chunks and how many bytes each is stored as (the format byte included). + pub async fn recorded_chunk_sizes(&self, ids: &[ChunkId]) -> Result, MetaError> { + let mut found = HashMap::new(); for part in ids.chunks(ROWS_PER_STATEMENT) { - let sql = format!("SELECT id FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, part.len())); + let sql = format!("SELECT id, stored_size FROM blobs WHERE kind = $1 AND id IN ({})", placeholders(2, part.len())); let mut query = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(BlobKind::Chunk.code()); for id in part { query = query.bind(id.as_bytes().to_vec()); } for row in query.fetch_all(&self.pool).await? { - found.insert(ChunkId::from_bytes(hash_from(&row.try_get::, _>("id")?)?)); + found.insert(ChunkId::from_bytes(hash_from(&row.try_get::, _>("id")?)?), to_u64(row.try_get("stored_size")?)?); } } Ok(found) diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index c0ceeed6..817863d9 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -30,7 +30,7 @@ use tokio::sync::{Mutex, OwnedMutexGuard}; pub use commit::{CommitOutcome, RestoreOutcome, TrashRestoreOutcome}; pub use craft_cloud_proto::BlobKind; pub use history::HistoryPage; -pub use index::{EdgeCheck, EdgeProblem, EdgeProblemKind, Finding, INDEX_BATCH_ROWS, IndexAudit, IndexState, Validation}; +pub use index::{BlobRecord, EdgeCheck, EdgeProblem, EdgeProblemKind, Finding, INDEX_BATCH_ROWS, IndexAudit, IndexState, Validation}; pub use join::{InviteFor, Redeemed}; pub use live::{ Appended, DEFAULT_LIVE_BYTES_PER_SPACE, LiveDoc, LiveUpdate, MAX_LIVE_DOC_BYTES, MAX_LIVE_DOC_ROWS_PER_SPACE, MAX_LIVE_DOCS_PER_SPACE, @@ -87,6 +87,9 @@ pub enum MetaError { Invalid(&'static str), #[error("stored data is damaged: {0}")] Corrupt(&'static str), + /// A database that was to exist already ([`Meta::open_existing`]) is not there or not ready; nothing was created. + #[error("{0}")] + Missing(&'static str), } /// How long a SQLite connection waits for a lock another connection or program holds before giving up. @@ -127,6 +130,18 @@ impl Meta { /// Connects and brings the schema up to date. `url` is `sqlite:path/to/meta.db` (created if missing), /// `sqlite::memory:`, or `postgres://user:pass@host/db`. pub async fn open(url: &str) -> Result { + Self::start(url, false).await + } + + /// Connects to a database that already exists and creates nothing: a SQLite file that is not there is an error + /// (`mode=rw`, whatever options the URL had), not a new empty database, and a database whose schema is not the + /// current one is refused, not upgraded. For a command that only reads, so that a wrong path is reported instead of + /// being taken for an empty database that has no problems. `sqlite::memory:` has nothing to exist. + pub async fn open_existing(url: &str) -> Result { + Self::start(url, true).await + } + + async fn start(url: &str, existing: bool) -> Result { sqlx::any::install_default_drivers(); let dialect = if url.starts_with("sqlite:") { Dialect::Sqlite @@ -136,6 +151,8 @@ impl Meta { return Err(MetaError::UnsupportedUrl); }; let url = match dialect { + Dialect::Sqlite if existing && url.contains(":memory:") => return Err(MetaError::Missing("an in-memory database has no existing data")), + Dialect::Sqlite if existing => format!("{}?mode=rw", url.split_once('?').map_or(url, |(path, _)| path)), Dialect::Sqlite if !url.contains('?') && !url.contains(":memory:") => format!("{url}?mode=rwc"), _ => url.to_owned(), }; @@ -162,10 +179,14 @@ impl Meta { Dialect::Postgres => pool_options, }; let pool = pool_options.connect_with(options).await?; - if dialect == Dialect::Sqlite && !url.contains(":memory:") { - sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + if existing { + schema::check_current(&pool, dialect).await?; + } else { + if dialect == Dialect::Sqlite && !url.contains(":memory:") { + sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await?; + } + schema::migrate(&pool, dialect).await?; } - schema::migrate(&pool, dialect).await?; Ok(Self { pool, dialect, diff --git a/crates/meta/src/schema.rs b/crates/meta/src/schema.rs index 7b6be4e1..b4481bf2 100644 --- a/crates/meta/src/schema.rs +++ b/crates/meta/src/schema.rs @@ -32,6 +32,37 @@ const POSTGRES: &[&str] = &[ /// database. Every server must use the same key; no other code takes this lock. const MIGRATION_LOCK_KEY: i64 = 7240318150661; +/// Checks that the database has the current schema, and changes nothing: for a database that must exist already +/// ([`Meta::open_existing`](crate::Meta::open_existing)). +pub(crate) async fn check_current(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { + let migrations = match dialect { + Dialect::Sqlite => SQLITE, + Dialect::Postgres => POSTGRES, + }; + let known = i64::try_from(migrations.len()).map_err(|_| MetaError::Invalid("too many migrations"))?; + let table = match dialect { + Dialect::Sqlite => "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'", + Dialect::Postgres => { + "SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'schema_version'" + } + }; + let tables: i64 = sqlx::query(table).fetch_one(pool).await?.try_get("n")?; + if tables == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + let current: i64 = sqlx::query("SELECT COALESCE(MAX(version), 0) AS v FROM schema_version").fetch_one(pool).await?.try_get("v")?; + if current == 0 { + return Err(MetaError::Missing("the database holds no Craft Cloud data (is the database path right?)")); + } + if current > known { + return Err(MetaError::Corrupt("the database was created by a newer Craft Cloud")); + } + if current < known { + return Err(MetaError::Missing("the database schema is older than this program; start the server once (`serve`) to upgrade it")); + } + Ok(()) +} + pub(crate) async fn migrate(pool: &AnyPool, dialect: Dialect) -> Result<(), MetaError> { // One connection for the whole run. The Postgres advisory lock belongs to this session, so the lock, // the work and the unlock must all use it. diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 43f37475..7cf95831 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -126,6 +126,12 @@ impl Store { /// Reads a blob and checks it. A blob whose bytes no longer match its ID is an error, never served. pub async fn get(&self, key: BlobKey) -> Result { + self.get_sized(key).await.map(|(data, _)| data) + } + + /// Like [`Store::get`], and also says how many bytes the object is stored as (the format byte included), which is + /// what the metadata records as the blob's stored size. + pub async fn get_sized(&self, key: BlobKey) -> Result<(Bytes, u64), StoreError> { let object = self.backend.get_raw(&key.path()).await?.ok_or(StoreError::NotFound(key))?; let Some((&format, _)) = object.split_first() else { return Err(StoreError::Corrupt { key, reason: "empty object" }); @@ -133,6 +139,7 @@ impl Store { if format != FORMAT_PLAIN { return Err(StoreError::Corrupt { key, reason: "unknown format byte" }); } + let stored = u64::try_from(object.len()).unwrap_or(u64::MAX); let data = object.slice(1..); if data.len() > max_len(key.kind) { return Err(StoreError::Corrupt { key, reason: "larger than the limit" }); @@ -140,7 +147,7 @@ impl Store { if *blake3::hash(&data).as_bytes() != key.id { return Err(StoreError::Corrupt { key, reason: "hash mismatch" }); } - Ok(data) + Ok((data, stored)) } pub async fn delete(&self, key: BlobKey) -> Result<(), StoreError> { diff --git a/docs/design/sync.md b/docs/design/sync.md index cc278802..0384fbfb 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -721,13 +721,15 @@ each from the store and checking it against its ID first; it resumes where it st A manifest is marked complete only after every edge was compared, ordinal by ordinal, with the manifest it was made from: equal counts are not enough, and repeated chunks, wrong IDs, swapped order, missing and extra edges are all found. An index that does not match its manifest while it is still being built is thrown away and written again; a complete index -that is wrong is only reported. Nothing in this slice frees anything: no deletion, no claims, no collection, no retention. +that is wrong is only reported. Throwing the edges away is itself done in batches of the same size (a marker on the index row says it is under way, so an interrupted run resumes), never in one transaction; an edge at a negative ordinal is a mismatch like any other. Nothing in this slice frees anything: no deletion, no claims, no collection, no retention. -`craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing (opening the database -brings its schema up to date, as every command does). It lists problems (which make it exit with an error): a manifest +`craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing: the database and the +storage directory must exist already (a wrong path is an error, not a new empty server with "no problems found"), and a +database whose schema is older than the program is refused, not upgraded (start the server once to upgrade it). It lists +problems (which make it exit with an error): a manifest object that is missing, damaged or not a manifest; a complete index that does not equal its manifest; edges whose manifest has no index row; an index row without a recorded manifest; an edge or a manifest that names a chunk that is not recorded; -a chunk object that is missing or damaged; a version, snapshot or import snapshot that names a manifest that is not +a chunk object that is missing or damaged; a size the database records that is not what the object has (the stored length of a chunk or manifest object, a chunk length the manifest names against the length recorded for that chunk, the file size recorded for a manifest); a version, snapshot or import snapshot that names a manifest that is not recorded; blob memberships without a blob. It also gives notices (which do not): manifests whose index is not complete, manifests no version or snapshot names, and chunks no complete manifest names. **The manifests no version or snapshot names are not garbage.** A live document's binary assets (images and the like, uploaded through the agent's blob From c8725af06482c1d64c564b6e31e0cf6973906df8 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 20:15:31 +0100 Subject: [PATCH 21/27] test: cover reference index edge cases --- apps/craft-cloud-server/tests/cli.rs | 48 ++++++-- crates/meta/tests/behaviour.rs | 165 +++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 8 deletions(-) diff --git a/apps/craft-cloud-server/tests/cli.rs b/apps/craft-cloud-server/tests/cli.rs index 49279e8a..a7bf42c8 100644 --- a/apps/craft-cloud-server/tests/cli.rs +++ b/apps/craft-cloud-server/tests/cli.rs @@ -305,7 +305,7 @@ struct Seeded { db_url: String, chunks: Vec, manifest: craft_cloud_proto::ManifestId, - meta: craft_cloud_meta::Meta, + meta: Option, _keep: tempfile::TempDir, } @@ -351,7 +351,7 @@ async fn seeded() -> Seeded { dir, chunks, manifest: manifest.id(), - meta, + meta: Some(meta), _keep: keep, } } @@ -361,6 +361,17 @@ async fn raw_sqlite(url: &str) -> sqlx::SqliteConnection { sqlx::SqliteConnection::connect(url).await.unwrap() } +async fn sqlite_snapshot(url: &str, db_path: &std::path::Path) -> Vec>> { + use sqlx::Connection; + let mut conn = raw_sqlite(url).await; + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&mut conn).await.unwrap(); + conn.close().await.unwrap(); + [db_path.to_owned(), db_path.with_extension("db-wal"), db_path.with_extension("db-shm")] + .into_iter() + .map(|path| path.exists().then(|| std::fs::read(path).unwrap())) + .collect() +} + fn object_path(dir: &std::path::Path, kind: &str, id: &[u8; 32]) -> std::path::PathBuf { let hex: String = id.iter().map(|b| format!("{b:02x}")).collect(); dir.join("blobs").join(kind).join(&hex[0..2]).join(&hex[2..4]).join(&hex) @@ -385,7 +396,7 @@ async fn verify_json(c: &str, extra: &[&str]) -> (serde_json::Value, bool) { #[tokio::test] async fn verify_passes_on_a_sound_server_and_changes_nothing() { let s = seeded().await; - let before = s.meta.index_edge_count().await.unwrap(); + let before = s.meta.as_ref().unwrap().index_edge_count().await.unwrap(); let out = cli(&["verify", "--config", &s.config], "").await.unwrap(); assert!(out.contains("no problems found"), "{out}"); assert!(out.contains("checked 1 manifest(s) and 2 chunk(s) (chunk objects read and hashed)"), "{out}"); @@ -397,8 +408,8 @@ async fn verify_passes_on_a_sound_server_and_changes_nothing() { (Some(1), Some(1), Some(true)) ); assert_eq!(json["problems"]["total"], 0); - assert_eq!(s.meta.index_edge_count().await.unwrap(), before); - assert_eq!(s.meta.index_edges(&s.manifest).await.unwrap().len(), 3); + assert_eq!(s.meta.as_ref().unwrap().index_edge_count().await.unwrap(), before); + assert_eq!(s.meta.as_ref().unwrap().index_edges(&s.manifest).await.unwrap().len(), 3); } #[tokio::test] @@ -490,7 +501,7 @@ async fn an_index_built_after_the_fact_is_exact_and_verify_notices_until_then() let out = cli(&["index", "build", "--config", &s.config, "--batch-rows", "2"], "").await.unwrap(); assert!(out.contains("indexed 1 manifest(s)"), "{out}"); assert_eq!( - s.meta.index_edges(&s.manifest).await.unwrap().iter().map(|(_, c)| *c).collect::>(), + s.meta.as_ref().unwrap().index_edges(&s.manifest).await.unwrap().iter().map(|(_, c)| *c).collect::>(), vec![s.chunks[0], s.chunks[1], s.chunks[0]] ); let (json, ok) = verify_json(&s.config, &[]).await; @@ -506,8 +517,8 @@ async fn an_index_built_after_the_fact_is_exact_and_verify_notices_until_then() std::fs::remove_file(object_path(&s.dir, "manifests", s.manifest.as_bytes())).unwrap(); let built = cli(&["index", "build", "--config", &s.config], "").await; assert!(matches!(built, Err(CliError::Inconsistent(_))), "{built:?}"); - assert_eq!(s.meta.index_state(&s.manifest).await.unwrap(), craft_cloud_meta::IndexState::Absent); - assert!(s.meta.index_edges(&s.manifest).await.unwrap().is_empty()); + assert_eq!(s.meta.as_ref().unwrap().index_state(&s.manifest).await.unwrap(), craft_cloud_meta::IndexState::Absent); + assert!(s.meta.as_ref().unwrap().index_edges(&s.manifest).await.unwrap().is_empty()); } #[tokio::test] @@ -554,6 +565,27 @@ async fn verify_of_a_wrong_database_path_is_an_error_and_creates_nothing() { assert_eq!(std::fs::metadata(&empty).unwrap().len(), 0, "and left as it was"); } +#[tokio::test] +async fn verify_refuses_older_and_newer_schemas_without_changing_the_database() { + let mut s = seeded().await; + let db_path = std::path::PathBuf::from(s.db_url.strip_prefix("sqlite:").unwrap().split('?').next().unwrap()); + let mut conn = raw_sqlite(&s.db_url).await; + let current: i64 = sqlx::query_scalar("SELECT MAX(version) FROM schema_version").fetch_one(&mut conn).await.unwrap(); + use sqlx::Connection; + conn.close().await.unwrap(); + drop(s.meta.take()); + for (version, message) in [(current - 1, "start the server once"), (current + 1, "newer Craft Cloud")] { + let mut conn = raw_sqlite(&s.db_url).await; + sqlx::query("DELETE FROM schema_version").execute(&mut conn).await.unwrap(); + sqlx::query("INSERT INTO schema_version (version) VALUES ($1)").bind(version).execute(&mut conn).await.unwrap(); + conn.close().await.unwrap(); + let before = sqlite_snapshot(&s.db_url, &db_path).await; + let error = cli(&["verify", "--config", &s.config], "").await.expect_err("verify must refuse a non-current schema"); + assert!(error.to_string().contains(message), "{error}"); + assert!(sqlite_snapshot(&s.db_url, &db_path).await == before, "verify changed the database, WAL, or SHM for schema version {version}"); + } +} + #[tokio::test] async fn verify_of_a_wrong_storage_path_is_an_error_and_creates_nothing() { let s = seeded().await; diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index ff9ff316..1c08d68a 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -3335,6 +3335,132 @@ async fn a_build_resets_a_wrong_index_in_batches_on(url: &str) { assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); } +/// An edge at `i64::MAX` follows valid pages and keeps a manifest from validating without overflowing the cursor. +async fn an_edge_at_i64_max_is_reported_on(url: &str) { + use craft_cloud_meta::{EdgeCheck, EdgeProblemKind}; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(3); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + sqlx::query("INSERT INTO manifest_chunks (manifest, ordinal, chunk) VALUES ($1, $2, $3)") + .bind(m.id().as_bytes().to_vec()) + .bind(i64::MAX) + .bind(random_chunk().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + let expected = EdgeCheck::Mismatch(craft_cloud_meta::EdgeProblem { ordinal: 3, kind: EdgeProblemKind::MissingEdge }); + assert_eq!(meta.index_check(&m).await.unwrap(), expected); + assert_eq!(meta.index_check(&m).await.unwrap(), expected, "the later page is stable, with no cursor overflow"); + let edges = meta.index_edges(&m.id()).await.unwrap(); + assert_eq!(edges.len(), 4); + assert_eq!(edges.last().unwrap().0, i64::MAX as u64, "the final page contains the maximum ordinal"); +} + +/// A reset of 12 edges at budget 3 takes four full batches and one empty batch before returning to building state 0. +async fn an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(url: &str) { + use craft_cloud_meta::IndexState; + let meta = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&meta, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&meta, &ana, &chunks).await; + meta.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + meta.index_complete(&m).await.unwrap(); + let before = meta.index_edge_count().await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + for batch in 1..=4 { + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + assert_eq!(meta.index_edge_count().await.unwrap(), before - batch * 3); + } + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 2, "four full batches leave reset state 2 until an empty batch proves completion"); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 0, total: 12 }); + let state: i64 = sqlx::query_scalar("SELECT state FROM manifest_index WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!(state, 0); + assert_eq!(meta.index_edge_count().await.unwrap(), before - 12); + assert_eq!(meta.index_stage(&m).await.unwrap(), IndexState::Building { done: 3, total: 12 }, "the next call writes instead of looping"); + meta.index_complete(&m).await.unwrap(); + assert_eq!(meta.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); +} + +/// Two independent Postgres handles finish one damaged manifest reset without duplicate or missing ordinals. +async fn concurrent_index_completes_reset_one_manifest_on(url: &str) { + use craft_cloud_meta::IndexState; + let left = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let right = Meta::open(url).await.unwrap().with_index_batch_rows(3); + let ana = user(&left, "ana").await; + let chunks = fresh_chunks(12); + let m = manifest_over(&left, &ana, &chunks).await; + left.add_manifest(ana.space, &m, 100, NOW).await.unwrap(); + left.index_complete(&m).await.unwrap(); + let mut conn = raw(url).await; + let changed = sqlx::query("UPDATE manifest_chunks SET chunk = $1 WHERE manifest = $2 AND ordinal = 5") + .bind(random_chunk().as_bytes().to_vec()) + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1, "the index damage must hit one existing edge"); + sqlx::query("UPDATE manifest_index SET indexed_ordinals = 0, state = 2 WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .execute(&mut conn) + .await + .unwrap(); + + let left_manifest = m.clone(); + let right_manifest = m.clone(); + let left_builder = left.clone(); + let mut first = tokio::spawn(async move { left_builder.index_complete(&left_manifest).await }); + let mut second = tokio::spawn(async move { right.index_complete(&right_manifest).await }); + let joined = tokio::time::timeout(std::time::Duration::from_secs(30), async { + (&mut first).await.unwrap().unwrap(); + (&mut second).await.unwrap().unwrap(); + }) + .await; + if joined.is_err() { + first.abort(); + second.abort(); + } + assert!(joined.is_ok(), "concurrent index completion must finish within 30 seconds"); + + assert_eq!(left.index_state(&m.id()).await.unwrap(), IndexState::Complete { total: 12 }); + assert_eq!(left.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); + let (edges, ordinals): (i64, i64) = sqlx::query_as("SELECT COUNT(*), COUNT(DISTINCT ordinal) FROM manifest_chunks WHERE manifest = $1") + .bind(m.id().as_bytes().to_vec()) + .fetch_one(&mut conn) + .await + .unwrap(); + assert_eq!((edges, ordinals), (12, 12)); +} + #[tokio::test] async fn an_edge_at_a_negative_ordinal_keeps_an_index_from_completing_on_sqlite() { let dir = tempfile::tempdir().unwrap(); @@ -3379,3 +3505,42 @@ async fn a_build_resets_a_wrong_index_in_batches_on_postgres() { }; a_build_resets_a_wrong_index_in_batches_on(&url).await; } + +#[tokio::test] +async fn an_edge_at_i64_max_is_reported_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_edge_at_i64_max_is_reported_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_edge_at_i64_max_is_reported_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_edge_at_i64_max_is_reported_on(&url).await; +} + +#[tokio::test] +async fn an_exact_multiple_index_reset_finishes_on_sqlite() { + let dir = tempfile::tempdir().unwrap(); + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_exact_multiple_index_reset_finishes_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + an_exact_multiple_index_reset_finishes_after_one_empty_batch_on(&url).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_index_completes_reset_one_manifest_on_postgres() { + let Some((url, _guard)) = postgres_database().await else { + eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); + return; + }; + concurrent_index_completes_reset_one_manifest_on(&url).await; +} From 1f9610a245d1fec5f12e95ad6d774d3f5eb8d8c8 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Wed, 4 Nov 2026 20:36:10 +0100 Subject: [PATCH 22/27] meta, server: make the reference index edge-case tests prove what they claim The verify schema test now snapshots the database and WAL straight from disk, with no SQLite connection opened between the setup checkpoint and the check. The concurrent index reset test holds the manifest_index row lock, proves both builders wait on it, and only then releases it. The i64::MAX edge test is renamed for what it pins; the cursor overflow guard is not reachable through the public API. --- apps/craft-cloud-server/tests/cli.rs | 23 ++++++------- crates/meta/tests/behaviour.rs | 49 +++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/apps/craft-cloud-server/tests/cli.rs b/apps/craft-cloud-server/tests/cli.rs index a7bf42c8..cc0b6a8c 100644 --- a/apps/craft-cloud-server/tests/cli.rs +++ b/apps/craft-cloud-server/tests/cli.rs @@ -361,15 +361,14 @@ async fn raw_sqlite(url: &str) -> sqlx::SqliteConnection { sqlx::SqliteConnection::connect(url).await.unwrap() } -async fn sqlite_snapshot(url: &str, db_path: &std::path::Path) -> Vec>> { - use sqlx::Connection; - let mut conn = raw_sqlite(url).await; - sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&mut conn).await.unwrap(); - conn.close().await.unwrap(); - [db_path.to_owned(), db_path.with_extension("db-wal"), db_path.with_extension("db-shm")] - .into_iter() - .map(|path| path.exists().then(|| std::fs::read(path).unwrap())) - .collect() +/// The bytes of the database file and of its WAL, read straight from disk without opening SQLite, so that nothing here +/// can checkpoint away a change the command under test made. A WAL that is not there counts as an empty one: opening a +/// WAL-mode database that has no open connection makes an empty `-wal` and a `-shm` file, which changes no data. The +/// `-shm` file is a rebuilt shared-memory index of the WAL, not data, so it is not compared; a change to the data +/// shows up in the database file or in the WAL's bytes. +fn sqlite_snapshot(db_path: &std::path::Path) -> (Vec, Vec) { + let read = |path: std::path::PathBuf| if path.exists() { std::fs::read(path).unwrap() } else { Vec::new() }; + (read(db_path.to_owned()), read(db_path.with_extension("db-wal"))) } fn object_path(dir: &std::path::Path, kind: &str, id: &[u8; 32]) -> std::path::PathBuf { @@ -578,11 +577,13 @@ async fn verify_refuses_older_and_newer_schemas_without_changing_the_database() let mut conn = raw_sqlite(&s.db_url).await; sqlx::query("DELETE FROM schema_version").execute(&mut conn).await.unwrap(); sqlx::query("INSERT INTO schema_version (version) VALUES ($1)").bind(version).execute(&mut conn).await.unwrap(); + // Checkpoint during setup only, then close every connection, so the snapshots below are taken of files at rest. + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&mut conn).await.unwrap(); conn.close().await.unwrap(); - let before = sqlite_snapshot(&s.db_url, &db_path).await; + let before = sqlite_snapshot(&db_path); let error = cli(&["verify", "--config", &s.config], "").await.expect_err("verify must refuse a non-current schema"); assert!(error.to_string().contains(message), "{error}"); - assert!(sqlite_snapshot(&s.db_url, &db_path).await == before, "verify changed the database, WAL, or SHM for schema version {version}"); + assert!(sqlite_snapshot(&db_path) == before, "verify changed the database or its WAL for schema version {version}"); } } diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index 1c08d68a..a985af8a 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -3335,8 +3335,15 @@ async fn a_build_resets_a_wrong_index_in_batches_on(url: &str) { assert_eq!(meta.index_edges(&m.id()).await.unwrap(), edge_list(&chunks)); } -/// An edge at `i64::MAX` follows valid pages and keeps a manifest from validating without overflowing the cursor. -async fn an_edge_at_i64_max_is_reported_on(url: &str) { +/// An extra edge at the highest possible ordinal, `i64::MAX`, after valid ones is reported as a missing edge at the first +/// ordinal that is not there, and the report is stable. +/// +/// This does not reach the `raw.checked_add(1)` that guards the page cursor in `compare`. An edge at `i64::MAX` is +/// returned as `MissingEdge` as soon as it is seen, because its ordinal is above the next expected one, and no +/// manifest has the 2^63 chunks that would make it the expected one, so the cursor step at `i64::MAX` cannot be +/// reached through the public meta API. The test pins that a high-ordinal edge is rejected without an error or an +/// overflow, not the checked addition. +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(url: &str) { use craft_cloud_meta::{EdgeCheck, EdgeProblemKind}; let meta = Meta::open(url).await.unwrap().with_index_batch_rows(2); let ana = user(&meta, "ana").await; @@ -3355,10 +3362,10 @@ async fn an_edge_at_i64_max_is_reported_on(url: &str) { let expected = EdgeCheck::Mismatch(craft_cloud_meta::EdgeProblem { ordinal: 3, kind: EdgeProblemKind::MissingEdge }); assert_eq!(meta.index_check(&m).await.unwrap(), expected); - assert_eq!(meta.index_check(&m).await.unwrap(), expected, "the later page is stable, with no cursor overflow"); + assert_eq!(meta.index_check(&m).await.unwrap(), expected, "the report is stable"); let edges = meta.index_edges(&m.id()).await.unwrap(); assert_eq!(edges.len(), 4); - assert_eq!(edges.last().unwrap().0, i64::MAX as u64, "the final page contains the maximum ordinal"); + assert_eq!(edges.last().unwrap().0, i64::MAX as u64, "the stored edge at the maximum ordinal is still there"); } /// A reset of 12 edges at budget 3 takes four full batches and one empty batch before returning to building state 0. @@ -3435,11 +3442,37 @@ async fn concurrent_index_completes_reset_one_manifest_on(url: &str) { .await .unwrap(); + // Force the overlap: a third connection holds the row lock of the manifest's index row, both builders start and are + // proven to wait on it, and only then is it released, so both reach the reset together. + let mut holder = raw(url).await; + sqlx::query("BEGIN").execute(&mut holder).await.unwrap(); + sqlx::query("SELECT 1 FROM manifest_index WHERE manifest = $1 FOR UPDATE").bind(m.id().as_bytes().to_vec()).execute(&mut holder).await.unwrap(); + let left_manifest = m.clone(); let right_manifest = m.clone(); let left_builder = left.clone(); let mut first = tokio::spawn(async move { left_builder.index_complete(&left_manifest).await }); let mut second = tokio::spawn(async move { right.index_complete(&right_manifest).await }); + let mut waiting = 0i64; + for _ in 0..300 { + waiting = sqlx::query_scalar( + "SELECT COUNT(*) FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock' AND query LIKE '%manifest_index%FOR UPDATE%'", + ) + .fetch_one(&mut conn) + .await + .unwrap(); + if waiting >= 2 || first.is_finished() || second.is_finished() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + let overlapped = waiting == 2 && !first.is_finished() && !second.is_finished(); + sqlx::query("ROLLBACK").execute(&mut holder).await.unwrap(); + if !overlapped { + first.abort(); + second.abort(); + } + assert!(overlapped, "both builders must be blocked on the manifest's index row at once (waiting backends: {waiting})"); let joined = tokio::time::timeout(std::time::Duration::from_secs(30), async { (&mut first).await.unwrap().unwrap(); (&mut second).await.unwrap().unwrap(); @@ -3507,18 +3540,18 @@ async fn a_build_resets_a_wrong_index_in_batches_on_postgres() { } #[tokio::test] -async fn an_edge_at_i64_max_is_reported_on_sqlite() { +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_sqlite() { let dir = tempfile::tempdir().unwrap(); - an_edge_at_i64_max_is_reported_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&format!("sqlite:{}", dir.path().join("meta.db").display())).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn an_edge_at_i64_max_is_reported_on_postgres() { +async fn an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on_postgres() { let Some((url, _guard)) = postgres_database().await else { eprintln!("skipped: CRAFT_CLOUD_TEST_PG not set"); return; }; - an_edge_at_i64_max_is_reported_on(&url).await; + an_extra_edge_at_the_highest_ordinal_is_reported_as_missing_on(&url).await; } #[tokio::test] From 84599b5a19866d29bd98352e814454aba9a98833 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Thu, 5 Nov 2026 01:09:31 +0100 Subject: [PATCH 23/27] meta, server: open an existing SQLite database read-only so verify cannot checkpoint a crashed WAL Closing the last read-write connection to a WAL-mode database checkpoints the WAL of a crashed process into the database file and removes it. Whether that finished before verify exited was a race, and a command that only reads must not make that write. Meta::open_existing now opens SQLite with mode=ro, which still reads the WAL's committed data. The tests run verify as a process of its own and compare the database file, the WAL and the storage tree after it has exited, with a row that only the WAL holds and with the -shm file gone. --- apps/craft-cloud-server/tests/cli.rs | 89 +++++++++++++++++++++++++++- crates/meta/src/lib.rs | 9 ++- docs/design/sync.md | 2 +- docs/server.md | 4 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/apps/craft-cloud-server/tests/cli.rs b/apps/craft-cloud-server/tests/cli.rs index cc0b6a8c..744aad19 100644 --- a/apps/craft-cloud-server/tests/cli.rs +++ b/apps/craft-cloud-server/tests/cli.rs @@ -371,6 +371,91 @@ fn sqlite_snapshot(db_path: &std::path::Path) -> (Vec, Vec) { (read(db_path.to_owned()), read(db_path.with_extension("db-wal"))) } +/// Runs the real `craft-cloud-server` binary with the arguments and waits for it to exit, so that every connection it +/// opened is closed, and every write a close or a checkpoint makes is on disk, when this returns: whether it exited with +/// success, its standard output and its standard error. +fn run_binary(args: &[&str]) -> (bool, String, String) { + let output = Process::new(env!("CARGO_BIN_EXE_craft-cloud-server")).args(args).stdin(Stdio::null()).output().unwrap(); + (output.status.success(), String::from_utf8(output.stdout).unwrap(), String::from_utf8(output.stderr).unwrap()) +} + +/// Not a test of its own: run as a child of the crashed-WAL tests, it commits a chunk row and ends without closing the +/// database, which leaves the row in the WAL and in no checkpoint. +#[test] +fn verify_test_leave_a_chunk_in_the_wal() { + let Ok(url) = std::env::var("CRAFT_CLOUD_TEST_WAL_CHUNK_URL") else { return }; + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + runtime.block_on(async { + let meta = craft_cloud_meta::Meta::open(&url).await.unwrap(); + let ana = meta.user_by_name("ana").await.unwrap().unwrap(); + let id = craft_cloud_proto::ChunkId::from_bytes(*blake3::hash(b"a chunk only the WAL knows").as_bytes()); + meta.add_chunk(ana.space, &id, 9, 0).await.unwrap(); + std::process::exit(0); + }); +} + +/// A database a crashed process left with a row only in its WAL: `verify` (a process of its own, finished before +/// anything is compared) reads that row, and the database file, the WAL and the storage tree keep their bytes. With +/// `delete_shm`, the `-shm` file the crash left is gone before the run, in a directory that can be written. +async fn verify_of_a_crashed_wal(delete_shm: bool) { + let parent = tempfile::tempdir().unwrap(); + let dir = parent.path().join("data"); + cli(&["init", "--dir", dir.to_str().unwrap(), "--admin", "ana"], "admin password 1\n").await.unwrap(); + let config = dir.join("config.toml"); + let loaded = Config::load(&config).unwrap(); + let mut conn = raw_sqlite(&loaded.database.url).await; + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&mut conn).await.unwrap(); + sqlx::Connection::close(conn).await.unwrap(); + let status = Process::new(std::env::current_exe().unwrap()) + .args(["--exact", "verify_test_leave_a_chunk_in_the_wal", "--nocapture"]) + .env("CRAFT_CLOUD_TEST_WAL_CHUNK_URL", &loaded.database.url) + .status() + .unwrap(); + assert!(status.success()); + let db_path = dir.join("meta.db"); + if delete_shm { + let _ = std::fs::remove_file(dir.join("meta.db-shm")); + } + let (db_before, wal_before) = sqlite_snapshot(&db_path); + assert!(!wal_before.is_empty(), "test requires a non-empty WAL"); + let tree_before = tree_snapshot(&dir.join("blobs")); + + let (ok, out, err) = run_binary(&["verify", "--json", "--config", config.to_str().unwrap()]); + assert!(!ok, "the chunk's object is not stored, so verify must find a problem: {out}{err}"); + let report: serde_json::Value = serde_json::from_str(&out).unwrap_or_else(|e| panic!("{e}: {out}{err}")); + assert_eq!(report["problems"]["by_kind"]["chunk_object_missing"], 1, "verify must see the row that only the WAL holds: {out}"); + let (db_after, wal_after) = sqlite_snapshot(&db_path); + assert!(db_before == db_after, "database file changed: {} -> {} bytes", db_before.len(), db_after.len()); + assert!(wal_before == wal_after, "WAL changed: {} -> {} bytes", wal_before.len(), wal_after.len()); + assert!(tree_before == tree_snapshot(&dir.join("blobs")), "the storage changed"); +} + +#[tokio::test] +async fn verify_reads_a_crashed_wal_and_changes_no_data() { + verify_of_a_crashed_wal(false).await; +} + +#[tokio::test] +async fn verify_reads_a_crashed_wal_whose_shm_file_is_gone() { + verify_of_a_crashed_wal(true).await; +} + +fn tree_snapshot(root: &std::path::Path) -> std::collections::BTreeMap> { + fn visit(root: &std::path::Path, dir: &std::path::Path, out: &mut std::collections::BTreeMap>) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, out); + } else { + out.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(&path).unwrap()); + } + } + } + let mut out = std::collections::BTreeMap::new(); + visit(root, root, &mut out); + out +} + fn object_path(dir: &std::path::Path, kind: &str, id: &[u8; 32]) -> std::path::PathBuf { let hex: String = id.iter().map(|b| format!("{b:02x}")).collect(); dir.join("blobs").join(kind).join(&hex[0..2]).join(&hex[2..4]).join(&hex) @@ -581,8 +666,8 @@ async fn verify_refuses_older_and_newer_schemas_without_changing_the_database() sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&mut conn).await.unwrap(); conn.close().await.unwrap(); let before = sqlite_snapshot(&db_path); - let error = cli(&["verify", "--config", &s.config], "").await.expect_err("verify must refuse a non-current schema"); - assert!(error.to_string().contains(message), "{error}"); + let (ok, _, error) = run_binary(&["verify", "--config", &s.config]); + assert!(!ok && error.contains(message), "verify must refuse a non-current schema: {error}"); assert!(sqlite_snapshot(&db_path) == before, "verify changed the database or its WAL for schema version {version}"); } } diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 817863d9..acea8c5d 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -134,9 +134,14 @@ impl Meta { } /// Connects to a database that already exists and creates nothing: a SQLite file that is not there is an error - /// (`mode=rw`, whatever options the URL had), not a new empty database, and a database whose schema is not the + /// (`mode=ro`, whatever options the URL had), not a new empty database, and a database whose schema is not the /// current one is refused, not upgraded. For a command that only reads, so that a wrong path is reported instead of /// being taken for an empty database that has no problems. `sqlite::memory:` has nothing to exist. + /// + /// A read-only SQLite connection never writes the database file or the WAL, and cannot checkpoint: closing the last + /// connection of a read-write one checkpoints the WAL of a crashed process into the database file and removes it, + /// which is a write a read-only command must not make. It still reads the WAL's committed data. Opening may leave + /// an empty `-wal` and a `-shm` file (SQLite's rebuilt index of the WAL, no data) when there were none. pub async fn open_existing(url: &str) -> Result { Self::start(url, true).await } @@ -152,7 +157,7 @@ impl Meta { }; let url = match dialect { Dialect::Sqlite if existing && url.contains(":memory:") => return Err(MetaError::Missing("an in-memory database has no existing data")), - Dialect::Sqlite if existing => format!("{}?mode=rw", url.split_once('?').map_or(url, |(path, _)| path)), + Dialect::Sqlite if existing => format!("{}?mode=ro", url.split_once('?').map_or(url, |(path, _)| path)), Dialect::Sqlite if !url.contains('?') && !url.contains(":memory:") => format!("{url}?mode=rwc"), _ => url.to_owned(), }; diff --git a/docs/design/sync.md b/docs/design/sync.md index 0384fbfb..57bae47a 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -725,7 +725,7 @@ that is wrong is only reported. Throwing the edges away is itself done in batche `craft-cloud-server verify [--json] [--skip-chunk-reads]` reads and reports and changes nothing: the database and the storage directory must exist already (a wrong path is an error, not a new empty server with "no problems found"), and a -database whose schema is older than the program is refused, not upgraded (start the server once to upgrade it). It lists +database whose schema is older than the program is refused, not upgraded (start the server once to upgrade it). A SQLite database is opened read-only (`Meta::open_existing`, `mode=ro`): the database file and the WAL keep their bytes, also when a crashed process left committed data in the WAL, which the command reads; it may leave an empty `-wal` and a `-shm` file (the rebuilt index of the WAL, not data). It lists problems (which make it exit with an error): a manifest object that is missing, damaged or not a manifest; a complete index that does not equal its manifest; edges whose manifest has no index row; an index row without a recorded manifest; an edge or a manifest that names a chunk that is not recorded; diff --git a/docs/server.md b/docs/server.md index 2c353537..316a08de 100644 --- a/docs/server.md +++ b/docs/server.md @@ -87,7 +87,9 @@ upgraded database and rejects the `[retention]` section, so take a backup before `craft-cloud-server verify --config cc-data/config.toml` checks the database and the blob storage against each other and prints what it finds; `--json` prints a machine-readable report and `--skip-chunk-reads` skips reading every chunk object (which hashes it, and takes as long as reading all your data). It changes nothing and exits with an error if -it found a problem. The report also says how many manifests have a complete reference index, which is what a later +it found a problem. It opens a SQLite database read-only: the database file and its WAL keep their bytes, also after a +crash that left committed data in the WAL, which it reads. Opening may leave an empty `-wal` and a `-shm` file behind +(SQLite's rebuilt index of the WAL, not data). The report also says how many manifests have a complete reference index, which is what a later cleanup would need before it could free anything. After upgrading, run `craft-cloud-server index build --config cc-data/config.toml` once: it indexes the manifests stored before the index existed, writing only that index, in small steps, and it can be stopped and run again. Nothing in the server frees storage yet. From 1fe8c6c97d2536fd1f59a3b996b3c48ee5377967 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Fri, 6 Nov 2026 20:49:15 +0100 Subject: [PATCH 24/27] meta, server: a folder comes back and goes with everything of it in the trash Restoring a folder from the trash now also brings back every node below it that is in the trash (deleted and not removed from the trash), however long before it was deleted, in the same transaction; the trash list shows such a node only through its folder; and removing a folder from the trash marks them too. A folder with more than 10,000 such nodes below it is refused whole as invalid for both, and Meta::with_trash_tree_cap lowers the limit for tests. --- apps/craft-cloud-agent/tests/end_to_end.rs | 37 ++++ apps/craft-cloud-server/tests/api.rs | 61 +++++++ crates/meta/src/commit.rs | 53 +++++- crates/meta/src/lib.rs | 16 ++ crates/meta/src/trash.rs | 48 ++++- crates/meta/tests/behaviour.rs | 193 +++++++++++++++++++++ docs/design/sync.md | 17 +- docs/server.md | 6 +- 8 files changed, 421 insertions(+), 10 deletions(-) diff --git a/apps/craft-cloud-agent/tests/end_to_end.rs b/apps/craft-cloud-agent/tests/end_to_end.rs index 92d2fd97..d65ad932 100644 --- a/apps/craft-cloud-agent/tests/end_to_end.rs +++ b/apps/craft-cloud-agent/tests/end_to_end.rs @@ -163,6 +163,43 @@ async fn a_first_sync_goes_both_ways() { assert_eq!(a.files(), b.files()); } +/// Deleting a folder with files in it and restoring it from the trash: the folder and everything that was in the trash +/// below it, also a file deleted on its own before, come back to the disk of both computers by their normal pass. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_folder_restored_from_the_trash_comes_back_with_its_files() { + let server = server().await; + let a = Agent::signed_in(&server, "desktop").await; + a.write("dir1/child.txt", "deleted with the folder"); + a.write("dir1/early.txt", "deleted first"); + a.write("dir1/sub/deep.txt", "deep"); + a.write("keep.txt", "stays"); + assert_eq!(a.sync().await, Exit::Done); + let b = Agent::signed_in(&server, "laptop").await; + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(b.files().len(), 4); + + std::fs::remove_file(a.path("dir1/early.txt")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + std::fs::remove_dir_all(a.path("dir1")).unwrap(); + assert_eq!(a.sync().await, Exit::Done); + assert_eq!(b.sync().await, Exit::Done); + assert_eq!(a.files(), ["keep.txt"]); + assert_eq!(b.files(), ["keep.txt"]); + + let ana = server.meta.user_by_name("ana").await.unwrap().unwrap(); + let listed = server.meta.trash(ana.space, None, 200, 0).await.unwrap().entries; + assert_eq!(listed.iter().map(|e| e.name.as_str()).collect::>(), ["dir1"], "the folder is listed alone"); + let device = server.meta.list_devices(ana.id).await.unwrap().into_iter().find(|d| d.name == "desktop").unwrap().id; + server.meta.restore_from_trash(ana.space, device, listed[0].id, 0).await.unwrap(); + + for agent in [&a, &b] { + assert_eq!(agent.sync().await, Exit::Done); + assert_eq!(agent.files(), ["dir1/child.txt", "dir1/early.txt", "dir1/sub/deep.txt", "keep.txt"]); + assert_eq!(agent.read("dir1/sub/deep.txt").as_deref(), Some("deep")); + assert_eq!(agent.read("dir1/early.txt").as_deref(), Some("deleted first")); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn big_files_travel_in_chunks_and_keep_their_exec_bit() { let server = server().await; diff --git a/apps/craft-cloud-server/tests/api.rs b/apps/craft-cloud-server/tests/api.rs index f111ca4c..6e48a1aa 100644 --- a/apps/craft-cloud-server/tests/api.rs +++ b/apps/craft-cloud-server/tests/api.rs @@ -163,6 +163,7 @@ on_every_backend!( restoring_an_old_version_makes_a_new_one, parallel_restores_against_one_head_let_one_win, the_trash_lists_restores_and_removes_over_http, + a_deleted_folder_comes_back_and_goes_with_what_was_deleted_in_it, every_rejection_is_a_json_error, one_request_cant_list_unbounded_missing_chunks, long_polls_are_capped_per_space, @@ -1755,6 +1756,66 @@ async fn the_trash_lists_restores_and_removes_over_http(s: &Server) { assert!(body.message.contains("/trash/"), "{}", body.message); } +/// A deleted folder is listed alone and is restored (or removed) with everything of it that is in the trash, also a +/// file that was deleted on its own before. +async fn a_deleted_folder_comes_back_and_goes_with_what_was_deleted_in_it(s: &Server) { + use craft_cloud_proto::api::{TrashResponse, TrashRestoreResponse}; + add_user(s, "ana").await; + let ana = login(s, "ana").await; + let r = root(s, &ana).await; + let (dir1, sub1, dir2, sub2) = (NodeId::generate(), NodeId::generate(), NodeId::generate(), NodeId::generate()); + for (name, dir, sub) in [("dir1", dir1, sub1), ("dir2", dir2, sub2)] { + let ops = [ + Op::Mkdir { node: dir, parent: r, name: Name::new(name).unwrap() }, + Op::Mkdir { node: sub, parent: dir, name: Name::new("sub").unwrap() }, + ]; + assert_eq!(commit(s, &ana, &ops).await.status(), StatusCode::OK); + } + let mut kids = Vec::new(); + for (dir, sub) in [(dir1, sub1), (dir2, sub2)] { + let (early, early_v) = new_file(s, &ana, dir, "early.txt", b"deleted on its own first").await; + delete_node(s, &ana, early, early_v).await; + let (child, child_v) = new_file(s, &ana, dir, "child.txt", b"deleted with the folder").await; + let (deep, deep_v) = new_file(s, &ana, sub, "deep.txt", b"deep").await; + let ops = [ + Op::Delete { node: child, expected: Some(child_v) }, + Op::Delete { node: deep, expected: Some(deep_v) }, + Op::Delete { node: sub, expected: None }, + Op::Delete { node: dir, expected: None }, + ]; + assert_eq!(commit(s, &ana, &ops).await.status(), StatusCode::OK); + kids.push([early, child, deep, sub]); + } + let listed: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + let mut ids: Vec<_> = listed.items.iter().map(|i| i.id).collect(); + ids.sort(); + let mut tops = vec![dir1, dir2]; + tops.sort(); + assert_eq!(ids, tops, "only the folders are listed"); + + // Restoring one folder brings all that was in the trash below it, and nothing of the other. + let before = space_of(s, &ana).await; + let done: TrashRestoreResponse = trash_restore(s, &ana, dir1).await.json().await.unwrap(); + assert_eq!((done.node, done.name.as_str(), done.renamed, done.seq), (dir1, "dir1", false, before.seq + 1)); + let tree: ChangeSet = s.client.get(format!("{}/tree", s.base)).bearer_auth(&ana).send().await.unwrap().json().await.unwrap(); + let live: Vec<_> = tree.nodes.iter().map(|n| n.id).collect(); + for node in kids[0].iter().chain([&dir1]) { + assert!(live.contains(node), "{node} is live again"); + } + for node in kids[1].iter().chain([&dir2]) { + assert!(!live.contains(node), "{node} is still deleted"); + } + + // Removing the other folder takes what is below it out of the trash too. + assert_eq!(trash_remove(s, &ana, dir2).await.status(), StatusCode::NO_CONTENT); + let left: TrashResponse = trash_page(s, &ana, "").await.json().await.unwrap(); + assert!(left.items.is_empty(), "{:?}", left.items); + for node in kids[1] { + let (status, body) = error(trash_restore(s, &ana, node).await).await; + assert_eq!((status, body.error.as_str()), (StatusCode::NOT_FOUND, "not_found"), "{node}"); + } +} + /// A server with another `retention.trash_days` (SQLite and disk). async fn sqlite_disk_with_trash_days(days: u32) -> Server { let db = tempfile::tempdir().unwrap(); diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index 7419cd62..1317d2dc 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -22,6 +22,7 @@ use sqlx::Row; use crate::blobs::manifest_file_size; use crate::codec::{flag, hash_from, to_i64, uuid_bytes, uuid_from}; +use crate::trash::TRASH_BELOW; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; @@ -441,7 +442,7 @@ impl Ctx<'_> { /// and what is below it moves in. Where a name is taken by a file, or the node's own name is taken by anything, the /// node comes back under a conflict name. Never overwrites, never creates a node. A conflict that can't be solved /// is an `Err(MetaError::Rejected)`, and the caller drops the transaction. - async fn restore_from_trash(&mut self, node: NodeId) -> Result { + async fn restore_from_trash(&mut self, node: NodeId, tree_cap: u64) -> Result { let found = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") .bind(self.space()) .bind(uuid_bytes(node.as_uuid())) @@ -507,12 +508,58 @@ impl Ctx<'_> { // server refuses that delete as a version mismatch, and the machine's engine already treats a node that came // back at another version as one whose delete went through. let version = if is_dir { None } else { Some(self.renew_version(node).await?) }; + if is_dir { + self.restore_below(node, tree_cap, &device).await?; + } let placed = self.node(node).await?.ok_or(MetaError::Corrupt("a restored node vanished"))?; let parent = placed.parent.ok_or(MetaError::Corrupt("a restored node has no parent"))?; let original = up.first().map(|a| a.name.as_str()).unwrap_or_default(); Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, version, seq: 0, epoch: self.epoch }) } + /// Brings back everything in the trash below the folder `top`, which is live again: its deleted children and theirs, + /// whenever they were deleted, parents first. A child keeps its name; one whose name another child that came back + /// took (a name deleted twice, the newest deletion first) gets a conflict name. A removed node, and what is only + /// below a removed folder, stays deleted. More than `cap` nodes: `Invalid`, and the caller drops the transaction. + async fn restore_below(&mut self, top: NodeId, cap: u64, device: &str) -> Result<(), MetaError> { + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let limit = i64::try_from(cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?.saturating_add(1); + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} SELECT n.id, n.parent, n.name, n.is_dir FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id \ + ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $4" + ))) + .bind(self.space()) + .bind(uuid_bytes(top.as_uuid())) + .bind(bound) + .bind(limit) + .fetch_all(&mut *self.conn) + .await?; + if u64::try_from(rows.len()).map_err(|_| MetaError::Corrupt("count out of range"))? > cap { + return Err(MetaError::Invalid(crate::trash::TRASH_TREE_TOO_LARGE)); + } + for row in &rows { + let id = NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?); + let parent = NodeId::from_uuid(uuid_from(&row.try_get::, _>("parent")?)?); + let is_dir = flag(row, "is_dir")?; + let own = Name::new(&row.try_get::("name")?).map_err(|_| MetaError::Corrupt("a stored name is invalid"))?; + let name = match self.live_child(parent, &own).await? { + None => own, + Some(existing) => match self.free_conflict_name(parent, &own, is_dir, device).await? { + Some(candidate) => candidate, + None => return Err(rejected(ConflictKind::NameTaken { existing })), + }, + }; + if is_dir && let Err(kind) = self.check_placement(parent, None).await? { + return Err(rejected(kind)); + } + self.revive(id, parent, &name).await?; + if !is_dir { + self.renew_version(id).await?; + } + } + Ok(()) + } + /// Gives the file `node` a new current version with the manifest, modification time and executable bit of its last /// one (no save record), after it. Returns the new version's ID. async fn renew_version(&mut self, node: NodeId) -> Result { @@ -622,14 +669,14 @@ impl Meta { } /// Brings the node `node` of the trash back in `space`, as one change of the space (one sequence number, all or - /// nothing): see `Ctx::restore_from_trash` for the rules. `NotFound` for a node that is not in the trash (live, + /// nothing): see `Ctx::restore_from_trash` for the rules. A folder brings back what is in the trash below it too. `NotFound` for a node that is not in the trash (live, /// unknown, another space's, or removed from the trash); `Rejected` (`NameTaken`) when no conflict name is free. pub async fn restore_from_trash(&self, space: SpaceId, device: DeviceId, node: NodeId, now_ms: i64) -> Result { let mut w = self.begin_write().await?; let info = space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; let seq = info.seq.checked_add(1).ok_or(MetaError::Corrupt("sequence overflow"))?; let mut ctx = Ctx { conn: &mut w.tx, space, seq: to_i64(seq)?, epoch: info.epoch, device, now_ms, failed_restores: HashMap::new() }; - let outcome = ctx.restore_from_trash(node).await?; + let outcome = ctx.restore_from_trash(node, self.trash_tree_cap).await?; sqlx::query("UPDATE spaces SET seq = $1 WHERE id = $2").bind(to_i64(seq)?).bind(uuid_bytes(space.as_uuid())).execute(&mut *w.tx).await?; w.commit().await?; Ok(TrashRestoreOutcome { seq, ..outcome }) diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index 292bc274..d1181cfd 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -90,6 +90,10 @@ pub enum MetaError { /// How long a SQLite connection waits for a lock another connection or program holds before giving up. pub const SQLITE_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +/// The most deleted items below one folder that restoring or removing the folder from the trash takes with it. A +/// folder with more is refused as a whole, so a restore or a removal is never partial. +pub const MAX_TRASH_TREE_NODES: u64 = 10_000; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Dialect { Sqlite, @@ -117,6 +121,9 @@ pub struct Meta { live_held_documents_cap: u64, /// The most bytes all of one space's live documents may hold ([`Meta::with_live_bytes_cap`]). live_bytes_cap: u64, + /// The most deleted items below one folder that restoring or removing the folder from the trash takes with it + /// ([`Meta::with_trash_tree_cap`]). + trash_tree_cap: u64, } impl Meta { @@ -173,9 +180,18 @@ impl Meta { live_documents_cap: MAX_LIVE_DOCUMENTS_PER_SPACE, live_held_documents_cap: MAX_LIVE_DOCUMENTS_HOLDING_UPDATES, live_bytes_cap: DEFAULT_LIVE_BYTES_PER_SPACE, + trash_tree_cap: MAX_TRASH_TREE_NODES, }) } + /// The store with another cap on the deleted items below one folder that restoring or removing the folder from the + /// trash takes with it (the default is [`MAX_TRASH_TREE_NODES`]); a folder with more is refused with + /// [`MetaError::Invalid`] and nothing is changed. For tests. + pub fn with_trash_tree_cap(mut self, cap: u64) -> Self { + self.trash_tree_cap = cap; + self + } + /// The store with another cap on the bytes of updates one live document's log may hold (the default is /// [`MAX_LIVE_DOC_BYTES`]); an append over it is refused with [`MetaError::DocFull`]. For tests. pub fn with_live_doc_cap(mut self, cap: u64) -> Self { diff --git a/crates/meta/src/trash.rs b/crates/meta/src/trash.rs index 8d7dab83..8485c373 100644 --- a/crates/meta/src/trash.rs +++ b/crates/meta/src/trash.rs @@ -4,8 +4,12 @@ //! with the time of the deletion. Bringing a node back is [`Meta::restore_from_trash`] in `commit.rs`, because it //! shares the commit path's rules for names. Removing a node from the trash here only marks it: nothing is freed, //! so versions, chunks and the tombstone stay as they were. +//! +//! A node is in the trash when it is deleted and its removal was not asked for. What is below a folder in the trash +//! is listed only through the folder (the folder's restore and removal take it along), whenever it was deleted. use craft_cloud_proto::api::{TRASH_MAX_LIMIT, TrashCursor}; +use craft_cloud_proto::limits::MAX_TREE_DEPTH; use craft_cloud_proto::{NodeId, SpaceId}; use sqlx::Row; @@ -13,6 +17,10 @@ use crate::codec::{flag, to_u64, uuid_bytes, uuid_from}; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; +/// Why a folder with too many deleted items below it is refused (`invalid`), for a restore and for a removal. +pub(crate) const TRASH_TREE_TOO_LARGE: &str = + "this folder holds too many deleted items to restore or remove at once; do it for the folders inside it first"; + /// A deleted node in the trash. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TrashEntry { @@ -36,6 +44,15 @@ pub struct TrashPage { pub repaired: u64, } +/// The deleted nodes below the node `$2` of space `$1` that are in the trash, found through nodes that are in the trash +/// themselves (a folder that was removed from the trash hides what is below it); `depth` is 1 for a child. `$3` bounds +/// the depth. A statement continues after it with `$4` and up. +pub(crate) const TRASH_BELOW: &str = "WITH RECURSIVE sub(id, depth) AS (\ + SELECT id, 1 FROM nodes WHERE space_id = $1 AND parent = $2 AND deleted = 1 AND purge_requested_ms IS NULL \ + UNION ALL \ + SELECT n.id, sub.depth + 1 FROM nodes n JOIN sub ON n.parent = sub.id \ + WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND sub.depth < $3) "; + impl Meta { /// One page of the trash of `space`: its deleted nodes that were not removed from the trash, newest deletion /// first by `(deleted_ms, id)`, after `after` (exclusive). `Invalid` for a `limit` outside 1 to @@ -58,6 +75,7 @@ impl Meta { "SELECT n.id, n.parent, n.name, n.is_dir, n.deleted_ms, v.size AS v_size \ FROM nodes n LEFT JOIN versions v ON v.space_id = n.space_id AND v.id = n.version \ WHERE n.space_id = $1 AND n.deleted = 1 AND n.purge_requested_ms IS NULL AND (n.deleted_ms, n.id) < ($2, $3) \ + AND NOT EXISTS (SELECT 1 FROM nodes p WHERE p.space_id = n.space_id AND p.id = n.parent AND p.deleted = 1 AND p.purge_requested_ms IS NULL) \ ORDER BY n.deleted_ms DESC, n.id DESC LIMIT $4", ) .bind(uuid_bytes(space.as_uuid())) @@ -111,12 +129,13 @@ impl Meta { /// Takes the deleted node `node` out of the trash: it is no longer listed and can't be restored from the trash. /// Only marks it (`purge_requested_ms`): no version, chunk or tombstone is touched, so the change feed and the /// sequence number stay as they are. Asking again changes nothing. `NotFound` when `node` is not a deleted node of - /// `space` in the trash. + /// `space` in the trash. Removing a folder marks the nodes in the trash below it too (all, whenever they were + /// deleted); `Invalid` and nothing marked when there are more than [`Meta::with_trash_tree_cap`] of them. pub async fn request_purge(&self, space: SpaceId, node: NodeId, now_ms: i64) -> Result<(), MetaError> { let mut w = self.begin_write().await?; // On Postgres this row lock orders the mark with commits and restores of the space. space_info(&mut w.tx, space, self.dialect == Dialect::Postgres).await?; - let row = sqlx::query("SELECT deleted, purge_requested_ms FROM nodes WHERE space_id = $1 AND id = $2") + let row = sqlx::query("SELECT deleted, purge_requested_ms, is_dir FROM nodes WHERE space_id = $1 AND id = $2") .bind(uuid_bytes(space.as_uuid())) .bind(uuid_bytes(node.as_uuid())) .fetch_optional(&mut *w.tx) @@ -128,6 +147,31 @@ impl Meta { if row.try_get::, _>("purge_requested_ms")?.is_some() { return Ok(()); } + if flag(&row, "is_dir")? { + // What is in the trash below the folder goes with it, or it would stay hidden and unmarked. A folder with + // more than the cap is refused whole. + let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; + let cap = i64::try_from(self.trash_tree_cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; + let counted = sqlx::query(sqlx::AssertSqlSafe(format!("{TRASH_BELOW} SELECT COUNT(*) AS c FROM (SELECT id FROM sub LIMIT $4) AS below"))) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(bound) + .bind(cap.saturating_add(1)) + .fetch_one(&mut *w.tx) + .await?; + if counted.try_get::("c")? > cap { + return Err(MetaError::Invalid(TRASH_TREE_TOO_LARGE)); + } + sqlx::query(sqlx::AssertSqlSafe(format!( + "{TRASH_BELOW} UPDATE nodes SET purge_requested_ms = $4 WHERE space_id = $1 AND id IN (SELECT id FROM sub)" + ))) + .bind(uuid_bytes(space.as_uuid())) + .bind(uuid_bytes(node.as_uuid())) + .bind(bound) + .bind(now_ms) + .execute(&mut *w.tx) + .await?; + } sqlx::query("UPDATE nodes SET purge_requested_ms = $1 WHERE space_id = $2 AND id = $3") .bind(now_ms) .bind(uuid_bytes(space.as_uuid())) diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index 385f0e9f..d2b518fb 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -114,6 +114,11 @@ on_every_backend!( a_restore_brings_the_deleted_folders_back_with_their_ids, a_restore_merges_into_live_folders_and_renames_around_taken_names, a_restore_with_no_free_name_changes_nothing, + the_trash_lists_a_folder_without_what_is_below_it, + a_folder_restore_brings_back_everything_in_the_trash_below_it, + a_folder_restore_gives_children_of_one_name_a_conflict_name, + removing_a_folder_from_the_trash_takes_what_is_below_it, + a_folder_with_too_much_below_it_is_refused_whole, only_nodes_in_the_trash_can_be_restored_or_removed, removing_from_the_trash_frees_nothing, a_trash_restore_and_a_commit_for_the_same_name_agree, @@ -2337,6 +2342,194 @@ async fn a_restore_merges_into_live_folders_and_renames_around_taken_names(meta: assert_eq!(live_name(meta, &u, live_d).await.1, "d"); } +/// A folder in the trash is listed alone: what is in the trash below it is listed only through it, however long before +/// it was deleted, and pages never split or repeat the top items. +async fn the_trash_lists_a_folder_without_what_is_below_it(meta: &Meta) { + use craft_cloud_proto::api::TrashCursor; + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let mut tops = Vec::new(); + for i in 0..7i64 { + let dir = folder(meta, &u, dev, r, &format!("d{i}"), NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub, "deep.txt", NOW).await; + // The child goes long before its folder, the deep file with the folders. + delete_file(meta, &u, dev, child, cv.id, NOW + 100 * i).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, sub, NOW + 1_000 + 100 * i).await; + delete_folder(meta, &u, dev, dir, NOW + 1_000 + 100 * i).await; + tops.push(dir); + } + for i in 0..3 { + let (f, v) = file_in(meta, &u, dev, r, &format!("loose{i}.txt"), NOW).await; + delete_file(meta, &u, dev, f, v.id, NOW + 500 + i64::from(i)).await; + tops.push(f); + } + tops.sort(); + for limit in [1u32, 2, 3, 200] { + let (mut seen, mut after) = (Vec::new(), None::); + loop { + let page = meta.trash(u.space, after, limit, NOW).await.unwrap(); + assert!(page.entries.len() <= limit as usize && !page.entries.is_empty(), "limit {limit}"); + seen.extend(page.entries.iter().map(|e| e.id)); + match page.next { + Some(next) => { + // A page is full whenever another follows: the hidden nodes take no room in it. + assert_eq!(page.entries.len(), limit as usize, "limit {limit}"); + after = Some(next); + } + None => break, + } + } + seen.sort(); + assert_eq!(seen, tops, "every top item exactly once, limit {limit}"); + } +} + +/// Restoring a folder brings back all that is in the trash below it, whenever it was deleted, as one change of the space +/// with each node revived like a single restore; a node removed from the trash, and what hangs from it, stays deleted. +async fn a_folder_restore_brings_back_everything_in_the_trash_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir1", NOW).await; + let (child, cv) = file_in(meta, &u, dev, dir, "child.txt", NOW).await; + let (early, ev) = file_in(meta, &u, dev, dir, "early.txt", NOW).await; + let sub = folder(meta, &u, dev, dir, "sub", NOW).await; + let sub2 = folder(meta, &u, dev, sub, "sub2", NOW).await; + let (deep, dv) = file_in(meta, &u, dev, sub2, "deep.txt", NOW).await; + let gone = folder(meta, &u, dev, dir, "gone", NOW).await; + let (hung, hv) = file_in(meta, &u, dev, gone, "hung.txt", NOW).await; + let (kept, kv) = file_in(meta, &u, dev, dir, "kept.txt", NOW).await; + // `early` was deleted on its own long before; `gone` and `kept` are removed from the trash; the rest goes with + // the folder. + delete_file(meta, &u, dev, early, ev.id, NOW + 10).await; + delete_file(meta, &u, dev, hung, hv.id, NOW + 1_000).await; + delete_folder(meta, &u, dev, gone, NOW + 1_001).await; + delete_file(meta, &u, dev, kept, kv.id, NOW + 1_002).await; + delete_file(meta, &u, dev, deep, dv.id, NOW + 1_003).await; + delete_folder(meta, &u, dev, sub2, NOW + 1_004).await; + delete_folder(meta, &u, dev, sub, NOW + 1_005).await; + delete_file(meta, &u, dev, child, cv.id, NOW + 1_006).await; + delete_folder(meta, &u, dev, dir, NOW + 1_007).await; + meta.request_purge(u.space, kept, NOW + 2_000).await.unwrap(); + meta.request_purge(u.space, gone, NOW + 2_001).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![dir], "the folder alone"); + let before = meta.space_info(u.space).await.unwrap(); + + let done = meta.restore_from_trash(u.space, dev, dir, NOW + 3_000).await.unwrap(); + assert_eq!((done.node, done.parent, done.name.as_str(), done.renamed, done.version), (dir, r, "dir1", false, None)); + assert_eq!(done.seq, before.seq + 1); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq + 1, "one change of the space"); + assert!(trash_ids(meta, &u).await.is_empty(), "nothing is left in the trash: the removed nodes are no longer listed"); + for (node, parent, node_name) in + [(dir, r, "dir1"), (child, dir, "child.txt"), (early, dir, "early.txt"), (sub, dir, "sub"), (sub2, sub, "sub2"), (deep, sub2, "deep.txt")] + { + assert_eq!(live_name(meta, &u, node).await, (Some(parent), node_name.to_owned(), false), "{node_name}"); + } + for node in [gone, hung, kept] { + assert!(live_name(meta, &u, node).await.2, "a removed node stays deleted"); + } + + // The feed reports exactly the nodes that came back, each at the new sequence number; the files as new versions. + let feed = meta.changes(u.space, before.seq, None, MAX_PAGE_NODES).await.unwrap(); + let mut changed: Vec<_> = feed.nodes.iter().map(|n| (n.id, n.deleted)).collect(); + changed.sort(); + let mut expected = vec![(dir, false), (child, false), (early, false), (sub, false), (sub2, false), (deep, false)]; + expected.sort(); + assert_eq!(changed, expected); + for (node, old) in [(child, &cv), (early, &ev), (deep, &dv)] { + let NodeKind::File(state) = &find(&feed.nodes, node).kind else { panic!("a file") }; + assert_eq!(state.manifest, old.manifest, "the content is what it was"); + assert_ne!(state.version, old.id, "as a new version"); + } +} + +/// Two deleted children of one name come back together: the newest deletion keeps the name, the other gets a conflict +/// name, and the folder itself is renamed only where its own name is taken. +async fn a_folder_restore_gives_children_of_one_name_a_conflict_name(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (old, ov) = file_in(meta, &u, dev, dir, "a.txt", NOW).await; + delete_file(meta, &u, dev, old, ov.id, NOW + 1).await; + let (new, nv) = file_in(meta, &u, dev, dir, "a.txt", NOW + 2).await; + delete_file(meta, &u, dev, new, nv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let (_squatter, _) = file_in(meta, &u, dev, r, "dir", NOW + 5).await; + + let done = meta.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.renamed, done.name.clone()), (true, conflict_of("dir", true, 1))); + assert_eq!(live_name(meta, &u, new).await, (Some(dir), "a.txt".to_owned(), false), "the newest deletion keeps the name"); + assert_eq!(live_name(meta, &u, old).await, (Some(dir), conflict_of("a.txt", false, 1), false)); +} + +/// Removing a folder from the trash removes what is in the trash below it too, so nothing stays listed or restorable +/// alone; a folder with more below it than the cap is refused whole, for a restore and for a removal. +async fn removing_a_folder_from_the_trash_takes_what_is_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let (c, cv) = file_in(meta, &u, dev, dir, "c.txt", NOW).await; + let s = folder(meta, &u, dev, dir, "s", NOW).await; + let (d, dv) = file_in(meta, &u, dev, s, "d.txt", NOW).await; + delete_file(meta, &u, dev, c, cv.id, NOW + 1).await; + delete_file(meta, &u, dev, d, dv.id, NOW + 2).await; + delete_folder(meta, &u, dev, s, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + let before = meta.space_info(u.space).await.unwrap(); + meta.request_purge(u.space, dir, NOW + 5).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + for node in [dir, c, s, d] { + assert!(matches!(meta.restore_from_trash(u.space, dev, node, NOW).await, Err(MetaError::NotFound(_))), "{node}"); + assert!(live_name(meta, &u, node).await.2); + } +} + +async fn folder_with_deleted_children(meta: &Meta, u: &UserRecord, dev: DeviceId, folder_name: &str, children: usize, base: i64) -> NodeId { + let r = root(meta, u).await; + let dir = folder(meta, u, dev, r, folder_name, base).await; + for i in 0..children { + let (f, v) = file_in(meta, u, dev, dir, &format!("f{i}.txt"), base).await; + delete_file(meta, u, dev, f, v.id, base + 1).await; + } + delete_folder(meta, u, dev, dir, base + 2).await; + dir +} + +async fn a_folder_with_too_much_below_it_is_refused_whole(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let big = folder_with_deleted_children(meta, &u, dev, "big", 4, NOW).await; + let exact = folder_with_deleted_children(meta, &u, dev, "exact", 3, NOW + 10).await; + let before = meta.space_info(u.space).await.unwrap(); + + // Over the cap: refused, nothing marked, nothing restored. + assert!(matches!(small.restore_from_trash(u.space, dev, big, NOW).await, Err(MetaError::Invalid(_)))); + assert!(matches!(small.request_purge(u.space, big, NOW).await, Err(MetaError::Invalid(_)))); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); + let mut listed = trash_ids(meta, &u).await; + listed.sort(); + let mut both = vec![big, exact]; + both.sort(); + assert_eq!(listed, both, "both folders are still in the trash"); + assert!(live_name(meta, &u, big).await.2); + + // At the cap: restored (so nothing of `big` was marked before: it restores in full under the default cap). + small.restore_from_trash(u.space, dev, exact, NOW).await.unwrap(); + assert!(!live_name(meta, &u, exact).await.2); + meta.restore_from_trash(u.space, dev, big, NOW).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + let feed = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); + assert!(feed.nodes.iter().all(|n| !n.deleted), "every node came back"); +} + /// When every conflict name is taken too, the restore is refused as a conflict and the space is exactly as it was: no /// sequence number, no revived node, the node still in the trash. async fn a_restore_with_no_free_name_changes_nothing(meta: &Meta) { diff --git a/docs/design/sync.md b/docs/design/sync.md index 02cf2de7..ce41e4d1 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -657,9 +657,9 @@ methods answer `not_found` / `invalid` (405) the same way: | `POST /commit` | batch of version/mkdir/move/delete operations, one transaction; `409 conflict` listing every conflicting operation (a version whose manifest isn't uploaded is a `missing_manifest` conflict) | | `GET /nodes/{id}/history?cursor=…&limit=…` | the versions of one file of your space, newest first by `(seq, id)`, in pages (default 50, at most 200): `{versions: [{id, parent_version, epoch, seq, size, mtime_ms, device, created_ms, save, pruned}], next?}`; `next` is an opaque cursor for the next page; a folder is `invalid`, another account's or an unknown node `not_found`; a deleted file keeps its history | | `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | -| `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash, newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | -| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | -| `DELETE /trash/{id}` | takes a node out of the trash (204): only marks it for removal, nothing is freed yet | +| `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash and whose folder is not in the trash itself (those are listed through the folder), newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | +| `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders and, for a folder, all that is in the trash below it, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | +| `DELETE /trash/{id}` | takes a node out of the trash (204), and, for a folder, all that is in the trash below it: only marks them for removal, nothing is freed yet; more than 10,000 below a folder is `invalid` and changes nothing | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | @@ -689,8 +689,17 @@ deleted node without a time (damaged data) is dated at the next read of the tras counts as expired. `DELETE /trash/{id}` frees nothing: versions, manifests, chunks and the tombstone stay, the change feed and the sequence number do not change, and the node's history stays readable; the node only leaves the trash list and can no longer be restored from it. (An edit that brings it back still wins, as for any deleted file.) Asking twice is fine. +A node is *in the trash* when it is deleted and not marked by `purge_requested_ms`. The list shows a node only if its +parent is not in the trash, so a deleted folder stands for everything below it. (A folder can only be deleted once it is +empty, so what is below it was deleted before it, in the same change or earlier; the rule does not depend on the times.) A restore from the trash brings back the node asked for and the deleted folders above it, root-most first, keeping their -IDs; it creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: +IDs; for a folder it also brings back every node below it that is in the trash, recursively, whenever it was deleted +(a file deleted on its own earlier inside the folder comes back with it); a node marked for removal, and what hangs +only from it, stays deleted. Each of them is revived like a single restore (new version for a file, `deleted_ms` and +the mark cleared, `changed_seq` set); a node below the folder keeps its name, except that of two deleted nodes of one +name in one folder the one deleted last keeps it and the other gets a conflict name. A removal of a folder marks +everything in the trash below it the same way. Both are refused as a whole with `invalid` above 10,000 nodes below the +folder. It creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is diff --git a/docs/server.md b/docs/server.md index 3eb26827..35be9bf1 100644 --- a/docs/server.md +++ b/docs/server.md @@ -69,7 +69,11 @@ A deleted file or folder stays in the trash, and in the server's records, until device lists the trash (`GET /api/v1/trash`), brings a node back (`POST /api/v1/trash/{id}/restore`, which names it differently if its name has been taken meanwhile) or takes it out of the trash (`DELETE /api/v1/trash/{id}`). Taking it out only marks it: **nothing is deleted from the server yet**, no version or chunk is freed, and the node can't be -brought back from the trash afterwards. The only setting is the number of days a deleted node waits, shown to clients as +brought back from the trash afterwards. A folder in the trash is listed alone: everything that is in the trash below it +(deleted and not removed from the trash) comes back with it when you bring the folder back, and goes with it when you +take the folder out, **also a file that was deleted on its own, earlier, inside that folder**. Those items are listed +only through the folder. A folder with more than 10,000 such items below it is refused (`invalid`) for both, and nothing +changes; bring back or remove the folders inside it first. The only setting is the number of days a deleted node waits, shown to clients as `purge_after_ms`; it removes nothing yet: ```toml From 1dfad983039be81c85bb1373f96414bb83d10e99 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Fri, 6 Nov 2026 21:01:15 +0100 Subject: [PATCH 25/27] sync: tests keep a folder alone by taking what is below it out of the trash first Restoring a folder from the trash now brings back all that is in the trash below it, so the lost-delete tests that restored a folder alone and expected its file to stay deleted take that file (or sub folder) out of the trash first. --- crates/sync/tests/engine.rs | 44 ++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/crates/sync/tests/engine.rs b/crates/sync/tests/engine.rs index 945227c1..71d1ecf6 100644 --- a/crates/sync/tests/engine.rs +++ b/crates/sync/tests/engine.rs @@ -3171,6 +3171,23 @@ async fn restore_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord server.meta.restore_from_trash(user.space, device, node, 1_760_000_000_000).await.unwrap(); } +/// Takes `node` out of the trash (marks it for removal), as the web or an admin tool would. +async fn remove_from_trash(server: &Server, user: &craft_cloud_meta::UserRecord, node: craft_cloud_proto::NodeId) { + server.meta.request_purge(user.space, node, 1_760_000_000_000).await.unwrap(); +} + +/// Brings the nodes named in `steps` back from the trash, in order; a name with a `-` in front is taken out of the trash +/// instead (and with it what is below it). Restoring a folder brings back all that is in the trash below it, so a test +/// that wants a folder alone removes the rest first. +async fn trash_steps(server: &Server, user: &craft_cloud_meta::UserRecord, steps: &[&str]) { + for step in steps { + match step.strip_prefix('-') { + Some(name) => remove_from_trash(server, user, node_named(server, user, name).await).await, + None => restore_from_trash(server, user, node_named(server, user, step).await).await, + } + } +} + /// A machine deleted a file, the server applied it, the answer was lost; the file is brought back from the trash /// before the machine syncs again. The machine's delete must not be sent again: the file comes back, on both machines. #[tokio::test] @@ -3314,9 +3331,7 @@ async fn lost_delete_of_a_folder_with_a_file(restore: &[&str]) -> (common::Machi a.fs_mut().remove_dir(&path("d")).unwrap(); a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); assert!(a.sync_once().await.is_err()); - for name in restore { - restore_from_trash(&server, &ana, node_named(&server, &ana, name).await).await; - } + trash_steps(&server, &ana, restore).await; settle(&mut [&mut a, &mut b], 6).await; settle(&mut [&mut a, &mut b], 3).await; (a, b, server, ana) @@ -3324,7 +3339,7 @@ async fn lost_delete_of_a_folder_with_a_file(restore: &[&str]) -> (common::Machi #[tokio::test] async fn a_non_empty_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { - let (a, b, server, ana) = lost_delete_of_a_folder_with_a_file(&["d"]).await; + let (a, b, server, ana) = lost_delete_of_a_folder_with_a_file(&["-f.txt", "d"]).await; assert_eq!(a.fs().dirs(), vec![path("d")]); assert_eq!(b.fs().dirs(), vec![path("d")]); assert!(files(&a).is_empty() && files(&b).is_empty(), "the file stays in the trash"); @@ -3333,9 +3348,10 @@ async fn a_non_empty_folder_restored_alone_after_a_lost_delete_answer_is_not_del assert_eq!(c.fs().dirs(), vec![path("d")], "the server still has the folder"); } +/// A folder restored brings back the file that was deleted with it, which is not deleted a second time either. #[tokio::test] -async fn a_folder_and_its_file_both_restored_after_a_lost_delete_answer_come_back() { - let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["d", "f.txt"]).await; +async fn a_folder_restored_after_a_lost_delete_answer_comes_back_with_its_file() { + let (a, b, _server, _ana) = lost_delete_of_a_folder_with_a_file(&["d"]).await; let expected = vec![("d/f.txt".to_owned(), b"keep me".to_vec())]; assert_eq!(files(&a), expected); assert_eq!(files(&b), expected); @@ -3373,9 +3389,7 @@ async fn lost_delete_of_a_nested_tree( } a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); assert!(a.sync_once().await.is_err()); - for name in restore { - restore_from_trash(&server, &ana, node_named(&server, &ana, name).await).await; - } + trash_steps(&server, &ana, restore).await; settle(&mut [&mut a, &mut b], 6).await; settle(&mut [&mut a, &mut b], 3).await; (a, b, server, ana) @@ -3395,13 +3409,13 @@ async fn assert_only_dirs(m: (common::Machine, common::Machine, Server, craft_cl #[tokio::test] async fn a_nested_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { - let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["d"]).await; + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-sub", "d"]).await; assert_only_dirs(m, &["d"]).await; } #[tokio::test] async fn a_nested_folder_and_its_subfolder_restored_after_a_lost_delete_answer_come_back() { - let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["d", "sub"]).await; + let m = lost_delete_of_a_nested_tree(&["d", "d/sub"], "d/sub/f.txt", &["-f.txt", "d"]).await; assert_only_dirs(m, &["d", "d/sub"]).await; } @@ -3416,19 +3430,19 @@ async fn only_the_deepest_file_restored_after_a_lost_delete_answer_brings_the_wh #[tokio::test] async fn a_three_deep_folder_restored_alone_after_a_lost_delete_answer_is_not_deleted_again() { - let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["d"]).await; + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-a", "d"]).await; assert_only_dirs(m, &["d"]).await; } #[tokio::test] async fn a_three_deep_folder_and_its_middle_folder_restored_after_a_lost_delete_answer_come_back() { - let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["d", "a"]).await; + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-b", "d"]).await; assert_only_dirs(m, &["d", "d/a"]).await; } #[tokio::test] async fn only_the_deepest_folder_restored_after_a_lost_delete_answer_brings_its_parents_back() { - let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["b"]).await; + let m = lost_delete_of_a_nested_tree(&["d", "d/a", "d/a/b"], "d/a/b/f.txt", &["-f.txt", "b"]).await; assert_only_dirs(m, &["d", "d/a", "d/a/b"]).await; } @@ -3501,7 +3515,7 @@ async fn a_nested_relist_keeps_only_the_restored_parent() { a.fs_mut().remove_dir(&path("d")).unwrap(); a.transport().faults.lose_commit_reply.store(true, Ordering::Relaxed); assert!(a.sync_once().await.is_err()); - restore_from_trash(&server, &ana, node_named(&server, &ana, "d").await).await; + trash_steps(&server, &ana, &["-sub", "d"]).await; a.transport().faults.resync_next_changes.store(true, Ordering::Relaxed); settle(&mut [&mut a, &mut b], 6).await; From b61fcd640fdc766baee0e3348dfc171eb9dae348 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Fri, 6 Nov 2026 21:53:34 +0100 Subject: [PATCH 26/27] meta: a folder restore or removal takes at most the cap below it, the rest is listed Instead of refusing a folder with more than 10,000 items in the trash below it (which left it impossible to restore or remove from a list that hides what is below a trashed folder), a restore or removal now takes the folder and at most the cap of items below it, shallowest first. The rest stays in the trash and is listed on its own because its folder is live or removed, so a further call takes it. The per-row work under the write lock is bounded by the cap. --- crates/meta/src/commit.rs | 16 ++--- crates/meta/src/lib.rs | 9 +-- crates/meta/src/trash.rs | 27 +++---- crates/meta/tests/behaviour.rs | 128 ++++++++++++++++++++++++++------- docs/design/sync.md | 9 ++- docs/server.md | 5 +- 6 files changed, 132 insertions(+), 62 deletions(-) diff --git a/crates/meta/src/commit.rs b/crates/meta/src/commit.rs index 1317d2dc..6828d9a7 100644 --- a/crates/meta/src/commit.rs +++ b/crates/meta/src/commit.rs @@ -517,13 +517,16 @@ impl Ctx<'_> { Ok(TrashRestoreOutcome { node, parent, renamed: placed.name != original, name: placed.name, version, seq: 0, epoch: self.epoch }) } - /// Brings back everything in the trash below the folder `top`, which is live again: its deleted children and theirs, - /// whenever they were deleted, parents first. A child keeps its name; one whose name another child that came back - /// took (a name deleted twice, the newest deletion first) gets a conflict name. A removed node, and what is only - /// below a removed folder, stays deleted. More than `cap` nodes: `Invalid`, and the caller drops the transaction. + /// Brings back what is in the trash below the folder `top`, which is live again: its deleted children and theirs, + /// whenever they were deleted, at most `cap` of them, shallowest first (then newest deletion first, then by ID), so a + /// node never comes back without its folder. The rest stays in the trash; its folder is live or still in the trash + /// under a live one, so it is listed on its own and a further restore brings it back. A child keeps its name; one + /// whose name another child that came back took (a name deleted twice, the newest deletion first) gets a conflict + /// name. A removed node, and what is only below a removed folder, stays deleted. The work done row by row under the + /// space's write lock is bounded by `cap`. async fn restore_below(&mut self, top: NodeId, cap: u64, device: &str) -> Result<(), MetaError> { let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; - let limit = i64::try_from(cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?.saturating_add(1); + let limit = i64::try_from(cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; let rows = sqlx::query(sqlx::AssertSqlSafe(format!( "{TRASH_BELOW} SELECT n.id, n.parent, n.name, n.is_dir FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id \ ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $4" @@ -534,9 +537,6 @@ impl Ctx<'_> { .bind(limit) .fetch_all(&mut *self.conn) .await?; - if u64::try_from(rows.len()).map_err(|_| MetaError::Corrupt("count out of range"))? > cap { - return Err(MetaError::Invalid(crate::trash::TRASH_TREE_TOO_LARGE)); - } for row in &rows { let id = NodeId::from_uuid(uuid_from(&row.try_get::, _>("id")?)?); let parent = NodeId::from_uuid(uuid_from(&row.try_get::, _>("parent")?)?); diff --git a/crates/meta/src/lib.rs b/crates/meta/src/lib.rs index d1181cfd..5ee7726e 100644 --- a/crates/meta/src/lib.rs +++ b/crates/meta/src/lib.rs @@ -90,8 +90,10 @@ pub enum MetaError { /// How long a SQLite connection waits for a lock another connection or program holds before giving up. pub const SQLITE_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); -/// The most deleted items below one folder that restoring or removing the folder from the trash takes with it. A -/// folder with more is refused as a whole, so a restore or a removal is never partial. +/// The most deleted items below one folder that restoring or removing the folder from the trash takes with it in one +/// call, shallowest first. What is beyond stays in the trash and, once its folder is live or removed from the trash, +/// appears in the trash list on its own, to be restored or removed by a further call. The bound keeps the work done +/// under the space's write lock by one call small. pub const MAX_TRASH_TREE_NODES: u64 = 10_000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -185,8 +187,7 @@ impl Meta { } /// The store with another cap on the deleted items below one folder that restoring or removing the folder from the - /// trash takes with it (the default is [`MAX_TRASH_TREE_NODES`]); a folder with more is refused with - /// [`MetaError::Invalid`] and nothing is changed. For tests. + /// trash takes with it in one call (the default is [`MAX_TRASH_TREE_NODES`]); the rest stays in the trash. For tests. pub fn with_trash_tree_cap(mut self, cap: u64) -> Self { self.trash_tree_cap = cap; self diff --git a/crates/meta/src/trash.rs b/crates/meta/src/trash.rs index 8485c373..f53c5868 100644 --- a/crates/meta/src/trash.rs +++ b/crates/meta/src/trash.rs @@ -17,10 +17,6 @@ use crate::codec::{flag, to_u64, uuid_bytes, uuid_from}; use crate::tree::space_info; use crate::{Dialect, Meta, MetaError}; -/// Why a folder with too many deleted items below it is refused (`invalid`), for a restore and for a removal. -pub(crate) const TRASH_TREE_TOO_LARGE: &str = - "this folder holds too many deleted items to restore or remove at once; do it for the folders inside it first"; - /// A deleted node in the trash. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TrashEntry { @@ -129,8 +125,9 @@ impl Meta { /// Takes the deleted node `node` out of the trash: it is no longer listed and can't be restored from the trash. /// Only marks it (`purge_requested_ms`): no version, chunk or tombstone is touched, so the change feed and the /// sequence number stay as they are. Asking again changes nothing. `NotFound` when `node` is not a deleted node of - /// `space` in the trash. Removing a folder marks the nodes in the trash below it too (all, whenever they were - /// deleted); `Invalid` and nothing marked when there are more than [`Meta::with_trash_tree_cap`] of them. + /// `space` in the trash. Removing a folder marks the nodes in the trash below it too, whenever they were deleted: at + /// most [`crate::MAX_TRASH_TREE_NODES`] of them (see [`Meta::with_trash_tree_cap`]), shallowest first, in the order of a + /// restore. The rest is no longer below a folder in the trash, so it is listed and can be removed by a further call. pub async fn request_purge(&self, space: SpaceId, node: NodeId, now_ms: i64) -> Result<(), MetaError> { let mut w = self.begin_write().await?; // On Postgres this row lock orders the mark with commits and restores of the space. @@ -148,27 +145,19 @@ impl Meta { return Ok(()); } if flag(&row, "is_dir")? { - // What is in the trash below the folder goes with it, or it would stay hidden and unmarked. A folder with - // more than the cap is refused whole. + // What is in the trash below the folder goes with it, so it does not stay hidden and unmarked: up to the cap, + // shallowest first (so a marked node's folder is always marked too). Marking is bounded by the cap under + // the write lock. What is left is listed once the folder is marked. let bound = i64::try_from(MAX_TREE_DEPTH + 2).map_err(|_| MetaError::Corrupt("depth limit out of range"))?; let cap = i64::try_from(self.trash_tree_cap).map_err(|_| MetaError::Corrupt("trash cap out of range"))?; - let counted = sqlx::query(sqlx::AssertSqlSafe(format!("{TRASH_BELOW} SELECT COUNT(*) AS c FROM (SELECT id FROM sub LIMIT $4) AS below"))) - .bind(uuid_bytes(space.as_uuid())) - .bind(uuid_bytes(node.as_uuid())) - .bind(bound) - .bind(cap.saturating_add(1)) - .fetch_one(&mut *w.tx) - .await?; - if counted.try_get::("c")? > cap { - return Err(MetaError::Invalid(TRASH_TREE_TOO_LARGE)); - } sqlx::query(sqlx::AssertSqlSafe(format!( - "{TRASH_BELOW} UPDATE nodes SET purge_requested_ms = $4 WHERE space_id = $1 AND id IN (SELECT id FROM sub)" + "{TRASH_BELOW} UPDATE nodes SET purge_requested_ms = $4 WHERE space_id = $1 AND id IN ( SELECT n.id FROM sub JOIN nodes n ON n.space_id = $1 AND n.id = sub.id ORDER BY sub.depth, n.deleted_ms DESC, n.id LIMIT $5)" ))) .bind(uuid_bytes(space.as_uuid())) .bind(uuid_bytes(node.as_uuid())) .bind(bound) .bind(now_ms) + .bind(cap) .execute(&mut *w.tx) .await?; } diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index d2b518fb..a9b57f9f 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -118,7 +118,9 @@ on_every_backend!( a_folder_restore_brings_back_everything_in_the_trash_below_it, a_folder_restore_gives_children_of_one_name_a_conflict_name, removing_a_folder_from_the_trash_takes_what_is_below_it, - a_folder_with_too_much_below_it_is_refused_whole, + a_folder_restore_takes_at_most_the_cap_below_it, + a_folder_restore_leaves_the_rest_listed_through_its_own_folder, + removing_a_folder_takes_at_most_the_cap_below_it, only_nodes_in_the_trash_can_be_restored_or_removed, removing_from_the_trash_frees_nothing, a_trash_restore_and_a_commit_for_the_same_name_agree, @@ -2468,7 +2470,7 @@ async fn a_folder_restore_gives_children_of_one_name_a_conflict_name(meta: &Meta } /// Removing a folder from the trash removes what is in the trash below it too, so nothing stays listed or restorable -/// alone; a folder with more below it than the cap is refused whole, for a restore and for a removal. +/// alone (up to the cap, see below). async fn removing_a_folder_from_the_trash_takes_what_is_below_it(meta: &Meta) { let u = user(meta, "ana").await; let dev = device(meta, &u).await; @@ -2491,43 +2493,117 @@ async fn removing_a_folder_from_the_trash_takes_what_is_below_it(meta: &Meta) { } } -async fn folder_with_deleted_children(meta: &Meta, u: &UserRecord, dev: DeviceId, folder_name: &str, children: usize, base: i64) -> NodeId { +/// A folder `name` in the root with `count` files, the file `i` deleted at `base + 1 + i` and the folder at `base + 20`: +/// the folder and its files, the newest deletion last. +async fn folder_with_deleted_files(meta: &Meta, u: &UserRecord, dev: DeviceId, folder_name: &str, count: i64, base: i64) -> (NodeId, Vec) { let r = root(meta, u).await; let dir = folder(meta, u, dev, r, folder_name, base).await; - for i in 0..children { + let mut files = Vec::new(); + for i in 0..count { let (f, v) = file_in(meta, u, dev, dir, &format!("f{i}.txt"), base).await; - delete_file(meta, u, dev, f, v.id, base + 1).await; + delete_file(meta, u, dev, f, v.id, base + 1 + i).await; + files.push(f); } - delete_folder(meta, u, dev, dir, base + 2).await; - dir + delete_folder(meta, u, dev, dir, base + 20).await; + (dir, files) } -async fn a_folder_with_too_much_below_it_is_refused_whole(meta: &Meta) { +fn sorted(mut ids: Vec) -> Vec { + ids.sort(); + ids +} + +/// A folder brings back at most the cap of what is in the trash below it, the newest deletions first among equals in +/// depth; the rest stays in the trash, is listed on its own (its folder is live) and comes back by a further restore. A +/// folder with exactly the cap below it brings everything. +async fn a_folder_restore_takes_at_most_the_cap_below_it(meta: &Meta) { let u = user(meta, "ana").await; let dev = device(meta, &u).await; let small = meta.clone().with_trash_tree_cap(3); - let big = folder_with_deleted_children(meta, &u, dev, "big", 4, NOW).await; - let exact = folder_with_deleted_children(meta, &u, dev, "exact", 3, NOW + 10).await; + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let (exact, exact_files) = folder_with_deleted_files(meta, &u, dev, "exact", 3, NOW + 100).await; + assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![dir, exact])); + let before = meta.space_info(u.space).await.unwrap(); + let done = small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert_eq!((done.node, done.seq), (dir, before.seq + 1), "one change of the space"); + assert!(!live_name(meta, &u, dir).await.2); + for f in &files[2..] { + assert!(!live_name(meta, &u, *f).await.2, "the three newest deletions came back"); + } + for f in &files[..2] { + assert!(live_name(meta, &u, *f).await.2, "the oldest two stay in the trash"); + } + assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![exact, files[0], files[1]]), "the rest is listed on its own"); + small.restore_from_trash(u.space, dev, files[1], NOW).await.unwrap(); + assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![exact, files[0]])); - // Over the cap: refused, nothing marked, nothing restored. - assert!(matches!(small.restore_from_trash(u.space, dev, big, NOW).await, Err(MetaError::Invalid(_)))); - assert!(matches!(small.request_purge(u.space, big, NOW).await, Err(MetaError::Invalid(_)))); - assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq); - let mut listed = trash_ids(meta, &u).await; - listed.sort(); - let mut both = vec![big, exact]; - both.sort(); - assert_eq!(listed, both, "both folders are still in the trash"); - assert!(live_name(meta, &u, big).await.2); - - // At the cap: restored (so nothing of `big` was marked before: it restores in full under the default cap). small.restore_from_trash(u.space, dev, exact, NOW).await.unwrap(); - assert!(!live_name(meta, &u, exact).await.2); - meta.restore_from_trash(u.space, dev, big, NOW).await.unwrap(); + for f in &exact_files { + assert!(!live_name(meta, &u, *f).await.2, "exactly the cap brings everything"); + } + assert_eq!(trash_ids(meta, &u).await, vec![files[0]]); +} + +/// Of the nodes below a folder the shallow ones come first, and what is left is listed only if its own folder is not in +/// the trash: a left-over folder is listed, what is in it is hidden behind it until it is restored. +async fn a_folder_restore_leaves_the_rest_listed_through_its_own_folder(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(1); + let dir = folder(meta, &u, dev, r, "dir", NOW).await; + let a = folder(meta, &u, dev, dir, "a", NOW).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW).await; + let (b, bv) = file_in(meta, &u, dev, dir, "b.txt", NOW).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 1).await; + delete_folder(meta, &u, dev, a, NOW + 2).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 3).await; + delete_folder(meta, &u, dev, dir, NOW + 4).await; + assert_eq!(trash_ids(meta, &u).await, vec![dir]); + + small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); + assert!(!live_name(meta, &u, dir).await.2 && !live_name(meta, &u, b).await.2, "the folder and its newest child"); + assert!(live_name(meta, &u, a).await.2 && live_name(meta, &u, a1).await.2); + assert_eq!(trash_ids(meta, &u).await, vec![a], "the folder is listed, what is in it is not"); + small.restore_from_trash(u.space, dev, a, NOW).await.unwrap(); + assert!(!live_name(meta, &u, a).await.2 && !live_name(meta, &u, a1).await.2); assert!(trash_ids(meta, &u).await.is_empty()); - let feed = meta.changes(u.space, 0, None, MAX_PAGE_NODES).await.unwrap(); - assert!(feed.nodes.iter().all(|n| !n.deleted), "every node came back"); +} + +/// Removing a folder marks it and at most the cap below it, shallowest first; the rest is listed on its own (its folder +/// is no longer in the trash) and goes by a further removal. +async fn removing_a_folder_takes_at_most_the_cap_below_it(meta: &Meta) { + let u = user(meta, "ana").await; + let dev = device(meta, &u).await; + let r = root(meta, &u).await; + let small = meta.clone().with_trash_tree_cap(3); + let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; + let before = meta.space_info(u.space).await.unwrap(); + small.request_purge(u.space, dir, NOW + 50).await.unwrap(); + assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); + assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![files[0], files[1]])); + for f in &files[2..] { + assert!(matches!(meta.restore_from_trash(u.space, dev, *f, NOW).await, Err(MetaError::NotFound(_))), "removed with the folder"); + } + small.request_purge(u.space, files[0], NOW + 51).await.unwrap(); + small.request_purge(u.space, files[1], NOW + 51).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + + // A left-over folder is listed and what is in it stays hidden behind it. + let tree = folder(meta, &u, dev, r, "tree", NOW + 100).await; + let a = folder(meta, &u, dev, tree, "a", NOW + 100).await; + let (a1, a1v) = file_in(meta, &u, dev, a, "a1.txt", NOW + 100).await; + let (b, bv) = file_in(meta, &u, dev, tree, "b.txt", NOW + 100).await; + delete_file(meta, &u, dev, a1, a1v.id, NOW + 101).await; + delete_folder(meta, &u, dev, a, NOW + 102).await; + delete_file(meta, &u, dev, b, bv.id, NOW + 103).await; + delete_folder(meta, &u, dev, tree, NOW + 104).await; + meta.clone().with_trash_tree_cap(1).request_purge(u.space, tree, NOW + 105).await.unwrap(); + assert_eq!(trash_ids(meta, &u).await, vec![a], "only the left-over folder is listed"); + meta.clone().with_trash_tree_cap(1).request_purge(u.space, a, NOW + 106).await.unwrap(); + assert!(trash_ids(meta, &u).await.is_empty()); + assert!(matches!(meta.restore_from_trash(u.space, dev, a1, NOW).await, Err(MetaError::NotFound(_)))); } /// When every conflict name is taken too, the restore is refused as a conflict and the space is exactly as it was: no diff --git a/docs/design/sync.md b/docs/design/sync.md index ce41e4d1..b44a0dcb 100644 --- a/docs/design/sync.md +++ b/docs/design/sync.md @@ -659,7 +659,7 @@ methods answer `not_found` / `invalid` (405) the same way: | `POST /nodes/{id}/restore` | `{version, expected_version}` → `{version, parent_version, seq, epoch}`: makes an old version's content the file's content again as a new version (see below); `409 conflict` when `expected_version` is not the current version | | `GET /trash?cursor=…&limit=…` | the deleted nodes of your space that were not removed from the trash and whose folder is not in the trash itself (those are listed through the folder), newest deletion first by `(deleted_ms, id)`, in pages (default 50, at most 200): `{items: [{id, parent, name, is_dir, size?, deleted_ms, purge_after_ms}], next?}`; `purge_after_ms` is `deleted_ms` plus `[retention] trash_days` | | `POST /trash/{id}/restore` | brings a node of the trash back, with its deleted folders and, for a folder, all that is in the trash below it, as one change: `{node, parent, name, renamed, version?, seq, epoch}` (a restored file has a new version); a taken name gets a conflict name (see below), a node that is not in the trash is `not_found` | -| `DELETE /trash/{id}` | takes a node out of the trash (204), and, for a folder, all that is in the trash below it: only marks them for removal, nothing is freed yet; more than 10,000 below a folder is `invalid` and changes nothing | +| `DELETE /trash/{id}` | takes a node out of the trash (204), and, for a folder, all that is in the trash below it: only marks them for removal, nothing is freed yet; up to 10,000 items below a folder at once, shallowest first; the rest is then listed and goes by a further call | | `GET /devices`, `DELETE /devices/{id}` | your devices | | `POST /pairings` | a new add-a-computer code from this device → `{id, code, expires_ms}` (the code shown once, normalised) | | `GET /pairings/{id}`, `DELETE /pairings/{id}` | one of your pairings: `{state: waiting\|used\|expired, device_name?}` / cancel it (204) | @@ -698,8 +698,11 @@ IDs; for a folder it also brings back every node below it that is in the trash, only from it, stays deleted. Each of them is revived like a single restore (new version for a file, `deleted_ms` and the mark cleared, `changed_seq` set); a node below the folder keeps its name, except that of two deleted nodes of one name in one folder the one deleted last keeps it and the other gets a conflict name. A removal of a folder marks -everything in the trash below it the same way. Both are refused as a whole with `invalid` above 10,000 nodes below the -folder. It creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: +everything in the trash below it the same way. Both take at most 10,000 nodes below the folder per call, in the order +depth, then newest deletion first, then ID, so a node never comes back or is marked without its folder; there is no error +for more. What is beyond has its folder live (restore) or marked (removal), or hangs under a folder that does, so it is +listed on its own and a further call takes it, bounded the same way. A restore also makes the per-row work under the write +lock at most that many nodes. It creates no node and overwrites none. Under one write lock and in one transaction, with one sequence number: where a deleted folder's name is held by a live folder, that folder is used and what is below moves into it (the deleted folder stays deleted); where it is held by a file, or where the node's own name is held by anything, the node comes back as `name (conflict, , )` like the engine's conflict copies, with `, 2` to `, 100` if that is diff --git a/docs/server.md b/docs/server.md index 35be9bf1..f0cd7eb7 100644 --- a/docs/server.md +++ b/docs/server.md @@ -72,8 +72,9 @@ it out only marks it: **nothing is deleted from the server yet**, no version or brought back from the trash afterwards. A folder in the trash is listed alone: everything that is in the trash below it (deleted and not removed from the trash) comes back with it when you bring the folder back, and goes with it when you take the folder out, **also a file that was deleted on its own, earlier, inside that folder**. Those items are listed -only through the folder. A folder with more than 10,000 such items below it is refused (`invalid`) for both, and nothing -changes; bring back or remove the folders inside it first. The only setting is the number of days a deleted node waits, shown to clients as +only through the folder. A folder brings back, or takes out, up to 10,000 items below it at once, the shallowest first; if +there are more, the rest then appears in the trash list on its own (its folder is live again or out of the trash), and +you bring it back or remove it by a further call. The only setting is the number of days a deleted node waits, shown to clients as `purge_after_ms`; it removes nothing yet: ```toml From 98020a2b2a346533f75cfd45960a131d3d0adda0 Mon Sep 17 00:00:00 2001 From: Skyfuller1303 Date: Fri, 6 Nov 2026 22:10:06 +0100 Subject: [PATCH 27/27] meta tests: name the trash tests' sorting helper sorted_nodes Another test helper in a later branch is called sorted; a distinct name keeps the trash tests compiling wherever they are merged. --- crates/meta/tests/behaviour.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meta/tests/behaviour.rs b/crates/meta/tests/behaviour.rs index a9b57f9f..b618b71a 100644 --- a/crates/meta/tests/behaviour.rs +++ b/crates/meta/tests/behaviour.rs @@ -2508,7 +2508,7 @@ async fn folder_with_deleted_files(meta: &Meta, u: &UserRecord, dev: DeviceId, f (dir, files) } -fn sorted(mut ids: Vec) -> Vec { +fn sorted_nodes(mut ids: Vec) -> Vec { ids.sort(); ids } @@ -2522,7 +2522,7 @@ async fn a_folder_restore_takes_at_most_the_cap_below_it(meta: &Meta) { let small = meta.clone().with_trash_tree_cap(3); let (dir, files) = folder_with_deleted_files(meta, &u, dev, "flat", 5, NOW).await; let (exact, exact_files) = folder_with_deleted_files(meta, &u, dev, "exact", 3, NOW + 100).await; - assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![dir, exact])); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![dir, exact])); let before = meta.space_info(u.space).await.unwrap(); let done = small.restore_from_trash(u.space, dev, dir, NOW).await.unwrap(); @@ -2534,9 +2534,9 @@ async fn a_folder_restore_takes_at_most_the_cap_below_it(meta: &Meta) { for f in &files[..2] { assert!(live_name(meta, &u, *f).await.2, "the oldest two stay in the trash"); } - assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![exact, files[0], files[1]]), "the rest is listed on its own"); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0], files[1]]), "the rest is listed on its own"); small.restore_from_trash(u.space, dev, files[1], NOW).await.unwrap(); - assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![exact, files[0]])); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![exact, files[0]])); small.restore_from_trash(u.space, dev, exact, NOW).await.unwrap(); for f in &exact_files { @@ -2582,7 +2582,7 @@ async fn removing_a_folder_takes_at_most_the_cap_below_it(meta: &Meta) { let before = meta.space_info(u.space).await.unwrap(); small.request_purge(u.space, dir, NOW + 50).await.unwrap(); assert_eq!(meta.space_info(u.space).await.unwrap().seq, before.seq, "no change of the space"); - assert_eq!(sorted(trash_ids(meta, &u).await), sorted(vec![files[0], files[1]])); + assert_eq!(sorted_nodes(trash_ids(meta, &u).await), sorted_nodes(vec![files[0], files[1]])); for f in &files[2..] { assert!(matches!(meta.restore_from_trash(u.space, dev, *f, NOW).await, Err(MetaError::NotFound(_))), "removed with the folder"); }