From 80a4e707c3b61aee05b983c8109e78e3822b8952 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 00:17:18 +0000 Subject: [PATCH 01/10] Keep less memory per file a mount is asked about Box the rarely set removed_from of each inode, keep only directories in the core's path-to-entry cache (files are one listing lookup further), share that cache's keys with the entries' own, shrink the inode table once the kernel forgets most of it, and cap a git mount's recent whole file reads at 64 MiB. A 200k-file walk goes from 147 MB to 89 MB with the same walk times. With files out of the path-entry cache, a recheck of a shrunk file changes only its listing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs-core/src/workspace.rs | 43 +++++++++++++++++++---------- crates/sandfs/src/fs.rs | 14 +++++++--- docs/changelog.md | 1 + 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/crates/sandfs-core/src/workspace.rs b/crates/sandfs-core/src/workspace.rs index 07e49db..f295cd8 100644 --- a/crates/sandfs-core/src/workspace.rs +++ b/crates/sandfs-core/src/workspace.rs @@ -54,10 +54,11 @@ type CacheKey = (u64, SourceKey); struct Cache { dirs: Mutex>>>>, blobs: Mutex>>>, - /// Entries already found, by mount id and path below the mount, so resolving a - /// path costs one lookup instead of one listing lookup per component. Only + /// Directories already found, by mount id and path below the mount, so resolving a + /// path costs a lookup or two instead of one listing lookup per component. Only /// entries that exist are kept, so probing for missing names costs no memory. - entries: Mutex, SourceEntry>>>, + /// A path that is its entry's key, as in a folder, shares the key's bytes. + entries: Mutex>>, /// Directories already prefetched, and whether recursively, so each is asked once. prefetched: Mutex>, /// Base files each directory needed fetched in the current look: sizes missing @@ -93,7 +94,8 @@ impl Cache { } fn remember_entry(&self, mount: u64, rel: &str, entry: SourceEntry) { - self.entries.lock().expect("cache lock").entry(mount).or_default().insert(rel.into(), entry); + let rel = if *entry.key == *rel { entry.key.clone() } else { rel.into() }; + self.entries.lock().expect("cache lock").entry(mount).or_default().insert(rel, entry); } /// Replace a cached listing with a newer answer from the same source. @@ -109,9 +111,8 @@ impl Cache { /// Put `entry` in the cached listing `key` under `name`, in place of the entry with /// the same source key, and say whether that changed it. The listing's own lock is /// held throughout, so two files of one directory changed at once both stay changed. - /// `then` runs under that lock too, so what it caches from `entry` changes in the - /// same order as the listing. - fn patch_listing(&self, key: &CacheKey, name: &str, entry: SourceEntry, then: impl FnOnce(SourceEntry)) -> bool { + /// Files aren't in the path-entry cache, so the listing is the one place to change. + fn patch_listing(&self, key: &CacheKey, name: &str, entry: SourceEntry) -> bool { let Some(slot) = self.dirs.lock().expect("cache lock").get(key).cloned() else { return false }; let mut listing = slot.lock().expect("cache slot lock"); let Some(current) = listing.as_ref() else { return false }; @@ -119,9 +120,8 @@ impl Cache { return false; } let mut fresh = (**current).clone(); - fresh.insert(name.to_owned(), entry.clone()); + fresh.insert(name.to_owned(), entry); *listing = Some(Arc::new(fresh)); - then(entry); true } @@ -472,10 +472,7 @@ impl Workspace { if let (Some(parent), Some(name)) = (path.parent(), path.name()) && let Some(dir) = self.base_dir(&parent, &mount)? && self.cached_listing(&mount, &dir)?.get(name).is_some_and(|entry| entry.key == key) - && let Some((mount_path, _)) = self.mounts.iter().find(|(_, owner)| owner.id == mount.id) - && self.cache.patch_listing(&(mount.id, dir), name, now, |now| { - self.cache.remember_entry(mount.id, &path.as_str()[mount_path.as_str().len() + 1..], now); - }) + && self.cache.patch_listing(&(mount.id, dir), name, now) { changed = true; } @@ -1038,13 +1035,29 @@ impl Workspace { if let Some(entry) = self.cache.entry(mount.id, rel) { return Ok(Some(entry)); } - let mut parts = rel.split('/').peekable(); + // Only directories are remembered, as files far outnumber them: a file is found + // in its directory's listing, one lookup more. let mut key = mount.root.clone(); + let mut done = 0; + if let Some((dir, _)) = rel.rsplit_once('/') + && let Some(found) = self.cache.entry(mount.id, dir) + { + if !found.kind.is_dir() { + return Ok(None); + } + key = found.key; + done = dir.len() + 1; + } + let mut parts = rel[done..].split('/').peekable(); while let Some(part) = parts.next() { let listing = self.cached_listing(mount, &key)?; let Some(entry) = listing.get(part) else { return Ok(None) }; + done += part.len(); + if entry.kind.is_dir() { + self.cache.remember_entry(mount.id, &rel[..done], entry.clone()); + } + done += 1; if parts.peek().is_none() { - self.cache.remember_entry(mount.id, rel, entry.clone()); return Ok(Some(entry.clone())); } if !entry.kind.is_dir() { diff --git a/crates/sandfs/src/fs.rs b/crates/sandfs/src/fs.rs index 29734a9..5fcea62 100644 --- a/crates/sandfs/src/fs.rs +++ b/crates/sandfs/src/fs.rs @@ -78,7 +78,7 @@ const MAX_MEMORY_FILE: u64 = 4 << 30; /// can't read a range on its own. The kernel asks for a file in chunks and caches /// what it gets, so only the few files being read right now are worth keeping. const RECENT_FILES: usize = 16; -const RECENT_BYTES: usize = 256 << 20; +const RECENT_BYTES: usize = 64 << 20; /// How long a file goes without a write before its buffer is committed and saved. /// Nothing tells this filesystem when a file is closed, so a quiet spell stands in. pub const SETTLE: Duration = Duration::from_millis(250); @@ -175,7 +175,8 @@ struct Node { sized: Cell, /// For a removed file, the workspace as it was before: the kernel may still have /// it open, and an open file stays readable until the kernel forgets the inode. - removed_from: Option, + /// Boxed, as few nodes have one and every node pays for its room. + removed_from: Option>, } /// An open directory, listed once so `readdir` pages through a stable list. @@ -865,6 +866,11 @@ impl Inner { } } self.buffers.remove(&ino); + // A table keeps its room once grown: after a walk of a big folder the + // kernel forgets nearly all of it, and the room goes too. + if self.nodes.capacity() > 4096 && self.nodes.len() < self.nodes.capacity() / 4 { + self.nodes.shrink_to(self.nodes.len() * 2); + } } } @@ -886,7 +892,7 @@ impl Inner { } if let Some(node) = self.nodes.get_mut(&ino) { node.linked = false; - node.removed_from = before.filter(|_| self.keep_removed); + node.removed_from = before.filter(|_| self.keep_removed).map(Box::new); // Without removed files kept readable (NFS), nothing ever says a client is // done with an inode, so a removed one goes now; a client still holding its // handle gets an error. @@ -1232,7 +1238,7 @@ impl Inner { fn contents(&self, ino: u64) -> FsResult<(&Workspace, &VPath)> { match self.nodes.get(&ino) { Some(Node { linked: true, path, .. }) => Ok((&self.ws, path)), - Some(Node { removed_from: Some(before), path, .. }) => Ok((before, path)), + Some(Node { removed_from: Some(before), path, .. }) => Ok((&**before, path)), _ => Err(Errno::ENOENT), } } diff --git a/docs/changelog.md b/docs/changelog.md index 8812078..8fb3f8e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. - **sandfs and sandfs-core, a file that shrank in the folder under a mount**: a mount of a folder lists each file once and reads its bytes from the folder as they are. When a file got shorter in the folder after that (a copy onto it truncates it first), a read came back short of the listed size, and macOS's NFS client filled the rest with zeros: copying files from a mount onto the folder under it wrote zeros into them. Now a read that comes back short makes the mount ask the folder again for that file's size and time alone. On Linux the read is the file's end and a later `stat` gives the new size. On macOS, whose client won't take a smaller size from a read, the read fails ("Stale NFS file handle") and the file gets a new inode, which the client looks up within a second or so and then reads as the file is now. A read never gives zeros. In `sandfs-core` this is `Workspace::recheck` and a new `Source::entry_now` with a default that changes nothing, which `DirSource` implements. Errors from a folder now name the path they happened at ("source error: PATH: Permission denied"). - **sandfs and sandfs-cli, real folders on Linux**: `sandfs fork` of a git clone (or of a fork kept as a folder) into a folder on a disk that clones files, btrfs or XFS, now keeps the fork there as a real folder of clones of the clone's files, as on macOS, instead of mounting it. It takes no space for the files, about 30 ms to make, and builds and git run in it at the disk's own speed, where a mount ran a build after an edit 25-40% slower and created small files 2-5 times slower. sandfs finds out by cloning one file next to the folder and removing it. On ext4, or with `--read-only` or `--mount`, the fork is mounted as before. From e45fa31ffaf23bc0458d786fe35429f88bb987bb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:35:25 +0000 Subject: [PATCH 02/10] Copy a folder on another disk onto that disk instead of refusing it Files clone only within one disk, so a fork of a plain folder on another disk used to fail with CrossDevice unless allow_copy made a full copy. Its frozen copy now goes in .sandfs/copies at the top of the folder's own disk, registered in the home's copies-elsewhere file, and is cleaned up with the home's other copies. The .sandfs folder is left out of forks of that disk's top. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- README.md | 2 +- crates/sandfs-cli/src/main.rs | 8 +- crates/sandfs-cli/tests/mount.rs | 36 +++++++ crates/sandfs/src/gitfile.rs | 25 ++++- crates/sandfs/src/lib.rs | 18 ++-- crates/sandfs/src/store.rs | 173 ++++++++++++++++++++++++------- docs/changelog.md | 1 + 7 files changed, 214 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 20a1993..34cb468 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ sandfs rm --unmount fix-login # done with the copy On macOS such a copy of a clone is a real folder rather than a mount, because macOS reaches a mount over NFS, which makes work on many small files several times slower than on the disk. Its files are APFS clones of the clone's, so it still takes no space, and programs and git run in it at the disk's own speed. Linux does the same where the clone's disk can clone files into the copy's folder (btrfs or XFS): a Linux mount is closer to the disk, but a build after an edit still ran 25-40% slower in one, and creating many small files 2-5 times slower. On ext4, which can't clone files, the copy stays a mount. sandfs reads its changes from the folder when asked (`changes`, `fork`), and `rm --unmount` deletes it. What only a mount can do, it can't: `--read-only` paths and `sandfs shell` need one, so with `--read-only`, or with `--mount`, the copy is mounted. `--real-folder` makes one anywhere (on a disk that can't clone files, its files are copied). -A folder that isn't in git forks too: the copy is the folder as it is now, its files cloned rather than copied (on APFS, btrfs or XFS), and `changes --copy-to` brings the work back, refusing to overwrite a file edited in the folder meanwhile. Cloning works only within one disk that can clone, so a folder on another disk than sandfs's home, or on ext4, is refused, unless `--allow-copy` says to copy its files in full: +A folder that isn't in git forks too: the copy is the folder as it is now, its files cloned rather than copied (on APFS, btrfs or XFS), and `changes --copy-to` brings the work back, refusing to overwrite a file edited in the folder meanwhile. Cloning works only within one disk, so a folder on another disk than sandfs's home, such as an external volume, is cloned onto its own disk, into `.sandfs/copies` at that disk's top. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. On a disk that can't clone, such as ext4, the folder is refused, unless `--allow-copy` says to copy its files in full: ```bash sandfs fork ~/notes ~/tasks/tidy-notes diff --git a/crates/sandfs-cli/src/main.rs b/crates/sandfs-cli/src/main.rs index 70d9b6e..74c34ca 100644 --- a/crates/sandfs-cli/src/main.rs +++ b/crates/sandfs-cli/src/main.rs @@ -87,10 +87,10 @@ enum Cmd { /// Mount the fork at DIR even where it would be kept as a real folder. #[arg(long)] mount: bool, - /// For a plain folder whose files can't be cloned into $SANDFS_HOME, because it - /// is on another disk or on one that doesn't clone files (such as ext4), copy - /// them in full instead of refusing. That takes as much space as the folder and - /// the time to copy it, and `update` copies the same way. + /// For a plain folder whose files can't be cloned, because its disk doesn't clone + /// files (such as ext4) or sandfs can't keep copies on it, copy them in full + /// instead of refusing. That takes as much space as the folder and the time to + /// copy it, and `update` copies the same way. #[arg(long)] allow_copy: bool, /// Serve in the foreground instead of in the background. diff --git a/crates/sandfs-cli/tests/mount.rs b/crates/sandfs-cli/tests/mount.rs index d166e77..0ee3c1a 100644 --- a/crates/sandfs-cli/tests/mount.rs +++ b/crates/sandfs-cli/tests/mount.rs @@ -775,6 +775,42 @@ fn a_linked_worktree_in_a_folder_commits_inside_the_workspace_or_not_at_all() { sandfs.unmount(&task); } +#[test] +fn a_fork_of_a_folder_on_another_disk_is_a_copy_on_that_disk() { + // /dev/shm is a disk of its own on Linux. + let shm = Path::new("/dev/shm"); + if !fuse_available() || !shm.is_dir() { + return; + } + let mut sandfs = Sandfs::new(); + let data = tempfile::tempdir_in(shm).unwrap(); + let folder = data.path().join("notes"); + std::fs::create_dir_all(&folder).unwrap(); + std::fs::write(folder.join("todo.md"), "a\n").unwrap(); + let task = sandfs.home.path().join("task"); + // tmpfs doesn't clone files, so they are copied; on that disk all the same. + let output = sandfs.try_run(&["fork", folder.to_str().unwrap(), task.to_str().unwrap()]); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success() && stderr.contains("can't clone"), "{stderr}"); + sandfs.run(&["fork", "--allow-copy", folder.to_str().unwrap(), task.to_str().unwrap()]); + sandfs.mounts.push(task.clone()); + let ours = std::fs::read_to_string(sandfs.home.path().join("copies-elsewhere")).unwrap(); + let ours = PathBuf::from(ours.trim()); + assert!(ours.starts_with(shm.join(".sandfs/copies")), "{}", ours.display()); + // One copy, beside the file that holds it for the mount. + assert_eq!(std::fs::read_dir(&ours).unwrap().flatten().filter(|entry| entry.path().is_dir()).count(), 1); + std::fs::write(folder.join("todo.md"), "a, later\n").unwrap(); + assert_eq!(std::fs::read_to_string(task.join("todo.md")).unwrap(), "a\n"); + std::fs::write(task.join("todo.md"), "b\n").unwrap(); + std::fs::write(folder.join("todo.md"), "a\n").unwrap(); + sandfs.run(&["changes", "task", "--copy-to", folder.to_str().unwrap()]); + assert_eq!(std::fs::read_to_string(folder.join("todo.md")).unwrap(), "b\n"); + // With the workspace, its copy and this home's folder on that disk go. + sandfs.run(&["rm", "--unmount", "task"]); + sandfs.mounts.clear(); + assert!(!ours.exists()); +} + #[test] fn a_fork_of_a_plain_folder_keeps_it_as_it_was() { use std::os::unix::fs::PermissionsExt; diff --git a/crates/sandfs/src/gitfile.rs b/crates/sandfs/src/gitfile.rs index 5216471..39dbd8e 100644 --- a/crates/sandfs/src/gitfile.rs +++ b/crates/sandfs/src/gitfile.rs @@ -76,7 +76,8 @@ fn relative(from: &Path, to: &Path) -> String { } /// A folder on disk as a mount shows it: [`DirSource`], with its linked worktrees' -/// `.git` files rewritten (see [`rewrite`]). +/// `.git` files rewritten (see [`rewrite`]), and at the top of a disk, without the +/// copies sandfs keeps there (see `Home::freeze`). #[derive(Debug)] pub struct FolderSource { files: DirSource, @@ -126,6 +127,13 @@ impl Source for FolderSource { fn read_dir(&self, key: &str) -> Result { let mut listing = self.files.read_dir(key)?; + // At the top of a disk, the copies sandfs keeps of its folders (see `Home::freeze`). + if listing.contains_key(crate::store::SHARED) + && *self.files.root()? == *key + && self.top.join(crate::store::SHARED).join("copies").is_dir() + { + listing.remove(crate::store::SHARED); + } if let Some(entry) = listing.remove(".git") { listing.insert(".git".to_owned(), self.sized(entry)); } @@ -204,6 +212,21 @@ mod tests { assert_eq!(rewrite(&[0xff, 0xfe], Path::new(""), top), None); } + #[test] + fn the_copies_at_a_disk_s_top_are_left_out_of_its_listing() { + let top = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(top.path().join(".sandfs/copies")).unwrap(); + std::fs::create_dir_all(top.path().join("sub/.sandfs/copies")).unwrap(); + let source = FolderSource::new(top.path()).unwrap(); + let root = source.root().unwrap(); + assert!(!source.read_dir(&root).unwrap().contains_key(".sandfs")); + let sub = source.read_dir(&root).unwrap()["sub"].key.clone(); + assert!(source.read_dir(&sub).unwrap().contains_key(".sandfs")); + // A folder of that name that holds no copies is the folder's own. + std::fs::remove_dir(top.path().join(".sandfs/copies")).unwrap(); + assert!(source.read_dir(&root).unwrap().contains_key(".sandfs")); + } + #[cfg(unix)] #[test] fn macos_names_tmp_with_or_without_private() { diff --git a/crates/sandfs/src/lib.rs b/crates/sandfs/src/lib.rs index b14edd2..4092603 100644 --- a/crates/sandfs/src/lib.rs +++ b/crates/sandfs/src/lib.rs @@ -258,18 +258,20 @@ pub struct ForkOptions { /// [`Home::fork`] read it as they read a mount; [`Home::remove`] with `unmount` /// deletes it. It can't be mounted, updated, served or confined. pub real_folder: Option, - /// For a plain folder whose files can't be cloned into the home, because it is on - /// another disk or on one that doesn't clone files (such as ext4), copy them in - /// full instead of refusing. That takes as much space as the folder, and the time - /// to copy it; [`Home::update`] copies the same way. Files that can be cloned still - /// are. + /// For a plain folder whose files can't be cloned, because its disk doesn't clone + /// files (such as ext4) or sandfs can't keep copies on it (see [`CrossDevice`]), + /// copy them in full instead of refusing. That takes as much space as the folder, + /// and the time to copy it; [`Home::update`] copies the same way. Files that can be + /// cloned still are. pub allow_copy: bool, } /// [`Home::fork`] refused to freeze a plain folder, since `folder` is on another disk -/// than the home, and a file is cloned only within one disk. Set -/// [`ForkOptions::allow_copy`] to copy the files instead, or mount the folder itself -/// with [`Home::mount`], which shows its later changes too. Find it with +/// than the copy, and a file is cloned only within one disk. A folder on another disk +/// than the home is copied onto its own disk, into `.sandfs/copies` at that disk's top, +/// so this is a folder mounted inside the one forked, or a disk whose top sandfs can't +/// write. Set [`ForkOptions::allow_copy`] to copy the files instead, or mount the +/// folder itself with [`Home::mount`], which shows its later changes too. Find it with /// `err.downcast_ref::()`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CrossDevice { diff --git a/crates/sandfs/src/store.rs b/crates/sandfs/src/store.rs index 56359d8..6067ef1 100644 --- a/crates/sandfs/src/store.rs +++ b/crates/sandfs/src/store.rs @@ -14,6 +14,7 @@ //! workspaces//lock held by the one process that may save it //! workspaces//holder who holds the lock, for whoever it refuses //! copies// a plain folder as `fork` found it, cloned (see `Home::freeze`) +//! copies-elsewhere this home's folders for copies on other disks, one a line //! run/ the process serving every mount (see `server`) //! ``` //! @@ -490,9 +491,11 @@ impl Home { /// The record for a new workspace over the plain folder `folder` as it is now (see /// `fork`): its files are cloned into `copies/`, which costs no time or space for - /// their contents on a disk that clones files (APFS, btrfs, XFS). Where the disk - /// can't, or `folder` is on another disk than the home, it is refused rather than - /// copied in full, unless `allow_copy`. Pipes, sockets and devices are left out. + /// their contents on a disk that clones files (APFS, btrfs, XFS). A file is cloned + /// only within one disk, so a folder on another disk than the home is cloned into + /// `.sandfs/copies` at the top of its own disk (see [`Home::copies_for`]). Where the + /// disk can't clone, it is refused rather than copied in full, unless `allow_copy`. + /// Pipes, sockets and devices are left out. pub fn freeze(&self, folder: &Path, allow_copy: bool) -> Result { let created = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); if crate::resolve_new(&self.root)?.starts_with(folder) { @@ -502,13 +505,15 @@ impl Home { self.root.display() ); } - let copies = self.copies_dir(); - std::fs::create_dir_all(&copies).with_context(|| format!("creating {}", copies.display()))?; + let copies = self.copies_for(folder)?; let copy = tempfile::Builder::new() .prefix("copy-") .tempdir_in(&copies) .with_context(|| format!("making room in {}", copies.display()))?; - clone_tree(folder, copy.path(), &self.root, allow_copy)?; + // The copies at the top of the disk, when that is the folder: never in a copy. + let own = folder.canonicalize().map(|folder| folder.join(SHARED)).ok(); + let skip = own.filter(|own| copies.starts_with(own)).map(|_| folder.join(SHARED)); + clone_tree(folder, copy.path(), &self.root, allow_copy, skip.as_deref())?; let path = copy.keep(); let source = folder.display().to_string(); Ok(Record { @@ -526,13 +531,50 @@ impl Home { /// home was named when it was made. pub fn holds_copy(&self, path: &Path) -> bool { let canonical = |path: &Path| path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); - canonical(path).starts_with(canonical(&self.copies_dir())) + let path = canonical(path); + self.all_copies_dirs().iter().any(|dir| path.starts_with(canonical(dir))) } fn copies_dir(&self) -> PathBuf { self.root.join("copies") } + /// The home's copies folder, then its folders on other disks. + fn all_copies_dirs(&self) -> Vec { + let mut dirs = vec![self.copies_dir()]; + dirs.extend(self.copies_elsewhere()); + dirs + } + + /// This home's folders for copies of folders on other disks (see [`Home::copies_for`]). + fn copies_elsewhere(&self) -> Vec { + let listed = std::fs::read_to_string(self.root.join("copies-elsewhere")).unwrap_or_default(); + listed.lines().filter(|line| !line.is_empty()).map(PathBuf::from).collect() + } + + /// The folder a copy of `folder` goes in, made if need be: the home's `copies/`, or, + /// for a folder on another disk, since a file is cloned only within one disk, this + /// home's own folder in `.sandfs/copies` at the top of that disk, where every home + /// gets one. Where that can't be made, the home's, which refuses the folder as on + /// another disk ([`CrossDevice`]). + fn copies_for(&self, folder: &Path) -> Result { + let home = self.copies_dir(); + std::fs::create_dir_all(&home).with_context(|| format!("creating {}", home.display()))?; + let Some(disk) = device(folder).filter(|disk| device(&home) != Some(*disk)) else { return Ok(home) }; + if let Some(dir) = self.copies_elsewhere().into_iter().find(|dir| device(dir) == Some(disk)) { + return Ok(dir); + } + let Some(top) = disk_top(folder) else { return Ok(home) }; + let shared = top.join(SHARED).join("copies"); + let made = std::fs::create_dir_all(&shared) + .and_then(|()| tempfile::Builder::new().prefix("home-").tempdir_in(&shared)) + .map(tempfile::TempDir::keep); + let Ok(dir) = made else { return Ok(home) }; + let mut listed = File::options().create(true).append(true).open(self.root.join("copies-elsewhere"))?; + std::io::Write::write_all(&mut listed, format!("{}\n", dir.display()).as_bytes())?; + Ok(dir) + } + /// Keep the copy of a folder at `path` while the returned file is open, though no /// workspace is over it any more: a mount that moved off it may still read files /// it has open there. @@ -547,7 +589,6 @@ impl Home { /// holds (see [`Home::hold_copy`]). The caller holds [`Home::repos_lock`] /// exclusively, so no workspace over one is being made. pub fn remove_unused_copies(&self) -> Result<()> { - let Ok(entries) = std::fs::read_dir(self.copies_dir()) else { return Ok(()) }; // Compared as the disk sees them, since the home may have been named another way // (through a link, as /tmp is on macOS) when a workspace was made. let mut used = Vec::new(); @@ -558,19 +599,44 @@ impl Home { used.push(path); } } - for entry in entries { - let path = entry?.path(); - if path.extension().is_some_and(|ext| ext == "held") || used.contains(&path.canonicalize()?) { - continue; - } - let held = held_path(&path); - let hold = File::options().write(true).open(&held); - match hold { - Ok(hold) if hold.try_lock().is_err() => continue, - Ok(_) => std::fs::remove_file(&held)?, - Err(_) => {} + for dir in self.all_copies_dirs() { + let Ok(entries) = std::fs::read_dir(&dir) else { continue }; + for entry in entries { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "held") || used.contains(&path.canonicalize()?) { + continue; + } + let held = held_path(&path); + let hold = File::options().write(true).open(&held); + match hold { + Ok(hold) if hold.try_lock().is_err() => continue, + Ok(_) => std::fs::remove_file(&held)?, + Err(_) => {} + } + remove_copy(&path)?; } - remove_copy(&path)?; + } + // A folder on another disk goes once it holds no copy, and `.sandfs` with it + // once no home has one there, so that disk keeps nothing of sandfs's. + let elsewhere = self.copies_elsewhere(); + let kept: Vec<_> = elsewhere + .iter() + .filter(|dir| { + // Gone, unless its disk is: an unplugged disk keeps its copies listed. + let Some(shared) = dir.parent() else { return true }; + let gone = std::fs::remove_dir(dir).is_ok() || (shared.exists() && !dir.exists()); + if gone { + let _ = std::fs::remove_dir(shared) + .and_then(|()| std::fs::remove_dir(shared.parent().unwrap_or(shared))); + } + !gone + }) + .collect(); + if kept.len() != elsewhere.len() { + let listed: String = kept.iter().map(|dir| format!("{}\n", dir.display())).collect(); + let (file, tmp) = (self.root.join("copies-elsewhere"), self.root.join("copies-elsewhere.tmp")); + std::fs::write(&tmp, listed).context("writing copies-elsewhere")?; + std::fs::rename(&tmp, file).context("writing copies-elsewhere")?; } Ok(()) } @@ -645,15 +711,18 @@ impl Home { /// Clone every file, link and folder in `top` into the empty folder `to`, keeping /// modes and modification times, so a build sees the files as it left them. A file /// that can't be cloned is copied if `allow_copy`, and refused otherwise. -fn clone_tree(top: &Path, to: &Path, home: &Path, allow_copy: bool) -> Result<()> { +fn clone_tree(top: &Path, to: &Path, home: &Path, allow_copy: bool, skip: Option<&Path>) -> Result<()> { let mut folders = vec![(top.to_path_buf(), to.to_path_buf())]; + let copy_disk = device(to); let mut made = Vec::new(); while let Some((from, to)) = folders.pop() { for entry in std::fs::read_dir(&from).with_context(|| format!("reading {}", from.display()))? { let entry = entry?; let (path, copy) = (entry.path(), to.join(entry.file_name())); let kind = entry.file_type()?; - if kind.is_dir() { + if skip == Some(path.as_path()) { + continue; + } else if kind.is_dir() { std::fs::create_dir(©).with_context(|| format!("creating {}", copy.display()))?; folders.push((path, copy)); } else if kind.is_symlink() { @@ -674,7 +743,7 @@ fn clone_tree(top: &Path, to: &Path, home: &Path, allow_copy: bool) -> Result<() cloned.map_err(|err| -> anyhow::Error { if err.kind() == std::io::ErrorKind::CrossesDevices { // The folder itself, unless only a folder mounted in it is elsewhere. - let same = device(top).is_some() && device(top) == device(home); + let same = device(top).is_some() && device(top) == copy_disk; let folder = if same { &from } else { top }; return CrossDevice { folder: folder.to_path_buf(), home: home.to_path_buf() }.into(); } @@ -714,6 +783,16 @@ fn linked_worktree(top: &Path, folder: &Path, path: &Path) -> Result Option { + let path = path.canonicalize().ok()?; + let disk = device(&path)?; + path.ancestors().take_while(|dir| device(dir) == Some(disk)).last().map(Path::to_path_buf) +} + /// The disk `path` is on, where the system says. #[cfg(unix)] fn device(path: &Path) -> Option { @@ -923,7 +1002,7 @@ mod tests { #[cfg(unix)] #[test] - fn a_folder_on_another_disk_is_refused_by_kind_or_copied() { + fn a_folder_on_another_disk_is_copied_on_that_disk() { use std::os::unix::fs::PermissionsExt; let state = tempfile::tempdir().unwrap(); @@ -936,23 +1015,47 @@ mod tests { std::fs::set_permissions(folder.join("run.sh"), std::fs::Permissions::from_mode(0o755)).unwrap(); let old = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000_000); File::options().write(true).open(folder.join("run.sh")).unwrap().set_modified(old).unwrap(); - - // Refused, naming both, with nothing left behind. - let err = home.freeze(&folder, false).unwrap_err(); - let refused = err.downcast_ref::().expect("a CrossDevice error"); - assert_eq!(refused, &CrossDevice { folder: folder.clone(), home: state.path().to_path_buf() }); - assert!(err.to_string().contains("is on another disk than sandfs's home"), "{err}"); - assert_eq!(std::fs::read_dir(home.copies_dir()).unwrap().count(), 0); - - // Copied in full when allowed, with modes and times, and the record says so. - let record = home.freeze(&folder, true).unwrap(); - assert!(record.allow_copy); + let top = disk_top(&folder).unwrap(); + let ours = |home: &Home| home.copies_elsewhere().into_iter().next(); + + // Cloned on the folder's own disk, or where that disk can't clone, as tmpfs and + // RAM disks can't, copied in full only when allowed. + let record = match home.freeze(&folder, false) { + Ok(record) => record, + Err(err) => { + assert!(err.downcast_ref::().is_none(), "{err}"); + assert!(err.to_string().contains("can't clone"), "{err}"); + assert_eq!(std::fs::read_dir(ours(&home).unwrap()).unwrap().count(), 0); + home.freeze(&folder, true).unwrap() + } + }; let Base::Dir { path } = &record.base else { panic!("{:?}", record.base) }; + assert!(path.starts_with(top.join(SHARED).join("copies")), "{}", path.display()); + assert_eq!(device(path), device(&folder)); assert_eq!(std::fs::read_to_string(path.join("drafts/plan.md")).unwrap(), "plan\n"); let meta = std::fs::metadata(path.join("run.sh")).unwrap(); assert_eq!(meta.permissions().mode() & 0o777, 0o755); assert_eq!(meta.modified().unwrap(), old); assert!(home.holds_copy(path)); + + // With no workspace over it, the copy goes, and this home's folder on that disk. + let dir = ours(&home).unwrap(); + home.remove_unused_copies().unwrap(); + assert!(!path.exists() && !dir.exists()); + assert_eq!(ours(&home), None); + } + + #[test] + fn a_folder_s_copies_of_its_disk_are_never_copied() { + let (data, copy) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + std::fs::create_dir_all(data.path().join(".sandfs/copies/home-x/copy-y")).unwrap(); + std::fs::create_dir_all(data.path().join("sub/.sandfs")).unwrap(); + std::fs::write(data.path().join("notes.md"), "notes\n").unwrap(); + let skip = data.path().join(SHARED); + clone_tree(data.path(), copy.path(), data.path(), true, Some(&skip)).unwrap(); + assert!(!copy.path().join(SHARED).exists()); + assert!(copy.path().join("sub/.sandfs").is_dir()); + assert!(copy.path().join("notes.md").is_file()); } #[test] diff --git a/docs/changelog.md b/docs/changelog.md index 8fb3f8e..be8493d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,7 @@ ## Unreleased +- **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't write. - **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. - **sandfs and sandfs-core, a file that shrank in the folder under a mount**: a mount of a folder lists each file once and reads its bytes from the folder as they are. When a file got shorter in the folder after that (a copy onto it truncates it first), a read came back short of the listed size, and macOS's NFS client filled the rest with zeros: copying files from a mount onto the folder under it wrote zeros into them. Now a read that comes back short makes the mount ask the folder again for that file's size and time alone. On Linux the read is the file's end and a later `stat` gives the new size. On macOS, whose client won't take a smaller size from a read, the read fails ("Stale NFS file handle") and the file gets a new inode, which the client looks up within a second or so and then reads as the file is now. A read never gives zeros. In `sandfs-core` this is `Workspace::recheck` and a new `Source::entry_now` with a default that changes nothing, which `DirSource` implements. Errors from a folder now name the path they happened at ("source error: PATH: Permission denied"). From 76cf0c653e81d6693d97b25db8c6a65f12bd7af1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:40:22 +0000 Subject: [PATCH 03/10] Set a copied file's time on Windows through a handle that may write attributes A full copy (allow_copy) opened each copied file read-only to set its modification time, which Windows refuses, and opened folders without backup semantics, which Windows can't. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs/src/store.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/sandfs/src/store.rs b/crates/sandfs/src/store.rs index 6067ef1..36edc52 100644 --- a/crates/sandfs/src/store.rs +++ b/crates/sandfs/src/store.rs @@ -834,12 +834,27 @@ pub fn remove_copy(path: &Path) -> Result<()> { /// Give `copy` the mode and modification time of `path`. pub(crate) fn keep_times(path: &Path, copy: &Path) -> Result<()> { let meta = std::fs::metadata(path)?; - let file = File::open(copy)?; + let file = open_to_set_times(copy)?; file.set_modified(meta.modified()?).with_context(|| format!("setting the time of {}", copy.display()))?; std::fs::set_permissions(copy, meta.permissions()) .with_context(|| format!("setting the mode of {}", copy.display())) } +#[cfg(not(windows))] +fn open_to_set_times(path: &Path) -> std::io::Result { + File::open(path) +} + +/// Windows sets a time only through a handle that may write attributes, +/// and opens a folder only with backup semantics. +#[cfg(windows)] +fn open_to_set_times(path: &Path) -> std::io::Result { + use std::os::windows::fs::OpenOptionsExt; + const FILE_WRITE_ATTRIBUTES: u32 = 0x100; + const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000; + std::fs::OpenOptions::new().access_mode(FILE_WRITE_ATTRIBUTES).custom_flags(FILE_FLAG_BACKUP_SEMANTICS).open(path) +} + #[cfg(unix)] pub(crate) fn symlink(target: &Path, path: &Path) -> Result<()> { std::os::unix::fs::symlink(target, path).with_context(|| format!("linking {}", path.display())) From 60966cd3049de5769d977969f914efbacc0736ce Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:42:41 +0000 Subject: [PATCH 04/10] Never keep copies through a .sandfs link, and hide .sandfs only at a disk's top A .sandfs or .sandfs/copies at a disk's top that is a link could point into the folder being copied, which would then copy into itself; such a disk's copies stay in the home. A folder's own .sandfs/copies, away from a disk's top, is listed as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs/src/gitfile.rs | 29 ++++++++++++++++++++++++----- crates/sandfs/src/store.rs | 7 ++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/sandfs/src/gitfile.rs b/crates/sandfs/src/gitfile.rs index 39dbd8e..0fbb739 100644 --- a/crates/sandfs/src/gitfile.rs +++ b/crates/sandfs/src/gitfile.rs @@ -131,6 +131,7 @@ impl Source for FolderSource { if listing.contains_key(crate::store::SHARED) && *self.files.root()? == *key && self.top.join(crate::store::SHARED).join("copies").is_dir() + && crate::store::disk_top(&self.top).is_some_and(|top| Some(top) == self.top.canonicalize().ok()) { listing.remove(crate::store::SHARED); } @@ -213,18 +214,36 @@ mod tests { } #[test] - fn the_copies_at_a_disk_s_top_are_left_out_of_its_listing() { + fn a_folder_s_own_sandfs_folder_is_listed_away_from_a_disk_s_top() { let top = tempfile::tempdir().unwrap(); std::fs::create_dir_all(top.path().join(".sandfs/copies")).unwrap(); std::fs::create_dir_all(top.path().join("sub/.sandfs/copies")).unwrap(); let source = FolderSource::new(top.path()).unwrap(); let root = source.root().unwrap(); - assert!(!source.read_dir(&root).unwrap().contains_key(".sandfs")); + assert!(source.read_dir(&root).unwrap().contains_key(".sandfs")); let sub = source.read_dir(&root).unwrap()["sub"].key.clone(); assert!(source.read_dir(&sub).unwrap().contains_key(".sandfs")); - // A folder of that name that holds no copies is the folder's own. - std::fs::remove_dir(top.path().join(".sandfs/copies")).unwrap(); - assert!(source.read_dir(&root).unwrap().contains_key(".sandfs")); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_copies_at_a_disk_s_top_are_left_out_of_its_listing() { + // /dev/shm is a disk of its own; the copies sandfs keeps there stay. + let top = Path::new("/dev/shm"); + if crate::store::disk_top(top).as_deref() != Some(top) { + return; + } + let copies = top.join(".sandfs/copies"); + // Other tests clean up an empty copies folder there as this one makes it. + let _home = (0..5) + .find_map(|_| { + std::fs::create_dir_all(&copies).ok()?; + tempfile::Builder::new().prefix("home-").tempdir_in(&copies).ok() + }) + .unwrap(); + let source = FolderSource::new(top).unwrap(); + let root = source.root().unwrap(); + assert!(!source.read_dir(&root).unwrap().contains_key(".sandfs")); } #[cfg(unix)] diff --git a/crates/sandfs/src/store.rs b/crates/sandfs/src/store.rs index 36edc52..3551960 100644 --- a/crates/sandfs/src/store.rs +++ b/crates/sandfs/src/store.rs @@ -566,6 +566,11 @@ impl Home { } let Some(top) = disk_top(folder) else { return Ok(home) }; let shared = top.join(SHARED).join("copies"); + // Through a link, the copies could land in the folder being copied. + let real = |dir: &Path| std::fs::symlink_metadata(dir).map_or(true, |meta| meta.is_dir()); + if !real(&top.join(SHARED)) || !real(&shared) { + return Ok(home); + } let made = std::fs::create_dir_all(&shared) .and_then(|()| tempfile::Builder::new().prefix("home-").tempdir_in(&shared)) .map(tempfile::TempDir::keep); @@ -787,7 +792,7 @@ fn linked_worktree(top: &Path, folder: &Path, path: &Path) -> Result Option { +pub(crate) fn disk_top(path: &Path) -> Option { let path = path.canonicalize().ok()?; let disk = device(&path)?; path.ancestors().take_while(|dir| device(dir) == Some(disk)).last().map(Path::to_path_buf) From 42074e76c6b8fe8b14aad8a12dc418348cb4988d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 01:51:01 +0000 Subject: [PATCH 05/10] Keep copies on another disk only where no one else can move their folder Anyone who could write .sandfs or .sandfs/copies at a disk's top, or rename them there, could swap a home's folder for a link, and copies would be written, and deleted, wherever it led. Those folders must be the home owner's and writable by no one else, at a top that is not writable by others or is sticky; otherwise the copy stays in the home, and cleanup leaves such a folder alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs/src/store.rs | 74 ++++++++++++++++++++++++++++++++++---- docs/changelog.md | 2 +- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/crates/sandfs/src/store.rs b/crates/sandfs/src/store.rs index 3551960..d66cb06 100644 --- a/crates/sandfs/src/store.rs +++ b/crates/sandfs/src/store.rs @@ -561,25 +561,52 @@ impl Home { let home = self.copies_dir(); std::fs::create_dir_all(&home).with_context(|| format!("creating {}", home.display()))?; let Some(disk) = device(folder).filter(|disk| device(&home) != Some(*disk)) else { return Ok(home) }; - if let Some(dir) = self.copies_elsewhere().into_iter().find(|dir| device(dir) == Some(disk)) { + if let Some(dir) = + self.copies_elsewhere().into_iter().find(|dir| device(dir) == Some(disk) && self.private(dir)) + { return Ok(dir); } let Some(top) = disk_top(folder) else { return Ok(home) }; let shared = top.join(SHARED).join("copies"); - // Through a link, the copies could land in the folder being copied. - let real = |dir: &Path| std::fs::symlink_metadata(dir).map_or(true, |meta| meta.is_dir()); - if !real(&top.join(SHARED)) || !real(&shared) { + if std::fs::create_dir_all(&shared).is_err() || !self.private(&shared.join("home")) { return Ok(home); } - let made = std::fs::create_dir_all(&shared) - .and_then(|()| tempfile::Builder::new().prefix("home-").tempdir_in(&shared)) - .map(tempfile::TempDir::keep); + let made = tempfile::Builder::new().prefix("home-").tempdir_in(&shared).map(tempfile::TempDir::keep); let Ok(dir) = made else { return Ok(home) }; let mut listed = File::options().create(true).append(true).open(self.root.join("copies-elsewhere"))?; std::io::Write::write_all(&mut listed, format!("{}\n", dir.display()).as_bytes())?; Ok(dir) } + /// Whether no one but this home's owner can change where the copies folder `dir`, on + /// another disk, leads: its `.sandfs/copies` and `.sandfs` are folders of the owner's + /// that no one else may write, at a disk's top where no one else may rename them, and + /// it is a folder, not a link. Anyone else could swap one for a link, and copies would + /// be written, and deleted, wherever it led. + #[cfg(unix)] + fn private(&self, dir: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + let Ok(owner) = std::fs::metadata(&self.root).map(|meta| meta.uid()) else { return false }; + let ours = |dir: &Path| { + std::fs::symlink_metadata(dir) + .is_ok_and(|meta| meta.is_dir() && meta.uid() == owner && meta.mode() & 0o022 == 0) + }; + let (Some(copies), Some(shared), Some(top)) = + (dir.parent(), dir.parent().and_then(Path::parent), dir.ancestors().nth(3)) + else { + return false; + }; + // A sticky top, as /dev/shm and /tmp are, lets others rename only their own. + let top = std::fs::metadata(top).is_ok_and(|meta| meta.mode() & 0o022 == 0 || meta.mode() & 0o1000 != 0); + let dir = std::fs::symlink_metadata(dir).map_or(true, |meta| meta.is_dir()); + dir && ours(copies) && ours(shared) && top + } + + #[cfg(not(unix))] + fn private(&self, _dir: &Path) -> bool { + false + } + /// Keep the copy of a folder at `path` while the returned file is open, though no /// workspace is over it any more: a mount that moved off it may still read files /// it has open there. @@ -605,6 +632,9 @@ impl Home { } } for dir in self.all_copies_dirs() { + if dir != self.copies_dir() && !self.private(&dir) { + continue; + } let Ok(entries) = std::fs::read_dir(&dir) else { continue }; for entry in entries { let path = entry?.path(); @@ -1065,6 +1095,36 @@ mod tests { assert_eq!(ours(&home), None); } + #[cfg(unix)] + #[test] + fn copies_go_on_another_disk_only_where_no_one_else_can_move_them() { + use std::os::unix::fs::PermissionsExt; + + let (state, top) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); + let home = Home::at(state.path()); + std::fs::create_dir_all(state.path()).unwrap(); + let copies = top.path().join(".sandfs/copies"); + std::fs::create_dir_all(&copies).unwrap(); + let mode = |path: &Path, mode| std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + mode(top.path(), 0o755); + mode(&copies, 0o755); + mode(&top.path().join(".sandfs"), 0o755); + let dir = copies.join("home-x"); + assert!(home.private(&dir)); + // Anyone may write the copies folder, or rename what is in the disk's top. + mode(&copies, 0o777); + assert!(!home.private(&dir)); + mode(&copies, 0o755); + mode(top.path(), 0o777); + assert!(!home.private(&dir)); + mode(top.path(), 0o1777); + assert!(home.private(&dir)); + // A link where the folder was. + std::fs::rename(top.path().join(".sandfs"), top.path().join("moved")).unwrap(); + std::os::unix::fs::symlink(top.path().join("moved"), top.path().join(".sandfs")).unwrap(); + assert!(!home.private(&dir)); + } + #[test] fn a_folder_s_copies_of_its_disk_are_never_copied() { let (data, copy) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()); diff --git a/docs/changelog.md b/docs/changelog.md index be8493d..484a26a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,7 +2,7 @@ ## Unreleased -- **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't write. +- **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. It does so only where no one else can move that folder: `.sandfs` and `.sandfs/copies` are this user's and writable by no one else, at a top where others can't rename them (a sticky one, as /dev/shm is, will do). `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't use that way. - **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. - **sandfs and sandfs-core, a file that shrank in the folder under a mount**: a mount of a folder lists each file once and reads its bytes from the folder as they are. When a file got shorter in the folder after that (a copy onto it truncates it first), a read came back short of the listed size, and macOS's NFS client filled the rest with zeros: copying files from a mount onto the folder under it wrote zeros into them. Now a read that comes back short makes the mount ask the folder again for that file's size and time alone. On Linux the read is the file's end and a later `stat` gives the new size. On macOS, whose client won't take a smaller size from a read, the read fails ("Stale NFS file handle") and the file gets a new inode, which the client looks up within a second or so and then reads as the file is now. A read never gives zeros. In `sandfs-core` this is `Workspace::recheck` and a new `Source::entry_now` with a default that changes nothing, which `DirSource` implements. Errors from a folder now name the path they happened at ("source error: PATH: Permission denied"). From 1132f1cf062662217dc0403237bb21591ccb27cd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 02:32:55 +0000 Subject: [PATCH 06/10] Release 0.9.0 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- Cargo.lock | 14 +++++++------- Cargo.toml | 12 ++++++------ crates/sandfs/README.md | 2 +- crates/sandfs/src/lib.rs | 4 ++-- docs/changelog.md | 9 +++++++++ 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5e415b..72068fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -684,7 +684,7 @@ dependencies = [ [[package]] name = "sandfs" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -708,7 +708,7 @@ dependencies = [ [[package]] name = "sandfs-bench" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "bytes", @@ -722,7 +722,7 @@ dependencies = [ [[package]] name = "sandfs-cli" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "clap", @@ -734,7 +734,7 @@ dependencies = [ [[package]] name = "sandfs-core" -version = "0.8.0" +version = "0.9.0" dependencies = [ "bytes", "imbl", @@ -744,7 +744,7 @@ dependencies = [ [[package]] name = "sandfs-git" -version = "0.8.0" +version = "0.9.0" dependencies = [ "bytes", "sandfs-core", @@ -753,7 +753,7 @@ dependencies = [ [[package]] name = "sandfs-mcp" -version = "0.8.0" +version = "0.9.0" dependencies = [ "base64", "bytes", @@ -766,7 +766,7 @@ dependencies = [ [[package]] name = "sandfs-py" -version = "0.8.0" +version = "0.9.0" dependencies = [ "bytes", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 14ac29f..5d69a45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/sandfs", "crates/sandfs-bench", "crates/sandfs-cli", "crates/sandfs-core", "crates/sandfs-git", "crates/sandfs-mcp", "crates/sandfs-py"] [workspace.package] -version = "0.8.0" +version = "0.9.0" edition = "2024" rust-version = "1.85" license = "MIT" @@ -14,11 +14,11 @@ homepage = "https://github.com/lydakis/sandfs" bytes = "1.12" imbl = "7.0" thiserror = "2.0" -sandfs-core = { path = "crates/sandfs-core", version = "0.8.0" } -sandfs-git = { path = "crates/sandfs-git", version = "0.8.0" } -sandfs-mcp = { path = "crates/sandfs-mcp", version = "0.8.0" } -sandfs = { path = "crates/sandfs", version = "0.8.0", default-features = false } -sandfs-cli = { path = "crates/sandfs-cli", version = "0.8.0" } +sandfs-core = { path = "crates/sandfs-core", version = "0.9.0" } +sandfs-git = { path = "crates/sandfs-git", version = "0.9.0" } +sandfs-mcp = { path = "crates/sandfs-mcp", version = "0.9.0" } +sandfs = { path = "crates/sandfs", version = "0.9.0", default-features = false } +sandfs-cli = { path = "crates/sandfs-cli", version = "0.9.0" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sandfs/README.md b/crates/sandfs/README.md index af5a7a5..9f9e9cb 100644 --- a/crates/sandfs/README.md +++ b/crates/sandfs/README.md @@ -50,7 +50,7 @@ The home is `$SANDFS_HOME`, else `$XDG_DATA_HOME/sandfs`, else `~/.local/share/s ## Before you build on it -- **The API will change.** sandfs is pre-1.0, so a minor release (0.8 to 0.9) may change this crate's API; the [changelog](https://github.com/lydakis/sandfs/blob/main/docs/changelog.md) says how each time. Depend on one minor version, such as `sandfs = "0.8"`. +- **The API will change.** sandfs is pre-1.0, so a minor release (0.9 to 0.10) may change this crate's API; the [changelog](https://github.com/lydakis/sandfs/blob/main/docs/changelog.md) says how each time. Depend on one minor version, such as `sandfs = "0.9"`. - **Writing in a mount costs more than writing to a plain folder, most of all on macOS.** Every file operation in a mount goes through the mount's process. On macOS each write is also a round trip over localhost before it returns, so write-heavy work is where a Mac feels it: compiling ripgrep from scratch took about 18 s in a mount against 13 s in a plain folder on the same Mac (October 2026). - **Linux needs FUSE 3** (`fusermount3`, package `fuse3`) to mount. - **sandfs isn't an execution sandbox.** A program working in a mount can still reach anything else its user can; `Home::shell` runs one confined to the mount by the operating system's own sandbox. diff --git a/crates/sandfs/src/lib.rs b/crates/sandfs/src/lib.rs index 4092603..e917cfa 100644 --- a/crates/sandfs/src/lib.rs +++ b/crates/sandfs/src/lib.rs @@ -58,8 +58,8 @@ //! //! # Before you build on it //! -//! - sandfs is pre-1.0: a minor release (0.8 to 0.9) may change this API, and the -//! changelog says how. Depend on one minor version, such as `sandfs = "0.8"`. +//! - sandfs is pre-1.0: a minor release (0.9 to 0.10) may change this API, and the +//! changelog says how. Depend on one minor version, such as `sandfs = "0.9"`. //! - Every file operation in a mount goes through the mount's process, and on macOS //! each write is also a round trip over localhost before it returns, so write-heavy //! work runs slower there than in a plain folder: compiling ripgrep from scratch diff --git a/docs/changelog.md b/docs/changelog.md index 484a26a..1854c62 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,15 @@ ## Unreleased +## 0.9.0 (2026-10-28) + +The main changes: +- A fork of a plain folder on another disk than sandfs's home is a frozen copy on that disk, in `.sandfs/copies` at its top, instead of being refused. +- A file that shrinks in a folder under a mount is never shown padded with zeros, and git in a linked worktree of a mounted or forked folder stays inside the workspace. +- A mount keeps less memory per file it is asked about, and a fork of a git clone on a Linux disk that clones files is a real folder. + +API changes in the `sandfs` crate: `Changes` has a new field, `committed`, so code that builds a `Changes` itself needs it. `CrossDevice` comes up far less often, now that a folder on another disk is copied there. In `sandfs-core`, `Source` has a new method, `entry_now`, with a default, and `Workspace` has `recheck`. + - **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. It does so only where no one else can move that folder: `.sandfs` and `.sandfs/copies` are this user's and writable by no one else, at a top where others can't rename them (a sticky one, as /dev/shm is, will do). `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't use that way. - **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. From dda3968ddd531767fb28fc49880b5897a99ee153 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 03:05:53 +0000 Subject: [PATCH 07/10] Keep a mount's caches to a budget and give memory back Folder listings, directories found and file bytes a workspace keeps stay within a budget, least used first out. A mount unused for a minute, or asked with `sandfs trim` / `Home::trim`, saves its changes, drops those caches, ends idle git readers and asks the allocator to hand freed memory back to the system, as it now does after every unmount. Paths in a mount's table of files are held once instead of twice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- Cargo.toml | 4 +- crates/sandfs-cli/README.md | 3 +- crates/sandfs-cli/src/main.rs | 18 ++ crates/sandfs-core/src/path.rs | 28 +-- crates/sandfs-core/src/source.rs | 5 + crates/sandfs-core/src/workspace.rs | 243 +++++++++++++++++++++++--- crates/sandfs-core/tests/workspace.rs | 36 ++++ crates/sandfs-git/src/lib.rs | 23 +++ crates/sandfs/src/fs.rs | 101 +++++++++-- crates/sandfs/src/lib.rs | 22 +++ crates/sandfs/src/memory.rs | 39 +++++ crates/sandfs/src/nfs.rs | 51 ++++++ crates/sandfs/src/ops.rs | 11 ++ crates/sandfs/src/server.rs | 108 ++++++++++-- crates/sandfs/src/store.rs | 12 +- docs/changelog.md | 4 +- 16 files changed, 645 insertions(+), 63 deletions(-) create mode 100644 crates/sandfs/src/memory.rs diff --git a/Cargo.toml b/Cargo.toml index 5d69a45..b0538ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,9 @@ sandfs = { path = "crates/sandfs", version = "0.9.0", default-features = false } sandfs-cli = { path = "crates/sandfs-cli", version = "0.9.0" } [workspace.lints.rust] -unsafe_code = "forbid" +# Deny rather than forbid: `memory::give_back` in the sandfs crate is the one audited +# exception, allowed on that call alone. +unsafe_code = "deny" missing_docs = "warn" [workspace.lints.clippy] diff --git a/crates/sandfs-cli/README.md b/crates/sandfs-cli/README.md index 19859fa..3779a98 100644 --- a/crates/sandfs-cli/README.md +++ b/crates/sandfs-cli/README.md @@ -23,6 +23,7 @@ sandfs update cleanup ~/datasets/survey-v2 # move a workspace onto a newer sandfs rm cleanup # forget a workspace and its changes sandfs rm --unmount ~/ws/survey # unmount it first if it is mounted sandfs clean # free the space downloaded repository files take +sandfs trim # mounts give back the memory they can do without ``` `--read-only PATH` (repeatable, relative to the source) makes a file or folder read-only in a new workspace and its forks: programs can read it, but writing, deleting, renaming or creating there fails as on a read-only file system (`EROFS`), and its files show without write permission. It is how a container or an agent given the mount gets to look at a folder without changing it even in its own copy. `serve --mcp SOURCE --read-only PATH` does the same for an agent over MCP. It is set when a workspace is created; mounting an existing workspace with it is refused. @@ -61,7 +62,7 @@ Read-only paths stay read-only under `srt`, and `changes` lists what the program One process serves a workspace at a time. A mount or an MCP session holds it until it ends, a second one is refused with the holder's name, and a crashed holder releases it. Parallel attempts each get their own fork. -Every mount runs in one background process, which the first mount starts and which exits a moment after the last `unmount`. It writes its log to `$SANDFS_HOME/run/server.log`. Mounts of one git commit, such as a workspace and its forks, share its git readers and every folder listing read from it. +Every mount runs in one background process, which the first mount starts and which exits a moment after the last `unmount`. It writes its log to `$SANDFS_HOME/run/server.log`. Mounts of one git commit, such as a workspace and its forks, share its git readers and every folder listing read from it. What a mount keeps in memory to answer faster, folder listings and files read through the library, stays within a budget (tens of megabytes) and is read again when needed. A mount nobody uses for a minute lets go of it, ends its idle git readers and hands the memory back to the system; `sandfs trim [NAME]` does that now. What stays is a small record of each file the system has asked about, which on macOS lasts until the unmount, since its NFS client never says it is done with a file. Changes are saved under `$SANDFS_HOME` (default `~/.local/share/sandfs`) once a file has gone a quarter of a second without a write. Closing a file doesn't save it, since the mount is never told about opens and closes. These saves are written to disk but not flushed, so they survive the mount's process crashing but not necessarily a power cut. `fsync` saves at once and flushes to disk before it returns, and `unmount`, `fork` and `changes` save and flush everything first. On macOS `fsync` doesn't: the Mac sends each write to the mount before the write returns, so a fork or `changes` sees it, and with that setting it never passes `fsync` on. There a write is flushed to disk only by `unmount`, `fork` or `changes`, which matters to programs that count on `fsync`, such as databases. A file being written goes straight into a file on disk under `$SANDFS_HOME`, not into memory, and once saved it is read from there through the kernel's page cache, so a mount's memory doesn't grow with what programs write in it and a file can be as large as the disk allows. Writing into part of a saved file first copies it, which takes no time on filesystems that clone files (APFS, btrfs, XFS) and is a full copy on others (ext4); a file from the source is copied on its first write. A fork on disk costs more than a fork in memory. `sandfs fork` never copies the source, but it waits for the workspace's latest changes to be saved, links each changed file into the fork (copying it where the disk can't link), and starts a new mount, which reads a host folder again rather than sharing what the first mount has read (mounts of one git commit do share it). Without a directory, `sandfs fork NAME --name SNAPSHOT` stops before the mount: it keeps a snapshot of the workspace as it is now, which `fork SNAPSHOT DIR` brings back as a new copy (or `mount SNAPSHOT DIR` mounts and changes the snapshot itself). With the files already saved it costs a link per changed file: 45 ms for a workspace holding a 1.1 GB build. A host folder is read as it is when each path is first touched, so mount one nothing else is changing. `git:[@rev]` mounts a commit instead of a folder (a URL works without `git:`); for a remote repository sandfs fetches only that commit and its folder listings, with no history, and fetches file contents when they are first read. Contents fetched from a remote repository are kept in one local copy per repository, shared by its workspaces, until `sandfs clean`: for each repository none of whose workspaces is mounted or served, it drops the file contents and keeps the commits and folder listings, so the workspaces still mount at once and fetch a file again when it is next read, and it deletes the copy of a repository no workspace uses any more. Offline, a commit id that was mounted before still mounts and the folders opened before still read; a branch or tag name is looked up on the server every time, so it doesn't mount offline. diff --git a/crates/sandfs-cli/src/main.rs b/crates/sandfs-cli/src/main.rs index 74c34ca..f445d78 100644 --- a/crates/sandfs-cli/src/main.rs +++ b/crates/sandfs-cli/src/main.rs @@ -145,6 +145,16 @@ enum Cmd { /// A workspace name or the directory it is mounted on. workspace: String, }, + /// Have mounts give back the memory they can do without. + /// + /// Saves the changes of the mounted workspace (every mounted one, without a name), + /// then lets go of the folder listings and file bytes it keeps and of idle git + /// helpers, and hands the memory freed back to the system. Nothing in the mount + /// changes. A mount left unused for a minute is trimmed on its own. + Trim { + /// A workspace name or the directory it is mounted on. + workspace: Option, + }, /// Delete an unmounted workspace and its changes. Rm { /// The workspace's name, or with --unmount the directory it is mounted on or @@ -304,6 +314,14 @@ fn run(cli: Cli) -> Result<()> { print_out(&out) } Cmd::Unmount { workspace } => home.unmount(&workspace), + Cmd::Trim { workspace } => { + let trimmed = home.trim(workspace.as_deref())?; + let mounts = if trimmed.mounts == 1 { "1 mount".to_owned() } else { format!("{} mounts", trimmed.mounts) }; + match trimmed.given_back { + Some(bytes) => print_out(&format!("trimmed {mounts}; {} given back to the system\n", size(bytes))), + None => print_out(&format!("trimmed {mounts}\n")), + } + } Cmd::Rm { name, unmount } => home.remove(&name, unmount), Cmd::Clean => { let mut out = String::new(); diff --git a/crates/sandfs-core/src/path.rs b/crates/sandfs-core/src/path.rs index 222fd6f..e2ee336 100644 --- a/crates/sandfs-core/src/path.rs +++ b/crates/sandfs-core/src/path.rs @@ -2,6 +2,7 @@ use std::borrow::Borrow; use std::fmt; +use std::sync::Arc; use crate::error::{Error, Result}; @@ -10,13 +11,16 @@ use crate::error::{Error, Result}; /// Parsing collapses repeated slashes and `.` components and resolves `..`; a `..` /// that would climb above the root is rejected rather than clamped, so a caller can /// never address something outside the workspace by accident. +/// +/// A path never changes once made, so a copy shares its bytes: a mount's table of +/// files and the path each file is known by hold one string between them. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct VPath(String); +pub struct VPath(Arc); impl VPath { /// The root path `/`. pub fn root() -> Self { - VPath("/".to_owned()) + VPath("/".into()) } /// Parse and normalize an absolute path. @@ -27,7 +31,7 @@ impl VPath { // Most paths arrive already normalized; keep them as they are. let normal = |part: &str| !part.is_empty() && part != "." && part != ".." && !part.contains('\0'); if raw == "/" || raw.split('/').skip(1).all(normal) { - return Ok(VPath(raw.to_owned())); + return Ok(VPath(raw.into())); } let mut parts: Vec<&str> = Vec::new(); for part in raw.split('/') { @@ -46,7 +50,7 @@ impl VPath { } } } - Ok(VPath(format!("/{}", parts.join("/")))) + Ok(VPath(format!("/{}", parts.join("/")).into())) } /// The path as a string. @@ -56,7 +60,7 @@ impl VPath { /// True for `/`. pub fn is_root(&self) -> bool { - self.0 == "/" + &*self.0 == "/" } /// The parent directory, or `None` for the root. @@ -65,7 +69,7 @@ impl VPath { return None; } let idx = self.0.rfind('/').unwrap_or(0); - Some(if idx == 0 { VPath::root() } else { VPath(self.0[..idx].to_owned()) }) + Some(if idx == 0 { VPath::root() } else { VPath(self.0[..idx].into()) }) } /// The last component, or `None` for the root. @@ -78,7 +82,7 @@ impl VPath { if name.is_empty() || name == "." || name == ".." || name.contains('/') { return Err(Error::InvalidPath(format!("{}/{}", self.0, name))); } - Ok(if self.is_root() { VPath(format!("/{name}")) } else { VPath(format!("{}/{name}", self.0)) }) + Ok(if self.is_root() { VPath(format!("/{name}").into()) } else { VPath(format!("{}/{name}", self.0).into()) }) } /// The components after the root, in order. @@ -95,7 +99,7 @@ impl VPath { pub fn starts_with(&self, prefix: &VPath) -> bool { prefix.is_root() || self.0 == prefix.0 - || (self.0.starts_with(&prefix.0) && self.0.as_bytes().get(prefix.0.len()) == Some(&b'/')) + || (self.0.starts_with(&*prefix.0) && self.0.as_bytes().get(prefix.0.len()) == Some(&b'/')) } /// The components of `self` below `prefix`, if `self` starts with it. @@ -121,7 +125,7 @@ impl VPath { /// Every proper ancestor from the root down, as slices of this path, so walking /// them allocates nothing. pub(crate) fn ancestor_strs(&self) -> impl Iterator { - let path = self.0.as_str(); + let path = &*self.0; let root = (!self.is_root()).then_some("/"); // Each '/' after the leading one ends an ancestor. root.into_iter().chain(path.match_indices('/').skip(1).map(move |(idx, _)| &path[..idx])) @@ -131,15 +135,15 @@ impl VPath { pub(crate) fn descendants_end(&self) -> VPath { if self.is_root() { // '0' is the byte after '/', so it bounds every absolute path. - VPath("0".to_owned()) + VPath("0".into()) } else { - VPath(format!("{}0", self.0)) + VPath(format!("{}0", self.0).into()) } } /// The smallest string that sorts before every descendant of `self`. pub(crate) fn descendants_start(&self) -> VPath { - VPath(if self.is_root() { "/".to_owned() } else { format!("{}/", self.0) }) + VPath(if self.is_root() { "/".into() } else { format!("{}/", self.0).into() }) } } diff --git a/crates/sandfs-core/src/source.rs b/crates/sandfs-core/src/source.rs index 68661d1..3f66d68 100644 --- a/crates/sandfs-core/src/source.rs +++ b/crates/sandfs-core/src/source.rs @@ -140,6 +140,11 @@ pub trait Source: Send + Sync + fmt::Debug { let _ = key; Ok(None) } + + /// Let go of what the source keeps only to answer faster, such as helper processes + /// sitting idle, when a workspace over it is trimmed ([`crate::Workspace::trim`]). + /// Every answer stays the same. The default keeps nothing, so does nothing. + fn trim(&self) {} } /// The part of `bytes` a ranged read asks for, clamped to its end. diff --git a/crates/sandfs-core/src/workspace.rs b/crates/sandfs-core/src/workspace.rs index f295cd8..2b404e4 100644 --- a/crates/sandfs-core/src/workspace.rs +++ b/crates/sandfs-core/src/workspace.rs @@ -49,16 +49,128 @@ type Slot = Mutex>; /// A `(mount id, source key)` pair. type CacheKey = (u64, SourceKey); +/// The bytes the cache keeps of each kind, at most (about): what it holds can always +/// be fetched again, so past these the least used goes. Listings are what a walk of a +/// large folder fills; a file's bytes are kept only when read whole through the API, +/// since a mount reads in ranges and the system caches what it read. +const LISTING_BYTES: usize = 32 << 20; +const SIZED_BYTES: usize = 8 << 20; +const ENTRY_BYTES: usize = 8 << 20; +const BLOB_BYTES: usize = 16 << 20; +/// What a cached entry costs besides its strings: the map's room for it, its own +/// fields and the allocations' bookkeeping. +const ENTRY_COST: usize = 96; +/// Directories noted for prefetching and for reads, at most, before the notes start +/// over: they only steer prefetching. +const NOTED_DIRS: usize = 1 << 16; + +/// Entries that can be fetched again, kept to about `budget` bytes in two halves. New +/// entries go in the young half; once it holds half the budget it becomes the old half, +/// and the old half goes. An entry used from the old half moves back to the young one, +/// so what is in use stays and the rest goes within two turns, at no cost per lookup. +struct Halves { + young: HashMap, + old: HashMap, + /// The bytes the young half holds. + bytes: usize, + budget: usize, + /// A half that just went, for the caller to drop once it lets go of the lock. + gone: Option>, +} + +impl Halves { + fn new(budget: usize) -> Self { + Halves { young: HashMap::new(), old: HashMap::new(), bytes: 0, budget, gone: None } + } + + fn get(&mut self, key: &Q) -> Option + where + K: std::borrow::Borrow, + { + if let Some((value, _)) = self.young.get(key) { + return Some(value.clone()); + } + let (key, (value, bytes)) = self.old.remove_entry(key)?; + self.insert(key, value.clone(), bytes); + Some(value) + } + + fn insert(&mut self, key: K, value: V, bytes: usize) { + if self.bytes + bytes > self.budget / 2 && !self.young.is_empty() { + self.turn(); + } + self.bytes += bytes; + if let Some((_, before)) = self.young.insert(key, (value, bytes)) { + self.bytes -= before; + } + } + + /// Count `bytes` more for the young entry `key`, once loaded. + fn charge(&mut self, key: &K, bytes: usize) { + if let Some((_, held)) = self.young.get_mut(key) { + *held += bytes; + self.bytes += bytes; + if self.bytes > self.budget / 2 { + self.turn(); + } + } + } + + fn remove(&mut self, key: &Q) + where + K: std::borrow::Borrow, + { + if let Some((_, bytes)) = self.young.remove(key) { + self.bytes -= bytes; + } + self.old.remove(key); + } + + /// The young half becomes the old one. + fn turn(&mut self) { + let old = std::mem::replace(&mut self.old, std::mem::take(&mut self.young)); + self.gone = Some(old); + self.bytes = 0; + } + + /// Everything, taken out to be dropped once the lock is let go. + fn clear(&mut self) -> [HashMap; 2] { + self.bytes = 0; + self.gone = None; + [std::mem::take(&mut self.young), std::mem::take(&mut self.old)] + } +} + +/// Run `op` on the map behind `lock`, dropping what it let go after the lock is let go, +/// so other threads don't wait for that. +fn with(lock: &Mutex>, op: impl FnOnce(&mut Halves) -> T) -> T { + let (out, gone) = { + let mut map = lock.lock().expect("cache lock"); + let out = op(&mut map); + (out, map.gone.take()) + }; + drop(gone); + out +} + +/// What a listing costs to keep. +fn listing_bytes(listing: &Listing) -> usize { + listing.iter().map(|(name, entry)| name.len() + entry.key.len() + ENTRY_COST).sum::() + ENTRY_COST +} + +type Slots = Mutex>>>; + /// Listings and blobs fetched from sources; keys are immutable, so forks share it. -#[derive(Default)] +/// Everything here can be fetched again, so each kind is kept to a budget, and +/// [`Workspace::trim`] drops it all. struct Cache { - dirs: Mutex>>>>, - blobs: Mutex>>>, + dirs: Slots>, + blobs: Slots, /// Directories already found, by mount id and path below the mount, so resolving a /// path costs a lookup or two instead of one listing lookup per component. Only /// entries that exist are kept, so probing for missing names costs no memory. /// A path that is its entry's key, as in a folder, shares the key's bytes. - entries: Mutex>>, + entries: Mutex>>, /// Directories already prefetched, and whether recursively, so each is asked once. prefetched: Mutex>, /// Base files each directory needed fetched in the current look: sizes missing @@ -66,46 +178,95 @@ struct Cache { /// so a file read in pieces counts once. touched: Mutex>>, /// Directories listed again for their files' sizes, at most once each. - sized: Mutex>>>>, + sized: Slots>, +} + +impl Default for Cache { + fn default() -> Self { + Cache { + dirs: Mutex::new(Halves::new(LISTING_BYTES)), + blobs: Mutex::new(Halves::new(BLOB_BYTES)), + entries: Mutex::default(), + prefetched: Mutex::default(), + touched: Mutex::default(), + sized: Mutex::new(Halves::new(SIZED_BYTES)), + } + } +} + +/// What a cached value costs to keep. +trait Weigh { + fn bytes(&self) -> usize; +} + +impl Weigh for Arc { + fn bytes(&self) -> usize { + listing_bytes(self) + } +} + +impl Weigh for Bytes { + fn bytes(&self) -> usize { + self.len() + ENTRY_COST + } } impl Cache { /// The cached value for `key`, loading it with `fetch` on a miss. Concurrent /// misses on one key fetch once. A failed fetch caches nothing, so it is retried. - fn get_or_load( - map: &Mutex>>>, - key: CacheKey, - fetch: impl FnOnce() -> Result, - ) -> Result { + fn get_or_load(map: &Slots, key: CacheKey, fetch: impl FnOnce() -> Result) -> Result { // The map lock is held only to find the slot, so loads of different keys // never wait for each other. - let slot = map.lock().expect("cache lock").entry(key).or_default().clone(); + let slot = with(map, |map| { + map.get(&key).unwrap_or_else(|| { + let slot = Arc::>::default(); + map.insert(key.clone(), slot.clone(), ENTRY_COST + key.1.len()); + slot + }) + }); let mut value = slot.lock().expect("cache slot lock"); if let Some(value) = value.as_ref() { return Ok(value.clone()); } let fetched = fetch()?; *value = Some(fetched.clone()); + drop(value); + with(map, |map| map.charge(&key, fetched.bytes())); Ok(fetched) } fn entry(&self, mount: u64, rel: &str) -> Option { - self.entries.lock().expect("cache lock").get(&mount)?.get(rel).cloned() + let (found, gone) = { + let mut entries = self.entries.lock().expect("cache lock"); + let found = entries.get_mut(&mount)?; + (found.get(rel), found.gone.take()) + }; + drop(gone); + found } fn remember_entry(&self, mount: u64, rel: &str, entry: SourceEntry) { - let rel = if *entry.key == *rel { entry.key.clone() } else { rel.into() }; - self.entries.lock().expect("cache lock").entry(mount).or_default().insert(rel, entry); + let shared = *entry.key == *rel; + let bytes = ENTRY_COST + entry.key.len() + if shared { 0 } else { rel.len() }; + let rel = if shared { entry.key.clone() } else { rel.into() }; + let gone = { + let mut entries = self.entries.lock().expect("cache lock"); + let found = entries.entry(mount).or_insert_with(|| Halves::new(ENTRY_BYTES)); + found.insert(rel, entry, bytes); + found.gone.take() + }; + drop(gone); } /// Replace a cached listing with a newer answer from the same source. fn replace_listing(&self, key: CacheKey, listing: Arc) { - self.dirs.lock().expect("cache lock").insert(key, Arc::new(Mutex::new(Some(listing)))); + let bytes = listing.bytes(); + with(&self.dirs, |dirs| dirs.insert(key, Arc::new(Mutex::new(Some(listing))), bytes)); } /// Drop the cached blob for `key`, whose file changed in its source. fn forget_blob(&self, key: &CacheKey) { - self.blobs.lock().expect("cache lock").remove(key); + with(&self.blobs, |blobs| blobs.remove(key)); } /// Put `entry` in the cached listing `key` under `name`, in place of the entry with @@ -113,7 +274,7 @@ impl Cache { /// held throughout, so two files of one directory changed at once both stay changed. /// Files aren't in the path-entry cache, so the listing is the one place to change. fn patch_listing(&self, key: &CacheKey, name: &str, entry: SourceEntry) -> bool { - let Some(slot) = self.dirs.lock().expect("cache lock").get(key).cloned() else { return false }; + let Some(slot) = with(&self.dirs, |dirs| dirs.get(key)) else { return false }; let mut listing = slot.lock().expect("cache slot lock"); let Some(current) = listing.as_ref() else { return false }; if current.get(name).is_none_or(|listed| listed.key != entry.key || *listed == entry) { @@ -127,9 +288,29 @@ impl Cache { /// The cached blob for `key`, if it is loaded and not being loaded right now. fn peek_blob(&self, key: &CacheKey) -> Option { - let slot = self.blobs.lock().expect("cache lock").get(key)?.clone(); + let slot = with(&self.blobs, |blobs| blobs.get(key))?; slot.try_lock().ok()?.clone() } + + /// Note `dir` as prefetched, and whether recursively; false if it was already. + fn note_prefetched(&self, dir: CacheKey, recursive: bool) -> bool { + let mut prefetched = self.prefetched.lock().expect("cache lock"); + if prefetched.len() >= NOTED_DIRS { + prefetched.clear(); + } + prefetched.insert((dir, recursive)) + } + + /// Drop everything: it is all fetched again when next needed. + fn clear(&self) { + let dirs = self.dirs.lock().expect("cache lock").clear(); + let blobs = self.blobs.lock().expect("cache lock").clear(); + let sized = self.sized.lock().expect("cache lock").clear(); + let entries = std::mem::take(&mut *self.entries.lock().expect("cache lock")); + let prefetched = std::mem::take(&mut *self.prefetched.lock().expect("cache lock")); + let touched = std::mem::take(&mut *self.touched.lock().expect("cache lock")); + drop((dirs, blobs, sized, entries, prefetched, touched)); + } } /// File content: bytes written into the overlay, a reference to a base file that @@ -482,6 +663,20 @@ impl Workspace { Ok(changed) } + /// Let go of everything this workspace and its forks keep that can be fetched + /// again: listings, file bytes, the directories found and what steers prefetching. + /// Nothing they show changes; the next look fetches what it needs again. The cache + /// keeps to a budget on its own (tens of megabytes), so this is for giving back + /// even that, such as when the workspace goes unused for a while. + pub fn trim(&self) { + self.cache.clear(); + for mount in self.mounts.values() { + mount.source.trim(); + } + let walks = std::mem::take(&mut *self.walks.lock().expect("walks lock")); + drop(walks); + } + /// [`Metadata::modified`] alone: the source's time for a file nobody wrote, found /// without reading the file even when the source left its size out. pub fn modified(&self, path: &str) -> Result> { @@ -1089,6 +1284,11 @@ impl Workspace { return; } let mut walks = self.walks.lock().expect("walks lock"); + // Only the latest walks matter: past this many directories, start over. + if walks.listed.len() >= NOTED_DIRS { + walks.listed.clear(); + walks.starts.clear(); + } walks.listed.insert(path.clone()); let Some(parent) = path.parent().filter(|parent| walks.listed.contains(parent)) else { return }; let start = walks.starts.entry(parent.clone()).or_insert(parent).clone(); @@ -1143,6 +1343,9 @@ impl Workspace { let fresh = read && !walking && !steady; let (count, first) = { let mut touched = self.cache.touched.lock().expect("cache lock"); + if touched.len() >= NOTED_DIRS && !touched.contains_key(&(mount.id, dir.clone())) { + touched.clear(); + } let files = touched.entry((mount.id, dir.clone())).or_default(); if fresh { files.clear(); @@ -1199,7 +1402,7 @@ impl Workspace { }; let Some(below) = below else { return Ok(steady) }; if let Some(key) = self.base_dir(&below, mount)? { - if self.cache.prefetched.lock().expect("cache lock").insert(((mount.id, key.clone()), true)) { + if self.cache.note_prefetched((mount.id, key.clone()), true) { mount.source.prefetch(&key, true); } } @@ -1210,7 +1413,7 @@ impl Workspace { /// It is asked even when everything below a parent was: a source may still be /// fetching a large tree, and this asks for these files first. fn prefetch_dir(&self, mount: &Arc, dir: &SourceKey) { - if self.cache.prefetched.lock().expect("cache lock").insert(((mount.id, dir.clone()), false)) { + if self.cache.note_prefetched((mount.id, dir.clone()), false) { mount.source.prefetch(dir, false); } } diff --git a/crates/sandfs-core/tests/workspace.rs b/crates/sandfs-core/tests/workspace.rs index 501b624..9cd3fa7 100644 --- a/crates/sandfs-core/tests/workspace.rs +++ b/crates/sandfs-core/tests/workspace.rs @@ -1327,3 +1327,39 @@ fn a_read_only_path_that_does_not_exist_yet_cannot_be_covered_by_a_file() { refused(ws.hard_link("/books/budget.csv", "/books/future")); ws.mkdir("/books/future", false).unwrap(); } + +#[test] +fn trimming_fetches_again_what_is_next_needed_and_shows_the_same() { + let (ws, source) = workspace(); + assert_eq!(ws.read_text("/repo/src/lib.rs").unwrap(), "pub fn one() -> u32 { 1 }\n"); + let read = source.reads(); + ws.read_text("/repo/src/lib.rs").unwrap(); + assert_eq!(source.reads(), read, "the cache answers"); + + let mut fork = ws.fork(); + fork.write("/repo/src/new.rs", "new").unwrap(); + ws.trim(); + assert_eq!(fork.read_text("/repo/src/lib.rs").unwrap(), "pub fn one() -> u32 { 1 }\n"); + assert!(source.reads().0 > read.0 && source.reads().1 > read.1, "forks share the cache, and it was let go"); + assert_eq!(names(&fork, "/repo/src"), ["lib.rs", "new.rs", "util"]); + assert_eq!(fork.read_text("/repo/src/new.rs").unwrap(), "new", "what was written stays"); +} + +#[test] +fn listings_are_kept_to_a_budget() { + // Far more listings than the cache keeps: each directory lists 200 long names. + let files = + (0..400).flat_map(|dir| (0..200).map(move |file| (format!("d{dir:03}/{file:03}-{}", "x".repeat(200)), ""))); + let source = Counting::new(MemorySource::from_files(files).unwrap()); + let mut ws = Workspace::new(); + ws.mount("/repo", source.clone()).unwrap(); + for dir in 0..400 { + assert_eq!(ws.list(&format!("/repo/d{dir:03}")).unwrap().len(), 200); + } + let listed = source.reads().0; + // The latest stay; the first ones went and are listed again. + ws.list("/repo/d399").unwrap(); + assert_eq!(source.reads().0, listed); + ws.list("/repo/d000").unwrap(); + assert_eq!(source.reads().0, listed + 1); +} diff --git a/crates/sandfs-git/src/lib.rs b/crates/sandfs-git/src/lib.rs index 9726f08..81530cf 100644 --- a/crates/sandfs-git/src/lib.rs +++ b/crates/sandfs-git/src/lib.rs @@ -548,6 +548,14 @@ impl Source for GitSource { } } } + + /// The `cat-file` processes not reading right now exit: each keeps a cache of its + /// own, and the next read starts one again. + fn trim(&self) { + self.readers.trim(); + let sizes = self.sizes.as_ref().and_then(|sizes| sizes.lock().ok().and_then(|mut sizes| sizes.take())); + drop(sizes); + } } /// See [`GitSource::missing`]. @@ -920,6 +928,21 @@ impl Readers { read } + /// End the processes not reading right now. + fn trim(&self) { + let idle = match self.idle.lock() { + Ok(mut idle) => { + let batches = std::mem::take(&mut idle.0); + idle.1 -= batches.len(); + batches + } + Err(_) => return, + }; + self.returned.notify_all(); + // Dropping one waits for it to exit, so outside the lock. + drop(idle); + } + /// An idle process; else a new one, if there is room; else the next to come free. fn take(&self) -> Result { let mut idle = self.idle.lock().map_err(poisoned)?; diff --git a/crates/sandfs/src/fs.rs b/crates/sandfs/src/fs.rs index 5fcea62..5cefc77 100644 --- a/crates/sandfs/src/fs.rs +++ b/crates/sandfs/src/fs.rs @@ -74,6 +74,13 @@ pub const MAX_FILE_SIZE: usize = i64::MAX as usize; const COPY_PIECE: u64 = 8 << 20; /// The largest file kept in memory while it is written, where there are no drafts. const MAX_MEMORY_FILE: u64 = 4 << 30; +/// Milliseconds since this process first asked, from 1, for times kept in atomics: 0 +/// stands for never. +fn since_start() -> u64 { + static START: OnceLock = OnceLock::new(); + START.get_or_init(Instant::now).elapsed().as_millis() as u64 + 1 +} + /// How many whole files, and bytes of them, [`Inner::reading`] keeps for sources that /// can't read a range on its own. The kernel asks for a file in chunks and caches /// what it gets, so only the few files being read right now are worth keeping. @@ -322,8 +329,9 @@ pub enum Stale { pub struct Inner { ws: Workspace, nodes: HashMap, - /// Path to inode, ordered so a renamed directory's entries are one range. - inos: BTreeMap, + /// Path to inode, ordered so a renamed directory's entries are one range. A path + /// here shares its bytes with its node's. + inos: BTreeMap, next_ino: u64, handles: HashMap, next_fh: u64, @@ -432,7 +440,7 @@ impl Inner { moved_off: Vec::new(), skip_bookkeeping: false, }; - inner.inos.insert(root.to_string(), ROOT_INO); + inner.inos.insert(root.clone(), ROOT_INO); inner.nodes.insert( ROOT_INO, Node { path: root.clone(), lookups: 0, linked: true, sized: Cell::new(false), removed_from: None }, @@ -440,6 +448,19 @@ impl Inner { inner } + /// Let go of what can be had again: the cache of listings and file bytes, shared + /// with the workspace's forks (see [`Workspace::trim`]), files read lately and the + /// room tables kept after growing. Writes not yet saved stay. + pub fn trim(&mut self) { + self.ws.trim(); + let recent = std::mem::take(&mut *self.recent.borrow_mut()); + drop(recent); + self.nodes.shrink_to_fit(); + self.handles.shrink_to_fit(); + self.open.shrink_to_fit(); + self.buffers.shrink_to_fit(); + } + /// A copy of what a save needs; the workspace part is an O(1) fork. Files written /// within `settle` stay out (see [`Inner::commit_written`]). pub fn snapshot(&mut self, settle: Duration) -> Snapshot { @@ -486,7 +507,7 @@ impl Inner { // with everything either version lists in each such folder. let mut paths = changed.cloned().unwrap_or_default(); let in_whole = |path: &VPath| changed.is_none() || whole.iter().any(|folder| path.starts_with(folder)); - let known: Vec = self.inos.keys().filter_map(|path| VPath::parse(path).ok()).collect(); + let known: Vec = self.inos.keys().cloned().collect(); for path in known.iter().filter(|path| in_whole(path)) { paths.insert(path.to_string()); for ws in [&old, &self.ws] { @@ -527,7 +548,6 @@ impl Inner { stale.push(Stale::Inode(parent)); self.times.extend(name.parent().map(|parent| (parent.to_string(), now))); for gone in subtree_keys(&self.inos, &name) { - let Ok(gone) = VPath::parse(&gone) else { continue }; let Some(ino) = self.inos.get(gone.as_str()).copied() else { continue }; // A name the newer workspace still links to it keeps it. let others = self.other_names(&gone); @@ -555,9 +575,9 @@ impl Inner { // Another name of an inode that went, which the newer workspace no longer links // to it, is looked up anew rather than reaching the removed inode. let unlinked = |ino: &u64| moved.contains(ino) && self.nodes.get(ino).is_none_or(|node| !node.linked); - let dangling: Vec = + let dangling: Vec = self.inos.iter().filter(|(_, ino)| unlinked(ino)).map(|(path, _)| path.clone()).collect(); - for name in dangling.iter().filter_map(|name| VPath::parse(name).ok()) { + for name in dangling { self.inos.remove(name.as_str()); let parent = name.parent().and_then(|parent| self.inos.get(parent.as_str()).copied()); if let Some(parent) = parent { @@ -666,12 +686,12 @@ impl Inner { // Every name of a hard-linked file is one inode, so the kernel caches its // pages once and a write through one name is read through the others. if let Some(ino) = self.linked_ino(path) { - self.inos.insert(path.to_string(), ino); + self.inos.insert(path.clone(), ino); return ino; } let ino = self.next_ino; self.next_ino += 1; - self.inos.insert(path.to_string(), ino); + self.inos.insert(path.clone(), ino); self.nodes.insert( ino, Node { path: path.clone(), lookups: 0, linked: true, sized: Cell::new(false), removed_from: None }, @@ -887,7 +907,7 @@ impl Inner { { node.path = other.clone(); } - self.inos.entry(other.to_string()).or_insert(ino); + self.inos.entry(other.clone()).or_insert(ino); return; } if let Some(node) = self.nodes.get_mut(&ino) { @@ -1009,7 +1029,7 @@ impl Inner { let from = self.path(ino)?; let to = self.child(new_parent, new_name)?; self.ws.hard_link(from.as_str(), to.as_str()).fs()?; - self.inos.insert(to.to_string(), ino); + self.inos.insert(to.clone(), ino); // The file keeps its time; only the directory it gained a name in changes. self.keep_source_time(&from); match self.times.get(from.as_str()).copied() { @@ -1137,11 +1157,11 @@ impl Inner { let moved = |map_keys: Vec| -> Vec<(String, String)> { map_keys.into_iter().map(|key| (key.clone(), format!("{to}{}", &key[from.as_str().len()..]))).collect() }; - let ino_keys = moved(subtree_keys(&self.inos, from)); - for (old, new) in ino_keys { + for old in subtree_keys(&self.inos, from) { + let Ok(new) = VPath::parse(&format!("{to}{}", &old.as_str()[from.as_str().len()..])) else { continue }; if let Some(ino) = self.inos.remove(&old) { - if let (Some(node), Ok(path)) = (self.nodes.get_mut(&ino), VPath::parse(&new)) { - node.path = path; + if let Some(node) = self.nodes.get_mut(&ino) { + node.path = new.clone(); } self.inos.insert(new, ino); } @@ -1624,6 +1644,10 @@ pub struct Shared { /// Whether a background save of this mount is waiting in [`Savers`]. scheduled: AtomicBool, stop: AtomicBool, + /// When the mount was last asked anything, and when it was last trimmed, as + /// [`since_start`] gives them. + used: AtomicU64, + trimmed: AtomicU64, /// Whether the kernel took no-open mode, so opens and flushes can be refused. #[cfg(fuse)] no_open: AtomicBool, @@ -1644,6 +1668,8 @@ impl Shared { me: me.clone(), scheduled: AtomicBool::new(false), stop: AtomicBool::new(false), + used: AtomicU64::new(since_start()), + trimmed: AtomicU64::new(0), #[cfg(fuse)] no_open: AtomicBool::new(false), #[cfg(fuse)] @@ -1654,9 +1680,26 @@ impl Shared { } pub fn lock(&self) -> MutexGuard<'_, Inner> { + self.used.store(since_start(), Ordering::Relaxed); self.inner.lock().unwrap_or_else(PoisonError::into_inner) } + /// Save every change, then let go of what the mount keeps that it can get again + /// (see [`Inner::trim`]). What it serves stays the same. + pub fn trim(&self) -> anyhow::Result<()> { + self.save(false)?; + self.inner.lock().unwrap_or_else(PoisonError::into_inner).trim(); + self.trimmed.store(since_start(), Ordering::Relaxed); + Ok(()) + } + + /// Whether the mount has gone unasked for `idle` since it was last used, and wasn't + /// trimmed since. + pub fn idle_untrimmed(&self, idle: Duration) -> bool { + let used = self.used.load(Ordering::Relaxed); + used > self.trimmed.load(Ordering::Relaxed) && since_start().saturating_sub(used) >= idle.as_millis() as u64 + } + /// Save the workspace's changes now, including every write; `durable` also /// flushes them to disk, as `fsync` promises. pub fn save(&self, durable: bool) -> anyhow::Result<()> { @@ -2718,6 +2761,34 @@ mod tests { assert_eq!(restored.read_text("/mnt/new.md").unwrap(), "NEW\n"); } + #[test] + fn a_trim_saves_writes_and_keeps_what_the_mount_serves() { + let state = tempfile::tempdir().unwrap(); + let saved = Saved::new(state.path()); + let mut inner = fs(); + inner.drafts = Some(saved.drafts()); + let shared = Shared::new(inner, saved, VPath::parse("/mnt").unwrap()); + let readme = shared.lock().lookup(ROOT_INO, name("README.md")).unwrap().ino.0; + let src = shared.lock().lookup(ROOT_INO, name("src")).unwrap().ino.0; + shared.lock().write(readme, 0, b"HE").unwrap(); + assert!(shared.idle_untrimmed(Duration::ZERO), "used, never trimmed"); + shared.trim().unwrap(); + assert!(!shared.idle_untrimmed(Duration::ZERO), "trimmed since last used"); + std::thread::sleep(Duration::from_millis(2)); + + // The write was saved, and the same inodes serve the same files. + let mut restored = fs().ws; + Saved::new(state.path()).load(&mut restored, &VPath::parse("/mnt").unwrap()).unwrap(); + assert_eq!(restored.read_text("/mnt/README.md").unwrap(), "HEllo\n"); + let mut inner = shared.lock(); + assert_eq!(&inner.read(readme, 0, 100).unwrap()[..], b"HEllo\n"); + assert_eq!(inner.lookup(ROOT_INO, name("src")).unwrap().ino.0, src); + let lib = inner.lookup(src, name("lib.rs")).unwrap().ino.0; + assert_eq!(&inner.read(lib, 0, 100).unwrap()[..], b"fn one() {}\n"); + drop(inner); + assert!(shared.idle_untrimmed(Duration::ZERO), "used since the trim"); + } + #[test] fn writes_are_saved_in_the_background_until_the_mount_stops() { let state = tempfile::tempdir().unwrap(); diff --git a/crates/sandfs/src/lib.rs b/crates/sandfs/src/lib.rs index e917cfa..dddd610 100644 --- a/crates/sandfs/src/lib.rs +++ b/crates/sandfs/src/lib.rs @@ -83,6 +83,8 @@ mod gitfile; mod github; mod host; mod mcp; +#[cfg(mount)] +mod memory; #[cfg(any(nfs, all(test, fuse)))] mod nfs; #[cfg(any(nfs, all(test, fuse)))] @@ -196,6 +198,16 @@ impl Home { ops::unmount_workspace(&self.inner, workspace) } + /// Have the mount of `workspace` (every mount of the home, when `None`) give back + /// the memory it can do without: its changes are saved, then the folder listings + /// and file bytes it keeps and idle git helpers go, and the memory freed goes back + /// to the system. Nothing it serves changes; what it needs next is read again. A + /// mount left unused for a minute is trimmed on its own, so this is for giving + /// memory back now, such as when an agent's task goes idle. + pub fn trim(&self, workspace: Option<&str>) -> Result { + ops::trim(&self.inner, workspace) + } + /// Delete the workspace `name` and its changes. A mounted one is refused, unless /// `unmount` says to unmount it first; then `name` may also be its folder. pub fn remove(&self, name: &str, unmount: bool) -> Result<()> { @@ -384,6 +396,16 @@ pub struct WorkspaceInfo { pub mounted: Option, } +/// What [`Home::trim`] let go of. +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct Trimmed { + /// The mounts trimmed. + pub mounts: usize, + /// The bytes of freed memory the allocator handed back to the system, where it says + /// (macOS). + pub given_back: Option, +} + /// What [`Home::clean`] did with one local copy of a remote repository. #[derive(Debug, Clone)] pub struct Cleaned { diff --git a/crates/sandfs/src/memory.rs b/crates/sandfs/src/memory.rs new file mode 100644 index 0000000..c08cc0f --- /dev/null +++ b/crates/sandfs/src/memory.rs @@ -0,0 +1,39 @@ +//! Handing memory the process freed back to the system. +//! +//! Freeing memory gives it back to the allocator, which keeps it for the next +//! allocation instead of returning it: after a mount of a large folder ends, macOS's +//! allocator still holds hundreds of megabytes of freed pages for the process, and +//! glibc's holds some too. Only the allocator can let them go, when asked to. +//! +//! This is the one place sandfs calls code Rust can't check: each call below takes no +//! pointer of ours (a null zone means every zone) and only returns unused pages to the +//! system, so it can't touch memory in use. + +/// Ask the allocator to return the memory it holds unused, after a trim or once a mount +/// is gone. Returns the bytes given back, where the allocator says. +#[allow(unsafe_code)] +pub fn give_back() -> Option { + #[cfg(target_os = "macos")] + { + unsafe extern "C" { + /// ``: returns the free pages of `zone` (every zone when + /// null) to the system, until `goal` bytes are freed (every one when 0). + fn malloc_zone_pressure_relief(zone: *mut std::ffi::c_void, goal: usize) -> usize; + } + // SAFETY: a null zone and a goal of 0 are documented to mean every zone and + // every free page; nothing else is passed, and only unused pages are released. + let released = unsafe { malloc_zone_pressure_relief(std::ptr::null_mut(), 0) }; + Some(released) + } + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + // SAFETY: `malloc_trim` takes no pointer; 0 keeps no padding at the heap's top. + unsafe { + libc::malloc_trim(0); + } + None + } + // musl's allocator returns freed memory on its own. + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))] + None +} diff --git a/crates/sandfs/src/nfs.rs b/crates/sandfs/src/nfs.rs index 5456a45..adad934 100644 --- a/crates/sandfs/src/nfs.rs +++ b/crates/sandfs/src/nfs.rs @@ -1474,4 +1474,55 @@ mod tests { assert!(options.split(',').any(|option| option == "sync"), "{options}"); assert!(options.split(',').any(|option| option == "nobrowse"), "{options}"); } + + /// What a walk of a big folder costs the server, the way macOS's client walks one: + /// every folder listed with attributes, every name looked up and its attributes + /// asked for, and nothing ever forgotten. Run under a heap profiler, such as + /// `valgrind --tool=massif`, with SANDFS_WALK_FILES set to the number of files. + #[test] + #[ignore] + fn a_walk_of_a_big_folder() { + let files: usize = std::env::var("SANDFS_WALK_FILES").ok().and_then(|n| n.parse().ok()).unwrap_or(20_000); + let top = + tempfile::tempdir_in(std::env::var_os("SANDFS_WALK_IN").unwrap_or(std::env::temp_dir().into())).unwrap(); + for i in 0..files { + let dir = top.path().join(format!("pkg_{:02}/module_{:03}", i / 10_000, (i / 100) % 100)); + if i % 100 == 0 { + std::fs::create_dir_all(&dir).unwrap(); + } + std::fs::write(dir.join(format!("source_file_number_{i}.rs")), "").unwrap(); + } + let (nfs, _saved) = serve(Arc::new(crate::gitfile::FolderSource::new(top.path()).unwrap())); + let walk = || { + block(async { + let mut dirs = vec![ROOT_INO]; + while let Some(dir) = dirs.pop() { + let mut after = 0; + loop { + let page = nfs.readdir(dir, after, 512).await.unwrap(); + for entry in &page.entries { + let id = nfs.lookup(dir, &entry.name).await.unwrap(); + let attr = nfs.getattr(id).await.unwrap(); + if matches!(attr.ftype, ftype3::NF3DIR) { + dirs.push(id); + } + after = entry.cookie; + } + if page.end { + break; + } + } + } + }) + }; + walk(); + eprintln!("walked {files} files, {} inodes known", nfs.shared.lock().node_count()); + // A trim lets go of what can be read again, and the next walk reads it. + nfs.shared.trim().unwrap(); + crate::memory::give_back(); + eprintln!("trimmed"); + walk(); + nfs.shared.trim().unwrap(); + eprintln!("walked again and trimmed"); + } } diff --git a/crates/sandfs/src/ops.rs b/crates/sandfs/src/ops.rs index c89e468..c317901 100644 --- a/crates/sandfs/src/ops.rs +++ b/crates/sandfs/src/ops.rs @@ -257,6 +257,17 @@ pub(crate) fn unmount_workspace(home: &Home, workspace: &str) -> Result<()> { unmount(home, &name, &info) } +pub(crate) fn trim(home: &Home, workspace: Option<&str>) -> Result { + let name = workspace.map(|workspace| home.find(workspace)).transpose()?; + #[cfg(mount)] + return server::trim(home, name.as_deref().unwrap_or_default()); + #[cfg(not(mount))] + { + let _ = name; + Ok(crate::Trimmed::default()) + } +} + pub(crate) fn remove(home: &Home, name: &str, first: bool) -> Result<()> { let name = if first { home.find(name)? } else { name.to_owned() }; if let Some(info) = home.attached_mount(&name) { diff --git a/crates/sandfs/src/server.rs b/crates/sandfs/src/server.rs index b4267f4..738821e 100644 --- a/crates/sandfs/src/server.rs +++ b/crates/sandfs/src/server.rs @@ -4,7 +4,9 @@ //! owner can reach, and start it first when none is running. Each mount runs on //! threads of its own, and workspaces over the same commit share it (see [`Bases`]). //! The process exits once its last mount is gone, so nothing stays running. -//! `update` asks it to move a workspace it serves onto a newer source in place. +//! `update` asks it to move a workspace it serves onto a newer source in place, and +//! `trim` to let go of memory its mounts can do without; a mount left unused for a +//! minute is trimmed anyway. use std::collections::HashMap; use std::fs::File; @@ -22,6 +24,7 @@ use nix::sys::signal::kill; use nix::unistd::{Pid, setsid}; use serde::{Deserialize, Serialize}; +use crate::Trimmed; use crate::fs::{Replacement, Shared}; use crate::store::{Base, Bases, Home, Lease, ROOT, Record}; use crate::update::{self, Updated}; @@ -32,6 +35,8 @@ const LINGER: Duration = Duration::from_secs(2); const START: Duration = Duration::from_secs(10); /// How long a mount may take, which includes fetching a remote commit's folders. const MOUNT: Duration = Duration::from_secs(120); +/// How long a mount goes unused before it is trimmed (see [`trim`]). +const IDLE: Duration = Duration::from_secs(60); /// The memory maps kept free when taking another mount, for what mounts already served /// need: threads that answer requests, large reads (each read of a large file maps /// memory of its own) and git helpers. A mount takes about 10. @@ -39,14 +44,17 @@ const MOUNT: Duration = Duration::from_secs(120); const MAPS_KEPT: usize = 1024; /// A request to mount the workspace `name` at `dir`, which exists and is canonical, -/// or to move the mounted workspace `name` onto a newer source. The directory goes as -/// its bytes, since a path needn't be UTF-8. +/// to move the mounted workspace `name` onto a newer source, or to trim the mount of +/// `name` (every mount when it is empty). The directory goes as its bytes, since a +/// path needn't be UTF-8. #[derive(Serialize, Deserialize)] struct Request { name: String, dir: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] update: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + trim: bool, } /// The source to move a workspace onto, found by the command that asked, and whether @@ -63,12 +71,15 @@ impl Request { } } -/// The answer, once the mount is up or has failed, or the workspace has moved. +/// The answer, once the mount is up or has failed, the workspace has moved, or the +/// mounts are trimmed. #[derive(Serialize, Deserialize)] struct Reply { error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] updated: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + trimmed: Option, } /// Where the background process writes what it reports. @@ -93,7 +104,7 @@ fn run_dir(home: &Home) -> Result { /// none is running, and wait until the mount is up. pub fn mount(home: &Home, name: &str, dir: &Path) -> Result<()> { let dir = dir.as_os_str().as_bytes().to_vec(); - let request = serde_json::to_string(&Request { name: name.to_owned(), dir, update: None })?; + let request = serde_json::to_string(&Request { name: name.to_owned(), dir, update: None, trim: false })?; let started = Instant::now(); let mut server: Option = None; // A running process answers at once, and a new one within a few milliseconds. @@ -122,7 +133,8 @@ pub fn mount(home: &Home, name: &str, dir: &Path) -> Result<()> { /// names it from now on if `given`. pub fn update(home: &Home, name: &str, next: &Record, given: bool) -> Result { let update = Move { next: next.clone(), given }; - let request = serde_json::to_string(&Request { name: name.to_owned(), dir: Vec::new(), update: Some(update) })?; + let request = + serde_json::to_string(&Request { name: name.to_owned(), dir: Vec::new(), update: Some(update), trim: false })?; let unserved = || anyhow!("{name} is mounted by a process that can't move it; unmount it first (sandfs unmount {name})"); let stream = UnixStream::connect(socket_path(home)).map_err(|_| unserved())?; @@ -134,6 +146,63 @@ pub fn update(home: &Home, name: &str, next: &Record, given: bool) -> Result Result { + let request = serde_json::to_string(&Request { name: name.to_owned(), dir: Vec::new(), update: None, trim: true })?; + let unserved = || anyhow!("{name} isn't mounted by this home's background process"); + let Ok(stream) = UnixStream::connect(socket_path(home)) else { + return if name.is_empty() { Ok(Trimmed::default()) } else { Err(unserved()) }; + }; + match ask(stream, &request)? { + Some(Reply { error: Some(err), .. }) => Err(anyhow!(err)), + Some(Reply { trimmed: Some(trimmed), .. }) => Ok(trimmed), + Some(_) => Err(anyhow!("the background process can't trim; unmount and mount again to update it")), + None if name.is_empty() => Ok(Trimmed::default()), + None => Err(unserved()), + } +} + +/// Trim the mount of `name`, or every mount this process serves when it is empty. +fn trim_served(name: &str) -> Result { + let served: Vec> = served_list() + .iter() + .filter(|(served, _)| name.is_empty() || *served == name) + .filter_map(|(_, served)| served.upgrade()) + .collect(); + if !name.is_empty() && served.is_empty() { + bail!("{name} isn't mounted by this home's background process"); + } + let mut failed = None; + for mount in &served { + if let Err(err) = mount.shared.trim() { + failed.get_or_insert(err); + } + } + let given_back = crate::memory::give_back().map(|bytes| bytes as u64); + match failed { + Some(err) => Err(err.context("saving before trimming")), + None => Ok(Trimmed { mounts: served.len(), given_back }), + } +} + +/// Trim the mounts no one has asked anything for `idle`, once each until used again. +fn trim_idle(idle: Duration) { + let idle: Vec> = + served_list().values().filter_map(Weak::upgrade).filter(|served| served.shared.idle_untrimmed(idle)).collect(); + if idle.is_empty() { + return; + } + for mount in &idle { + if let Err(err) = mount.shared.trim() { + eprintln!("sandfs: trimming an unused mount: {err:#}"); + } + } + crate::memory::give_back(); +} + /// Send `request` and read the reply, or `None` if the connection closes first. fn ask(mut stream: UnixStream, request: &str) -> Result> { stream.set_read_timeout(Some(MOUNT))?; @@ -271,6 +340,12 @@ pub fn run(home: Home) -> Result<()> { std::thread::spawn(move || answer(&stream, &home, &bases, ticket)); } }); + std::thread::spawn(|| { + loop { + std::thread::sleep(IDLE / 4); + trim_idle(IDLE); + } + }); state.wait_until_idle(LINGER); // The socket stays: connecting to it fails once this process is gone, and the next // process replaces it. Removing it here could remove a successor's instead, when @@ -283,10 +358,20 @@ pub fn run(home: Home) -> Result<()> { /// or move the workspace and answer once it has moved. fn answer(mut stream: &UnixStream, home: &Home, bases: &Arc, ticket: Ticket) { let mut request = read_request(stream); + if let Ok(Request { name, trim: true, .. }) = &request { + let reply = match trim_served(name) { + Ok(trimmed) => Reply { error: None, updated: None, trimmed: Some(trimmed) }, + Err(err) => Reply { error: Some(format!("{err:#}")), updated: None, trimmed: None }, + }; + if let Ok(reply) = serde_json::to_string(&reply) { + let _ = stream.write_all(format!("{reply}\n").as_bytes()); + } + return; + } if let Ok(Request { name, update: Some(next), .. }) = &mut request { let reply = match move_served(home, bases, name, next) { - Ok(updated) => Reply { error: None, updated: Some(updated) }, - Err(err) => Reply { error: Some(format!("{err:#}")), updated: None }, + Ok(updated) => Reply { error: None, updated: Some(updated), trimmed: None }, + Err(err) => Reply { error: Some(format!("{err:#}")), updated: None, trimmed: None }, }; if let Ok(reply) = serde_json::to_string(&reply) { let _ = stream.write_all(format!("{reply}\n").as_bytes()); @@ -296,7 +381,7 @@ fn answer(mut stream: &UnixStream, home: &Home, bases: &Arc, ticket: Tick let dir = request.as_ref().ok().map(Request::dir); let mounted = request.and_then(|request| start_mount(home, bases, request, ticket)); let up = mounted.is_ok(); - let reply = Reply { error: mounted.err().map(|err| format!("{err:#}")), updated: None }; + let reply = Reply { error: mounted.err().map(|err| format!("{err:#}")), updated: None, trimmed: None }; let told = serde_json::to_string(&reply).is_ok_and(|reply| stream.write_all(format!("{reply}\n").as_bytes()).is_ok()); // A requester that gave up (timed out, or was interrupted) has reported a failure, @@ -343,6 +428,8 @@ fn start_mount(home: &Home, bases: &Arc, request: Request, ticket: Ticket if let Err(err) = &served { eprintln!("sandfs: {}: {err:#}", request.name); } + // What the mount took is free now: hand it back to the system. + crate::memory::give_back(); // A mount that never came up reports why; for one that did, this goes unread. let _ = done.send(served); }) @@ -520,7 +607,8 @@ mod tests { #[test] fn requests_carry_any_directory_name() { let dir = PathBuf::from(std::ffi::OsStr::from_bytes(b"/tmp/caf\xe9")); - let request = Request { name: "ws".into(), dir: dir.as_os_str().as_bytes().to_vec(), update: None }; + let request = + Request { name: "ws".into(), dir: dir.as_os_str().as_bytes().to_vec(), update: None, trim: false }; let sent: Request = serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap(); assert_eq!(sent.dir(), dir); } diff --git a/crates/sandfs/src/store.rs b/crates/sandfs/src/store.rs index d66cb06..cfceee2 100644 --- a/crates/sandfs/src/store.rs +++ b/crates/sandfs/src/store.rs @@ -992,11 +992,17 @@ pub fn check_name(name: &str) -> Result<()> { } /// The keys of `map` equal to `prefix` or below it. -pub(crate) fn subtree_keys(map: &BTreeMap, prefix: &VPath) -> Vec { +pub(crate) fn subtree_keys, T>( + map: &BTreeMap, + prefix: &VPath, +) -> Vec { + use std::ops::Bound; let p = prefix.as_str(); - let mut keys: Vec = map.contains_key(p).then(|| p.to_owned()).into_iter().collect(); + let mut keys: Vec = map.get_key_value(p).map(|(key, _)| key.clone()).into_iter().collect(); // '0' is the byte after '/', so this range is exactly the paths below `prefix`. - keys.extend(map.range(format!("{p}/")..format!("{p}0")).map(|(key, _)| key.clone())); + let (start, end) = (format!("{p}/"), format!("{p}0")); + let below = (Bound::Included(start.as_str()), Bound::Excluded(end.as_str())); + keys.extend(map.range::(below).map(|(key, _)| key.clone())); keys } diff --git a/docs/changelog.md b/docs/changelog.md index 1854c62..9640375 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,11 +7,13 @@ The main changes: - A fork of a plain folder on another disk than sandfs's home is a frozen copy on that disk, in `.sandfs/copies` at its top, instead of being refused. - A file that shrinks in a folder under a mount is never shown padded with zeros, and git in a linked worktree of a mounted or forked folder stays inside the workspace. +- A mount keeps its caches within a budget and gives memory back: after a minute unused, on `sandfs trim`, and when unmounted, freed memory goes back to the system instead of staying with the process. - A mount keeps less memory per file it is asked about, and a fork of a git clone on a Linux disk that clones files is a real folder. -API changes in the `sandfs` crate: `Changes` has a new field, `committed`, so code that builds a `Changes` itself needs it. `CrossDevice` comes up far less often, now that a folder on another disk is copied there. In `sandfs-core`, `Source` has a new method, `entry_now`, with a default, and `Workspace` has `recheck`. +API changes in the `sandfs` crate: `Changes` has a new field, `committed`, so code that builds a `Changes` itself needs it. `CrossDevice` comes up far less often, now that a folder on another disk is copied there. `Home::trim` and `Trimmed` are new. In `sandfs-core`, `Source` has two new methods with defaults, `entry_now` and `trim`, and `Workspace` has `recheck` and `trim`. - **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. It does so only where no one else can move that folder: `.sandfs` and `.sandfs/copies` are this user's and writable by no one else, at a top where others can't rename them (a sticky one, as /dev/shm is, will do). `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't use that way. +- **sandfs, sandfs-core and sandfs-git, giving memory back**: the process serving mounts grew with every file a walk asked about and never shrank: about 1 GB after a walk of a 700,000-file folder on macOS, and even after the unmount it held 337 MB, freed but kept by macOS's allocator. Now the folder listings, directories found and file bytes a workspace keeps to answer faster stay within a budget (32 MiB of listings, 16 MiB of file bytes), the least used going first, and are read again when needed. A mount nobody has used for a minute is trimmed: its changes are saved, those caches go, idle `git cat-file` readers exit, and the allocator is asked to hand the freed memory back to the system (on macOS `malloc_zone_pressure_relief`, on glibc `malloc_trim`; this one call is the only code sandfs runs that Rust can't check). `sandfs trim [NAME]` trims at once, and in the library `Home::trim` does, for a program to call when a task goes idle. The memory is handed back after every unmount too. On Linux, a walk of 50,000 files peaked at 25 MB where it peaked at 31 MB, and a trim took it to 13 MB. What stays is a record of each file the system asked about: a mount's table of files now holds each path once, not twice, and on macOS, whose NFS client never says it is done with a file, that table lasts until the unmount. In `sandfs-core` this is `Workspace::trim` and `Source::trim`, with a default that does nothing, which `GitSource` implements. - **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. - **sandfs and sandfs-core, a file that shrank in the folder under a mount**: a mount of a folder lists each file once and reads its bytes from the folder as they are. When a file got shorter in the folder after that (a copy onto it truncates it first), a read came back short of the listed size, and macOS's NFS client filled the rest with zeros: copying files from a mount onto the folder under it wrote zeros into them. Now a read that comes back short makes the mount ask the folder again for that file's size and time alone. On Linux the read is the file's end and a later `stat` gives the new size. On macOS, whose client won't take a smaller size from a read, the read fails ("Stale NFS file handle") and the file gets a new inode, which the client looks up within a second or so and then reads as the file is now. A read never gives zeros. In `sandfs-core` this is `Workspace::recheck` and a new `Source::entry_now` with a default that changes nothing, which `DirSource` implements. Errors from a folder now name the path they happened at ("source error: PATH: Permission denied"). From f038e790b2e1e41647001a057e71a19f59ad7bc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 03:09:05 +0000 Subject: [PATCH 08/10] Count a process tree's peaks on a tree of idle processes The test summed this test process's own peak with its child's and compared that with the tree's, read a moment later. Linux updates a process's peak lazily, so once memory is freed (as other tests, and now a mount's hand-back, do) the later reading can be lower. A shell with two sleeping children frees nothing meanwhile. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs-bench/src/sys.rs | 36 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/crates/sandfs-bench/src/sys.rs b/crates/sandfs-bench/src/sys.rs index 5fb7bff..15ecf2f 100644 --- a/crates/sandfs-bench/src/sys.rs +++ b/crates/sandfs-bench/src/sys.rs @@ -164,7 +164,7 @@ fn parse_size(text: &str) -> Option { #[cfg(test)] mod tests { - use super::{children_peak_rss, current_rss, disk_usage, parse_size, peak_rss, tree_peak_rss}; + use super::{children_peak_rss, current_rss, disk_usage, parents, parse_size, peak_rss, tree_peak_rss}; #[test] fn sizes_parse_as_vmmap_prints_them() { @@ -177,24 +177,30 @@ mod tests { #[test] fn a_process_tree_counts_its_running_children() { - let mut child = std::process::Command::new("sleep").arg("30").spawn().unwrap(); - // Until the child has become `sleep`, it is a copy of this process with its peak. - let name = || { - let output = - std::process::Command::new("ps").args(["-o", "comm=", "-p"]).arg(child.id().to_string()).output(); + // A tree of idle processes: this test's own process frees memory as other tests + // run, and Linux may then report a lower peak than it did a moment before. + let mut shell = std::process::Command::new("sh").args(["-c", "sleep 30 & sleep 30 & wait"]).spawn().unwrap(); + let name = |pid: u32| { + let output = std::process::Command::new("ps").args(["-o", "comm=", "-p"]).arg(pid.to_string()).output(); String::from_utf8_lossy(&output.unwrap().stdout).trim().to_owned() }; - while !name().ends_with("sleep") { + // Until a child has become `sleep`, it is a copy of the shell. + let sleepers = loop { + let children: Vec = + parents().unwrap().into_iter().filter(|&(_, parent)| parent == shell.id()).map(|(c, _)| c).collect(); + if children.len() == 2 && children.iter().all(|&child| name(child).ends_with("sleep")) { + break children; + } std::thread::yield_now(); + }; + let parts = peak_rss(shell.id()).unwrap() + sleepers.iter().map(|&child| peak_rss(child).unwrap()).sum::(); + let tree = tree_peak_rss(shell.id()).unwrap(); + for &child in &sleepers { + let _ = std::process::Command::new("kill").arg(child.to_string()).status(); } - // Peaks only grow, so reading the parts before the whole keeps the sum below it. - let alone = peak_rss(std::process::id()).unwrap(); - let sleeping = peak_rss(child.id()).unwrap(); - let tree = tree_peak_rss(std::process::id()).unwrap(); - child.kill().unwrap(); - child.wait().unwrap(); - assert!(sleeping > 0); - assert!(tree >= alone + sleeping, "tree {tree} < {alone} + {sleeping}"); + shell.kill().unwrap(); + shell.wait().unwrap(); + assert!(tree >= parts && parts > 0, "tree {tree} < {parts}"); } #[test] From e2ec7534da64bbcdb2820142e5ff5027a3508088 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 03:19:18 +0000 Subject: [PATCH 09/10] Say a trim's given-back bytes only when the allocator names some, and log idle trims On macOS 27 the allocator answered 0 while it handed the pages back over the next seconds, so "0 B given back" read as a failure. The walk test also times each walk now, against the cache budget. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- crates/sandfs-cli/src/main.rs | 3 ++- crates/sandfs/src/nfs.rs | 6 +++++- crates/sandfs/src/server.rs | 1 + docs/changelog.md | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/sandfs-cli/src/main.rs b/crates/sandfs-cli/src/main.rs index f445d78..a346655 100644 --- a/crates/sandfs-cli/src/main.rs +++ b/crates/sandfs-cli/src/main.rs @@ -317,7 +317,8 @@ fn run(cli: Cli) -> Result<()> { Cmd::Trim { workspace } => { let trimmed = home.trim(workspace.as_deref())?; let mounts = if trimmed.mounts == 1 { "1 mount".to_owned() } else { format!("{} mounts", trimmed.mounts) }; - match trimmed.given_back { + // macOS's allocator may hand pages back later rather than say so now. + match trimmed.given_back.filter(|&bytes| bytes > 0) { Some(bytes) => print_out(&format!("trimmed {mounts}; {} given back to the system\n", size(bytes))), None => print_out(&format!("trimmed {mounts}\n")), } diff --git a/crates/sandfs/src/nfs.rs b/crates/sandfs/src/nfs.rs index adad934..0cab89b 100644 --- a/crates/sandfs/src/nfs.rs +++ b/crates/sandfs/src/nfs.rs @@ -1515,8 +1515,12 @@ mod tests { } }) }; + let started = std::time::Instant::now(); walk(); - eprintln!("walked {files} files, {} inodes known", nfs.shared.lock().node_count()); + eprintln!("walked {files} files in {:?}, {} inodes known", started.elapsed(), nfs.shared.lock().node_count()); + let started = std::time::Instant::now(); + walk(); + eprintln!("walked again in {:?}", started.elapsed()); // A trim lets go of what can be read again, and the next walk reads it. nfs.shared.trim().unwrap(); crate::memory::give_back(); diff --git a/crates/sandfs/src/server.rs b/crates/sandfs/src/server.rs index 738821e..a053835 100644 --- a/crates/sandfs/src/server.rs +++ b/crates/sandfs/src/server.rs @@ -201,6 +201,7 @@ fn trim_idle(idle: Duration) { } } crate::memory::give_back(); + eprintln!("sandfs: trimmed {} mount(s) unused for {}s", idle.len(), IDLE.as_secs()); } /// Send `request` and read the reply, or `None` if the connection closes first. diff --git a/docs/changelog.md b/docs/changelog.md index 9640375..8e6efd7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,7 +13,7 @@ The main changes: API changes in the `sandfs` crate: `Changes` has a new field, `committed`, so code that builds a `Changes` itself needs it. `CrossDevice` comes up far less often, now that a folder on another disk is copied there. `Home::trim` and `Trimmed` are new. In `sandfs-core`, `Source` has two new methods with defaults, `entry_now` and `trim`, and `Workspace` has `recheck` and `trim`. - **sandfs, folders on another disk**: `sandfs fork` of a plain folder on another disk than sandfs's home, such as an external volume, was refused (`CrossDevice`), since a file is cloned only within one disk, and a program fell back to mounting the folder itself, which shows the folder as it changes and has written zeros over files. The copy is now made on the folder's own disk, in `.sandfs/copies` at that disk's top: a frozen copy as on the home's disk, which takes no space until a file in it changes. sandfs makes that folder the first time, leaves it out of every copy and mount of the disk, and deletes it once it holds no copy. It does so only where no one else can move that folder: `.sandfs` and `.sandfs/copies` are this user's and writable by no one else, at a top where others can't rename them (a sticky one, as /dev/shm is, will do). `CrossDevice` is left for a folder mounted inside the one forked, or a disk whose top sandfs can't use that way. -- **sandfs, sandfs-core and sandfs-git, giving memory back**: the process serving mounts grew with every file a walk asked about and never shrank: about 1 GB after a walk of a 700,000-file folder on macOS, and even after the unmount it held 337 MB, freed but kept by macOS's allocator. Now the folder listings, directories found and file bytes a workspace keeps to answer faster stay within a budget (32 MiB of listings, 16 MiB of file bytes), the least used going first, and are read again when needed. A mount nobody has used for a minute is trimmed: its changes are saved, those caches go, idle `git cat-file` readers exit, and the allocator is asked to hand the freed memory back to the system (on macOS `malloc_zone_pressure_relief`, on glibc `malloc_trim`; this one call is the only code sandfs runs that Rust can't check). `sandfs trim [NAME]` trims at once, and in the library `Home::trim` does, for a program to call when a task goes idle. The memory is handed back after every unmount too. On Linux, a walk of 50,000 files peaked at 25 MB where it peaked at 31 MB, and a trim took it to 13 MB. What stays is a record of each file the system asked about: a mount's table of files now holds each path once, not twice, and on macOS, whose NFS client never says it is done with a file, that table lasts until the unmount. In `sandfs-core` this is `Workspace::trim` and `Source::trim`, with a default that does nothing, which `GitSource` implements. +- **sandfs, sandfs-core and sandfs-git, giving memory back**: the process serving mounts grew with every file a walk asked about and never shrank: about 1 GB after a walk of a 700,000-file folder on macOS, and even after the unmount it held 337 MB, freed but kept by macOS's allocator. Now the folder listings, directories found and file bytes a workspace keeps to answer faster stay within a budget (32 MiB of listings, 16 MiB of file bytes), the least used going first, and are read again when needed. A mount nobody has used for a minute is trimmed: its changes are saved, those caches go, idle `git cat-file` readers exit, and the allocator is asked to hand the freed memory back to the system (on macOS `malloc_zone_pressure_relief`, on glibc `malloc_trim`; this one call is the only code sandfs runs that Rust can't check). `sandfs trim [NAME]` trims at once, and in the library `Home::trim` does, for a program to call when a task goes idle. The memory is handed back after every unmount too. On macOS, walks (`find` and `ls -lR`) of a 706,605-file folder peaked at 404 MB where they peaked at 1,028 MB, and after the unmount the process kept 48 MB where it kept 337 MB. On Linux, a walk of 50,000 files peaked at 25 MB where it peaked at 31 MB, and a trim took it to 13 MB. The price is time on walks of folders whose listings don't fit the budget: a folder listed again is read again, so on macOS those two walks of 706,605 files took 59 s where they took 43 s; walks of smaller folders, and work within one part of a large one, are as fast as before. What stays is a record of each file the system asked about: a mount's table of files now holds each path once, not twice, and on macOS, whose NFS client never says it is done with a file, that table lasts until the unmount. In `sandfs-core` this is `Workspace::trim` and `Source::trim`, with a default that does nothing, which `GitSource` implements. - **sandfs and sandfs-core, less memory per file**: a mount keeps less for each file it is asked about. After a walk of 200,000 files (`find` and `ls -lR`), the process serving the mount held 89 MB where it held 147 MB, and walks take as long as before. Three changes do it. The record of each file the kernel knows is smaller. Files are found through their folder's listing, and only folders are kept in the table that resolves paths in one lookup. The table of files the kernel knows gives its room back once the kernel forgets most of them, which took a further 14 MB off after the kernel forgot the walk. A git mount also keeps at most 64 MiB of whole files read lately, down from 256 MiB. - **sandfs and sandfs-cli, linked worktrees in a folder**: a git worktree's `.git` is a file naming its repository by full path, so git in a mount or fork of a folder holding one wrote that repository on the disk: a commit landed in the real repository while the files it was made from stayed in the workspace, and the real worktree then looked changed. A mount and a fork now show such a file with the path the repository has there: relative, when the repository is in the folder too, so the commit stays in the workspace; otherwise a path that isn't there, so git stops instead of writing outside it, and the file says "this worktree's repository is outside this folder, at PATH". `sandfs changes` now says which folders hold a repository with commits made in the workspace, which copies leave out with the rest of `.git`; in the library that is the new `Changes::committed`. - **sandfs and sandfs-core, a file that shrank in the folder under a mount**: a mount of a folder lists each file once and reads its bytes from the folder as they are. When a file got shorter in the folder after that (a copy onto it truncates it first), a read came back short of the listed size, and macOS's NFS client filled the rest with zeros: copying files from a mount onto the folder under it wrote zeros into them. Now a read that comes back short makes the mount ask the folder again for that file's size and time alone. On Linux the read is the file's end and a later `stat` gives the new size. On macOS, whose client won't take a smaller size from a read, the read fails ("Stale NFS file handle") and the file gets a new inode, which the client looks up within a second or so and then reads as the file is now. A read never gives zeros. In `sandfs-core` this is `Workspace::recheck` and a new `Source::entry_now` with a default that changes nothing, which `DirSource` implements. Errors from a folder now name the path they happened at ("source error: PATH: Permission denied"). From 08191eb45eae4dc56a7f099f885a52181bf40820 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 03:23:35 +0000 Subject: [PATCH 10/10] Count a load that finishes after its cache half turned old, and note the unsafe exception A slow load whose slot moved to the old half had its bytes dropped, so once it moved back it counted as an empty slot and the cache could pass its budget. Its bytes now go on the old entry when it is the same slot. AGENTS.md names the one allowed unsafe function. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013jkdGETmgGUb8hcSEpppQK --- AGENTS.md | 2 +- crates/sandfs-core/src/workspace.rs | 48 ++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8531be7..a0588ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ - `pre-commit install` – enables automatic Ruff and MyPy hooks before every commit. ## Coding Style & Naming Conventions -- Rust: edition 2024, `rustfmt.toml` settings, no `unsafe`, doc comments on public items. Prefer small, composable functions over new concepts; a feature that can be built from mount, fork and changes should be. +- Rust: edition 2024, `rustfmt.toml` settings, no `unsafe`, doc comments on public items. The one exception, approved by the maintainer, is `give_back` in `crates/sandfs/src/memory.rs`: one allocator call per platform that hands freed memory back to the system, allowed on that function alone (the workspace lint is `deny`, not `forbid`, for it). Don't add another without the maintainer's approval. Prefer small, composable functions over new concepts; a feature that can be built from mount, fork and changes should be. - Python 3.11+, 4-space indentation, PEP 8 naming, type hints everywhere. - Keep comments meaningful: a brief rationale before non-trivial logic. Avoid repository-specific jargon unless defined. diff --git a/crates/sandfs-core/src/workspace.rs b/crates/sandfs-core/src/workspace.rs index 2b404e4..835e2a8 100644 --- a/crates/sandfs-core/src/workspace.rs +++ b/crates/sandfs-core/src/workspace.rs @@ -105,14 +105,22 @@ impl Halves { } } - /// Count `bytes` more for the young entry `key`, once loaded. - fn charge(&mut self, key: &K, bytes: usize) { - if let Some((_, held)) = self.young.get_mut(key) { - *held += bytes; - self.bytes += bytes; - if self.bytes > self.budget / 2 { - self.turn(); + /// Count `bytes` more for the entry `key`, once loaded, if it is still the one + /// `loaded` picks out. A load may finish after its half turned old: the old half's + /// entry then holds the bytes, which count once it moves back. + fn charge(&mut self, key: &K, bytes: usize, loaded: impl Fn(&V) -> bool) { + if let Some((value, held)) = self.young.get_mut(key) { + if loaded(value) { + *held += bytes; + self.bytes += bytes; + if self.bytes > self.budget / 2 { + self.turn(); + } } + } else if let Some((value, held)) = self.old.get_mut(key) + && loaded(value) + { + *held += bytes; } } @@ -231,7 +239,7 @@ impl Cache { let fetched = fetch()?; *value = Some(fetched.clone()); drop(value); - with(map, |map| map.charge(&key, fetched.bytes())); + with(map, |map| map.charge(&key, fetched.bytes(), |held| Arc::ptr_eq(held, &slot))); Ok(fetched) } @@ -1794,3 +1802,27 @@ fn content_size(content: &Content) -> Option { Content::Stored(stored) => Some(stored.size), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_load_that_finishes_after_its_half_turned_old_is_counted_when_it_moves_back() { + let mut halves: Halves<&str, Arc> = Halves::new(200); + let slow = Arc::new(1); + halves.insert("slow", slow.clone(), 10); + // Another load fills the young half meanwhile, which turns it old. + let fast = Arc::new(2); + halves.insert("fast", fast.clone(), 10); + halves.charge(&"fast", 90, |held| Arc::ptr_eq(held, &fast)); + assert!(halves.young.is_empty() && halves.old.contains_key("slow")); + halves.charge(&"slow", 90, |held| Arc::ptr_eq(held, &slow)); + assert_eq!(halves.old["slow"].1, 100); + // A charge for a slot that was replaced since counts for nothing. + halves.charge(&"slow", 1000, |held| Arc::ptr_eq(held, &fast)); + assert_eq!(halves.old["slow"].1, 100); + halves.get("slow"); + assert_eq!(halves.bytes, 100, "its bytes count in the young half again"); + } +}