Give memory back, and release 0.9.0

AGENTS.md

@@ -17,7 +17,7 @@
 - `pre-commit install` – enables automatic Ruff and MyPy hooks before every commit.
 
 ## Coding Style & Naming Conventions
-- Rust: edition 2024, `rustfmt.toml` settings, no `unsafe`, doc comments on public items. Prefer small, composable functions over new concepts; a feature that can be built from mount, fork and changes should be.
+- Rust: edition 2024, `rustfmt.toml` settings, no `unsafe`, doc comments on public items. The one exception, approved by the maintainer, is `give_back` in `crates/sandfs/src/memory.rs`: one allocator call per platform that hands freed memory back to the system, allowed on that function alone (the workspace lint is `deny`, not `forbid`, for it). Don't add another without the maintainer's approval. Prefer small, composable functions over new concepts; a feature that can be built from mount, fork and changes should be.
 - Python 3.11+, 4-space indentation, PEP 8 naming, type hints everywhere.
 - Keep comments meaningful: a brief rationale before non-trivial logic. Avoid repository-specific jargon unless defined.
 

Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/sandfs", "crates/sandfs-bench", "crates/sandfs-cli", "crates/sandfs-core", "crates/sandfs-git", "crates/sandfs-mcp", "crates/sandfs-py"]
 
 [workspace.package]
-version = "0.8.0"
+version = "0.9.0"
 edition = "2024"
 rust-version = "1.85"
 license = "MIT"
@@ -14,14 +14,16 @@ homepage = "https://github.com/lydakis/sandfs"
 bytes = "1.12"
 imbl = "7.0"
 thiserror = "2.0"
-sandfs-core = { path = "crates/sandfs-core", version = "0.8.0" }
-sandfs-git = { path = "crates/sandfs-git", version = "0.8.0" }
-sandfs-mcp = { path = "crates/sandfs-mcp", version = "0.8.0" }
-sandfs = { path = "crates/sandfs", version = "0.8.0", default-features = false }
-sandfs-cli = { path = "crates/sandfs-cli", version = "0.8.0" }
+sandfs-core = { path = "crates/sandfs-core", version = "0.9.0" }
+sandfs-git = { path = "crates/sandfs-git", version = "0.9.0" }
+sandfs-mcp = { path = "crates/sandfs-mcp", version = "0.9.0" }
+sandfs = { path = "crates/sandfs", version = "0.9.0", default-features = false }
+sandfs-cli = { path = "crates/sandfs-cli", version = "0.9.0" }
 
 [workspace.lints.rust]
-unsafe_code = "forbid"
+# Deny rather than forbid: `memory::give_back` in the sandfs crate is the one audited
+# exception, allowed on that call alone.
+unsafe_code = "deny"
 missing_docs = "warn"
 
 [workspace.lints.clippy]

crates/sandfs-bench/src/sys.rs

@@ -164,7 +164,7 @@ fn parse_size(text: &str) -> Option<u64> {
 
 #[cfg(test)]
 mod tests {
-    use super::{children_peak_rss, current_rss, disk_usage, parse_size, peak_rss, tree_peak_rss};
+    use super::{children_peak_rss, current_rss, disk_usage, parents, parse_size, peak_rss, tree_peak_rss};
 
     #[test]
     fn sizes_parse_as_vmmap_prints_them() {
@@ -177,24 +177,30 @@ mod tests {
 
     #[test]
     fn a_process_tree_counts_its_running_children() {
-        let mut child = std::process::Command::new("sleep").arg("30").spawn().unwrap();
-        // Until the child has become `sleep`, it is a copy of this process with its peak.
-        let name = || {
-            let output =
-                std::process::Command::new("ps").args(["-o", "comm=", "-p"]).arg(child.id().to_string()).output();
+        // A tree of idle processes: this test's own process frees memory as other tests
+        // run, and Linux may then report a lower peak than it did a moment before.
+        let mut shell = std::process::Command::new("sh").args(["-c", "sleep 30 & sleep 30 & wait"]).spawn().unwrap();
+        let name = |pid: u32| {
+            let output = std::process::Command::new("ps").args(["-o", "comm=", "-p"]).arg(pid.to_string()).output();
             String::from_utf8_lossy(&output.unwrap().stdout).trim().to_owned()
         };
-        while !name().ends_with("sleep") {
+        // Until a child has become `sleep`, it is a copy of the shell.
+        let sleepers = loop {
+            let children: Vec<u32> =
+                parents().unwrap().into_iter().filter(|&(_, parent)| parent == shell.id()).map(|(c, _)| c).collect();
+            if children.len() == 2 && children.iter().all(|&child| name(child).ends_with("sleep")) {
+                break children;
+            }
             std::thread::yield_now();
+        };
+        let parts = peak_rss(shell.id()).unwrap() + sleepers.iter().map(|&child| peak_rss(child).unwrap()).sum::<u64>();
+        let tree = tree_peak_rss(shell.id()).unwrap();
+        for &child in &sleepers {
+            let _ = std::process::Command::new("kill").arg(child.to_string()).status();
         }
-        // Peaks only grow, so reading the parts before the whole keeps the sum below it.
-        let alone = peak_rss(std::process::id()).unwrap();
-        let sleeping = peak_rss(child.id()).unwrap();
-        let tree = tree_peak_rss(std::process::id()).unwrap();
-        child.kill().unwrap();
-        child.wait().unwrap();
-        assert!(sleeping > 0);
-        assert!(tree >= alone + sleeping, "tree {tree} < {alone} + {sleeping}");
+        shell.kill().unwrap();
+        shell.wait().unwrap();
+        assert!(tree >= parts && parts > 0, "tree {tree} < {parts}");
     }
 
     #[test]

crates/sandfs-cli/README.md

@@ -23,6 +23,7 @@ sandfs update cleanup ~/datasets/survey-v2       # move a workspace onto a newer
 sandfs rm cleanup                                # forget a workspace and its changes
 sandfs rm --unmount ~/ws/survey                  # unmount it first if it is mounted
 sandfs clean                                     # free the space downloaded repository files take
+sandfs trim                                      # mounts give back the memory they can do without
 ```
 
 `--read-only PATH` (repeatable, relative to the source) makes a file or folder read-only in a new workspace and its forks: programs can read it, but writing, deleting, renaming or creating there fails as on a read-only file system (`EROFS`), and its files show without write permission. It is how a container or an agent given the mount gets to look at a folder without changing it even in its own copy. `serve --mcp SOURCE --read-only PATH` does the same for an agent over MCP. It is set when a workspace is created; mounting an existing workspace with it is refused.
@@ -61,7 +62,7 @@ Read-only paths stay read-only under `srt`, and `changes` lists what the program
 
 One process serves a workspace at a time. A mount or an MCP session holds it until it ends, a second one is refused with the holder's name, and a crashed holder releases it. Parallel attempts each get their own fork.
 
-Every mount runs in one background process, which the first mount starts and which exits a moment after the last `unmount`. It writes its log to `$SANDFS_HOME/run/server.log`. Mounts of one git commit, such as a workspace and its forks, share its git readers and every folder listing read from it.
+Every mount runs in one background process, which the first mount starts and which exits a moment after the last `unmount`. It writes its log to `$SANDFS_HOME/run/server.log`. Mounts of one git commit, such as a workspace and its forks, share its git readers and every folder listing read from it. What a mount keeps in memory to answer faster, folder listings and files read through the library, stays within a budget (tens of megabytes) and is read again when needed. A mount nobody uses for a minute lets go of it, ends its idle git readers and hands the memory back to the system; `sandfs trim [NAME]` does that now. What stays is a small record of each file the system has asked about, which on macOS lasts until the unmount, since its NFS client never says it is done with a file.
 
 Changes are saved under `$SANDFS_HOME` (default `~/.local/share/sandfs`) once a file has gone a quarter of a second without a write. Closing a file doesn't save it, since the mount is never told about opens and closes. These saves are written to disk but not flushed, so they survive the mount's process crashing but not necessarily a power cut. `fsync` saves at once and flushes to disk before it returns, and `unmount`, `fork` and `changes` save and flush everything first. On macOS `fsync` doesn't: the Mac sends each write to the mount before the write returns, so a fork or `changes` sees it, and with that setting it never passes `fsync` on. There a write is flushed to disk only by `unmount`, `fork` or `changes`, which matters to programs that count on `fsync`, such as databases. A file being written goes straight into a file on disk under `$SANDFS_HOME`, not into memory, and once saved it is read from there through the kernel's page cache, so a mount's memory doesn't grow with what programs write in it and a file can be as large as the disk allows. Writing into part of a saved file first copies it, which takes no time on filesystems that clone files (APFS, btrfs, XFS) and is a full copy on others (ext4); a file from the source is copied on its first write. A fork on disk costs more than a fork in memory. `sandfs fork` never copies the source, but it waits for the workspace's latest changes to be saved, links each changed file into the fork (copying it where the disk can't link), and starts a new mount, which reads a host folder again rather than sharing what the first mount has read (mounts of one git commit do share it). Without a directory, `sandfs fork NAME --name SNAPSHOT` stops before the mount: it keeps a snapshot of the workspace as it is now, which `fork SNAPSHOT DIR` brings back as a new copy (or `mount SNAPSHOT DIR` mounts and changes the snapshot itself). With the files already saved it costs a link per changed file: 45 ms for a workspace holding a 1.1 GB build. A host folder is read as it is when each path is first touched, so mount one nothing else is changing. `git:<path or url>[@rev]` mounts a commit instead of a folder (a URL works without `git:`); for a remote repository sandfs fetches only that commit and its folder listings, with no history, and fetches file contents when they are first read. Contents fetched from a remote repository are kept in one local copy per repository, shared by its workspaces, until `sandfs clean`: for each repository none of whose workspaces is mounted or served, it drops the file contents and keeps the commits and folder listings, so the workspaces still mount at once and fetch a file again when it is next read, and it deletes the copy of a repository no workspace uses any more. Offline, a commit id that was mounted before still mounts and the folders opened before still read; a branch or tag name is looked up on the server every time, so it doesn't mount offline.
 

crates/sandfs-cli/src/main.rs

@@ -145,6 +145,16 @@ enum Cmd {
         /// A workspace name or the directory it is mounted on.
         workspace: String,
     },
+    /// Have mounts give back the memory they can do without.
+    ///
+    /// Saves the changes of the mounted workspace (every mounted one, without a name),
+    /// then lets go of the folder listings and file bytes it keeps and of idle git
+    /// helpers, and hands the memory freed back to the system. Nothing in the mount
+    /// changes. A mount left unused for a minute is trimmed on its own.
+    Trim {
+        /// A workspace name or the directory it is mounted on.
+        workspace: Option<String>,
+    },
     /// Delete an unmounted workspace and its changes.
     Rm {
         /// The workspace's name, or with --unmount the directory it is mounted on or
@@ -304,6 +314,15 @@ fn run(cli: Cli) -> Result<()> {
             print_out(&out)
         }
         Cmd::Unmount { workspace } => home.unmount(&workspace),
+        Cmd::Trim { workspace } => {
+            let trimmed = home.trim(workspace.as_deref())?;
+            let mounts = if trimmed.mounts == 1 { "1 mount".to_owned() } else { format!("{} mounts", trimmed.mounts) };
+            // macOS's allocator may hand pages back later rather than say so now.
+            match trimmed.given_back.filter(|&bytes| bytes > 0) {
+                Some(bytes) => print_out(&format!("trimmed {mounts}; {} given back to the system\n", size(bytes))),
+                None => print_out(&format!("trimmed {mounts}\n")),
+            }
+        }
         Cmd::Rm { name, unmount } => home.remove(&name, unmount),
         Cmd::Clean => {
             let mut out = String::new();

crates/sandfs-core/src/path.rs

@@ -2,6 +2,7 @@
 
 use std::borrow::Borrow;
 use std::fmt;
+use std::sync::Arc;
 
 use crate::error::{Error, Result};
 
@@ -10,13 +11,16 @@ use crate::error::{Error, Result};
 /// Parsing collapses repeated slashes and `.` components and resolves `..`; a `..`
 /// that would climb above the root is rejected rather than clamped, so a caller can
 /// never address something outside the workspace by accident.
+///
+/// A path never changes once made, so a copy shares its bytes: a mount's table of
+/// files and the path each file is known by hold one string between them.
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct VPath(String);
+pub struct VPath(Arc<str>);
 
 impl VPath {
     /// The root path `/`.
     pub fn root() -> Self {
-        VPath("/".to_owned())
+        VPath("/".into())
     }
 
     /// Parse and normalize an absolute path.
@@ -27,7 +31,7 @@ impl VPath {
         // Most paths arrive already normalized; keep them as they are.
         let normal = |part: &str| !part.is_empty() && part != "." && part != ".." && !part.contains('\0');
         if raw == "/" || raw.split('/').skip(1).all(normal) {
-            return Ok(VPath(raw.to_owned()));
+            return Ok(VPath(raw.into()));
         }
         let mut parts: Vec<&str> = Vec::new();
         for part in raw.split('/') {
@@ -46,7 +50,7 @@ impl VPath {
                 }
             }
         }
-        Ok(VPath(format!("/{}", parts.join("/"))))
+        Ok(VPath(format!("/{}", parts.join("/")).into()))
     }
 
     /// The path as a string.
@@ -56,7 +60,7 @@ impl VPath {
 
     /// True for `/`.
     pub fn is_root(&self) -> bool {
-        self.0 == "/"
+        &*self.0 == "/"
     }
 
     /// The parent directory, or `None` for the root.
@@ -65,7 +69,7 @@ impl VPath {
             return None;
         }
         let idx = self.0.rfind('/').unwrap_or(0);
-        Some(if idx == 0 { VPath::root() } else { VPath(self.0[..idx].to_owned()) })
+        Some(if idx == 0 { VPath::root() } else { VPath(self.0[..idx].into()) })
     }
 
     /// The last component, or `None` for the root.
@@ -78,7 +82,7 @@ impl VPath {
         if name.is_empty() || name == "." || name == ".." || name.contains('/') {
             return Err(Error::InvalidPath(format!("{}/{}", self.0, name)));
         }
-        Ok(if self.is_root() { VPath(format!("/{name}")) } else { VPath(format!("{}/{name}", self.0)) })
+        Ok(if self.is_root() { VPath(format!("/{name}").into()) } else { VPath(format!("{}/{name}", self.0).into()) })
     }
 
     /// The components after the root, in order.
@@ -95,7 +99,7 @@ impl VPath {
     pub fn starts_with(&self, prefix: &VPath) -> bool {
         prefix.is_root()
             || self.0 == prefix.0
-            || (self.0.starts_with(&prefix.0) && self.0.as_bytes().get(prefix.0.len()) == Some(&b'/'))
+            || (self.0.starts_with(&*prefix.0) && self.0.as_bytes().get(prefix.0.len()) == Some(&b'/'))
     }
 
     /// The components of `self` below `prefix`, if `self` starts with it.
@@ -121,7 +125,7 @@ impl VPath {
     /// Every proper ancestor from the root down, as slices of this path, so walking
     /// them allocates nothing.
     pub(crate) fn ancestor_strs(&self) -> impl Iterator<Item = &str> {
-        let path = self.0.as_str();
+        let path = &*self.0;
         let root = (!self.is_root()).then_some("/");
         // Each '/' after the leading one ends an ancestor.
         root.into_iter().chain(path.match_indices('/').skip(1).map(move |(idx, _)| &path[..idx]))
@@ -131,15 +135,15 @@ impl VPath {
     pub(crate) fn descendants_end(&self) -> VPath {
         if self.is_root() {
             // '0' is the byte after '/', so it bounds every absolute path.
-            VPath("0".to_owned())
+            VPath("0".into())
         } else {
-            VPath(format!("{}0", self.0))
+            VPath(format!("{}0", self.0).into())
         }
     }
 
     /// The smallest string that sorts before every descendant of `self`.
     pub(crate) fn descendants_start(&self) -> VPath {
-        VPath(if self.is_root() { "/".to_owned() } else { format!("{}/", self.0) })
+        VPath(if self.is_root() { "/".into() } else { format!("{}/", self.0).into() })
     }
 }
 

crates/sandfs-core/src/source.rs

@@ -140,6 +140,11 @@ pub trait Source: Send + Sync + fmt::Debug {
         let _ = key;
         Ok(None)
     }
+
+    /// Let go of what the source keeps only to answer faster, such as helper processes
+    /// sitting idle, when a workspace over it is trimmed ([`crate::Workspace::trim`]).
+    /// Every answer stays the same. The default keeps nothing, so does nothing.
+    fn trim(&self) {}
 }
 
 /// The part of `bytes` a ranged read asks for, clamped to its end.