Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,12 @@ jobs:
CARGO_TARGET_DIR=target cargo test --manifest-path tests/tls_connectors/Cargo.toml
cargo fmt --manifest-path tests/tls_connectors/Cargo.toml --check
cargo clippy --manifest-path tests/tls_connectors/Cargo.toml --all-targets -- -D warnings
CARGO_TARGET_DIR=target cargo test --manifest-path tests/zstd_seekable/Cargo.toml
CARGO_TARGET_DIR=target cargo test --manifest-path tests/zstd_parquet/Cargo.toml
cargo fmt --manifest-path tests/zstd_seekable/Cargo.toml --check
cargo fmt --manifest-path tests/zstd_parquet/Cargo.toml --check
cargo clippy --manifest-path tests/zstd_seekable/Cargo.toml --all-targets -- -D warnings
cargo clippy --manifest-path tests/zstd_parquet/Cargo.toml --all-targets -- -D warnings
cargo test
cargo build --no-default-features
cargo test --no-default-features
Expand Down Expand Up @@ -207,6 +213,8 @@ jobs:
cache-on-failure: true
- run: cargo test --all-features
- run: cargo test --manifest-path tests/tls_connectors/Cargo.toml --target-dir target
- run: cargo test --manifest-path tests/zstd_seekable/Cargo.toml --target-dir target
- run: cargo test --manifest-path tests/zstd_parquet/Cargo.toml --target-dir target
- run: cargo build --no-default-features
- run: cargo test --no-default-features
- run: cargo test --no-default-features --features experimental
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/release-policy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,16 @@ jobs:
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@1.96.0
- name: Independent workspace release inputs
run: |
python3 -m unittest discover -s tests -p 'test_release_inputs.py'
python3 tests/check_release_inputs.py
- name: Independently resolved TLS dependency regressions
run: python3 .github/scripts/check_tls_dependencies.py
- name: Complete zstd dependency security floors
run: |
cargo generate-lockfile
python3 tests/check_zstd_dependencies.py
- uses: EmbarkStudios/cargo-deny-action@v2.1.1
with:
rust-version: "1.96.0"
Expand Down
20 changes: 10 additions & 10 deletions crates/casita-fskit/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion crates/casita/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ aws-sdk-s3 = { version = "=1.144.0", default-features = false, features = ["sigv
rustls = { version = "0.23.46", default-features = false, optional = true }
rustls-webpki = { version = "0.103.16", default-features = false, optional = true }
fastcdc = { version = "5", features = ["tokio"], optional = true }
zstd = { version = "0.13", optional = true }
# Security floor for RUSTSEC-2026-0391, RUSTSEC-2026-0398 and RUSTSEC-2026-0403.
zstd = { version = "0.14.3", optional = true }
tokio-util = { version = "0.7", features = ["io", "io-util"], optional = true }
tokio-stream = { version = "0.1", features = ["sync"], optional = true }
async-stream = { version = "0.3", optional = true }
Expand Down
1 change: 1 addition & 0 deletions crates/casita/examples/s3_sync/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
name = "casita-s3-sync"
version = "0.0.0"
edition = "2024"
license = "Apache-2.0"
publish = false

# Keep this consumer separate so workspace feature unification cannot enable
Expand Down
38 changes: 19 additions & 19 deletions crates/casita/src/blob/repairing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::BoxStream;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, AsyncWrite};
use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard, RwLock};
use tokio::sync::{Notify, OwnedRwLockReadGuard, RwLock};

use super::{
BlobBatchGuard, BlobGc, BlobIntegrityError, BlobReader, BlobStore, BlobStreamReader,
Expand Down Expand Up @@ -69,7 +69,9 @@ impl BlobRepairError {
pub struct RepairingBlobStore {
near: ChunkedBlobStore,
far: ChunkedBlobStore,
repairs: Arc<Mutex<HashMap<BlobId, Arc<RepairFlight>>>>,
// Registry critical sections contain no I/O or awaits. Cancellation must
// unlink its flight in Drop, without relying on another task being polled.
repairs: Arc<StdMutex<HashMap<BlobId, Arc<RepairFlight>>>>,
gate: Arc<RwLock<()>>,
}

Expand All @@ -79,7 +81,7 @@ impl RepairingBlobStore {
Self {
near,
far,
repairs: Arc::new(Mutex::new(HashMap::new())),
repairs: Arc::new(StdMutex::new(HashMap::new())),
gate: Arc::new(RwLock::new(())),
}
}
Expand All @@ -98,7 +100,7 @@ impl RepairingBlobStore {
pub(crate) async fn repair_waiters(&self, digest: &BlobId) -> usize {
self.repairs
.lock()
.await
.expect("repair registry mutex poisoned")
.get(digest)
.map_or(0, |flight| flight.waiters.load(Ordering::SeqCst))
}
Expand Down Expand Up @@ -159,7 +161,7 @@ impl RepairingBlobStore {
#[tracing::instrument(name = "blob.repairing.repair", skip_all, fields(blob = %digest))]
async fn repair_once(&self, digest: BlobId, near: String) -> Result<(), Error> {
let (flight, leader) = {
let mut repairs = self.repairs.lock().await;
let mut repairs = self.repairs.lock().expect("repair registry mutex poisoned");
match repairs.get(&digest) {
Some(flight) => (flight.clone(), false),
None => {
Expand All @@ -176,7 +178,7 @@ impl RepairingBlobStore {
RepairLeader::new(digest, near.clone(), flight.clone(), self.repairs.clone());
let outcome = self.perform_repair(digest, near).await;
flight.complete(outcome.clone());
remove_flight(&self.repairs, digest, &flight).await;
remove_flight(&self.repairs, digest, &flight);
completion.disarm();
return outcome.map_err(|error| Error::Backend(Box::new(error)));
}
Expand Down Expand Up @@ -629,12 +631,12 @@ impl Drop for RepairWaiter<'_> {
}
}

async fn remove_flight(
repairs: &Mutex<HashMap<BlobId, Arc<RepairFlight>>>,
fn remove_flight(
repairs: &StdMutex<HashMap<BlobId, Arc<RepairFlight>>>,
digest: BlobId,
flight: &Arc<RepairFlight>,
) {
let mut repairs = repairs.lock().await;
let mut repairs = repairs.lock().expect("repair registry mutex poisoned");
if repairs
.get(&digest)
.is_some_and(|current| Arc::ptr_eq(current, flight))
Expand All @@ -647,7 +649,7 @@ struct RepairLeader {
digest: BlobId,
near: String,
flight: Arc<RepairFlight>,
repairs: Arc<Mutex<HashMap<BlobId, Arc<RepairFlight>>>>,
repairs: Arc<StdMutex<HashMap<BlobId, Arc<RepairFlight>>>>,
armed: bool,
}

Expand All @@ -656,7 +658,7 @@ impl RepairLeader {
digest: BlobId,
near: String,
flight: Arc<RepairFlight>,
repairs: Arc<Mutex<HashMap<BlobId, Arc<RepairFlight>>>>,
repairs: Arc<StdMutex<HashMap<BlobId, Arc<RepairFlight>>>>,
) -> Self {
Self {
digest,
Expand All @@ -677,19 +679,14 @@ impl Drop for RepairLeader {
if !self.armed {
return;
}
// Retire the registration before waking existing waiters so a new
// reader can lead a repair even without a running executor.
remove_flight(&self.repairs, self.digest, &self.flight);
self.flight.complete(Err(BlobRepairError {
digest: self.digest,
near: self.near.clone(),
repair: "repair was cancelled before publication completed".to_owned(),
}));
let repairs = self.repairs.clone();
let flight = self.flight.clone();
let digest = self.digest;
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
runtime.spawn(async move {
remove_flight(&repairs, digest, &flight).await;
});
}
}
}

Expand Down Expand Up @@ -763,3 +760,6 @@ impl BlobWriter for GuardedWriter {
self.inner.close().await
}
}

#[cfg(test)]
mod tests;
155 changes: 155 additions & 0 deletions crates/casita/src/blob/repairing/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
use super::*;
#[cfg(feature = "native")]
use crate::test_util::native::{small_chunked_store, write_blob};

fn registered_leader() -> RepairLeader {
let digest = BlobId::new([7; 32].into());
let flight = Arc::new(RepairFlight::new());
let leader = RepairLeader::new(
digest,
"original near-tier corruption".to_owned(),
flight,
Arc::default(),
);
leader
.repairs
.try_lock()
.unwrap()
.insert(digest, leader.flight.clone());
leader
}

#[test]
fn cancelled_leader_releases_its_flight_without_a_runtime() {
let leader = registered_leader();
let digest = leader.digest;
let repairs = leader.repairs.clone();
drop(leader);

assert!(
!repairs.try_lock().unwrap().contains_key(&digest),
"cancelled repair stayed registered without a runtime to schedule cleanup"
);
}

#[cfg(feature = "native")]
#[test]
fn cancelled_registration_does_not_block_the_next_real_repair() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let (near, near_dir) = small_chunked_store();
let (far, _far_dir) = small_chunked_store();
let data = b"a cancelled repair must not strand these verified bytes".to_vec();
let digest = runtime.block_on(async {
let digest = write_blob(&near, &data).await;
assert_eq!(write_blob(&far, &data).await, digest);
let chunk = near.chunks(&digest).await.unwrap().unwrap()[0].digest;
let path = crate::blob::chunked::sharded_path(
&object_store::path::Path::default(),
"chunks",
chunk.as_digest(),
);
std::fs::write(near_dir.path().join(path.as_ref()), b"not zstd").unwrap();
assert!(
validated_reader(&near, &digest).await.is_err(),
"fixture must require a near-tier repair"
);
digest
});
let repairing = RepairingBlobStore::new(near.clone(), far);
let leader = RepairLeader::new(
digest,
"near-tier chunk is corrupt".to_owned(),
Arc::new(RepairFlight::new()),
repairing.repairs.clone(),
);
repairing
.repairs
.try_lock()
.unwrap()
.insert(digest, leader.flight.clone());

// A cancelled future may be dropped outside an entered runtime. Exercise
// the next actual read, including far verification and near publication.
drop(leader);
assert_eq!(
runtime.block_on(repairing.read_to_vec(&digest)).unwrap(),
Some(data.clone())
);
assert_eq!(
runtime.block_on(near.read_to_vec(&digest)).unwrap(),
Some(data)
);
}

#[tokio::test(flavor = "current_thread")]
async fn cancelled_leader_releases_its_flight_before_a_cleanup_task_runs() {
let leader = registered_leader();
let digest = leader.digest;
let repairs = leader.repairs.clone();
drop(leader);

// Do not yield: a new reader must not depend on the scheduler choosing an
// unrelated cleanup task before it can start a replacement repair.
assert!(
!repairs.try_lock().unwrap().contains_key(&digest),
"cancelled repair stayed registered until a cleanup task was polled"
);
}

#[tokio::test(flavor = "current_thread")]
async fn cancelled_leader_preserves_the_diagnostic_for_an_existing_waiter() {
let leader = registered_leader();
let digest = leader.digest;
let flight = leader.flight.clone();
let mut waiter = Box::pin(flight.wait());
assert!(futures::poll!(waiter.as_mut()).is_pending());
assert_eq!(flight.waiters.load(Ordering::SeqCst), 1);

drop(leader);

let Poll::Ready(Err(error)) = futures::poll!(waiter.as_mut()) else {
panic!("registered waiter did not receive the cancellation outcome");
};
assert_eq!(error.digest(), digest);
assert_eq!(error.near_error(), "original near-tier corruption");
assert_eq!(
error.repair_error(),
"repair was cancelled before publication completed"
);
assert_eq!(flight.waiters.load(Ordering::SeqCst), 0);
}

#[test]
fn cancelling_an_old_leader_preserves_a_replacement_flight() {
let leader = registered_leader();
let digest = leader.digest;
let repairs = leader.repairs.clone();
let replacement = Arc::new(RepairFlight::new());
repairs
.try_lock()
.unwrap()
.insert(digest, replacement.clone());

drop(leader);

assert!(Arc::ptr_eq(
repairs.try_lock().unwrap().get(&digest).unwrap(),
&replacement
));
assert!(replacement.outcome.lock().unwrap().is_none());
}

#[test]
fn disarmed_leader_preserves_the_completed_outcome() {
let mut leader = registered_leader();
let flight = leader.flight.clone();
flight.complete(Ok(()));
leader.disarm();

drop(leader);

assert!(matches!(*flight.outcome.lock().unwrap(), Some(Ok(()))));
}
Loading