diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 5fd5d216..5e4d9756 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -283,7 +283,9 @@ jobs: - name: Mutate supervision and system code run: > cargo mutants -p spawned-concurrency - -f concurrency/src/supervision.rs -f concurrency/src/supervision/nested.rs + -f concurrency/src/supervision.rs -f concurrency/src/supervision/child.rs + -f concurrency/src/supervision/state.rs -f concurrency/src/supervision/builder.rs + -f concurrency/src/supervision/run.rs -f concurrency/src/supervision/nested.rs -f concurrency/src/system.rs -f concurrency/src/system/group.rs -j 2 --timeout 120 || true - uses: actions/upload-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 610785c7..b4bff54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ 0.8 stays patch-compatible. Default builds are unchanged. The experiments stay runnable: `benchmarks/experiments/prepare.sh` applies the hooks to a scratch tree (see `benchmarks/README.md`). +- Split the largest source files without changing behaviour or API: + `tasks/actor.rs` and `threads/actor.rs` into `actor/{envelope,context,actor_ref,lifecycle}.rs` + plus one file per test module, `supervision.rs` into + `supervision/{child,state,builder,run,policy_tests}.rs`, and the macros + crate into one module per macro (`protocol`, `actor`, `lints`, + `send_handler`, `sim_test`). Test names are unchanged. - Clarify dispatch admission when a stop races a message, and check deferred token ownership before drop callbacks in debug builds. Release behavior is unchanged. - Keep the Loom models from drifting: a clippy ban (`concurrency/clippy.toml`, diff --git a/CLAUDE.md b/CLAUDE.md index 2dfa2fcc..7b50c91a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ spawned/ │ ├── tasks/ # Async version (requires tokio runtime) │ ├── threads/ # Sync version (native OS threads) │ ├── system.rs # ActorSystem, ExecutionGroup (v0.7) -│ ├── supervision.rs # ChildSpec, all three strategies, nested trees (v0.7) +│ ├── supervision.rs # ChildSpec, all three strategies (supervision/: core, runners, nested trees) │ ├── child_handle.rs # ChildHandle, ActorId, ExitReason │ ├── monitor.rs # MonitorRef, Down │ ├── link.rs # links, Exit, trap_exit diff --git a/benchmarks/experiments/backend-experiments.patch b/benchmarks/experiments/backend-experiments.patch index 94557302..a64df120 100644 --- a/benchmarks/experiments/backend-experiments.patch +++ b/benchmarks/experiments/backend-experiments.patch @@ -154,10 +154,57 @@ index fa86bbbb..91426709 100644 struct Node(Arc); impl Actor for Node {} diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs -index f99ecbbf..bda7e379 100644 +index 34b6c48a..d6a44a4d 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs -@@ -1769,6 +1769,38 @@ impl ActorRef { +@@ -42,6 +42,8 @@ mod context; + mod lifecycle; + #[cfg(test)] + use lifecycle::run_actor; ++#[cfg(all(test, feature = "backend-experiments"))] ++use lifecycle::MessageTurn; + + // --------------------------------------------------------------------------- + // Backend +@@ -444,7 +446,10 @@ mod warn_on_block { + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let type_id = std::any::type_name::(); ++ #[cfg(not(feature = "backend-experiments"))] + let task_id = rt::task_id(); ++ #[cfg(feature = "backend-experiments")] ++ let task_id = rt::experimental::try_task_id(); + let this = self.project(); + let now = Instant::now(); + let res = this.inner.poll(cx); +@@ -470,6 +475,9 @@ mod monitor_cleanup_regressions; + #[cfg(test)] + mod early_stop_regression; + ++#[cfg(all(test, feature = "backend-experiments"))] ++mod turn_regressions; ++ + #[cfg(test)] + mod idle_stop_regression; + +diff --git a/concurrency/src/tasks/actor/idle_stop_regression.rs b/concurrency/src/tasks/actor/idle_stop_regression.rs +index 81956895..c38ea120 100644 +--- a/concurrency/src/tasks/actor/idle_stop_regression.rs ++++ b/concurrency/src/tasks/actor/idle_stop_regression.rs +@@ -36,6 +36,8 @@ fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { + rx, + token.clone(), + &mut cleanup, ++ #[cfg(feature = "backend-experiments")] ++ None, + )); + // Start and park the actual actor loop, without allowing another + // executor task to run it between admission and stop below. +diff --git a/concurrency/src/tasks/actor/lifecycle.rs b/concurrency/src/tasks/actor/lifecycle.rs +index a88489ac..8790cefc 100644 +--- a/concurrency/src/tasks/actor/lifecycle.rs ++++ b/concurrency/src/tasks/actor/lifecycle.rs +@@ -93,6 +93,38 @@ impl ActorRef { // Recorded from here: the start_linked link is the model's initial one. crate::lifecycle_trace::life_spawn(id.raw(), parent_id); @@ -196,7 +243,7 @@ index f99ecbbf..bda7e379 100644 let cleanup_token = cancellation_token.clone(); let cleanup = crate::actor_cleanup::ActorCleanup::new( actor_ref.shared.id, -@@ -1794,7 +1826,20 @@ impl ActorRef { +@@ -118,7 +150,20 @@ impl ActorRef { // instance that can never answer. let inner_future = async move { let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); @@ -218,7 +265,7 @@ index f99ecbbf..bda7e379 100644 let reason = cleanup.die(); if let Some(scope) = &managed { scope.exited(reason); -@@ -1835,13 +1880,49 @@ impl ActorRef { +@@ -159,13 +204,49 @@ impl ActorRef { Backend::Async => { #[cfg(debug_assertions)] let inner_future = warn_on_block::WarnOnBlocking::new(inner_future); @@ -268,22 +315,22 @@ index f99ecbbf..bda7e379 100644 } } -@@ -1849,6 +1930,70 @@ impl ActorRef { +@@ -173,6 +254,70 @@ impl ActorRef { } } +// Batch only between completed handlers: never preempt or concurrently poll an actor. +#[cfg(feature = "backend-experiments")] -+struct MessageTurn { -+ limit: usize, -+ adaptive: bool, -+ slice: Option, -+ completed: usize, -+ began: Option, ++pub(super) struct MessageTurn { ++ pub(super) limit: usize, ++ pub(super) adaptive: bool, ++ pub(super) slice: Option, ++ pub(super) completed: usize, ++ pub(super) began: Option, +} +#[cfg(feature = "backend-experiments")] +impl MessageTurn { -+ fn from_env() -> Self { ++ pub(super) fn from_env() -> Self { + let batch: usize = std::env::var("SPAWNED_EXPERIMENT_BATCH") + .unwrap_or_else(|_| "16".into()) + .parse() @@ -302,19 +349,19 @@ index f99ecbbf..bda7e379 100644 + began: None, + } + } -+ fn parked(&mut self) { ++ pub(super) fn parked(&mut self) { + self.completed = 0; + if self.adaptive { + self.limit = 1; + } + self.began = None; + } -+ fn resumed(&mut self) { ++ pub(super) fn resumed(&mut self) { + if self.slice.is_some() { + self.began.get_or_insert_with(std::time::Instant::now); + } + } -+ async fn after_message(&mut self, empty: bool) { ++ pub(super) async fn after_message(&mut self, empty: bool) { + if empty { + self.parked(); + return; @@ -339,7 +386,7 @@ index f99ecbbf..bda7e379 100644 /// A pending user future of the lifecycle: `started()`, a turn (handler, /// `exit_received`, a job's `done`) or `stopped()`. If the runtime drops the /// lifecycle while it is pending (G1), its `Drop` reaches the death point -@@ -2009,6 +2154,7 @@ async fn run_actor( +@@ -333,6 +478,7 @@ pub(super) async fn run_actor( rx: mpsc::Receiver + Send>>, cancellation_token: StopToken, cleanup: &mut crate::actor_cleanup::ActorCleanup, @@ -347,7 +394,7 @@ index f99ecbbf..bda7e379 100644 ) { let request = ctx.shared.linked_reason.clone(); let mut owned = ExitOnDrop { -@@ -2022,6 +2168,8 @@ async fn run_actor( +@@ -346,6 +492,8 @@ pub(super) async fn run_actor( &mut owned.rx, &mut *owned.cleanup, cancellation_token, @@ -356,7 +403,7 @@ index f99ecbbf..bda7e379 100644 )) .catch_unwind() .await -@@ -2048,6 +2196,7 @@ async fn run_actor_lifecycle( +@@ -372,6 +520,7 @@ pub(super) async fn run_actor_lifecycle( rx: &mut mpsc::Receiver + Send>>, cleanup: &mut crate::actor_cleanup::ActorCleanup, cancellation_token: StopToken, @@ -364,7 +411,7 @@ index f99ecbbf..bda7e379 100644 ) { let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); let request = ctx.shared.linked_reason.clone(); -@@ -2115,7 +2264,7 @@ async fn run_actor_lifecycle( +@@ -439,7 +588,7 @@ pub(super) async fn run_actor_lifecycle( // mailbox acquire and retain a rejected item for shutdown. // The receive, check and trace event are one step // of the lifecycle trace: a recorded stop is seen here. @@ -373,7 +420,7 @@ index f99ecbbf..bda7e379 100644 #[cfg(all(test, not(feature = "lifecycle-trace"), not(spawned_sim)))] stop_receive_probe::before_receive(); let received = rx.poll_recv(cx); -@@ -2130,7 +2279,16 @@ async fn run_actor_lifecycle( +@@ -454,7 +603,16 @@ pub(super) async fn run_actor_lifecycle( }); } result @@ -391,7 +438,7 @@ index f99ecbbf..bda7e379 100644 })); let mut recv = recv; future::poll_fn(|cx| match recv.as_mut().poll(cx) { -@@ -2226,6 +2384,15 @@ async fn run_actor_lifecycle( +@@ -550,6 +708,15 @@ pub(super) async fn run_actor_lifecycle( }); break; } @@ -407,193 +454,171 @@ index f99ecbbf..bda7e379 100644 } None => break, } -@@ -2437,7 +2604,10 @@ mod warn_on_block { - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - let type_id = std::any::type_name::(); -+ #[cfg(not(feature = "backend-experiments"))] - let task_id = rt::task_id(); -+ #[cfg(feature = "backend-experiments")] -+ let task_id = rt::experimental::try_task_id(); - let this = self.project(); - let now = Instant::now(); - let res = this.inner.poll(cx); -@@ -4428,6 +4598,139 @@ mod early_stop_regression { - } - } - -+#[cfg(all(test, feature = "backend-experiments"))] -+mod turn_regressions { -+ use super::*; -+ use std::sync::atomic::{AtomicUsize, Ordering}; -+ #[test] -+ fn idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield() { -+ crate::test_wait::bounded( -+ concat!( -+ module_path!(), -+ "::idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield" -+ ), -+ move || { +diff --git a/concurrency/src/tasks/actor/stop_receive_probe.rs b/concurrency/src/tasks/actor/stop_receive_probe.rs +index 6155de04..03743a8d 100644 +--- a/concurrency/src/tasks/actor/stop_receive_probe.rs ++++ b/concurrency/src/tasks/actor/stop_receive_probe.rs +@@ -41,7 +41,16 @@ fn stop_between_check_and_receive_discards_post_stop_request() { + Some(stop_ctx.send_request(Ping, Deadline::Never).unwrap()); + })) + }); +- run_actor(Count(hits.clone()), ctx, rx, token, &mut cleanup).await; ++ run_actor( ++ Count(hits.clone()), ++ ctx, ++ rx, ++ token, ++ &mut cleanup, ++ #[cfg(feature = "backend-experiments")] ++ None, ++ ) ++ .await; + assert_eq!(cleanup.die(), ExitReason::Normal); + let response = response + .borrow_mut() +@@ -145,6 +154,8 @@ fn receive_rejected_by_stop_keeps_its_caller_until_the_death_point() { + rx, + token, + &mut cleanup, ++ #[cfg(feature = "backend-experiments")] ++ None, + )); + let expected = if park { + assert!(futures::poll!(actor.as_mut()).is_pending()); +diff --git a/concurrency/src/tasks/actor/turn_regressions.rs b/concurrency/src/tasks/actor/turn_regressions.rs +new file mode 100644 +index 00000000..924ee83d +--- /dev/null ++++ b/concurrency/src/tasks/actor/turn_regressions.rs +@@ -0,0 +1,128 @@ ++use super::*; ++use std::sync::atomic::{AtomicUsize, Ordering}; ++#[test] ++fn idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield() { ++ crate::test_wait::bounded( ++ concat!( ++ module_path!(), ++ "::idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield" ++ ), ++ move || { ++ let mut turn = MessageTurn { ++ limit: 16, ++ adaptive: false, ++ slice: Some(Duration::from_micros(50)), ++ completed: 0, ++ began: Some(std::time::Instant::now()), ++ }; ++ turn.parked(); ++ assert!(turn.began.is_none()); ++ let wake = std::time::Instant::now(); ++ turn.resumed(); ++ assert!(turn.began.unwrap() >= wake); ++ futures::executor::block_on(turn.after_message(true)); ++ assert!(turn.began.is_none()); ++ assert_eq!(turn.completed, 0); ++ }, ++ ); ++} ++#[test] ++fn count_only_turn_never_starts_a_clock_and_yields_at_its_count() { ++ crate::test_wait::bounded( ++ concat!( ++ module_path!(), ++ "::count_only_turn_never_starts_a_clock_and_yields_at_its_count" ++ ), ++ move || { ++ rt::Runtime::new().unwrap().block_on(async { + let mut turn = MessageTurn { -+ limit: 16, ++ limit: 4, + adaptive: false, -+ slice: Some(Duration::from_micros(50)), ++ slice: None, + completed: 0, -+ began: Some(std::time::Instant::now()), ++ began: None, + }; -+ turn.parked(); -+ assert!(turn.began.is_none()); -+ let wake = std::time::Instant::now(); + turn.resumed(); -+ assert!(turn.began.unwrap() >= wake); -+ futures::executor::block_on(turn.after_message(true)); ++ for _ in 0..3 { ++ let mut message = Box::pin(turn.after_message(false)); ++ assert!(futures::poll!(message.as_mut()).is_ready()); ++ } + assert!(turn.began.is_none()); ++ let mut fourth = Box::pin(turn.after_message(false)); ++ assert!(futures::poll!(fourth.as_mut()).is_pending()); ++ fourth.await; + assert_eq!(turn.completed, 0); -+ }, -+ ); -+ } -+ #[test] -+ fn count_only_turn_never_starts_a_clock_and_yields_at_its_count() { -+ crate::test_wait::bounded( -+ concat!( -+ module_path!(), -+ "::count_only_turn_never_starts_a_clock_and_yields_at_its_count" -+ ), -+ move || { -+ rt::Runtime::new().unwrap().block_on(async { -+ let mut turn = MessageTurn { -+ limit: 4, -+ adaptive: false, -+ slice: None, -+ completed: 0, -+ began: None, -+ }; -+ turn.resumed(); -+ for _ in 0..3 { -+ let mut message = Box::pin(turn.after_message(false)); -+ assert!(futures::poll!(message.as_mut()).is_ready()); -+ } -+ assert!(turn.began.is_none()); -+ let mut fourth = Box::pin(turn.after_message(false)); -+ assert!(futures::poll!(fourth.as_mut()).is_pending()); -+ fourth.await; -+ assert_eq!(turn.completed, 0); -+ assert!(turn.began.is_none()); -+ turn.parked(); -+ turn.resumed(); -+ assert!(turn.began.is_none()); -+ }); -+ }, -+ ); -+ } -+ #[test] -+ fn stop_during_batch_yield_does_not_execute_the_next_queued_message() { -+ crate::test_wait::bounded( -+ concat!( -+ module_path!(), -+ "::stop_during_batch_yield_does_not_execute_the_next_queued_message" -+ ), -+ move || { -+ struct Count(Arc); -+ impl Actor for Count {} -+ struct Inc; -+ impl Message for Inc { -+ type Result = (); ++ assert!(turn.began.is_none()); ++ turn.parked(); ++ turn.resumed(); ++ assert!(turn.began.is_none()); ++ }); ++ }, ++ ); ++} ++#[test] ++fn stop_during_batch_yield_does_not_execute_the_next_queued_message() { ++ crate::test_wait::bounded( ++ concat!( ++ module_path!(), ++ "::stop_during_batch_yield_does_not_execute_the_next_queued_message" ++ ), ++ move || { ++ struct Count(Arc); ++ impl Actor for Count {} ++ struct Inc; ++ impl Message for Inc { ++ type Result = (); ++ } ++ impl Handler for Count { ++ async fn handle( ++ &mut self, ++ _: Inc, ++ reply: crate::reply::PendingReply<'_, ()>, ++ _: &Context, ++ ) { ++ self.0.fetch_add(1, Ordering::SeqCst); ++ reply.reply(()); + } -+ impl Handler for Count { -+ async fn handle( -+ &mut self, -+ _: Inc, -+ reply: crate::reply::PendingReply<'_, ()>, -+ _: &Context, -+ ) { -+ self.0.fetch_add(1, Ordering::SeqCst); -+ reply.reply(()); -+ } ++ } ++ rt::Runtime::new().unwrap().block_on(async { ++ let (tx, rx) = mpsc::channel(); ++ let token = StopToken::new(); ++ let (ctx, mut cleanup) = test_context(&tx, &token); ++ let count = Arc::new(AtomicUsize::new(0)); ++ for _ in 0..2 { ++ tx.send(Box::new(MessageEnvelope { ++ msg: Some(Inc), ++ slot: None, ++ #[cfg(feature = "metrics")] ++ observation: None, ++ #[cfg(feature = "lifecycle-trace")] ++ from: 0, ++ }) as Box + Send>) ++ .unwrap(); + } -+ rt::Runtime::new().unwrap().block_on(async { -+ let (tx, rx) = mpsc::channel(); -+ let token = StopToken::new(); -+ let (ctx, mut cleanup) = test_context(&tx, &token); -+ let count = Arc::new(AtomicUsize::new(0)); -+ for _ in 0..2 { -+ tx.send(Box::new(MessageEnvelope { -+ msg: Some(Inc), -+ slot: None, -+ #[cfg(feature = "metrics")] -+ observation: None, -+ #[cfg(feature = "lifecycle-trace")] -+ from: 0, -+ }) -+ as Box + Send>) -+ .unwrap(); -+ } -+ let turn = MessageTurn { -+ limit: 1, -+ adaptive: false, -+ slice: Some(Duration::from_micros(50)), -+ completed: 0, -+ began: None, -+ }; -+ let mut job = Box::pin(run_actor( -+ Count(count.clone()), -+ ctx, -+ rx, -+ token.clone(), -+ &mut cleanup, -+ Some(turn), -+ )); -+ assert!(futures::poll!(job.as_mut()).is_pending()); -+ assert_eq!(count.load(Ordering::SeqCst), 1); -+ token.cancel(); -+ job.await; -+ assert_eq!(cleanup.die(), ExitReason::Normal); -+ assert_eq!(count.load(Ordering::SeqCst), 1); -+ }); -+ }, -+ ); -+ } ++ let turn = MessageTurn { ++ limit: 1, ++ adaptive: false, ++ slice: Some(Duration::from_micros(50)), ++ completed: 0, ++ began: None, ++ }; ++ let mut job = Box::pin(run_actor( ++ Count(count.clone()), ++ ctx, ++ rx, ++ token.clone(), ++ &mut cleanup, ++ Some(turn), ++ )); ++ assert!(futures::poll!(job.as_mut()).is_pending()); ++ assert_eq!(count.load(Ordering::SeqCst), 1); ++ token.cancel(); ++ job.await; ++ assert_eq!(cleanup.die(), ExitReason::Normal); ++ assert_eq!(count.load(Ordering::SeqCst), 1); ++ }); ++ }, ++ ); +} -+ - #[cfg(test)] - mod idle_stop_regression { - use super::*; -@@ -4468,6 +4771,8 @@ mod idle_stop_regression { - rx, - token.clone(), - &mut cleanup, -+ #[cfg(feature = "backend-experiments")] -+ None, - )); - // Start and park the actual actor loop, without allowing another - // executor task to run it between admission and stop below. -@@ -4889,7 +5194,16 @@ mod stop_receive_probe { - Some(stop_ctx.send_request(Ping, Deadline::Never).unwrap()); - })) - }); -- run_actor(Count(hits.clone()), ctx, rx, token, &mut cleanup).await; -+ run_actor( -+ Count(hits.clone()), -+ ctx, -+ rx, -+ token, -+ &mut cleanup, -+ #[cfg(feature = "backend-experiments")] -+ None, -+ ) -+ .await; - assert_eq!(cleanup.die(), ExitReason::Normal); - let response = response - .borrow_mut() -@@ -4998,6 +5312,8 @@ mod stop_receive_probe { - rx, - token, - &mut cleanup, -+ #[cfg(feature = "backend-experiments")] -+ None, - )); - let expected = if park { - assert!(futures::poll!(actor.as_mut()).is_pending()); diff --git a/rt/Cargo.toml b/rt/Cargo.toml index 7ce38903..ac506c8e 100644 --- a/rt/Cargo.toml diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index da460669..3ed4dfe6 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -31,6 +31,15 @@ use spawned_rt::time::Instant; mod nested; pub use nested::{SupervisedSupervisor, SupervisorSpec}; +mod child; +pub use child::*; +mod state; +pub(crate) use state::*; +mod builder; +pub use builder::*; +mod run; +use run::*; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RestartMode { Permanent, @@ -402,3166 +411,5 @@ impl ChildSpec { } } -struct RecoveryMeta { - id: u64, - origin: usize, - finished: std::sync::atomic::AtomicBool, -} -struct RecoverySlot { - meta: Arc, - stop: CancellationToken, - resume: CancellationToken, - cleaned: watch::Sender, - ready: watch::Sender, -} -impl RecoverySlot { - fn new(meta: Arc) -> Arc { - let (cleaned, _) = watch::channel(false); - let (ready, _) = watch::channel(false); - Arc::new(Self { - meta, - stop: CancellationToken::new(), - resume: CancellationToken::new(), - cleaned, - ready, - }) - } -} -pub(crate) struct RecoveryQueue { - pub(crate) pending: Vec, - pub(crate) covered: Vec, - pub(crate) next_id: u64, -} -impl RecoveryQueue { - /// Opens the next round at the lowest pending origin: clears pending from - /// `start` on, stamps those children as covered by the fresh round id and - /// returns `(origin, start, id)`. - pub(crate) fn select(&mut self, one_for_all: bool) -> Option<(usize, usize, u64)> { - let origin = self.pending.iter().position(|pending| *pending)?; - let start = if one_for_all { 0 } else { origin }; - for pending in &mut self.pending[start..] { - *pending = false; - } - let id = self.next_id; - self.next_id = id.checked_add(1).expect("recovery IDs exhausted"); - self.covered[start..].fill(id); - Some((origin, start, id)) - } - pub(crate) fn request(&mut self, index: usize, after: u64) -> bool { - if after < self.covered[index] { - return false; - } - self.pending[index] = true; - true - } -} -struct ViewData { - recovery: Option>, - snapshot: ChildSnapshot, - current: Option>, - scope: Option>, -} -struct View { - index: usize, - data: Mutex>, - gate: Arc, - failed: Arc, - changed: watch::Sender, - initial: watch::Sender, - stop_turn: CancellationToken, - done: watch::Sender, - shutdown_timeout: Duration, - /// 1-based declaration index: the model's child id in lifecycle traces. - trace_index: usize, -} -impl View { - fn unavailable(&self) -> ChildUnavailable { - if self.failed.load(std::sync::atomic::Ordering::Acquire) { - ChildUnavailable(ChildState::Failed) - } else { - let state = system::lock(&self.data).snapshot.state; - ChildUnavailable( - if matches!(state, ChildState::Stopped | ChildState::Failed) { - state - } else { - ChildState::Stopping - }, - ) - } - } - fn change(&self, f: impl FnOnce(&mut ViewData)) { - { - f(&mut system::lock(&self.data)); - } - self.changed.send_modify(|n| *n = n.wrapping_add(1)); - } - fn recovery(&self, after: u64) -> Option> { - system::lock(&self.data) - .recovery - .clone() - .filter(|r| r.meta.id > after) - } - async fn next_recovery(&self, after: u64) -> Arc { - let mut changes = self.changed.subscribe(); - loop { - changes.borrow_and_update(); - if let Some(round) = self.recovery(after) { - return round; - } - let _ = changes.changed().await; - } - } - async fn recovery_stopping(&self, after: u64) { - self.next_recovery(after).await.stop.cancelled().await; - } - fn finish_unless_recovering(&self, after: u64, failed: bool) -> bool { - { - let mut data = system::lock(&self.data); - if !failed && data.recovery.as_ref().is_some_and(|r| r.meta.id > after) { - return false; - } - data.current = None; - data.scope = None; - data.snapshot.state = if failed || data.snapshot.state == ChildState::Failed { - ChildState::Failed - } else { - ChildState::Stopped - }; - } - self.changed.send_modify(|n| *n = n.wrapping_add(1)); - self.done.send_replace(true); - true - } - /// Cleanup stuck past its deadlines: the snapshot settles as Failed, - /// which releases a waiting `start()` (S2), while the controller keeps - /// awaiting the real join. - fn mark_stuck(&self) { - self.change(|data| { - data.current = None; - data.snapshot.state = ChildState::Failed; - }); - } - /// Terminal, or settled as Failed by a stuck cleanup. - async fn settled(&self) { - let mut changes = self.changed.subscribe(); - loop { - changes.borrow_and_update(); - let state = system::lock(&self.data).snapshot.state; - if matches!(state, ChildState::Stopped | ChildState::Failed) || *self.done.borrow() { - return; - } - if changes.changed().await.is_err() { - return; - } - } - } - fn finish(&self, failed: bool) { - self.change(|data| { - data.current = None; - data.scope = None; - data.snapshot.state = if failed || data.snapshot.state == ChildState::Failed { - ChildState::Failed - } else { - ChildState::Stopped - }; - }); - self.done.send_replace(true); - } -} - -/// Explicit lookup of a replacement, not a rebound ActorRef. Resolution can -/// race with a subsequent crash; callers must still handle delivery errors. -pub struct SupervisedChild { - view: Arc>, -} -impl Clone for SupervisedChild { - fn clone(&self) -> Self { - Self { - view: self.view.clone(), - } - } -} -impl SupervisedChild { - pub fn snapshot(&self) -> ChildSnapshot { - system::lock(&self.view.data).snapshot.clone() - } - pub fn current(&self) -> Result, ChildUnavailable> { - self.view - .gate - .if_open(|| { - let data = system::lock(&self.view.data); - if data.scope.as_ref().is_some_and(|s| s.token.is_cancelled()) { - return Err(ChildUnavailable(ChildState::Stopping)); - } - data.current - .clone() - .ok_or(ChildUnavailable(data.snapshot.state)) - }) - .unwrap_or_else(|| Err(self.view.unavailable())) - } - pub async fn wait_ready(&self) -> Result, ChildUnavailable> { - self.wait_for_generation(0).await - } - /// Wait for a ready generation strictly greater than the supplied value. - pub async fn wait_for_generation(&self, after: u64) -> Result, ChildUnavailable> { - let mut changes = self.view.changed.subscribe(); - loop { - changes.borrow_and_update(); - let snapshot = self.snapshot(); - if snapshot.generation > after { - if let Ok(actor) = self.current() { - return Ok(actor); - } - } - if matches!(snapshot.state, ChildState::Stopped | ChildState::Failed) { - return Err(ChildUnavailable(snapshot.state)); - } - if !self.view.gate.is_open() { - return Err(self.view.unavailable()); - } - if changes.changed().await.is_err() { - return Err(ChildUnavailable(ChildState::Stopped)); - } - } - } -} - -trait ChildControl: Send + Sync { - fn assign_recovery(&self, slot: Arc) -> bool; - fn stop_generation(&self, slot: &RecoverySlot); - fn snapshot(&self) -> ChildSnapshot; - fn close_admission(&self); - fn stop(&self); - fn done(&self) -> watch::Receiver; - fn initial(&self) -> watch::Receiver; - fn shutdown_timeout(&self) -> Duration; - fn finish_unstarted(&self, failed: bool); - fn settled(&self) -> BoxFuture<'_, ()>; -} -impl ChildControl for View { - fn assign_recovery(&self, slot: Arc) -> bool { - let old = { - let mut data = system::lock(&self.data); - if matches!( - data.snapshot.state, - ChildState::Stopped | ChildState::Failed - ) { - return false; - } - let old = data.recovery.replace(slot); - data.current = None; - data.snapshot.state = ChildState::Stopping; - if let Some(scope) = &data.scope { - scope.close_admission(); - } - old - }; - drop(old); - self.changed.send_modify(|n| *n = n.wrapping_add(1)); - true - } - /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit - /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. - fn stop_generation(&self, slot: &RecoverySlot) { - slot.stop.cancel(); - let scope = system::lock(&self.data).scope.clone(); - if let Some(scope) = scope { - scope.signal_parent(ExitReason::Shutdown); - } - } - fn snapshot(&self) -> ChildSnapshot { - system::lock(&self.data).snapshot.clone() - } - fn close_admission(&self) { - let (scope, constructing) = { - let data = system::lock(&self.data); - (data.scope.clone(), data.current.is_none()) - }; - if let Some(scope) = scope { - scope.close_admission(); - if constructing { - scope.stop(ExitReason::Shutdown); - } - } - self.changed.send_modify(|n| *n = n.wrapping_add(1)); - } - /// Supervisor shutdown of this child (V9, reverse order): the parent - /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends - /// `Shutdown` without `stopped()`; a trapping child finishes its earlier - /// messages and runs `stopped()`. The controller kills it at - /// `shutdown_timeout`. - fn stop(&self) { - self.stop_turn.cancel(); - if let Some(scope) = system::lock(&self.data).scope.clone() { - scope.signal_parent(ExitReason::Shutdown); - } - } - fn done(&self) -> watch::Receiver { - self.done.subscribe() - } - fn initial(&self) -> watch::Receiver { - self.initial.subscribe() - } - fn shutdown_timeout(&self) -> Duration { - self.shutdown_timeout - } - fn finish_unstarted(&self, failed: bool) { - if self.snapshot().generation == 0 { - self.finish(failed); - } - } - fn settled(&self) -> BoxFuture<'_, ()> { - Box::pin(View::settled(self)) - } -} - -struct RetryBudget { - attempts: VecDeque, - reserved: usize, - max: usize, - window: Duration, -} -impl RetryBudget { - fn available(&self, now: Instant) -> bool { - self.attempts - .iter() - .filter(|&&time| now.saturating_duration_since(time) < self.window) - .count() - .saturating_add(self.reserved) - < self.max - } - fn take(&mut self, now: Instant) -> bool { - admit_in_window( - &mut self.attempts, - self.max.saturating_sub(self.reserved), - self.window, - now, - |now, t| now.saturating_duration_since(t), - ) - } -} - -/// Rolling-window rate limit: forget stamps at least `window` old, then admit -/// `now` only while fewer than `max` remain. `since(now, t)` is the elapsed -/// time from `t` to `now`, generic so the window can be checked without clocks. -pub(crate) fn admit_in_window( - attempts: &mut VecDeque, - max: usize, - window: Duration, - now: T, - since: impl Fn(T, T) -> Duration, -) -> bool { - while attempts.front().is_some_and(|t| since(now, *t) >= window) { - attempts.pop_front(); - } - if attempts.len() >= max { - return false; - } - attempts.push_back(now); - true -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FailureKind { - Other, - /// A child's first start failed (S1). - FirstStart, - Cleanup, - Coordinator, - RestartBudget, -} -#[derive(Debug, Clone)] -struct FailureOrigin { - child: String, - generation: u64, - reason: Option, - last_error: Option, -} -#[derive(Debug, Clone)] -struct Failure { - kind: FailureKind, - message: String, - origin: Option, - logged: bool, -} -impl Failure { - fn new(kind: FailureKind, message: String, origin: Option) -> Self { - Self { - kind, - message, - origin, - logged: false, - } - } -} -impl std::ops::Deref for Failure { - type Target = str; - fn deref(&self) -> &str { - &self.message - } -} - -struct Core { - recovery: Mutex, - recovery_changed: watch::Sender, - gate: Arc, - cancel: CancellationToken, - admission_closed: CancellationToken, - done: watch::Sender, - ready: watch::Sender, - failure: Mutex>, - failed: Arc, - cleanup_failure: Mutex>, - /// Every recorded cleanup failure, for the rollback list of a failed - /// start (S2). - cleanup_failures: Mutex>, - children: Mutex>>, - budget: Mutex, - config: SupervisorConfig, - trace: TraceId, - #[cfg(test)] - initial_ready_pause: Mutex)>>, - #[cfg(test)] - retry_admission_pause: Mutex)>>, - #[cfg(feature = "lifecycle-trace")] - deadline_recorded: std::sync::atomic::AtomicBool, -} -impl Core { - fn close_admission(&self) { - trace::atomic(|| { - if self.gate.is_open() { - self.cohort(|| Event::sup("CClose")); - trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); - } - #[cfg(feature = "lifecycle-trace")] - if self.gate.is_open() && !trace::close_suppressed() { - trace::emit(self.trace, || Event::sup("Close")); - } - self.gate.close(); - }); - self.admission_closed.cancel(); - let children = system::lock(&self.children).clone(); - for child in children { - child.close_admission(); - } - } - fn close(&self) { - trace::atomic(|| { - self.close_admission(); - if !self.cancel.is_cancelled() { - trace::tree(self.trace, |id| TreeEvent::new("TCancel", id)); - self.cancel.cancel(); - } - }); - } - fn cancel_unready_startup(&self) { - trace::atomic(|| { - let cancel = self.gate.close_unless( - || *self.ready.borrow(), - || { - self.cohort(|| Event::sup("CClose")); - trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); - #[cfg(feature = "lifecycle-trace")] - if !trace::close_suppressed() { - trace::emit(self.trace, || Event::sup("Close")); - } - }, - ); - if cancel { - self.close(); - } - }); - } - fn stop_remaining(&self) { - trace::atomic(|| { - self.close(); - #[cfg(feature = "lifecycle-trace")] - if !*self.done.borrow() - && !self - .deadline_recorded - .swap(true, std::sync::atomic::Ordering::Relaxed) - { - trace::emit(self.trace, || Event::sup("ShutdownDeadline")); - } - }); - let children = system::lock(&self.children).clone(); - for child in children.iter().rev() { - child.stop(); - } - } - #[cfg(test)] - fn fail(&self, error: String) { - self.fail_kind(FailureKind::Other, error); - } - fn fail_kind(&self, kind: FailureKind, error: String) { - self.fail_record(Failure::new(kind, error, None)); - } - fn fail_record(&self, record: Failure) { - let mut failure = system::lock(&self.failure); - if failure.is_none() { - *failure = Some(record); - } - self.failed - .store(true, std::sync::atomic::Ordering::Release); - drop(failure); - self.close(); - } - #[inline(always)] - fn cohort(&self, make: impl FnOnce() -> Event) { - #[cfg(feature = "lifecycle-trace")] - if self.config.strategy != SupervisorStrategy::OneForOne { - trace::emit(self.trace, make); - } - #[cfg(not(feature = "lifecycle-trace"))] - let _ = make; - } - fn record_cleanup_failure(&self, error: &str) { - system::lock(&self.cleanup_failures).push(error.to_string()); - let mut failure = system::lock(&self.cleanup_failure); - if failure.is_none() { - *failure = Some(error.to_string()); - } - } - fn fail_cleanup(&self, error: String) { - self.record_cleanup_failure(&error); - self.fail_kind(FailureKind::Cleanup, error); - } - fn fail_cleanup_for( - &self, - snapshot: ChildSnapshot, - error: String, - event: impl FnOnce() -> Option, - ) { - self.record_cleanup_failure(&error); - let record = snapshot_failure(snapshot, FailureKind::Cleanup, error); - trace::failing(|| { - trace::emit_some(self.trace, event); - self.fail_record(record); - }); - } - fn request_recovery(&self, index: usize, after: u64) { - trace::atomic(|| { - self.gate.if_open(|| { - let mut queue = system::lock(&self.recovery); - let accepted = queue.request(index, after); - self.cohort(|| { - Event::new("CRequest", index + 1, trace::flag(accepted)).round(after) - }); - if accepted { - self.recovery_changed - .send_modify(|n| *n = n.wrapping_add(1)); - } - }) - }); - } - /// `fail` as part of the model action `event` (whose `Fail` closes the - /// gate), so the gate flip is not traced as a separate `Close`. - fn fail_as_kind( - &self, - kind: FailureKind, - error: String, - event: impl FnOnce() -> Option, - ) { - trace::failing(|| { - trace::emit_some(self.trace, event); - self.fail_kind(kind, error); - }); - } - fn log_failure_once(&self) { - let failure = { - let mut slot = system::lock(&self.failure); - let Some(failure) = slot.as_mut() else { - return; - }; - if failure.logged { - return; - } - failure.logged = true; - failure.clone() - }; - let (child, generation, reason, last_error) = failure - .origin - .as_ref() - .map(|origin| { - ( - origin.child.as_str(), - origin.generation, - origin.reason.as_ref(), - origin.last_error.as_deref(), - ) - }) - .unwrap_or(("", 0, None, None)); - let event = if failure.kind == FailureKind::RestartBudget { - "restart_budget_exhausted" - } else { - "supervision_give_up" - }; - if let Err(payload) = std::panic::catch_unwind(AssertUnwindSafe(|| { - tracing::warn!(event, child, generation, ?reason, ?last_error, - kind = ?failure.kind, error = %failure.message, "supervisor failed"); - })) { - crate::mailbox::dispose_panic(payload); - } - } - /// A concurrent ordinary stop wins over a later child failure. The gate - /// lock, rather than optional trace instrumentation, commits this decision. - #[cfg(test)] - fn fail_if_open_as(&self, error: String, event: impl FnOnce() -> Option) -> bool { - self.fail_record_if_open_as(Failure::new(FailureKind::Other, error, None), event) - } - fn fail_record_if_open_as( - &self, - record: Failure, - event: impl FnOnce() -> Option, - ) -> bool { - trace::failing(|| { - let committed = self.gate.close_if_open(|| { - trace::emit_some(self.trace, event); - self.cohort(|| Event::sup("CClose")); - trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); - let mut failure = system::lock(&self.failure); - if failure.is_none() { - *failure = Some(record); - } - self.failed - .store(true, std::sync::atomic::Ordering::Release); - }); - if committed { - self.close(); - } - committed - }) - } - fn reserve_retry(&self, child: usize) -> Option> { - trace::atomic(|| { - self.gate - .if_open(|| { - let mut budget = system::lock(&self.budget); - let now = Instant::now(); - // Pending backoffs reserve capacity, so siblings cannot all sleep - // against the same final permit. Only actual grants enter the window. - while budget - .attempts - .front() - .is_some_and(|time| now.saturating_duration_since(*time) >= budget.window) - { - budget.attempts.pop_front(); - } - if !budget.available(now) { - return None; - } - budget.reserved += 1; - trace::emit(self.trace, || { - Event::new("RetryReserve", child, "") - .count(budget.attempts.len() + budget.reserved) - }); - Some(RetryReservation { - core: self, - child, - held: true, - }) - }) - .flatten() - }) - } - /// One OneForAll/RestForOne recovery round is one restart action (OTP's - /// `add_restart` runs once per restart, before the cohort is rebuilt): - /// take its grant now. The round's controllers then rebuild without a - /// grant of their own. Pending per-child reservations (retries during - /// the initial start) keep their capacity. Returns whether it was taken. - fn charge_round(&self) -> bool { - let mut budget = system::lock(&self.budget); - budget.take(Instant::now()) - } - /// Caller holds the admission gate and slot lock; charge atomically with - /// generation admission, so placement failure or supersession refunds it. - fn retry_under_gate(&self, reservation: &mut RetryReservation<'_>) -> bool { - let mut budget = system::lock(&self.budget); - budget.reserved -= 1; - reservation.held = false; - let taken = budget.take(Instant::now()); - debug_assert!(taken, "reserved retry lost its capacity"); - if taken { - trace::emit(self.trace, || { - Event::new("BackoffDone", reservation.child, "retry") - .count(budget.attempts.len() + budget.reserved) - }); - } - taken - } - #[cfg(test)] - fn retry(&self, reservation: &mut RetryReservation<'_>) -> bool { - self.gate - .if_open(|| self.retry_under_gate(reservation)) - .unwrap_or(false) - } -} -struct RetryReservation<'a> { - core: &'a Core, - child: usize, - held: bool, -} -impl Drop for RetryReservation<'_> { - fn drop(&mut self) { - if self.held { - trace::atomic(|| { - let mut budget = system::lock(&self.core.budget); - budget.reserved -= 1; - trace::emit(self.core.trace, || { - Event::new("RetryRelease", self.child, "") - .count(budget.attempts.len() + budget.reserved) - }); - }); - } - } -} -impl ShutdownUnit for Core { - fn close_admission(&self) { - Core::close_admission(self); - } - fn stop(&self) { - Core::close(self); - } - fn stop_remaining(&self) { - Core::stop_remaining(self); - } - fn joined(&self) -> BoxFuture<'_, ()> { - Box::pin(async move { - let _ = system::wait_true(self.done.subscribe()).await; - }) - } - fn is_done(&self) -> bool { - *self.done.borrow() - } - fn failure(&self) -> Option { - system::lock(&self.failure) - .as_ref() - .map(|f| f.message.clone()) - } -} - -trait Runner: Send { - fn run(self: Box, core: Arc) -> BoxFuture<'static, ()>; -} -struct TypedRunner { - spec: ChildSpec, - view: Arc>, -} -impl Runner for TypedRunner { - fn run(self: Box, core: Arc) -> BoxFuture<'static, ()> { - let view = self.view.clone(); - Box::pin(async move { - if let Err(payload) = AssertUnwindSafe(run_child(self.spec, self.view, core.clone())) - .catch_unwind() - .await - { - let message = format!( - "child controller panicked: {}", - crate::error::panic_message(&*payload) - ); - crate::mailbox::dispose_panic(payload); - core.fail_cleanup_for(view.snapshot(), message, || { - Some(Event::new( - "Unmodelled", - view.trace_index, - "ControllerPanicked", - )) - }); - let scope = system::lock(&view.data).scope.clone(); - if let Some(scope) = scope { - scope.stop(ExitReason::Shutdown); - scope.joined().await; - } - view.finish(true); - } - }) - } -} - -pub struct SupervisorBuilder { - owner: Weak, - core: Arc, - runners: Vec>, - started: bool, -} -impl SupervisorBuilder { - pub(crate) fn new( - system: &ActorSystem, - config: SupervisorConfig, - ) -> Result { - Self::from_owner(Arc::downgrade(&system.inner), config) - } - fn from_owner( - owner: Weak, - config: SupervisorConfig, - ) -> Result { - if config.restart_window.is_zero() - || config.max_children == 0 - || config.max_children > 65536 - || config.max_restarts > 65536 - || config.backoff.maximum < config.backoff.initial - { - return Err(SupervisorError::Configuration( - "invalid limits or backoff".into(), - )); - } - let (done, _) = watch::channel(false); - let (ready, _) = watch::channel(false); - let (recovery_changed, _) = watch::channel(0); - let core = Arc::new(Core { - recovery: Mutex::new(RecoveryQueue { - pending: Vec::new(), - covered: Vec::new(), - next_id: 1, - }), - recovery_changed, - gate: Admission::new(), - cancel: CancellationToken::new(), - admission_closed: CancellationToken::new(), - done, - ready, - failure: Mutex::new(None), - failed: Arc::new(std::sync::atomic::AtomicBool::new(false)), - cleanup_failure: Mutex::new(None), - cleanup_failures: Mutex::new(Vec::new()), - children: Mutex::new(Vec::new()), - budget: Mutex::new(RetryBudget { - attempts: VecDeque::new(), - reserved: 0, - max: config.max_restarts, - window: config.restart_window, - }), - config, - trace: TraceId::next(), - #[cfg(test)] - initial_ready_pause: Mutex::new(None), - #[cfg(test)] - retry_admission_pause: Mutex::new(None), - #[cfg(feature = "lifecycle-trace")] - deadline_recorded: Default::default(), - }); - #[cfg(feature = "lifecycle-trace")] - trace::describe( - core.trace, - match core.config.strategy { - SupervisorStrategy::OneForOne => "OneForOne", - SupervisorStrategy::OneForAll => "OneForAll", - SupervisorStrategy::RestForOne => "RestForOne", - }, - core.config.max_restarts, - ); - Ok(Self { - owner, - core, - runners: Vec::new(), - started: false, - }) - } - - /// Lifecycle trace id of this supervisor (`lifecycle_trace::take`). - #[cfg(feature = "lifecycle-trace")] - #[doc(hidden)] - pub fn trace_id(&self) -> u64 { - self.core.trace.0 - } - - pub fn add_supervisor( - &mut self, - spec: SupervisorSpec, - ) -> Result { - nested::add(self, spec) - } - pub fn add( - &mut self, - spec: ChildSpec, - ) -> Result, SupervisorError> { - self.owner - .upgrade() - .ok_or(SystemError::Closed)? - .accepts_group(&spec.group)?; - if spec.name.is_empty() - || spec.name.contains('\0') - || spec.startup_timeout.is_zero() - || spec.shutdown_timeout.is_zero() - { - return Err(SupervisorError::Configuration( - "invalid child name or timeout".into(), - )); - } - let mut children = system::lock(&self.core.children); - if children.len() >= self.core.config.max_children { - return Err(SupervisorError::Capacity); - } - if children.iter().any(|c| c.snapshot().name == spec.name) { - return Err(SupervisorError::DuplicateChild(spec.name)); - } - let (changed, _) = watch::channel(0); - let (initial, _) = watch::channel(false); - let (done, _) = watch::channel(false); - let view = Arc::new(View { - index: children.len(), - data: Mutex::new(ViewData { - recovery: None, - snapshot: ChildSnapshot { - name: spec.name.clone(), - state: ChildState::Waiting, - generation: 0, - actor_id: None, - last_exit: None, - last_error: None, - }, - current: None, - scope: None, - }), - gate: self.core.gate.clone(), - failed: self.core.failed.clone(), - changed, - initial, - done, - stop_turn: CancellationToken::new(), - shutdown_timeout: spec.shutdown_timeout, - trace_index: children.len() + 1, - }); - children.push(view.clone()); - #[cfg(feature = "lifecycle-trace")] - trace::describe_child( - self.core.trace, - match spec.restart { - RestartMode::Permanent => "Permanent", - RestartMode::Transient => "Transient", - RestartMode::Temporary => "Temporary", - }, - ); - { - let mut queue = system::lock(&self.core.recovery); - queue.pending.push(false); - queue.covered.push(0); - } - self.runners.push(Box::new(TypedRunner { - spec, - view: view.clone(), - })); - Ok(SupervisedChild { view }) - } - - /// Start children in declaration order. Dropping this future before readiness - /// commits requests cleanup. After readiness, ActorSystem owns the tree even - /// if this future is dropped before returning its Supervisor handle. - pub async fn start(self) -> Result { - self.start_owned(true).await - } - async fn start_owned(mut self, register_root: bool) -> Result { - let owner = self.owner.upgrade().ok_or(SystemError::Closed)?; - // Registration and controller dispatch happen without an intervening await. - if register_root { - owner.register_unit(self.core.clone())?; - } - trace::tree(self.core.trace, |id| { - TreeEvent::new("TCreate", id).peer(0, self.runners.len()) - }); - self.started = true; - let runners = std::mem::take(&mut self.runners); - let core = self.core.clone(); - owner - .control - .handle - .spawn(run_supervisor(core.clone(), runners)); - let mut guard = StartupGuard { - core: core.clone(), - armed: true, - }; - let ready = core.ready.subscribe(); - let cancelled = core.cancel.cancelled(); - let _ = select(Box::pin(cancelled), Box::pin(system::wait_true(ready))).await; - // Readiness is historical, just like ActorRef::ready(). A later system - // admission close must not turn successful startup into rollback. - let initialized = *core.ready.borrow(); - if initialized { - guard.armed = false; - return Ok(Supervisor { core: core.clone() }); - } - // S2: the error is returned only after the rollback finished: every - // child is terminal, or settled as Failed by its stuck cleanup. - let children = system::lock(&core.children).clone(); - for child in &children { - child.settled().await; - } - Err(core.start_error()) - } -} -impl Core { - /// The cause of a failed initial start (S2), with the rollback's cleanup - /// failures listed separately. - fn start_error(&self) -> SupervisorError { - let Some(failure) = system::lock(&self.failure).clone() else { - return SupervisorError::Stopped; - }; - let rollback_failures: Vec = system::lock(&self.cleanup_failures) - .iter() - .filter(|f| **f != failure.message) - .cloned() - .collect(); - match (failure.kind, failure.origin) { - (FailureKind::FirstStart, Some(origin)) => { - let prefix = format!("failed to start child {}: ", origin.child); - let cause = failure - .message - .strip_prefix(&prefix) - .unwrap_or(&failure.message) - .to_string(); - SupervisorError::FailedToStartChild { - child: origin.child, - cause, - rollback_failures, - } - } - (FailureKind::Cleanup, Some(origin)) => SupervisorError::StartupCleanupStuck { - child: origin.child, - cause: failure.message, - rollback_failures, - }, - _ => SupervisorError::Failed(failure.message), - } - } -} -impl Drop for SupervisorBuilder { - fn drop(&mut self) { - if !self.started { - self.core.close(); - for child in system::lock(&self.core.children).iter() { - child.finish_unstarted(self.core.failed.load(std::sync::atomic::Ordering::Acquire)); - } - trace::atomic(|| { - trace::tree(self.core.trace, |id| TreeEvent::new("TDone", id)); - self.core.done.send_replace(true); - }); - } - } -} -struct StartupGuard { - core: Arc, - armed: bool, -} -impl Drop for StartupGuard { - fn drop(&mut self) { - if self.armed { - self.core.cancel_unready_startup(); - } - } -} - -/// Owned by ActorSystem even if public handles are dropped. -#[derive(Clone)] -pub struct Supervisor { - core: Arc, -} -#[derive(Debug, Clone)] -#[non_exhaustive] -pub struct SupervisorReport { - pub complete: bool, - pub failure: Option, - pub children: Vec, -} -impl Supervisor { - pub fn stop(&self) { - self.core.close(); - } - /// Lifecycle trace id of this supervisor (`lifecycle_trace::take`). - #[cfg(feature = "lifecycle-trace")] - #[doc(hidden)] - pub fn trace_id(&self) -> u64 { - self.core.trace.0 - } - pub fn snapshots(&self) -> Vec { - system::lock(&self.core.children) - .iter() - .map(|c| c.snapshot()) - .collect() - } - pub fn failure(&self) -> Option { - system::lock(&self.core.failure) - .as_ref() - .map(|f| f.message.clone()) - } - pub async fn join(&self) { - let _ = system::wait_true(self.core.done.subscribe()).await; - } - pub async fn shutdown(&self, grace: Duration) -> SupervisorReport { - self.stop(); - let complete = rt::timeout(grace, system::wait_true(self.core.done.subscribe())) - .await - .unwrap_or(false); - if !complete { - self.core.stop_remaining(); - } - SupervisorReport { - complete, - failure: self.failure(), - children: self.snapshots(), - } - } -} - -async fn run_supervisor(core: Arc, runners: Vec>) { - // All coordinator phases share a boundary. Keep ownership and join progress - // outside it so a panic during startup, stopping, joining or final publication - // cannot detach a controller or lose the runtime/subtree completion signal. - let mut runners = Some(runners); - let mut tasks = Vec::new(); - let mut joined = 0; - let mut coordination = None; - let mut emergency = false; - loop { - let result = AssertUnwindSafe(async { - if let Some(runners) = runners.take() { - run_supervisor_body(core.clone(), runners, &mut tasks, &mut coordination).await; - } - if emergency { - let children = system::lock(&core.children).clone(); - let mut signalled = true; - for child in children.iter().rev() { - if *child.done().borrow() { - continue; - } - if let Err(payload) = std::panic::catch_unwind(AssertUnwindSafe(|| { - child.finish_unstarted(true); - child.stop(); - })) { - let message = crate::error::panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - core.record_cleanup_failure(&format!( - "emergency child stop panicked: {message}" - )); - signalled = false; - } - } - if !signalled { - return false; - } - } - while joined < tasks.len() { - let result = (&mut tasks[joined]).await; - joined += 1; // Never poll a consumed JoinHandle after a later panic. - if let Err(error) = result { - let message = format!("child controller failed: {error}"); - if error.is_panic() { - crate::mailbox::dispose_panic(error.into_panic()); - } - core.record_cleanup_failure(&message); - core.fail_as_kind(FailureKind::Coordinator, message, || { - Some(Event::new("Unmodelled", 0, "ControllerFailed")) - }); - } - } - if let Some(task) = coordination.as_mut() { - let result = task.await; - coordination = None; - if let Err(error) = result { - let message = format!("recovery coordinator failed: {error}"); - if error.is_panic() { - crate::mailbox::dispose_panic(error.into_panic()); - } - core.record_cleanup_failure(&message); - core.fail_kind(FailureKind::Coordinator, message); - } - } - let children = system::lock(&core.children).clone(); - for child in &children { - child.finish_unstarted(core.failed.load(std::sync::atomic::Ordering::Acquire)); - } - core.log_failure_once(); - trace::atomic(|| { - trace::emit(core.trace, || Event::sup("SupJoin")); - core.cohort(|| Event::sup("CDone")); - trace::tree(core.trace, |id| TreeEvent::new("TDone", id)); - // Copy the tapes before `done` lets a test `take` them. - #[cfg(feature = "lifecycle-trace")] - trace::dump_all(core.trace); - core.done.send_replace(true); - }); - true - }) - .catch_unwind() - .await; - match result { - Ok(true) => return, - Ok(false) => (), - Err(payload) => { - let message = crate::error::panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - let message = format!("supervisor coordinator panicked: {message}"); - core.record_cleanup_failure(&message); - core.fail_kind(FailureKind::Coordinator, message); - emergency = true; - } - } - rt::sleep(Duration::from_millis(10)).await; - } -} - -async fn run_supervisor_body( - core: Arc, - runners: Vec>, - tasks: &mut Vec>, - coordination: &mut Option>, -) { - let children = system::lock(&core.children).clone(); - *coordination = if core.config.strategy == SupervisorStrategy::OneForOne { - None - } else { - let coordinator = core.clone(); - Some(rt::spawn(async move { - if let Err(payload) = AssertUnwindSafe(run_recoveries(coordinator.clone())) - .catch_unwind() - .await - { - let message = format!( - "recovery coordinator panicked: {}", - crate::error::panic_message(&*payload) - ); - crate::mailbox::dispose_panic(payload); - coordinator.record_cleanup_failure(&message); - coordinator.fail_kind(FailureKind::Coordinator, message); - coordinator.stop_remaining(); - } - })) - }; - for (index, runner) in runners.into_iter().enumerate() { - let open = trace::atomic(|| { - let open = core.gate.is_open(); - if open { - trace::emit(core.trace, || Event::new("SupLaunch", index + 1, "")); - } - open - }); - if !open { - break; - } - tasks.push(rt::spawn(runner.run(core.clone()))); - let event = select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(system::wait_true(children[index].initial())), - ) - .await; - if !matches!(event, Either::Right((true, _))) { - break; - } - } - for child in &children { - child.finish_unstarted(core.failed.load(std::sync::atomic::Ordering::Acquire)); - } - let initialized = trace::atomic(|| { - let committed = core.gate.if_open(|| { - trace::emit(core.trace, || Event::new("SupStartDone", 0, "true")); - core.cohort(|| Event::sup("CInitialReady")); - core.ready.send_replace(true); - }); - if committed.is_none() { - trace::emit(core.trace, || Event::new("SupStartDone", 0, "false")); - } - committed.is_some() - }); - - #[cfg(test)] - let ready_probe = { - let pause = system::lock(&core.initial_ready_pause).take(); - if let Some((resume, passed)) = pause { - resume.cancelled().await; - Some(passed) - } else { - None - } - }; - // Once readiness commits, later admission closure is not a stop turn. - // ActorSystem owns cross-target ordering and cancels this tree in its turn. - if !initialized { - core.close(); - } - #[cfg(test)] - if let Some(passed) = ready_probe { - passed.send_replace(true); - } - core.cancel.cancelled().await; - trace::emit(core.trace, || Event::sup("SupCancelSeen")); - for (index, child) in children.iter().enumerate().rev() { - let c = index + 1; - trace::emit(core.trace, || Event::new("SupStopTurn", c, "")); - fail_point!("shutdown.before_stop_child", async); - child.stop(); - let finished = rt::timeout(child.shutdown_timeout(), system::wait_true(child.done())) - .await - .is_ok(); - // The child may finish between the deadline firing and this check; - // decide on its published completion, atomically with recording. - trace::atomic(|| { - if finished || *child.done().borrow() { - trace::emit(core.trace, || Event::new("SupChildDone", c, "")); - } else { - core.fail_cleanup_for( - child.snapshot(), - format!("child cleanup deadline exceeded: {}", child.snapshot().name), - || Some(Event::new("SupChildTimeout", c, "")), - ); - } - }); - } -} - -async fn run_recoveries(core: Arc) { - let children = system::lock(&core.children).clone(); - let mut changes = core.recovery_changed.subscribe(); - 'rounds: loop { - changes.borrow_and_update(); - if !core.gate.is_open() { - return; - } - let round = trace::atomic(|| { - let mut queue = system::lock(&core.recovery); - // The round is the restart action: charge it once, before any - // child of the cohort is stopped or rebuilt. - let origin = queue.pending.iter().position(|pending| *pending)?; - if !core.charge_round() { - return Some(Err(origin)); - } - queue - .select(core.config.strategy == SupervisorStrategy::OneForAll) - .map(|(origin, start, id)| { - let meta = Arc::new(RecoveryMeta { - id, - origin, - finished: std::sync::atomic::AtomicBool::new(false), - }); - core.cohort(|| { - Event::new("CSelect", origin + 1, "") - .round(id) - .count(start + 1) - }); - Ok((start, meta)) - }) - }); - if let Some(Err(origin)) = round { - // Exhausted: a concurrent stop wins; only an open supervisor fails. - let snapshot = children[origin].snapshot(); - let message = format!("restart budget exhausted by {}", snapshot.name); - core.fail_record_if_open_as( - snapshot_failure(snapshot, FailureKind::RestartBudget, message), - || None, - ); - core.log_failure_once(); - return; - } - let Some(Ok((start, meta))) = round else { - if matches!( - select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(changes.changed()) - ) - .await, - Either::Left(_) - ) { - return; - } - continue; - }; - let mut affected = Vec::new(); - for (affected_index, child) in children.iter().enumerate().skip(start) { - let slot = RecoverySlot::new(meta.clone()); - let accepted = trace::atomic(|| { - let accepted = child.assign_recovery(slot.clone()); - core.cohort(|| { - Event::new("CAssign", affected_index + 1, trace::flag(accepted)).round(meta.id) - }); - accepted - }); - if accepted { - affected.push((affected_index + 1, child.clone(), slot)); - } - } - for (index, child, slot) in affected.iter().rev() { - if !core.gate.is_open() { - return; - } - trace::atomic(|| { - child.stop_generation(slot); - core.cohort(|| Event::new("CStop", *index, "").round(meta.id)); - }); - match select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(rt::timeout( - child.shutdown_timeout(), - system::wait_true(slot.cleaned.subscribe()), - )), - ) - .await - { - Either::Left(_) => return, - Either::Right((Ok(true), _)) => (), - _ => { - core.fail_cleanup_for( - child.snapshot(), - format!("coordinated cleanup stuck for {}", child.snapshot().name), - || None, - ); - core.stop_remaining(); - return; - } - } - } - for (index, _, slot) in &affected { - if !core.gate.is_open() { - return; - } - if system::lock(&core.recovery) - .pending - .iter() - .any(|pending| *pending) - { - trace::atomic(|| { - core.cohort(|| Event::new("CEnd", 0, "superseded").round(meta.id)); - meta.finished - .store(true, std::sync::atomic::Ordering::Release); - }); - continue 'rounds; - } - trace::atomic(|| { - core.cohort(|| Event::new("CResume", *index, "").round(meta.id)); - slot.resume.cancel(); - }); - loop { - changes.borrow_and_update(); - // A ready upstream or previously unaffected dependency can fail - // while this child initializes. Supersede the round rather than - // await a dependent that cannot start without that upstream. - if system::lock(&core.recovery) - .pending - .iter() - .any(|pending| *pending) - { - trace::atomic(|| { - core.cohort(|| Event::new("CEnd", 0, "superseded").round(meta.id)); - meta.finished - .store(true, std::sync::atomic::Ordering::Release); - }); - continue 'rounds; - } - match select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(select( - Box::pin(system::wait_true(slot.ready.subscribe())), - Box::pin(changes.changed()), - )), - ) - .await - { - Either::Left(_) => return, - Either::Right((Either::Left((true, _)), _)) => break, - Either::Right((Either::Left((false, _)), _)) => return, - Either::Right((Either::Right(_), _)) => (), - } - } - } - trace::atomic(|| { - core.cohort(|| Event::new("CEnd", 0, "complete").round(meta.id)); - meta.finished - .store(true, std::sync::atomic::Ordering::Release); - }); - } -} - -async fn run_child(spec: ChildSpec, view: Arc>, core: Arc) { - let mut first = true; - let mut last_recovery = 0; - let mut collateral = false; - // This iteration rebuilds for a recovery round (already charged). - let mut in_round = false; - let mut failures = 0u32; - // Wall-clock entropy; the simulation seed's entropy stream under spawned_sim. - let seed = core - .config - .backoff - .seed - .unwrap_or_else(spawned_rt::time::entropy); - let mut rng = jitter_seed(seed, system::lock(&view.data).snapshot.name.as_bytes()); - let c = view.trace_index; - loop { - let open = trace::atomic(|| { - let open = core.gate.is_open(); - let branch = if !open { - "finish" - } else if first { - "attempt" - } else { - "backoff" - }; - trace::emit(core.trace, || Event::new("LoopTop", c, branch)); - open - }); - if !open { - break; - } - if let Some(round) = view.recovery(last_recovery) { - if matches!( - select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(round.stop.cancelled()) - ) - .await, - Either::Left(_) - ) { - break; - } - trace::atomic(|| { - core.cohort(|| Event::new("CClean", c, "").round(round.meta.id)); - round.cleaned.send_replace(true); - }); - match select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(select( - Box::pin(view.next_recovery(round.meta.id)), - Box::pin(round.resume.cancelled()), - )), - ) - .await - { - Either::Left(_) => break, - Either::Right((Either::Left(_), _)) => continue, - Either::Right((Either::Right(_), _)) => (), - } - last_recovery = round.meta.id; - if spec.restart == RestartMode::Temporary { - let failed = system::lock(&core.failure).is_some(); - // Publish terminal state under the same lock as assignment before - // acknowledging the round. A superseding coordinator must either - // skip this finished slot or be serviced by this controller. - if trace::atomic(|| { - let finished = view.finish_unless_recovering(last_recovery, failed); - if finished { - trace::emit(core.trace, || Event::new("Finish", c, "")); - core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); - } - finished - }) { - trace::atomic(|| { - core.cohort(|| Event::new("CAck", c, "").round(round.meta.id)); - round.ready.send_replace(true); - }); - core.log_failure_once(); - return; - } - continue; - } - collateral = round.meta.origin != view.index; - in_round = true; - } - let mut backoff = Duration::ZERO; - let mut retry_reservation = None; - // A recovery round was charged once when the coordinator selected - // it: its rebuilds (origin and collateral) take no grant of their own. - let round_rebuild = std::mem::take(&mut in_round); - if !first { - fail_point!("supervisor.before_restart", async); - let reservation = if round_rebuild { - Some(None) - } else { - core.reserve_retry(c).map(Some) - }; - let Some(reservation) = reservation else { - // A concurrent stop wins: only fail a supervisor that is still open. - trace::atomic(|| { - if core.gate.is_open() { - core.fail_record_if_open_as( - child_failure( - &spec, - &view, - FailureKind::RestartBudget, - format!("restart budget exhausted by {}", spec.name), - ), - || { - let budget = system::lock(&core.budget); - let held = budget.attempts.len() + budget.reserved; - Some(Event::new("BackoffDone", c, "exhausted").count(held)) - }, - ); - } else { - trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); - } - }); - core.log_failure_once(); - break; - }; - let delay = if collateral { - Duration::ZERO - } else { - core.config.backoff.delay(failures, &mut rng) - }; - backoff = delay; - let count_attempt = !collateral; - collateral = false; - view.change(|v| v.snapshot.state = ChildState::Backoff); - if matches!( - select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(select( - Box::pin(async { - if core.config.strategy == SupervisorStrategy::OneForOne { - futures::future::pending::<()>().await; - } else { - view.recovery_stopping(last_recovery).await; - } - }), - Box::pin(rt::sleep(delay)) - )) - ) - .await, - Either::Left(_) - ) { - drop(reservation); - trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); - break; - } - fail_point!("supervisor.during_backoff", async); - if view.recovery(last_recovery).is_some() { - continue; - } - if count_attempt { - failures = failures.saturating_add(1); - } - retry_reservation = reservation; - } - first = false; - let (scope, lease) = match spec.group.reserve(Some(core.gate.clone())) { - Ok(attempt) => attempt, - Err(e) => { - trace::atomic(|| { - let was_retry = retry_reservation.is_some(); - drop(retry_reservation.take()); - if core.gate.is_open() { - core.fail_record_if_open_as( - child_failure( - &spec, - &view, - if *view.initial.borrow() { - FailureKind::Other - } else { - FailureKind::FirstStart - }, - format!("placement failed for {}: {e}", spec.name), - ), - || Some(Event::new("Unmodelled", c, "PlacementFailed")), - ); - } else { - trace::emit(core.trace, || { - Event::new( - if was_retry { "BackoffDone" } else { "Attempt" }, - c, - if was_retry { "closed" } else { "false" }, - ) - }); - } - }); - break; - } - }; - #[cfg(test)] - if retry_reservation.is_some() || round_rebuild { - let pause = system::lock(&core.retry_admission_pause).take(); - if let Some((resume, entered)) = pause { - entered.send_replace(true); - resume.cancelled().await; - } - } - let generation = trace::atomic(|| { - let was_retry = retry_reservation.is_some(); - let generation = core - .gate - .if_open(|| { - let mut data = system::lock(&view.data); - if data - .recovery - .as_ref() - .is_some_and(|r| r.meta.id > last_recovery) - { - return None; - } - if let Some(reservation) = retry_reservation.as_mut() { - if !core.retry_under_gate(reservation) { - return None; - } - } - data.snapshot.generation += 1; - data.snapshot.state = ChildState::Starting; - data.snapshot.actor_id = None; - data.current = None; - data.scope = Some(scope.clone()); - Some(data.snapshot.generation) - }) - .flatten(); - if generation.is_none() && was_retry { - drop(retry_reservation.take()); - if !core.gate.is_open() { - trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); - } - } - if generation.is_some() || !was_retry { - trace::emit(core.trace, || { - Event::new("Attempt", c, trace::flag(generation.is_some())) - }); - } - if let Some(generation) = generation { - trace::tree(core.trace, |id| { - TreeEvent::new("TBegin", id).child(c, generation) - }); - core.cohort(|| { - Event::new("CBegin", c, "") - .round(last_recovery) - .count(generation as usize) - }); - } - generation - }); - let Some(generation) = generation else { - scope.stop(ExitReason::Shutdown); - drop(lease); - if core.gate.is_open() { - continue; - } else { - break; - } - }; - scope.set_name(&spec.name); - if generation > 1 { - let previous = view.snapshot(); - crate::actor_cleanup::log_safely(|| { - tracing::info!(event = "restart_scheduled", child = %spec.name, - generation = generation - 1, reason = ?previous.last_exit, - error = ?previous.last_error, ?backoff, "child retry granted after backoff"); - }); - crate::actor_cleanup::log_safely(|| { - tracing::info!( - event = "restart_started", - child = %spec.name, - generation, - reason = ?previous.last_exit, - error = ?previous.last_error, - "starting replacement child" - ); - }); - } - view.changed.send_modify(|n| *n = n.wrapping_add(1)); - let ctx = FactoryContext { - scope: scope.clone(), - generation, - }; - let factory = spec.factory.clone(); - scope.set_bounds(spec.bounds); - let attempt_scope = scope.clone(); - let construction = async move { - if !attempt_scope.can_start() { - return Err("scope closed before construction".into()); - } - let factory_work = AssertUnwindSafe(async move { - fail_point!("child.factory", async => return Err("failpoint child.factory".into())); - factory(ctx).await - }) - .catch_unwind(); - let result = match select( - Box::pin(attempt_scope.token.cancelled()), - Box::pin(factory_work), - ) - .await - { - Either::Left(_) => return Err("construction cancelled".into()), - Either::Right((Ok(result), _)) => result, - Either::Right((Err(payload), _)) => { - return Err(format!( - "factory panicked: {}", - crate::error::panic_message(&*payload) - )) - } - }; - let actor = result?; - let actor_lease = attempt_scope.actor_lease().map_err(|e| e.to_string())?; - ActorRef::spawn_managed(actor, attempt_scope.clone(), actor_lease) - .map_err(|e| e.to_string()) - }; - let task = scope.handle.spawn(Tracked { - future: construction, - lease, - }); - let startup = async { - let actor = task - .await - .map_err(|e| format!("construction task failed: {e}"))??; - actor.ready().await.map_err(|e| e.to_string())?; - Ok::<_, String>(actor) - }; - let mut startup_timed_out = false; - let began = match select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(rt::timeout(spec.startup_timeout, startup)), - ) - .await - { - Either::Left(_) => Err("supervisor stopping".to_string()), - Either::Right((Err(_), _)) => { - startup_timed_out = true; - Err("startup deadline exceeded".to_string()) - } - Either::Right((Ok(result), _)) => result, - }; - #[cfg(feature = "failpoints")] - let began = match began { - Ok(actor) => { - let mut began = Ok(actor); - fail_point!("child.after_ready_before_publish", async => { - began = Err("failpoint child.after_ready_before_publish".to_string()) - }); - began - } - failed => failed, - }; - let mut ready_at = None; - let mut exit = None; - let mut cleanup_until = None; - // Trace bookkeeping: the model's StartupFailed (whose reason is known - // only after cleanup) and CleanupStuck (which skips CleanupDone and - // Decide). - let startup_failed = began.is_err(); - let mut stuck = false; - match began { - Ok(actor) => { - let published = trace::atomic(|| { - let published = core - .gate - .if_open(|| { - let mut data = system::lock(&view.data); - if data - .recovery - .as_ref() - .is_some_and(|r| r.meta.id > last_recovery) - { - return false; - } - data.snapshot.state = ChildState::Running; - data.snapshot.actor_id = Some(actor.id()); - data.current = Some(actor.clone()); - true - }) - .unwrap_or(false); - trace::emit(core.trace, || { - Event::new("BecomeReady", c, trace::flag(published)) - }); - if published { - core.cohort(|| { - Event::new("CReady", c, "") - .round(last_recovery) - .count(generation as usize) - }); - } - published - }); - view.changed.send_modify(|n| *n = n.wrapping_add(1)); - if published { - ready_at = Some(Instant::now()); - view.initial.send_replace(true); - let round = system::lock(&view.data).recovery.clone(); - if let Some(round) = round.filter(|r| r.meta.id == last_recovery) { - trace::atomic(|| { - core.cohort(|| Event::new("CAck", c, "").round(round.meta.id)); - round.ready.send_replace(true); - }); - } - let child = actor.child_handle(); - // Cancellation precedes terminal publication when stopped() - // hangs. Start its cleanup deadline at stop observation, - // rather than waiting forever for the completion channel. - let ended = select( - Box::pin(core.cancel.cancelled()), - Box::pin(scope.token.cancelled()), - ) - .await; - let stop_turn = matches!(ended, Either::Left(_)); - if stop_turn { - view.stop_turn.cancelled().await; - trace::emit(core.trace, || Event::new("StopObserved", c, "")); - // ChildView::stop already sent the parent's Shutdown. - } - view.change(|v| { - v.current = None; - v.snapshot.state = ChildState::Stopping; - }); - let cleanup_started = Instant::now(); - cleanup_until = Some(cleanup_started); - // The child's jobs are joined within the same deadline. - scope.set_cleanup_timeout(spec.shutdown_timeout); - exit = Some( - match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()).await { - Ok(reason) => { - if !stop_turn { - let failed = child.cleanup_failure().is_some(); - trace::emit(core.trace, || { - Event::new("SelfExit", c, exit_class(&reason, failed)) - }); - } - reason - } - // E3: the supervisor stopped this child and it - // did not reach its death point within - // shutdown_timeout: the parent's kill, then a - // second interval before cleanup is stuck. - Err(_) if stop_turn => { - scope.kill(); - // The second interval runs from the kill. - cleanup_until = Some(Instant::now()); - match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) - .await - { - Ok(reason) => reason, - Err(_) => { - stuck = true; - // Reply traces: the cleanup deadline - // passed before the child's death - // point (DeferredReplies `CleanupDeadline`). - crate::lifecycle_trace::reply( - child.id().raw(), - "Stuck", - 0, - "", - ); - core.fail_cleanup_for( - view.snapshot(), - format!("actor cleanup stuck for {}", spec.name), - || Some(Event::new("CleanupStuck", c, "")), - ); - view.mark_stuck(); - child.wait_exit_async().await - } - } - } - Err(_) => { - if !stop_turn { - trace::emit(core.trace, || { - Event::new("SelfExit", c, "unknown") - }); - } - stuck = true; - crate::lifecycle_trace::reply(child.id().raw(), "Stuck", 0, ""); - core.fail_cleanup_for( - view.snapshot(), - format!("actor cleanup stuck for {}", spec.name), - || Some(Event::new("CleanupStuck", c, "")), - ); - view.mark_stuck(); - child.wait_exit_async().await - } - }, - ); - } - } - Err(error) => view.change(|v| v.snapshot.last_error = Some(error)), - } - let startup_reason = |exit: Option<&ExitReason>| { - if startup_timed_out { - "timeout" - } else { - trace::reason(exit) - } - }; - scope.stop(exit.clone().unwrap_or(ExitReason::Shutdown)); - view.change(|v| { - v.current = None; - v.snapshot.last_exit = exit.clone(); - // A stuck cleanup already settled the snapshot as Failed (S2). - if v.snapshot.state != ChildState::Failed { - v.snapshot.state = ChildState::Stopping; - } - }); - fail_point!("child.before_cleanup_join", async); - let remaining = cleanup_until - .map(|started| spec.shutdown_timeout.saturating_sub(started.elapsed())) - .unwrap_or(spec.shutdown_timeout); - if rt::timeout(remaining, scope.joined()).await.is_err() { - core.fail_cleanup_for( - view.snapshot(), - format!("owned cleanup stuck for {}", spec.name), - || { - if stuck { - return None; - } - if startup_failed { - let reason = match scope.child().map(|child| child.exit_reason()) { - Some(None) if !startup_timed_out => "unknown", - Some(reason) => startup_reason(reason.as_ref()), - None => startup_reason(None), - }; - trace::emit(core.trace, || Event::new("StartupFailed", c, reason)); - } - Some(Event::new("CleanupStuck", c, "")) - }, - ); - view.mark_stuck(); - scope.joined().await; - trace::atomic(|| { - trace::tree(core.trace, |id| { - TreeEvent::new("TJoined", id).child(c, generation) - }); - core.cohort(|| Event::new("CJoined", c, "").count(generation as usize)); - }); - break; - } - trace::atomic(|| { - trace::tree(core.trace, |id| { - TreeEvent::new("TJoined", id).child(c, generation) - }); - core.cohort(|| Event::new("CJoined", c, "").count(generation as usize)); - }); - if exit.is_none() { - exit = scope.child().and_then(|child| child.exit_reason()); - } - if !stuck { - trace::atomic(|| { - if startup_failed { - trace::emit(core.trace, || { - Event::new("StartupFailed", c, startup_reason(exit.as_ref())) - }); - } - trace::emit(core.trace, || Event::new("CleanupDone", c, "")); - }); - } - let cleanup_failure = scope.child().and_then(|child| child.cleanup_failure()); - view.change(|v| { - v.snapshot.last_exit = exit.clone(); - if cleanup_failure.is_some() { - v.snapshot.last_error = cleanup_failure.clone(); - } - if v.snapshot.actor_id.is_none() { - v.snapshot.actor_id = scope.child().map(|child| child.id()); - } - v.scope = None; - }); - // A stable generation resets its failure streak even when another - // child's recovery supersedes this controller during cleanup. - if scope.was_stable(core.config.backoff.stable_after) { - failures = 0; - } - if core.gate.is_open() && view.recovery(last_recovery).is_some() { - continue; - } - // S1: a child's first start that fails (any cause) aborts the - // supervisor's start whatever the restart type, as OTP's - // `start_children` returns `failed_to_start_child`. - let first_start = ready_at.is_none() && !*view.initial.borrow(); - // A generation that never became ready after the child's initial - // readiness is a failed restart attempt: retried within the budget - // whatever its exit reason, as OTP retries a failed restart - // (`try_again_restart`), unless Temporary. - let failed_restart = ready_at.is_none() && !first_start; - let restart = if first_start { - false - } else if startup_timed_out || failed_restart { - spec.restart != RestartMode::Temporary - } else { - spec.restart - .restarts(exit.as_ref(), cleanup_failure.is_some()) - }; - // S1: a ready child's exit during the initial start is recorded and - // processed once the tree commits ready, as OTP handles the queued - // 'EXIT' after `init` returns. If the start aborts instead, the - // exit is part of the rollback. Only the decision waits: cleanup - // already ran within its deadline. - let deferred = ready_at.is_some() - && trace::atomic(|| { - let defer = core.gate.is_open() && !*core.ready.borrow(); - if defer { - trace::emit(core.trace, || Event::new("DeferExit", c, "")); - } - defer - }); - if deferred { - let _ = select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(system::wait_true(core.ready.subscribe())), - ) - .await; - trace::emit(core.trace, || Event::new("Undefer", c, "")); - } - // Gate check, decision and any resulting failure happen in one step, - // so a concurrent stop is never followed by a failure of this child. - let again = trace::atomic(|| { - if !core.gate.is_open() { - if !stuck { - trace::emit(core.trace, || Event::new("Decide", c, "closed")); - } - return false; - } - if spec.group.is_closed() { - core.fail_record_if_open_as( - child_failure( - &spec, - &view, - if first_start { - FailureKind::FirstStart - } else { - FailureKind::Other - }, - format!("execution group closed for {}", spec.name), - ), - || Some(Event::new("Unmodelled", c, "GroupClosed")), - ); - return false; - } - if restart { - trace::emit(core.trace, || Event::new("Decide", c, "restart")); - return true; - } - if ready_at.is_some() { - trace::emit(core.trace, || Event::new("Decide", c, "stop")); - } else { - let (kind, message) = if first_start { - let snapshot = view.snapshot(); - let cause = snapshot - .last_error - .clone() - .or_else(|| exit.as_ref().map(|reason| format!("exited {reason}"))) - .unwrap_or_else(|| "unknown".into()); - ( - FailureKind::FirstStart, - format!("failed to start child {}: {cause}", spec.name), - ) - } else { - ( - FailureKind::Other, - format!("non-restartable child failed startup: {}", spec.name), - ) - }; - core.fail_record_if_open_as(child_failure(&spec, &view, kind, message), || { - Some(Event::new("Decide", c, "fail")) - }); - } - false - }); - if !again { - if core.config.strategy == SupervisorStrategy::OneForOne || !core.gate.is_open() { - break; - } - let failed = system::lock(&core.failure).is_some(); - if trace::atomic(|| { - let finished = view.finish_unless_recovering(last_recovery, failed); - if finished { - trace::emit(core.trace, || Event::new("Finish", c, "")); - core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); - } - finished - }) { - core.log_failure_once(); - return; - } - continue; - } - // A failed replacement inside a recovery round requests a new round - // like any other failure, so OneForAll stops and rebuilds the whole - // cohort again (and RestForOne the failed child and its later - // siblings), as OTP's `try_again_restart` re-runs the strategy. - if core.config.strategy != SupervisorStrategy::OneForOne && *core.ready.borrow() { - core.request_recovery(view.index, last_recovery); - if matches!( - select( - Box::pin(core.admission_closed.cancelled()), - Box::pin(view.next_recovery(last_recovery)) - ) - .await, - Either::Left(_) - ) { - break; - } - } - } - // Record Finish and publish completion together, so the supervisor's - // deadline check sees both or neither. - trace::atomic(|| { - trace::emit(core.trace, || Event::new("Finish", c, "")); - view.finish(system::lock(&core.failure).is_some()); - core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); - }); - core.log_failure_once(); -} - -fn child_failure( - spec: &ChildSpec, - view: &View, - kind: FailureKind, - message: String, -) -> Failure { - let mut snapshot = view.snapshot(); - snapshot.name = spec.name.clone(); - snapshot_failure(snapshot, kind, message) -} - -fn snapshot_failure(snapshot: ChildSnapshot, kind: FailureKind, message: String) -> Failure { - Failure::new( - kind, - message, - Some(FailureOrigin { - child: snapshot.name, - generation: snapshot.generation, - reason: snapshot.last_exit, - last_error: snapshot.last_error, - }), - ) -} - #[cfg(test)] -mod policy_tests { - use super::*; - #[test] - fn jitter_draw_covers_the_whole_range() { - // `nanos + 1` would overflow at the top of the range; the full range - // takes the stream value as is. - assert_eq!(jitter_draw(u64::MAX, 12345), 12345); - assert_eq!(jitter_draw(u64::MAX, u64::MAX), u64::MAX); - assert_eq!(jitter_draw(0, 12345), 0); - assert_eq!(jitter_draw(9, 12345), 12345 % 10); - let mut rng = 1; - assert!(jitter(Duration::MAX, &mut rng) <= Duration::from_nanos(u64::MAX)); - assert_ne!(rng, 1, "each draw advances the stream"); - } - #[test] - fn jitter_seed_separates_supervisors_and_siblings() { - // Kani proves injectivity in `seed`; this keeps the property under - // `cargo test` (and cargo-mutants), which do not run Kani. - assert_ne!(jitter_seed(1, b"a"), jitter_seed(2, b"a")); - assert_ne!(jitter_seed(7, b"left"), jitter_seed(7, b"right")); - assert_eq!(jitter_seed(7, b""), 7); - assert_eq!(jitter_seed(0, b"ab"), 97 * 31 + 98); - // The seed is mixed with `^`: `|` and `&` survived cargo-mutants - // (both still separate the pairs above). - assert_eq!(jitter_seed(1, b"a"), 1 ^ 97); - } - #[tokio::test] - async fn failure_classification_is_typed_and_terminal_failure_is_sticky() { - crate::test_wait::within( - concat!( - module_path!(), - "::failure_classification_is_typed_and_terminal_failure_is_sticky" - ), - async move { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("terminal-order", 1).unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let first = tree - .add(ChildSpec::new("first", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - first.view.finish(true); - first.view.finish_unstarted(false); // stale non-failed sweep races last - second.view.finish_unstarted(false); - second.view.finish(true); // failed controller races last - assert_eq!(first.snapshot().state, ChildState::Failed); - assert_eq!(second.snapshot().state, ChildState::Failed); - tree.core - .fail("restart budget exhausted by a user-provided message".into()); - assert_eq!( - system::lock(&tree.core.failure).as_ref().unwrap().kind, - FailureKind::Other - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); - }, - ) - .await; - } - #[tokio::test] - async fn initial_placement_failure_has_deterministic_failed_slots() { - crate::test_wait::within( - concat!( - module_path!(), - "::initial_placement_failure_has_deterministic_failed_slots" - ), - async move { - use crate::tasks::ActorStart; - for _ in 0..8 { - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("placement-failure", 1).unwrap(); - let occupied = CoordinatorWorker.start_on(&group).unwrap(); - rt::timeout(Duration::from_secs(10), occupied.ready()) - .await - .unwrap() - .unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let first = tree - .add(ChildSpec::new("first", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = tree.core.clone(); - assert!(tree.start().await.is_err()); - assert!(rt::timeout( - Duration::from_secs(10), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - for child in [first, second] { - assert_eq!(child.snapshot().generation, 0); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(matches!( - child.current(), - Err(ChildUnavailable(ChildState::Failed)) - )); - } - assert!(system.shutdown(Duration::from_secs(10)).await.complete); - } - }, - ) - .await; - } - struct PanicRunner; - impl Runner for PanicRunner { - fn run(self: Box, _: Arc) -> BoxFuture<'static, ()> { - panic!("injected supervisor coordinator fault"); - } - } - struct CoordinatorWorker; - impl Actor for CoordinatorWorker {} - #[tokio::test] - async fn coordinator_fault_joins_launched_controllers_and_helpers() { - crate::test_wait::within(concat!(module_path!(), "::coordinator_fault_joins_launched_controllers_and_helpers"), async move { - let system = ActorSystem::new().unwrap(); - let group = system - .async_group("supervisor-coordinator-fault", 1) - .unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let (release, gate) = std::sync::mpsc::channel(); - let gate = Arc::new(Mutex::new(Some(gate))); - tree.add( - ChildSpec::new("live", &group, move |ctx| { - let gate = system::lock(&gate).take().unwrap(); - async move { - ctx.spawn_blocking_helper(move || { - gate.recv_timeout(Duration::from_secs(10)).unwrap(); - }) - .unwrap(); - Ok::<_, String>(CoordinatorWorker) - } - }) - .shutdown_timeout(Duration::from_secs(10)), - ) - .unwrap(); - tree.add(ChildSpec::new("not-launched", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - tree.runners[1] = Box::new(PanicRunner); - let core = tree.core.clone(); - // v0.8 (S2): start() returns its error only after the rollback, which - // waits for the live child's owned helper. - let start = rt::spawn(tree.start()); - assert!(rt::timeout( - Duration::from_millis(30), - system::wait_true(core.done.subscribe()) - ) - .await - .is_err()); - release.send(()).unwrap(); - assert!( - matches!(start.await.unwrap(), Err(SupervisorError::Failed(e)) if e.contains("injected supervisor coordinator fault")) - ); - assert!(rt::timeout( - Duration::from_secs(10), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - assert!(system::lock(&core.children) - .iter() - .all(|child| *child.done().borrow())); - let report = system.shutdown(Duration::from_secs(10)).await; - assert!(report.complete); - assert!(report - .supervisor_failures - .iter() - .any(|e| e.contains("injected supervisor coordinator fault"))); - }).await; - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn stop_wins_over_late_child_failure_without_trace_serialization() { - crate::test_wait::within( - concat!( - module_path!(), - "::stop_wins_over_late_child_failure_without_trace_serialization" - ), - async move { - let system = ActorSystem::new().unwrap(); - let tree = system.supervisor(SupervisorConfig::default()).unwrap(); - tree.core.close(); - assert!(!tree - .core - .fail_if_open_as("late child failure".into(), || None)); - assert!(system::lock(&tree.core.failure).is_none()); - assert!(tree - .core - .gate - .if_open(|| tree.core.ready.send_replace(true)) - .is_none()); - assert!(!*tree.core.ready.borrow()); - // Cleanup failures remain observable after ordinary admission closure. - tree.core.fail_cleanup("late cleanup failure".into()); - assert_eq!( - system::lock(&tree.core.failure).as_deref(), - Some("late cleanup failure") - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(5)).await.complete); - }, - ) - .await; - } - #[test] - fn delayed_failure_notifications_are_absorbed_by_the_already_assigned_round() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::delayed_failure_notifications_are_absorbed_by_the_already_assigned_round" - ), - move || { - let mut queue = RecoveryQueue { - pending: vec![false; 3], - covered: vec![0; 3], - next_id: 1, - }; - assert!(queue.request(1, 0)); - queue.pending.fill(false); - // The coordinator publishes coverage before per-child assignment. - queue.covered[1..].fill(1); - assert!(!queue.request(2, 0)); - assert!(!queue.pending.iter().any(|value| *value)); - // A new failure in a reconstructed generation must still be admitted. - assert!(queue.request(2, 1)); - assert!(queue.request(0, 0)); // unaffected upstream in RestForOne - }, - ); - } - - #[tokio::test] - async fn admission_closure_after_readiness_preserves_fixed_system_order() { - crate::test_wait::within( - concat!( - module_path!(), - "::admission_closure_after_readiness_preserves_fixed_system_order" - ), - async move { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn started(&mut self, ctx: &crate::tasks::Context) { - // v0.8: stopped() records the order only for a trapping - // child (E3, decision 2). - ctx.trap_exit(true); - } - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - if let Some(release) = self.release.take() { - let _ = release.await; - } - } - } - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { - Ordered { - id, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - async move { - Ok::<_, String>(Ordered { - id: 100, - log, - entered: None, - release: None, - }) - } - })) - .unwrap(); - let resume = CancellationToken::new(); - let (passed, after_check) = watch::channel(false); - *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); - let tree = builder.start().await.unwrap(); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // The later standalone actor owns the current stop turn. Resume the - // supervisor exactly between readiness and its cancellation wait. - resume.cancel(); - assert!(rt::timeout(limit, system::wait_true(after_check)) - .await - .unwrap()); - assert!( - !tree.core.cancel.is_cancelled(), - "admission closure stole the supervisor's stop turn" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - }, - ) - .await; - } - - #[tokio::test] - async fn startup_completion_or_drop_after_readiness_preserves_system_order() { - crate::test_wait::within( - concat!( - module_path!(), - "::startup_completion_or_drop_after_readiness_preserves_system_order" - ), - async move { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn started(&mut self, ctx: &crate::tasks::Context) { - // v0.8: stopped() records the order only for a trapping - // child (E3, decision 2). - ctx.trap_exit(true); - } - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - if let Some(release) = self.release.take() { - let _ = release.await; - } - } - } - for complete_start in [true, false] { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { - Ordered { - id, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; - } - Ok::<_, String>(Ordered { - id: 100, - log, - entered: None, - release: None, - }) - } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) - .await - .unwrap() - ); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // Readiness committed, but the caller has not observed start's result. - // Admission now closes while the later actor still owns the stop turn. - if complete_start { - assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); - } else { - drop(startup); - } - assert!( - !core.cancel.is_cancelled(), - "completed readiness was rolled back" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - } - }, - ) - .await; - } - - #[tokio::test] - async fn startup_success_remains_historical_after_unobserved_ready_then_failure() { - crate::test_wait::within( - concat!( - module_path!(), - "::startup_success_remains_historical_after_unobserved_ready_then_failure" - ), - async move { - struct Idle; - impl Actor for Idle {} - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("historical-ready-failure", 1).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 0, - ..SupervisorConfig::default() - }) - .unwrap(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - let child = builder - .add(ChildSpec::new("ready-then-failed", &group, move |_| { - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; - } - Ok::<_, String>(Idle) - } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) - .await - .unwrap() - ); - child - .current() - .unwrap() - .context() - .stop_with_reason(ExitReason::Kill); - assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) - .await - .unwrap()); - let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); - assert!(tree.failure().unwrap().contains("budget exhausted")); - assert!(system.shutdown(limit).await.complete); - }, - ) - .await; - } - - #[tokio::test] - async fn started_failures_allow_n_restarts_and_escalate_the_next_failure() { - crate::test_wait::within( - concat!( - module_path!(), - "::started_failures_allow_n_restarts_and_escalate_the_next_failure" - ), - async move { - use std::sync::atomic::{AtomicUsize, Ordering}; - // v0.8 (S1): a first start is not retried, so the first generation - // is healthy and every replacement panics in started(). - struct FailsStarted(Arc, bool); - impl Actor for FailsStarted { - async fn started(&mut self, _: &crate::tasks::Context) { - if self.1 { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("injected started failure"); - } - } - } - for allowed in [0, 1, 3] { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("started-budget", 1).unwrap(); - let count = Arc::new(AtomicUsize::new(0)); - let attempts = count.clone(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: allowed, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("startup-failure", &group, move |ctx| { - let attempts = attempts.clone(); - let fail = ctx.generation() > 1; - async move { Ok::<_, String>(FailsStarted(attempts, fail)) } - })) - .unwrap(); - let core = builder.core.clone(); - rt::timeout(Duration::from_secs(30), builder.start()) - .await - .unwrap() - .unwrap(); - child.current().unwrap().child_handle().kill(); - assert!(rt::timeout( - Duration::from_secs(30), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - assert_eq!(count.load(Ordering::SeqCst), allowed); - assert_eq!(child.snapshot().generation as usize, allowed + 1); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); - } - }, - ) - .await; - } - - #[tokio::test] - async fn sliding_window_expires_only_old_grants_before_the_next_retry() { - crate::test_wait::within( - concat!( - module_path!(), - "::sliding_window_expires_only_old_grants_before_the_next_retry" - ), - async move { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("sliding-budget", 1).unwrap(); - let window = Duration::from_secs(3600); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 2, - restart_window: window, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("worker", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let mut actor = child.current().unwrap(); - for generation in 1..=2 { - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("retry".into())); - actor = rt::timeout( - Duration::from_secs(30), - child.wait_for_generation(generation), - ) - .await - .unwrap() - .unwrap(); - } - { - let mut budget = system::lock(&core.budget); - assert_eq!(budget.attempts.len(), 2); - // Control just the oldest timestamp; the real controller still owns - // admission and retry. The second grant remains inside the window. - budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); - } - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("expired oldest".into())); - let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) - .await - .unwrap() - .unwrap(); - assert_eq!(system::lock(&core.budget).attempts.len(), 2); - fourth - .child_handle() - .stop_with_reason(ExitReason::Panic("now exhausted".into())); - rt::timeout(Duration::from_secs(30), tree.join()) - .await - .unwrap(); - assert_eq!(child.snapshot().generation, 4); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); - }, - ) - .await; - } - - /// OTP charges `add_restart` when it decides a restart, before the - /// cohort is rebuilt: a superseded round keeps its charge, and the - /// superseding round is charged once more. - #[tokio::test] - async fn supersession_charges_each_selected_round_once() { - crate::test_wait::within( - concat!( - module_path!(), - "::supersession_charges_each_selected_round_once" - ), - async move { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system - .async_group("refund-superseded-admission", 2) - .unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::RestForOne, - max_restarts: 3, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for name in ["upstream", "middle", "tail"] { - children.push( - builder - .add(ChildSpec::new(name, &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(), - ); - } - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let resume = CancellationToken::new(); - let (entered, arrival) = watch::channel(false); - *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); - children[1] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("middle fails".into())); - assert!(rt::timeout(limit, system::wait_true(arrival)) - .await - .unwrap()); - let old_round = system::lock(&children[1].view.data) - .recovery - .as_ref() - .unwrap() - .meta - .id; - { - let budget = system::lock(&core.budget); - // The round was charged once, at selection; its rebuilds hold - // no reservation of their own. - assert_eq!(budget.attempts.len(), 1); - assert_eq!(budget.reserved, 0); - } - children[0] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); - let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) - .await - .unwrap(); - resume.cancel(); - for child in &children { - rt::timeout(limit, child.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); - assert_eq!(child.snapshot().generation, 2); - } - assert!(tree.failure().is_none()); - { - let budget = system::lock(&core.budget); - // Two restart actions: the superseded round and its successor, - // however many children each rebuilt. - assert_eq!(budget.attempts.len(), 2); - assert_eq!(budget.reserved, 0); - } - assert!(system.shutdown(limit).await.complete); - }, - ) - .await; - } - - #[tokio::test] - async fn cohort_round_takes_one_grant_for_all_rebuilt_children() { - crate::test_wait::within( - concat!( - module_path!(), - "::cohort_round_takes_one_grant_for_all_rebuilt_children" - ), - async move { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("cohort-budget-contract", 2).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 1, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for index in 0..6 { - children.push( - builder - .add(ChildSpec::new( - format!("worker-{index}"), - &group, - |_| async { Ok::<_, String>(CoordinatorWorker) }, - )) - .unwrap(), - ); - } - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - children[5] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("last child fails".into())); - // One grant rebuilds all six children (one restart action). - for child in &children { - rt::timeout(limit, child.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); - } - assert!(tree.failure().is_none()); - assert_eq!(system::lock(&core.budget).attempts.len(), 1); - children[5] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("second failure".into())); - rt::timeout(limit, tree.join()).await.unwrap(); - assert!(tree - .failure() - .unwrap() - .contains("budget exhausted by worker-5")); - assert_eq!( - children - .iter() - .map(|child| child.snapshot().generation) - .collect::>(), - vec![2; 6] - ); - assert!(system.shutdown(limit).await.complete); - }, - ) - .await; - } - - #[tokio::test] - async fn retry_reservations_exclude_siblings_and_refund_interrupted_backoff() { - crate::test_wait::within( - concat!( - module_path!(), - "::retry_reservations_exclude_siblings_and_refund_interrupted_backoff" - ), - async move { - let system = ActorSystem::new().unwrap(); - let tree = system - .supervisor(SupervisorConfig { - max_restarts: 1, - ..SupervisorConfig::default() - }) - .unwrap(); - let reservation = tree.core.reserve_retry(1).unwrap(); - assert!(tree.core.reserve_retry(2).is_none()); - assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); - drop(reservation); - let mut replacement = tree.core.reserve_retry(2).unwrap(); - tree.core.close(); - assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); - drop(replacement); - { - let budget = system::lock(&tree.core.budget); - assert!(budget.attempts.is_empty()); - assert_eq!(budget.reserved, 0); - } - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); - }, - ) - .await; - } - - #[test] - fn retry_budget_reservations_count_against_availability() { - // `available` (the reservation check) must agree with `take` at the - // limit: `< max` became `<= max` without any test failing. - let now = Instant::now(); - let mut budget = RetryBudget { - attempts: VecDeque::from([now]), - reserved: 0, - max: 2, - window: Duration::from_secs(10), - }; - assert!(budget.available(now)); - budget.reserved = 1; - assert!(!budget.available(now)); - budget.reserved = 0; - budget.attempts.push_back(now); - assert!(!budget.available(now)); - assert!(budget.available(now + Duration::from_secs(10))); - } - #[test] - fn retry_window_releases_capacity_at_boundary_and_does_not_bank_unused_retries() { - let now = Instant::now(); - let mut budget = RetryBudget { - attempts: VecDeque::new(), - reserved: 0, - max: 2, - window: Duration::from_secs(10), - }; - assert!(budget.take(now)); - assert!(budget.take(now + Duration::from_secs(1))); - assert!(!budget.take(now + Duration::from_secs(9))); - assert!(budget.take(now + Duration::from_secs(10))); - assert!(!budget.take(now + Duration::from_secs(10))); - assert!(budget.take(now + Duration::from_secs(100))); - assert!(budget.take(now + Duration::from_secs(100))); - assert!(!budget.take(now + Duration::from_secs(100))); - } - #[test] - fn capped_backoff_and_seeded_jitter_remain_bounded_after_long_failure_streaks() { - let mut policy = Backoff { - initial: Duration::from_millis(10), - maximum: Duration::from_millis(35), - jitter: false, - ..Backoff::default() - }; - let mut seed = 7; - assert_eq!( - [0, 1, 2, 32, u32::MAX].map(|n| policy.delay(n, &mut seed)), - [10, 20, 35, 35, 35].map(Duration::from_millis) - ); - policy.jitter = true; - let mut a = 42; - let mut b = 42; - let first: Vec<_> = (0..100).map(|n| policy.delay(n, &mut a)).collect(); - let replay: Vec<_> = (0..100).map(|n| policy.delay(n, &mut b)).collect(); - assert_eq!(first, replay); - assert!(first.iter().all(|d| *d <= policy.maximum)); - assert!(first.windows(2).any(|pair| pair[0] != pair[1])); - } -} +mod policy_tests; diff --git a/concurrency/src/supervision/builder.rs b/concurrency/src/supervision/builder.rs new file mode 100644 index 00000000..1c9bc5c6 --- /dev/null +++ b/concurrency/src/supervision/builder.rs @@ -0,0 +1,329 @@ +//! `SupervisorBuilder` (ordered startup, rollback) and the running `Supervisor` handle and report. +use super::*; + +pub struct SupervisorBuilder { + pub(super) owner: Weak, + pub(super) core: Arc, + pub(super) runners: Vec>, + pub(super) started: bool, +} +impl SupervisorBuilder { + pub(crate) fn new( + system: &ActorSystem, + config: SupervisorConfig, + ) -> Result { + Self::from_owner(Arc::downgrade(&system.inner), config) + } + pub(super) fn from_owner( + owner: Weak, + config: SupervisorConfig, + ) -> Result { + if config.restart_window.is_zero() + || config.max_children == 0 + || config.max_children > 65536 + || config.max_restarts > 65536 + || config.backoff.maximum < config.backoff.initial + { + return Err(SupervisorError::Configuration( + "invalid limits or backoff".into(), + )); + } + let (done, _) = watch::channel(false); + let (ready, _) = watch::channel(false); + let (recovery_changed, _) = watch::channel(0); + let core = Arc::new(Core { + recovery: Mutex::new(RecoveryQueue { + pending: Vec::new(), + covered: Vec::new(), + next_id: 1, + }), + recovery_changed, + gate: Admission::new(), + cancel: CancellationToken::new(), + admission_closed: CancellationToken::new(), + done, + ready, + failure: Mutex::new(None), + failed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + cleanup_failure: Mutex::new(None), + cleanup_failures: Mutex::new(Vec::new()), + children: Mutex::new(Vec::new()), + budget: Mutex::new(RetryBudget { + attempts: VecDeque::new(), + reserved: 0, + max: config.max_restarts, + window: config.restart_window, + }), + config, + trace: TraceId::next(), + #[cfg(test)] + initial_ready_pause: Mutex::new(None), + #[cfg(test)] + retry_admission_pause: Mutex::new(None), + #[cfg(feature = "lifecycle-trace")] + deadline_recorded: Default::default(), + }); + #[cfg(feature = "lifecycle-trace")] + trace::describe( + core.trace, + match core.config.strategy { + SupervisorStrategy::OneForOne => "OneForOne", + SupervisorStrategy::OneForAll => "OneForAll", + SupervisorStrategy::RestForOne => "RestForOne", + }, + core.config.max_restarts, + ); + Ok(Self { + owner, + core, + runners: Vec::new(), + started: false, + }) + } + + /// Lifecycle trace id of this supervisor (`lifecycle_trace::take`). + #[cfg(feature = "lifecycle-trace")] + #[doc(hidden)] + pub fn trace_id(&self) -> u64 { + self.core.trace.0 + } + + pub fn add_supervisor( + &mut self, + spec: SupervisorSpec, + ) -> Result { + nested::add(self, spec) + } + pub fn add( + &mut self, + spec: ChildSpec, + ) -> Result, SupervisorError> { + self.owner + .upgrade() + .ok_or(SystemError::Closed)? + .accepts_group(&spec.group)?; + if spec.name.is_empty() + || spec.name.contains('\0') + || spec.startup_timeout.is_zero() + || spec.shutdown_timeout.is_zero() + { + return Err(SupervisorError::Configuration( + "invalid child name or timeout".into(), + )); + } + let mut children = system::lock(&self.core.children); + if children.len() >= self.core.config.max_children { + return Err(SupervisorError::Capacity); + } + if children.iter().any(|c| c.snapshot().name == spec.name) { + return Err(SupervisorError::DuplicateChild(spec.name)); + } + let (changed, _) = watch::channel(0); + let (initial, _) = watch::channel(false); + let (done, _) = watch::channel(false); + let view = Arc::new(View { + index: children.len(), + data: Mutex::new(ViewData { + recovery: None, + snapshot: ChildSnapshot { + name: spec.name.clone(), + state: ChildState::Waiting, + generation: 0, + actor_id: None, + last_exit: None, + last_error: None, + }, + current: None, + scope: None, + }), + gate: self.core.gate.clone(), + failed: self.core.failed.clone(), + changed, + initial, + done, + stop_turn: CancellationToken::new(), + shutdown_timeout: spec.shutdown_timeout, + trace_index: children.len() + 1, + }); + children.push(view.clone()); + #[cfg(feature = "lifecycle-trace")] + trace::describe_child( + self.core.trace, + match spec.restart { + RestartMode::Permanent => "Permanent", + RestartMode::Transient => "Transient", + RestartMode::Temporary => "Temporary", + }, + ); + { + let mut queue = system::lock(&self.core.recovery); + queue.pending.push(false); + queue.covered.push(0); + } + self.runners.push(Box::new(TypedRunner { + spec, + view: view.clone(), + })); + Ok(SupervisedChild { view }) + } + + /// Start children in declaration order. Dropping this future before readiness + /// commits requests cleanup. After readiness, ActorSystem owns the tree even + /// if this future is dropped before returning its Supervisor handle. + pub async fn start(self) -> Result { + self.start_owned(true).await + } + pub(super) async fn start_owned( + mut self, + register_root: bool, + ) -> Result { + let owner = self.owner.upgrade().ok_or(SystemError::Closed)?; + // Registration and controller dispatch happen without an intervening await. + if register_root { + owner.register_unit(self.core.clone())?; + } + trace::tree(self.core.trace, |id| { + TreeEvent::new("TCreate", id).peer(0, self.runners.len()) + }); + self.started = true; + let runners = std::mem::take(&mut self.runners); + let core = self.core.clone(); + owner + .control + .handle + .spawn(run_supervisor(core.clone(), runners)); + let mut guard = StartupGuard { + core: core.clone(), + armed: true, + }; + let ready = core.ready.subscribe(); + let cancelled = core.cancel.cancelled(); + let _ = select(Box::pin(cancelled), Box::pin(system::wait_true(ready))).await; + // Readiness is historical, just like ActorRef::ready(). A later system + // admission close must not turn successful startup into rollback. + let initialized = *core.ready.borrow(); + if initialized { + guard.armed = false; + return Ok(Supervisor { core: core.clone() }); + } + // S2: the error is returned only after the rollback finished: every + // child is terminal, or settled as Failed by its stuck cleanup. + let children = system::lock(&core.children).clone(); + for child in &children { + child.settled().await; + } + Err(core.start_error()) + } +} +impl Core { + /// The cause of a failed initial start (S2), with the rollback's cleanup + /// failures listed separately. + pub(super) fn start_error(&self) -> SupervisorError { + let Some(failure) = system::lock(&self.failure).clone() else { + return SupervisorError::Stopped; + }; + let rollback_failures: Vec = system::lock(&self.cleanup_failures) + .iter() + .filter(|f| **f != failure.message) + .cloned() + .collect(); + match (failure.kind, failure.origin) { + (FailureKind::FirstStart, Some(origin)) => { + let prefix = format!("failed to start child {}: ", origin.child); + let cause = failure + .message + .strip_prefix(&prefix) + .unwrap_or(&failure.message) + .to_string(); + SupervisorError::FailedToStartChild { + child: origin.child, + cause, + rollback_failures, + } + } + (FailureKind::Cleanup, Some(origin)) => SupervisorError::StartupCleanupStuck { + child: origin.child, + cause: failure.message, + rollback_failures, + }, + _ => SupervisorError::Failed(failure.message), + } + } +} +impl Drop for SupervisorBuilder { + fn drop(&mut self) { + if !self.started { + self.core.close(); + for child in system::lock(&self.core.children).iter() { + child.finish_unstarted(self.core.failed.load(std::sync::atomic::Ordering::Acquire)); + } + trace::atomic(|| { + trace::tree(self.core.trace, |id| TreeEvent::new("TDone", id)); + self.core.done.send_replace(true); + }); + } + } +} +pub(super) struct StartupGuard { + pub(super) core: Arc, + pub(super) armed: bool, +} +impl Drop for StartupGuard { + fn drop(&mut self) { + if self.armed { + self.core.cancel_unready_startup(); + } + } +} + +/// Owned by ActorSystem even if public handles are dropped. +#[derive(Clone)] +pub struct Supervisor { + pub(super) core: Arc, +} +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct SupervisorReport { + pub complete: bool, + pub failure: Option, + pub children: Vec, +} +impl Supervisor { + pub fn stop(&self) { + self.core.close(); + } + /// Lifecycle trace id of this supervisor (`lifecycle_trace::take`). + #[cfg(feature = "lifecycle-trace")] + #[doc(hidden)] + pub fn trace_id(&self) -> u64 { + self.core.trace.0 + } + pub fn snapshots(&self) -> Vec { + system::lock(&self.core.children) + .iter() + .map(|c| c.snapshot()) + .collect() + } + pub fn failure(&self) -> Option { + system::lock(&self.core.failure) + .as_ref() + .map(|f| f.message.clone()) + } + pub async fn join(&self) { + let _ = system::wait_true(self.core.done.subscribe()).await; + } + pub async fn shutdown(&self, grace: Duration) -> SupervisorReport { + self.stop(); + let complete = rt::timeout(grace, system::wait_true(self.core.done.subscribe())) + .await + .unwrap_or(false); + if !complete { + self.core.stop_remaining(); + } + SupervisorReport { + complete, + failure: self.failure(), + children: self.snapshots(), + } + } +} diff --git a/concurrency/src/supervision/child.rs b/concurrency/src/supervision/child.rs new file mode 100644 index 00000000..1f7960b4 --- /dev/null +++ b/concurrency/src/supervision/child.rs @@ -0,0 +1,315 @@ +//! Per-child state: the recovery queue, the typed child view behind `SupervisedChild`, and the type-erased `ChildControl`. +use super::*; + +pub(super) struct RecoveryMeta { + pub(super) id: u64, + pub(super) origin: usize, + pub(super) finished: std::sync::atomic::AtomicBool, +} +pub(super) struct RecoverySlot { + pub(super) meta: Arc, + pub(super) stop: CancellationToken, + pub(super) resume: CancellationToken, + pub(super) cleaned: watch::Sender, + pub(super) ready: watch::Sender, +} +impl RecoverySlot { + pub(super) fn new(meta: Arc) -> Arc { + let (cleaned, _) = watch::channel(false); + let (ready, _) = watch::channel(false); + Arc::new(Self { + meta, + stop: CancellationToken::new(), + resume: CancellationToken::new(), + cleaned, + ready, + }) + } +} +pub(crate) struct RecoveryQueue { + pub(crate) pending: Vec, + pub(crate) covered: Vec, + pub(crate) next_id: u64, +} +impl RecoveryQueue { + /// Opens the next round at the lowest pending origin: clears pending from + /// `start` on, stamps those children as covered by the fresh round id and + /// returns `(origin, start, id)`. + pub(crate) fn select(&mut self, one_for_all: bool) -> Option<(usize, usize, u64)> { + let origin = self.pending.iter().position(|pending| *pending)?; + let start = if one_for_all { 0 } else { origin }; + for pending in &mut self.pending[start..] { + *pending = false; + } + let id = self.next_id; + self.next_id = id.checked_add(1).expect("recovery IDs exhausted"); + self.covered[start..].fill(id); + Some((origin, start, id)) + } + pub(crate) fn request(&mut self, index: usize, after: u64) -> bool { + if after < self.covered[index] { + return false; + } + self.pending[index] = true; + true + } +} +pub(super) struct ViewData { + pub(super) recovery: Option>, + pub(super) snapshot: ChildSnapshot, + pub(super) current: Option>, + pub(super) scope: Option>, +} +pub(super) struct View { + pub(super) index: usize, + pub(super) data: Mutex>, + pub(super) gate: Arc, + pub(super) failed: Arc, + pub(super) changed: watch::Sender, + pub(super) initial: watch::Sender, + pub(super) stop_turn: CancellationToken, + pub(super) done: watch::Sender, + pub(super) shutdown_timeout: Duration, + /// 1-based declaration index: the model's child id in lifecycle traces. + pub(super) trace_index: usize, +} +impl View { + pub(super) fn unavailable(&self) -> ChildUnavailable { + if self.failed.load(std::sync::atomic::Ordering::Acquire) { + ChildUnavailable(ChildState::Failed) + } else { + let state = system::lock(&self.data).snapshot.state; + ChildUnavailable( + if matches!(state, ChildState::Stopped | ChildState::Failed) { + state + } else { + ChildState::Stopping + }, + ) + } + } + pub(super) fn change(&self, f: impl FnOnce(&mut ViewData)) { + { + f(&mut system::lock(&self.data)); + } + self.changed.send_modify(|n| *n = n.wrapping_add(1)); + } + pub(super) fn recovery(&self, after: u64) -> Option> { + system::lock(&self.data) + .recovery + .clone() + .filter(|r| r.meta.id > after) + } + pub(super) async fn next_recovery(&self, after: u64) -> Arc { + let mut changes = self.changed.subscribe(); + loop { + changes.borrow_and_update(); + if let Some(round) = self.recovery(after) { + return round; + } + let _ = changes.changed().await; + } + } + pub(super) async fn recovery_stopping(&self, after: u64) { + self.next_recovery(after).await.stop.cancelled().await; + } + pub(super) fn finish_unless_recovering(&self, after: u64, failed: bool) -> bool { + { + let mut data = system::lock(&self.data); + if !failed && data.recovery.as_ref().is_some_and(|r| r.meta.id > after) { + return false; + } + data.current = None; + data.scope = None; + data.snapshot.state = if failed || data.snapshot.state == ChildState::Failed { + ChildState::Failed + } else { + ChildState::Stopped + }; + } + self.changed.send_modify(|n| *n = n.wrapping_add(1)); + self.done.send_replace(true); + true + } + /// Cleanup stuck past its deadlines: the snapshot settles as Failed, + /// which releases a waiting `start()` (S2), while the controller keeps + /// awaiting the real join. + pub(super) fn mark_stuck(&self) { + self.change(|data| { + data.current = None; + data.snapshot.state = ChildState::Failed; + }); + } + /// Terminal, or settled as Failed by a stuck cleanup. + pub(super) async fn settled(&self) { + let mut changes = self.changed.subscribe(); + loop { + changes.borrow_and_update(); + let state = system::lock(&self.data).snapshot.state; + if matches!(state, ChildState::Stopped | ChildState::Failed) || *self.done.borrow() { + return; + } + if changes.changed().await.is_err() { + return; + } + } + } + pub(super) fn finish(&self, failed: bool) { + self.change(|data| { + data.current = None; + data.scope = None; + data.snapshot.state = if failed || data.snapshot.state == ChildState::Failed { + ChildState::Failed + } else { + ChildState::Stopped + }; + }); + self.done.send_replace(true); + } +} + +/// Explicit lookup of a replacement, not a rebound ActorRef. Resolution can +/// race with a subsequent crash; callers must still handle delivery errors. +pub struct SupervisedChild { + pub(super) view: Arc>, +} +impl Clone for SupervisedChild { + fn clone(&self) -> Self { + Self { + view: self.view.clone(), + } + } +} +impl SupervisedChild { + pub fn snapshot(&self) -> ChildSnapshot { + system::lock(&self.view.data).snapshot.clone() + } + pub fn current(&self) -> Result, ChildUnavailable> { + self.view + .gate + .if_open(|| { + let data = system::lock(&self.view.data); + if data.scope.as_ref().is_some_and(|s| s.token.is_cancelled()) { + return Err(ChildUnavailable(ChildState::Stopping)); + } + data.current + .clone() + .ok_or(ChildUnavailable(data.snapshot.state)) + }) + .unwrap_or_else(|| Err(self.view.unavailable())) + } + pub async fn wait_ready(&self) -> Result, ChildUnavailable> { + self.wait_for_generation(0).await + } + /// Wait for a ready generation strictly greater than the supplied value. + pub async fn wait_for_generation(&self, after: u64) -> Result, ChildUnavailable> { + let mut changes = self.view.changed.subscribe(); + loop { + changes.borrow_and_update(); + let snapshot = self.snapshot(); + if snapshot.generation > after { + if let Ok(actor) = self.current() { + return Ok(actor); + } + } + if matches!(snapshot.state, ChildState::Stopped | ChildState::Failed) { + return Err(ChildUnavailable(snapshot.state)); + } + if !self.view.gate.is_open() { + return Err(self.view.unavailable()); + } + if changes.changed().await.is_err() { + return Err(ChildUnavailable(ChildState::Stopped)); + } + } + } +} + +pub(super) trait ChildControl: Send + Sync { + fn assign_recovery(&self, slot: Arc) -> bool; + fn stop_generation(&self, slot: &RecoverySlot); + fn snapshot(&self) -> ChildSnapshot; + fn close_admission(&self); + fn stop(&self); + fn done(&self) -> watch::Receiver; + fn initial(&self) -> watch::Receiver; + fn shutdown_timeout(&self) -> Duration; + fn finish_unstarted(&self, failed: bool); + fn settled(&self) -> BoxFuture<'_, ()>; +} +impl ChildControl for View { + fn assign_recovery(&self, slot: Arc) -> bool { + let old = { + let mut data = system::lock(&self.data); + if matches!( + data.snapshot.state, + ChildState::Stopped | ChildState::Failed + ) { + return false; + } + let old = data.recovery.replace(slot); + data.current = None; + data.snapshot.state = ChildState::Stopping; + if let Some(scope) = &data.scope { + scope.close_admission(); + } + old + }; + drop(old); + self.changed.send_modify(|n| *n = n.wrapping_add(1)); + true + } + /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit + /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. + fn stop_generation(&self, slot: &RecoverySlot) { + slot.stop.cancel(); + let scope = system::lock(&self.data).scope.clone(); + if let Some(scope) = scope { + scope.signal_parent(ExitReason::Shutdown); + } + } + fn snapshot(&self) -> ChildSnapshot { + system::lock(&self.data).snapshot.clone() + } + fn close_admission(&self) { + let (scope, constructing) = { + let data = system::lock(&self.data); + (data.scope.clone(), data.current.is_none()) + }; + if let Some(scope) = scope { + scope.close_admission(); + if constructing { + scope.stop(ExitReason::Shutdown); + } + } + self.changed.send_modify(|n| *n = n.wrapping_add(1)); + } + /// Supervisor shutdown of this child (V9, reverse order): the parent + /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends + /// `Shutdown` without `stopped()`; a trapping child finishes its earlier + /// messages and runs `stopped()`. The controller kills it at + /// `shutdown_timeout`. + fn stop(&self) { + self.stop_turn.cancel(); + if let Some(scope) = system::lock(&self.data).scope.clone() { + scope.signal_parent(ExitReason::Shutdown); + } + } + fn done(&self) -> watch::Receiver { + self.done.subscribe() + } + fn initial(&self) -> watch::Receiver { + self.initial.subscribe() + } + fn shutdown_timeout(&self) -> Duration { + self.shutdown_timeout + } + fn finish_unstarted(&self, failed: bool) { + if self.snapshot().generation == 0 { + self.finish(failed); + } + } + fn settled(&self) -> BoxFuture<'_, ()> { + Box::pin(View::settled(self)) + } +} diff --git a/concurrency/src/supervision/policy_tests.rs b/concurrency/src/supervision/policy_tests.rs new file mode 100644 index 00000000..ece00e1e --- /dev/null +++ b/concurrency/src/supervision/policy_tests.rs @@ -0,0 +1,952 @@ +use super::*; +#[test] +fn jitter_draw_covers_the_whole_range() { + // `nanos + 1` would overflow at the top of the range; the full range + // takes the stream value as is. + assert_eq!(jitter_draw(u64::MAX, 12345), 12345); + assert_eq!(jitter_draw(u64::MAX, u64::MAX), u64::MAX); + assert_eq!(jitter_draw(0, 12345), 0); + assert_eq!(jitter_draw(9, 12345), 12345 % 10); + let mut rng = 1; + assert!(jitter(Duration::MAX, &mut rng) <= Duration::from_nanos(u64::MAX)); + assert_ne!(rng, 1, "each draw advances the stream"); +} +#[test] +fn jitter_seed_separates_supervisors_and_siblings() { + // Kani proves injectivity in `seed`; this keeps the property under + // `cargo test` (and cargo-mutants), which do not run Kani. + assert_ne!(jitter_seed(1, b"a"), jitter_seed(2, b"a")); + assert_ne!(jitter_seed(7, b"left"), jitter_seed(7, b"right")); + assert_eq!(jitter_seed(7, b""), 7); + assert_eq!(jitter_seed(0, b"ab"), 97 * 31 + 98); + // The seed is mixed with `^`: `|` and `&` survived cargo-mutants + // (both still separate the pairs above). + assert_eq!(jitter_seed(1, b"a"), 1 ^ 97); +} +#[tokio::test] +async fn failure_classification_is_typed_and_terminal_failure_is_sticky() { + crate::test_wait::within( + concat!( + module_path!(), + "::failure_classification_is_typed_and_terminal_failure_is_sticky" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("terminal-order", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + first.view.finish(true); + first.view.finish_unstarted(false); // stale non-failed sweep races last + second.view.finish_unstarted(false); + second.view.finish(true); // failed controller races last + assert_eq!(first.snapshot().state, ChildState::Failed); + assert_eq!(second.snapshot().state, ChildState::Failed); + tree.core + .fail("restart budget exhausted by a user-provided message".into()); + assert_eq!( + system::lock(&tree.core.failure).as_ref().unwrap().kind, + FailureKind::Other + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; +} +#[tokio::test] +async fn initial_placement_failure_has_deterministic_failed_slots() { + crate::test_wait::within( + concat!( + module_path!(), + "::initial_placement_failure_has_deterministic_failed_slots" + ), + async move { + use crate::tasks::ActorStart; + for _ in 0..8 { + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("placement-failure", 1).unwrap(); + let occupied = CoordinatorWorker.start_on(&group).unwrap(); + rt::timeout(Duration::from_secs(10), occupied.ready()) + .await + .unwrap() + .unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = tree.core.clone(); + assert!(tree.start().await.is_err()); + assert!(rt::timeout( + Duration::from_secs(10), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + for child in [first, second] { + assert_eq!(child.snapshot().generation, 0); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(matches!( + child.current(), + Err(ChildUnavailable(ChildState::Failed)) + )); + } + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + } + }, + ) + .await; +} +struct PanicRunner; +impl Runner for PanicRunner { + fn run(self: Box, _: Arc) -> BoxFuture<'static, ()> { + panic!("injected supervisor coordinator fault"); + } +} +struct CoordinatorWorker; +impl Actor for CoordinatorWorker {} +#[tokio::test] +async fn coordinator_fault_joins_launched_controllers_and_helpers() { + crate::test_wait::within(concat!(module_path!(), "::coordinator_fault_joins_launched_controllers_and_helpers"), async move { + let system = ActorSystem::new().unwrap(); + let group = system + .async_group("supervisor-coordinator-fault", 1) + .unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let gate = Arc::new(Mutex::new(Some(gate))); + tree.add( + ChildSpec::new("live", &group, move |ctx| { + let gate = system::lock(&gate).take().unwrap(); + async move { + ctx.spawn_blocking_helper(move || { + gate.recv_timeout(Duration::from_secs(10)).unwrap(); + }) + .unwrap(); + Ok::<_, String>(CoordinatorWorker) + } + }) + .shutdown_timeout(Duration::from_secs(10)), + ) + .unwrap(); + tree.add(ChildSpec::new("not-launched", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + tree.runners[1] = Box::new(PanicRunner); + let core = tree.core.clone(); + // v0.8 (S2): start() returns its error only after the rollback, which + // waits for the live child's owned helper. + let start = rt::spawn(tree.start()); + assert!(rt::timeout( + Duration::from_millis(30), + system::wait_true(core.done.subscribe()) + ) + .await + .is_err()); + release.send(()).unwrap(); + assert!( + matches!(start.await.unwrap(), Err(SupervisorError::Failed(e)) if e.contains("injected supervisor coordinator fault")) + ); + assert!(rt::timeout( + Duration::from_secs(10), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + assert!(system::lock(&core.children) + .iter() + .all(|child| *child.done().borrow())); + let report = system.shutdown(Duration::from_secs(10)).await; + assert!(report.complete); + assert!(report + .supervisor_failures + .iter() + .any(|e| e.contains("injected supervisor coordinator fault"))); + }).await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_wins_over_late_child_failure_without_trace_serialization() { + crate::test_wait::within( + concat!( + module_path!(), + "::stop_wins_over_late_child_failure_without_trace_serialization" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system.supervisor(SupervisorConfig::default()).unwrap(); + tree.core.close(); + assert!(!tree + .core + .fail_if_open_as("late child failure".into(), || None)); + assert!(system::lock(&tree.core.failure).is_none()); + assert!(tree + .core + .gate + .if_open(|| tree.core.ready.send_replace(true)) + .is_none()); + assert!(!*tree.core.ready.borrow()); + // Cleanup failures remain observable after ordinary admission closure. + tree.core.fail_cleanup("late cleanup failure".into()); + assert_eq!( + system::lock(&tree.core.failure).as_deref(), + Some("late cleanup failure") + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, + ) + .await; +} +#[test] +fn delayed_failure_notifications_are_absorbed_by_the_already_assigned_round() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::delayed_failure_notifications_are_absorbed_by_the_already_assigned_round" + ), + move || { + let mut queue = RecoveryQueue { + pending: vec![false; 3], + covered: vec![0; 3], + next_id: 1, + }; + assert!(queue.request(1, 0)); + queue.pending.fill(false); + // The coordinator publishes coverage before per-child assignment. + queue.covered[1..].fill(1); + assert!(!queue.request(2, 0)); + assert!(!queue.pending.iter().any(|value| *value)); + // A new failure in a reconstructed generation must still be admitted. + assert!(queue.request(2, 1)); + assert!(queue.request(0, 0)); // unaffected upstream in RestForOne + }, + ); +} + +#[tokio::test] +async fn admission_closure_after_readiness_preserves_fixed_system_order() { + crate::test_wait::within( + concat!( + module_path!(), + "::admission_closure_after_readiness_preserves_fixed_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, + } + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; + } + } + } + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + } + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + async move { + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let resume = CancellationToken::new(); + let (passed, after_check) = watch::channel(false); + *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); + let tree = builder.start().await.unwrap(); + Ordered { + id: 200, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // The later standalone actor owns the current stop turn. Resume the + // supervisor exactly between readiness and its cancellation wait. + resume.cancel(); + assert!(rt::timeout(limit, system::wait_true(after_check)) + .await + .unwrap()); + assert!( + !tree.core.cancel.is_cancelled(), + "admission closure stole the supervisor's stop turn" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + }, + ) + .await; +} + +#[tokio::test] +async fn startup_completion_or_drop_after_readiness_preserves_system_order() { + crate::test_wait::within( + concat!( + module_path!(), + "::startup_completion_or_drop_after_readiness_preserves_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, + } + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; + } + } + } + for complete_start in [true, false] { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + } + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + Ordered { + id: 200, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // Readiness committed, but the caller has not observed start's result. + // Admission now closes while the later actor still owns the stop turn. + if complete_start { + assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); + } else { + drop(startup); + } + assert!( + !core.cancel.is_cancelled(), + "completed readiness was rolled back" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + } + }, + ) + .await; +} + +#[tokio::test] +async fn startup_success_remains_historical_after_unobserved_ready_then_failure() { + crate::test_wait::within( + concat!( + module_path!(), + "::startup_success_remains_historical_after_unobserved_ready_then_failure" + ), + async move { + struct Idle; + impl Actor for Idle {} + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("historical-ready-failure", 1).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 0, + ..SupervisorConfig::default() + }) + .unwrap(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + let child = builder + .add(ChildSpec::new("ready-then-failed", &group, move |_| { + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Idle) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + child + .current() + .unwrap() + .context() + .stop_with_reason(ExitReason::Kill); + assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) + .await + .unwrap()); + let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); + assert!(tree.failure().unwrap().contains("budget exhausted")); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; +} + +#[tokio::test] +async fn started_failures_allow_n_restarts_and_escalate_the_next_failure() { + crate::test_wait::within( + concat!( + module_path!(), + "::started_failures_allow_n_restarts_and_escalate_the_next_failure" + ), + async move { + use std::sync::atomic::{AtomicUsize, Ordering}; + // v0.8 (S1): a first start is not retried, so the first generation + // is healthy and every replacement panics in started(). + struct FailsStarted(Arc, bool); + impl Actor for FailsStarted { + async fn started(&mut self, _: &crate::tasks::Context) { + if self.1 { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("injected started failure"); + } + } + } + for allowed in [0, 1, 3] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("started-budget", 1).unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let attempts = count.clone(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: allowed, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("startup-failure", &group, move |ctx| { + let attempts = attempts.clone(); + let fail = ctx.generation() > 1; + async move { Ok::<_, String>(FailsStarted(attempts, fail)) } + })) + .unwrap(); + let core = builder.core.clone(); + rt::timeout(Duration::from_secs(30), builder.start()) + .await + .unwrap() + .unwrap(); + child.current().unwrap().child_handle().kill(); + assert!(rt::timeout( + Duration::from_secs(30), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + assert_eq!(count.load(Ordering::SeqCst), allowed); + assert_eq!(child.snapshot().generation as usize, allowed + 1); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + } + }, + ) + .await; +} + +#[tokio::test] +async fn sliding_window_expires_only_old_grants_before_the_next_retry() { + crate::test_wait::within( + concat!( + module_path!(), + "::sliding_window_expires_only_old_grants_before_the_next_retry" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sliding-budget", 1).unwrap(); + let window = Duration::from_secs(3600); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 2, + restart_window: window, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("worker", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let mut actor = child.current().unwrap(); + for generation in 1..=2 { + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("retry".into())); + actor = rt::timeout( + Duration::from_secs(30), + child.wait_for_generation(generation), + ) + .await + .unwrap() + .unwrap(); + } + { + let mut budget = system::lock(&core.budget); + assert_eq!(budget.attempts.len(), 2); + // Control just the oldest timestamp; the real controller still owns + // admission and retry. The second grant remains inside the window. + budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); + } + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("expired oldest".into())); + let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) + .await + .unwrap() + .unwrap(); + assert_eq!(system::lock(&core.budget).attempts.len(), 2); + fourth + .child_handle() + .stop_with_reason(ExitReason::Panic("now exhausted".into())); + rt::timeout(Duration::from_secs(30), tree.join()) + .await + .unwrap(); + assert_eq!(child.snapshot().generation, 4); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + }, + ) + .await; +} + +/// OTP charges `add_restart` when it decides a restart, before the +/// cohort is rebuilt: a superseded round keeps its charge, and the +/// superseding round is charged once more. +#[tokio::test] +async fn supersession_charges_each_selected_round_once() { + crate::test_wait::within( + concat!( + module_path!(), + "::supersession_charges_each_selected_round_once" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system + .async_group("refund-superseded-admission", 2) + .unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::RestForOne, + max_restarts: 3, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for name in ["upstream", "middle", "tail"] { + children.push( + builder + .add(ChildSpec::new(name, &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let resume = CancellationToken::new(); + let (entered, arrival) = watch::channel(false); + *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); + children[1] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("middle fails".into())); + assert!(rt::timeout(limit, system::wait_true(arrival)) + .await + .unwrap()); + let old_round = system::lock(&children[1].view.data) + .recovery + .as_ref() + .unwrap() + .meta + .id; + { + let budget = system::lock(&core.budget); + // The round was charged once, at selection; its rebuilds hold + // no reservation of their own. + assert_eq!(budget.attempts.len(), 1); + assert_eq!(budget.reserved, 0); + } + children[0] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); + let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) + .await + .unwrap(); + resume.cancel(); + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(child.snapshot().generation, 2); + } + assert!(tree.failure().is_none()); + { + let budget = system::lock(&core.budget); + // Two restart actions: the superseded round and its successor, + // however many children each rebuilt. + assert_eq!(budget.attempts.len(), 2); + assert_eq!(budget.reserved, 0); + } + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; +} + +#[tokio::test] +async fn cohort_round_takes_one_grant_for_all_rebuilt_children() { + crate::test_wait::within( + concat!( + module_path!(), + "::cohort_round_takes_one_grant_for_all_rebuilt_children" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("cohort-budget-contract", 2).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::OneForAll, + max_restarts: 1, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for index in 0..6 { + children.push( + builder + .add(ChildSpec::new( + format!("worker-{index}"), + &group, + |_| async { Ok::<_, String>(CoordinatorWorker) }, + )) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("last child fails".into())); + // One grant rebuilds all six children (one restart action). + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + } + assert!(tree.failure().is_none()); + assert_eq!(system::lock(&core.budget).attempts.len(), 1); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("second failure".into())); + rt::timeout(limit, tree.join()).await.unwrap(); + assert!(tree + .failure() + .unwrap() + .contains("budget exhausted by worker-5")); + assert_eq!( + children + .iter() + .map(|child| child.snapshot().generation) + .collect::>(), + vec![2; 6] + ); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; +} + +#[tokio::test] +async fn retry_reservations_exclude_siblings_and_refund_interrupted_backoff() { + crate::test_wait::within( + concat!( + module_path!(), + "::retry_reservations_exclude_siblings_and_refund_interrupted_backoff" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system + .supervisor(SupervisorConfig { + max_restarts: 1, + ..SupervisorConfig::default() + }) + .unwrap(); + let reservation = tree.core.reserve_retry(1).unwrap(); + assert!(tree.core.reserve_retry(2).is_none()); + assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); + drop(reservation); + let mut replacement = tree.core.reserve_retry(2).unwrap(); + tree.core.close(); + assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); + drop(replacement); + { + let budget = system::lock(&tree.core.budget); + assert!(budget.attempts.is_empty()); + assert_eq!(budget.reserved, 0); + } + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; +} + +#[test] +fn retry_budget_reservations_count_against_availability() { + // `available` (the reservation check) must agree with `take` at the + // limit: `< max` became `<= max` without any test failing. + let now = Instant::now(); + let mut budget = RetryBudget { + attempts: VecDeque::from([now]), + reserved: 0, + max: 2, + window: Duration::from_secs(10), + }; + assert!(budget.available(now)); + budget.reserved = 1; + assert!(!budget.available(now)); + budget.reserved = 0; + budget.attempts.push_back(now); + assert!(!budget.available(now)); + assert!(budget.available(now + Duration::from_secs(10))); +} +#[test] +fn retry_window_releases_capacity_at_boundary_and_does_not_bank_unused_retries() { + let now = Instant::now(); + let mut budget = RetryBudget { + attempts: VecDeque::new(), + reserved: 0, + max: 2, + window: Duration::from_secs(10), + }; + assert!(budget.take(now)); + assert!(budget.take(now + Duration::from_secs(1))); + assert!(!budget.take(now + Duration::from_secs(9))); + assert!(budget.take(now + Duration::from_secs(10))); + assert!(!budget.take(now + Duration::from_secs(10))); + assert!(budget.take(now + Duration::from_secs(100))); + assert!(budget.take(now + Duration::from_secs(100))); + assert!(!budget.take(now + Duration::from_secs(100))); +} +#[test] +fn capped_backoff_and_seeded_jitter_remain_bounded_after_long_failure_streaks() { + let mut policy = Backoff { + initial: Duration::from_millis(10), + maximum: Duration::from_millis(35), + jitter: false, + ..Backoff::default() + }; + let mut seed = 7; + assert_eq!( + [0, 1, 2, 32, u32::MAX].map(|n| policy.delay(n, &mut seed)), + [10, 20, 35, 35, 35].map(Duration::from_millis) + ); + policy.jitter = true; + let mut a = 42; + let mut b = 42; + let first: Vec<_> = (0..100).map(|n| policy.delay(n, &mut a)).collect(); + let replay: Vec<_> = (0..100).map(|n| policy.delay(n, &mut b)).collect(); + assert_eq!(first, replay); + assert!(first.iter().all(|d| *d <= policy.maximum)); + assert!(first.windows(2).any(|pair| pair[0] != pair[1])); +} diff --git a/concurrency/src/supervision/run.rs b/concurrency/src/supervision/run.rs new file mode 100644 index 00000000..e9d37e5d --- /dev/null +++ b/concurrency/src/supervision/run.rs @@ -0,0 +1,1114 @@ +//! The supervisor coordinator and the per-child runner loops (start, restart with backoff, cohort rounds, shutdown). +use super::*; + +pub(super) async fn run_supervisor(core: Arc, runners: Vec>) { + // All coordinator phases share a boundary. Keep ownership and join progress + // outside it so a panic during startup, stopping, joining or final publication + // cannot detach a controller or lose the runtime/subtree completion signal. + let mut runners = Some(runners); + let mut tasks = Vec::new(); + let mut joined = 0; + let mut coordination = None; + let mut emergency = false; + loop { + let result = AssertUnwindSafe(async { + if let Some(runners) = runners.take() { + run_supervisor_body(core.clone(), runners, &mut tasks, &mut coordination).await; + } + if emergency { + let children = system::lock(&core.children).clone(); + let mut signalled = true; + for child in children.iter().rev() { + if *child.done().borrow() { + continue; + } + if let Err(payload) = std::panic::catch_unwind(AssertUnwindSafe(|| { + child.finish_unstarted(true); + child.stop(); + })) { + let message = crate::error::panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + core.record_cleanup_failure(&format!( + "emergency child stop panicked: {message}" + )); + signalled = false; + } + } + if !signalled { + return false; + } + } + while joined < tasks.len() { + let result = (&mut tasks[joined]).await; + joined += 1; // Never poll a consumed JoinHandle after a later panic. + if let Err(error) = result { + let message = format!("child controller failed: {error}"); + if error.is_panic() { + crate::mailbox::dispose_panic(error.into_panic()); + } + core.record_cleanup_failure(&message); + core.fail_as_kind(FailureKind::Coordinator, message, || { + Some(Event::new("Unmodelled", 0, "ControllerFailed")) + }); + } + } + if let Some(task) = coordination.as_mut() { + let result = task.await; + coordination = None; + if let Err(error) = result { + let message = format!("recovery coordinator failed: {error}"); + if error.is_panic() { + crate::mailbox::dispose_panic(error.into_panic()); + } + core.record_cleanup_failure(&message); + core.fail_kind(FailureKind::Coordinator, message); + } + } + let children = system::lock(&core.children).clone(); + for child in &children { + child.finish_unstarted(core.failed.load(std::sync::atomic::Ordering::Acquire)); + } + core.log_failure_once(); + trace::atomic(|| { + trace::emit(core.trace, || Event::sup("SupJoin")); + core.cohort(|| Event::sup("CDone")); + trace::tree(core.trace, |id| TreeEvent::new("TDone", id)); + // Copy the tapes before `done` lets a test `take` them. + #[cfg(feature = "lifecycle-trace")] + trace::dump_all(core.trace); + core.done.send_replace(true); + }); + true + }) + .catch_unwind() + .await; + match result { + Ok(true) => return, + Ok(false) => (), + Err(payload) => { + let message = crate::error::panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + let message = format!("supervisor coordinator panicked: {message}"); + core.record_cleanup_failure(&message); + core.fail_kind(FailureKind::Coordinator, message); + emergency = true; + } + } + rt::sleep(Duration::from_millis(10)).await; + } +} + +pub(super) async fn run_supervisor_body( + core: Arc, + runners: Vec>, + tasks: &mut Vec>, + coordination: &mut Option>, +) { + let children = system::lock(&core.children).clone(); + *coordination = if core.config.strategy == SupervisorStrategy::OneForOne { + None + } else { + let coordinator = core.clone(); + Some(rt::spawn(async move { + if let Err(payload) = AssertUnwindSafe(run_recoveries(coordinator.clone())) + .catch_unwind() + .await + { + let message = format!( + "recovery coordinator panicked: {}", + crate::error::panic_message(&*payload) + ); + crate::mailbox::dispose_panic(payload); + coordinator.record_cleanup_failure(&message); + coordinator.fail_kind(FailureKind::Coordinator, message); + coordinator.stop_remaining(); + } + })) + }; + for (index, runner) in runners.into_iter().enumerate() { + let open = trace::atomic(|| { + let open = core.gate.is_open(); + if open { + trace::emit(core.trace, || Event::new("SupLaunch", index + 1, "")); + } + open + }); + if !open { + break; + } + tasks.push(rt::spawn(runner.run(core.clone()))); + let event = select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(system::wait_true(children[index].initial())), + ) + .await; + if !matches!(event, Either::Right((true, _))) { + break; + } + } + for child in &children { + child.finish_unstarted(core.failed.load(std::sync::atomic::Ordering::Acquire)); + } + let initialized = trace::atomic(|| { + let committed = core.gate.if_open(|| { + trace::emit(core.trace, || Event::new("SupStartDone", 0, "true")); + core.cohort(|| Event::sup("CInitialReady")); + core.ready.send_replace(true); + }); + if committed.is_none() { + trace::emit(core.trace, || Event::new("SupStartDone", 0, "false")); + } + committed.is_some() + }); + + #[cfg(test)] + let ready_probe = { + let pause = system::lock(&core.initial_ready_pause).take(); + if let Some((resume, passed)) = pause { + resume.cancelled().await; + Some(passed) + } else { + None + } + }; + // Once readiness commits, later admission closure is not a stop turn. + // ActorSystem owns cross-target ordering and cancels this tree in its turn. + if !initialized { + core.close(); + } + #[cfg(test)] + if let Some(passed) = ready_probe { + passed.send_replace(true); + } + core.cancel.cancelled().await; + trace::emit(core.trace, || Event::sup("SupCancelSeen")); + for (index, child) in children.iter().enumerate().rev() { + let c = index + 1; + trace::emit(core.trace, || Event::new("SupStopTurn", c, "")); + fail_point!("shutdown.before_stop_child", async); + child.stop(); + let finished = rt::timeout(child.shutdown_timeout(), system::wait_true(child.done())) + .await + .is_ok(); + // The child may finish between the deadline firing and this check; + // decide on its published completion, atomically with recording. + trace::atomic(|| { + if finished || *child.done().borrow() { + trace::emit(core.trace, || Event::new("SupChildDone", c, "")); + } else { + core.fail_cleanup_for( + child.snapshot(), + format!("child cleanup deadline exceeded: {}", child.snapshot().name), + || Some(Event::new("SupChildTimeout", c, "")), + ); + } + }); + } +} + +pub(super) async fn run_recoveries(core: Arc) { + let children = system::lock(&core.children).clone(); + let mut changes = core.recovery_changed.subscribe(); + 'rounds: loop { + changes.borrow_and_update(); + if !core.gate.is_open() { + return; + } + let round = trace::atomic(|| { + let mut queue = system::lock(&core.recovery); + // The round is the restart action: charge it once, before any + // child of the cohort is stopped or rebuilt. + let origin = queue.pending.iter().position(|pending| *pending)?; + if !core.charge_round() { + return Some(Err(origin)); + } + queue + .select(core.config.strategy == SupervisorStrategy::OneForAll) + .map(|(origin, start, id)| { + let meta = Arc::new(RecoveryMeta { + id, + origin, + finished: std::sync::atomic::AtomicBool::new(false), + }); + core.cohort(|| { + Event::new("CSelect", origin + 1, "") + .round(id) + .count(start + 1) + }); + Ok((start, meta)) + }) + }); + if let Some(Err(origin)) = round { + // Exhausted: a concurrent stop wins; only an open supervisor fails. + let snapshot = children[origin].snapshot(); + let message = format!("restart budget exhausted by {}", snapshot.name); + core.fail_record_if_open_as( + snapshot_failure(snapshot, FailureKind::RestartBudget, message), + || None, + ); + core.log_failure_once(); + return; + } + let Some(Ok((start, meta))) = round else { + if matches!( + select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(changes.changed()) + ) + .await, + Either::Left(_) + ) { + return; + } + continue; + }; + let mut affected = Vec::new(); + for (affected_index, child) in children.iter().enumerate().skip(start) { + let slot = RecoverySlot::new(meta.clone()); + let accepted = trace::atomic(|| { + let accepted = child.assign_recovery(slot.clone()); + core.cohort(|| { + Event::new("CAssign", affected_index + 1, trace::flag(accepted)).round(meta.id) + }); + accepted + }); + if accepted { + affected.push((affected_index + 1, child.clone(), slot)); + } + } + for (index, child, slot) in affected.iter().rev() { + if !core.gate.is_open() { + return; + } + trace::atomic(|| { + child.stop_generation(slot); + core.cohort(|| Event::new("CStop", *index, "").round(meta.id)); + }); + match select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(rt::timeout( + child.shutdown_timeout(), + system::wait_true(slot.cleaned.subscribe()), + )), + ) + .await + { + Either::Left(_) => return, + Either::Right((Ok(true), _)) => (), + _ => { + core.fail_cleanup_for( + child.snapshot(), + format!("coordinated cleanup stuck for {}", child.snapshot().name), + || None, + ); + core.stop_remaining(); + return; + } + } + } + for (index, _, slot) in &affected { + if !core.gate.is_open() { + return; + } + if system::lock(&core.recovery) + .pending + .iter() + .any(|pending| *pending) + { + trace::atomic(|| { + core.cohort(|| Event::new("CEnd", 0, "superseded").round(meta.id)); + meta.finished + .store(true, std::sync::atomic::Ordering::Release); + }); + continue 'rounds; + } + trace::atomic(|| { + core.cohort(|| Event::new("CResume", *index, "").round(meta.id)); + slot.resume.cancel(); + }); + loop { + changes.borrow_and_update(); + // A ready upstream or previously unaffected dependency can fail + // while this child initializes. Supersede the round rather than + // await a dependent that cannot start without that upstream. + if system::lock(&core.recovery) + .pending + .iter() + .any(|pending| *pending) + { + trace::atomic(|| { + core.cohort(|| Event::new("CEnd", 0, "superseded").round(meta.id)); + meta.finished + .store(true, std::sync::atomic::Ordering::Release); + }); + continue 'rounds; + } + match select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(select( + Box::pin(system::wait_true(slot.ready.subscribe())), + Box::pin(changes.changed()), + )), + ) + .await + { + Either::Left(_) => return, + Either::Right((Either::Left((true, _)), _)) => break, + Either::Right((Either::Left((false, _)), _)) => return, + Either::Right((Either::Right(_), _)) => (), + } + } + } + trace::atomic(|| { + core.cohort(|| Event::new("CEnd", 0, "complete").round(meta.id)); + meta.finished + .store(true, std::sync::atomic::Ordering::Release); + }); + } +} + +pub(super) async fn run_child(spec: ChildSpec, view: Arc>, core: Arc) { + let mut first = true; + let mut last_recovery = 0; + let mut collateral = false; + // This iteration rebuilds for a recovery round (already charged). + let mut in_round = false; + let mut failures = 0u32; + // Wall-clock entropy; the simulation seed's entropy stream under spawned_sim. + let seed = core + .config + .backoff + .seed + .unwrap_or_else(spawned_rt::time::entropy); + let mut rng = jitter_seed(seed, system::lock(&view.data).snapshot.name.as_bytes()); + let c = view.trace_index; + loop { + let open = trace::atomic(|| { + let open = core.gate.is_open(); + let branch = if !open { + "finish" + } else if first { + "attempt" + } else { + "backoff" + }; + trace::emit(core.trace, || Event::new("LoopTop", c, branch)); + open + }); + if !open { + break; + } + if let Some(round) = view.recovery(last_recovery) { + if matches!( + select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(round.stop.cancelled()) + ) + .await, + Either::Left(_) + ) { + break; + } + trace::atomic(|| { + core.cohort(|| Event::new("CClean", c, "").round(round.meta.id)); + round.cleaned.send_replace(true); + }); + match select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(select( + Box::pin(view.next_recovery(round.meta.id)), + Box::pin(round.resume.cancelled()), + )), + ) + .await + { + Either::Left(_) => break, + Either::Right((Either::Left(_), _)) => continue, + Either::Right((Either::Right(_), _)) => (), + } + last_recovery = round.meta.id; + if spec.restart == RestartMode::Temporary { + let failed = system::lock(&core.failure).is_some(); + // Publish terminal state under the same lock as assignment before + // acknowledging the round. A superseding coordinator must either + // skip this finished slot or be serviced by this controller. + if trace::atomic(|| { + let finished = view.finish_unless_recovering(last_recovery, failed); + if finished { + trace::emit(core.trace, || Event::new("Finish", c, "")); + core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); + } + finished + }) { + trace::atomic(|| { + core.cohort(|| Event::new("CAck", c, "").round(round.meta.id)); + round.ready.send_replace(true); + }); + core.log_failure_once(); + return; + } + continue; + } + collateral = round.meta.origin != view.index; + in_round = true; + } + let mut backoff = Duration::ZERO; + let mut retry_reservation = None; + // A recovery round was charged once when the coordinator selected + // it: its rebuilds (origin and collateral) take no grant of their own. + let round_rebuild = std::mem::take(&mut in_round); + if !first { + fail_point!("supervisor.before_restart", async); + let reservation = if round_rebuild { + Some(None) + } else { + core.reserve_retry(c).map(Some) + }; + let Some(reservation) = reservation else { + // A concurrent stop wins: only fail a supervisor that is still open. + trace::atomic(|| { + if core.gate.is_open() { + core.fail_record_if_open_as( + child_failure( + &spec, + &view, + FailureKind::RestartBudget, + format!("restart budget exhausted by {}", spec.name), + ), + || { + let budget = system::lock(&core.budget); + let held = budget.attempts.len() + budget.reserved; + Some(Event::new("BackoffDone", c, "exhausted").count(held)) + }, + ); + } else { + trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); + } + }); + core.log_failure_once(); + break; + }; + let delay = if collateral { + Duration::ZERO + } else { + core.config.backoff.delay(failures, &mut rng) + }; + backoff = delay; + let count_attempt = !collateral; + collateral = false; + view.change(|v| v.snapshot.state = ChildState::Backoff); + if matches!( + select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(select( + Box::pin(async { + if core.config.strategy == SupervisorStrategy::OneForOne { + futures::future::pending::<()>().await; + } else { + view.recovery_stopping(last_recovery).await; + } + }), + Box::pin(rt::sleep(delay)) + )) + ) + .await, + Either::Left(_) + ) { + drop(reservation); + trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); + break; + } + fail_point!("supervisor.during_backoff", async); + if view.recovery(last_recovery).is_some() { + continue; + } + if count_attempt { + failures = failures.saturating_add(1); + } + retry_reservation = reservation; + } + first = false; + let (scope, lease) = match spec.group.reserve(Some(core.gate.clone())) { + Ok(attempt) => attempt, + Err(e) => { + trace::atomic(|| { + let was_retry = retry_reservation.is_some(); + drop(retry_reservation.take()); + if core.gate.is_open() { + core.fail_record_if_open_as( + child_failure( + &spec, + &view, + if *view.initial.borrow() { + FailureKind::Other + } else { + FailureKind::FirstStart + }, + format!("placement failed for {}: {e}", spec.name), + ), + || Some(Event::new("Unmodelled", c, "PlacementFailed")), + ); + } else { + trace::emit(core.trace, || { + Event::new( + if was_retry { "BackoffDone" } else { "Attempt" }, + c, + if was_retry { "closed" } else { "false" }, + ) + }); + } + }); + break; + } + }; + #[cfg(test)] + if retry_reservation.is_some() || round_rebuild { + let pause = system::lock(&core.retry_admission_pause).take(); + if let Some((resume, entered)) = pause { + entered.send_replace(true); + resume.cancelled().await; + } + } + let generation = trace::atomic(|| { + let was_retry = retry_reservation.is_some(); + let generation = core + .gate + .if_open(|| { + let mut data = system::lock(&view.data); + if data + .recovery + .as_ref() + .is_some_and(|r| r.meta.id > last_recovery) + { + return None; + } + if let Some(reservation) = retry_reservation.as_mut() { + if !core.retry_under_gate(reservation) { + return None; + } + } + data.snapshot.generation += 1; + data.snapshot.state = ChildState::Starting; + data.snapshot.actor_id = None; + data.current = None; + data.scope = Some(scope.clone()); + Some(data.snapshot.generation) + }) + .flatten(); + if generation.is_none() && was_retry { + drop(retry_reservation.take()); + if !core.gate.is_open() { + trace::emit(core.trace, || Event::new("BackoffDone", c, "closed")); + } + } + if generation.is_some() || !was_retry { + trace::emit(core.trace, || { + Event::new("Attempt", c, trace::flag(generation.is_some())) + }); + } + if let Some(generation) = generation { + trace::tree(core.trace, |id| { + TreeEvent::new("TBegin", id).child(c, generation) + }); + core.cohort(|| { + Event::new("CBegin", c, "") + .round(last_recovery) + .count(generation as usize) + }); + } + generation + }); + let Some(generation) = generation else { + scope.stop(ExitReason::Shutdown); + drop(lease); + if core.gate.is_open() { + continue; + } else { + break; + } + }; + scope.set_name(&spec.name); + if generation > 1 { + let previous = view.snapshot(); + crate::actor_cleanup::log_safely(|| { + tracing::info!(event = "restart_scheduled", child = %spec.name, + generation = generation - 1, reason = ?previous.last_exit, + error = ?previous.last_error, ?backoff, "child retry granted after backoff"); + }); + crate::actor_cleanup::log_safely(|| { + tracing::info!( + event = "restart_started", + child = %spec.name, + generation, + reason = ?previous.last_exit, + error = ?previous.last_error, + "starting replacement child" + ); + }); + } + view.changed.send_modify(|n| *n = n.wrapping_add(1)); + let ctx = FactoryContext { + scope: scope.clone(), + generation, + }; + let factory = spec.factory.clone(); + scope.set_bounds(spec.bounds); + let attempt_scope = scope.clone(); + let construction = async move { + if !attempt_scope.can_start() { + return Err("scope closed before construction".into()); + } + let factory_work = AssertUnwindSafe(async move { + fail_point!("child.factory", async => return Err("failpoint child.factory".into())); + factory(ctx).await + }) + .catch_unwind(); + let result = match select( + Box::pin(attempt_scope.token.cancelled()), + Box::pin(factory_work), + ) + .await + { + Either::Left(_) => return Err("construction cancelled".into()), + Either::Right((Ok(result), _)) => result, + Either::Right((Err(payload), _)) => { + return Err(format!( + "factory panicked: {}", + crate::error::panic_message(&*payload) + )) + } + }; + let actor = result?; + let actor_lease = attempt_scope.actor_lease().map_err(|e| e.to_string())?; + ActorRef::spawn_managed(actor, attempt_scope.clone(), actor_lease) + .map_err(|e| e.to_string()) + }; + let task = scope.handle.spawn(Tracked { + future: construction, + lease, + }); + let startup = async { + let actor = task + .await + .map_err(|e| format!("construction task failed: {e}"))??; + actor.ready().await.map_err(|e| e.to_string())?; + Ok::<_, String>(actor) + }; + let mut startup_timed_out = false; + let began = match select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(rt::timeout(spec.startup_timeout, startup)), + ) + .await + { + Either::Left(_) => Err("supervisor stopping".to_string()), + Either::Right((Err(_), _)) => { + startup_timed_out = true; + Err("startup deadline exceeded".to_string()) + } + Either::Right((Ok(result), _)) => result, + }; + #[cfg(feature = "failpoints")] + let began = match began { + Ok(actor) => { + let mut began = Ok(actor); + fail_point!("child.after_ready_before_publish", async => { + began = Err("failpoint child.after_ready_before_publish".to_string()) + }); + began + } + failed => failed, + }; + let mut ready_at = None; + let mut exit = None; + let mut cleanup_until = None; + // Trace bookkeeping: the model's StartupFailed (whose reason is known + // only after cleanup) and CleanupStuck (which skips CleanupDone and + // Decide). + let startup_failed = began.is_err(); + let mut stuck = false; + match began { + Ok(actor) => { + let published = trace::atomic(|| { + let published = core + .gate + .if_open(|| { + let mut data = system::lock(&view.data); + if data + .recovery + .as_ref() + .is_some_and(|r| r.meta.id > last_recovery) + { + return false; + } + data.snapshot.state = ChildState::Running; + data.snapshot.actor_id = Some(actor.id()); + data.current = Some(actor.clone()); + true + }) + .unwrap_or(false); + trace::emit(core.trace, || { + Event::new("BecomeReady", c, trace::flag(published)) + }); + if published { + core.cohort(|| { + Event::new("CReady", c, "") + .round(last_recovery) + .count(generation as usize) + }); + } + published + }); + view.changed.send_modify(|n| *n = n.wrapping_add(1)); + if published { + ready_at = Some(Instant::now()); + view.initial.send_replace(true); + let round = system::lock(&view.data).recovery.clone(); + if let Some(round) = round.filter(|r| r.meta.id == last_recovery) { + trace::atomic(|| { + core.cohort(|| Event::new("CAck", c, "").round(round.meta.id)); + round.ready.send_replace(true); + }); + } + let child = actor.child_handle(); + // Cancellation precedes terminal publication when stopped() + // hangs. Start its cleanup deadline at stop observation, + // rather than waiting forever for the completion channel. + let ended = select( + Box::pin(core.cancel.cancelled()), + Box::pin(scope.token.cancelled()), + ) + .await; + let stop_turn = matches!(ended, Either::Left(_)); + if stop_turn { + view.stop_turn.cancelled().await; + trace::emit(core.trace, || Event::new("StopObserved", c, "")); + // ChildView::stop already sent the parent's Shutdown. + } + view.change(|v| { + v.current = None; + v.snapshot.state = ChildState::Stopping; + }); + let cleanup_started = Instant::now(); + cleanup_until = Some(cleanup_started); + // The child's jobs are joined within the same deadline. + scope.set_cleanup_timeout(spec.shutdown_timeout); + exit = Some( + match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()).await { + Ok(reason) => { + if !stop_turn { + let failed = child.cleanup_failure().is_some(); + trace::emit(core.trace, || { + Event::new("SelfExit", c, exit_class(&reason, failed)) + }); + } + reason + } + // E3: the supervisor stopped this child and it + // did not reach its death point within + // shutdown_timeout: the parent's kill, then a + // second interval before cleanup is stuck. + Err(_) if stop_turn => { + scope.kill(); + // The second interval runs from the kill. + cleanup_until = Some(Instant::now()); + match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) + .await + { + Ok(reason) => reason, + Err(_) => { + stuck = true; + // Reply traces: the cleanup deadline + // passed before the child's death + // point (DeferredReplies `CleanupDeadline`). + crate::lifecycle_trace::reply( + child.id().raw(), + "Stuck", + 0, + "", + ); + core.fail_cleanup_for( + view.snapshot(), + format!("actor cleanup stuck for {}", spec.name), + || Some(Event::new("CleanupStuck", c, "")), + ); + view.mark_stuck(); + child.wait_exit_async().await + } + } + } + Err(_) => { + if !stop_turn { + trace::emit(core.trace, || { + Event::new("SelfExit", c, "unknown") + }); + } + stuck = true; + crate::lifecycle_trace::reply(child.id().raw(), "Stuck", 0, ""); + core.fail_cleanup_for( + view.snapshot(), + format!("actor cleanup stuck for {}", spec.name), + || Some(Event::new("CleanupStuck", c, "")), + ); + view.mark_stuck(); + child.wait_exit_async().await + } + }, + ); + } + } + Err(error) => view.change(|v| v.snapshot.last_error = Some(error)), + } + let startup_reason = |exit: Option<&ExitReason>| { + if startup_timed_out { + "timeout" + } else { + trace::reason(exit) + } + }; + scope.stop(exit.clone().unwrap_or(ExitReason::Shutdown)); + view.change(|v| { + v.current = None; + v.snapshot.last_exit = exit.clone(); + // A stuck cleanup already settled the snapshot as Failed (S2). + if v.snapshot.state != ChildState::Failed { + v.snapshot.state = ChildState::Stopping; + } + }); + fail_point!("child.before_cleanup_join", async); + let remaining = cleanup_until + .map(|started| spec.shutdown_timeout.saturating_sub(started.elapsed())) + .unwrap_or(spec.shutdown_timeout); + if rt::timeout(remaining, scope.joined()).await.is_err() { + core.fail_cleanup_for( + view.snapshot(), + format!("owned cleanup stuck for {}", spec.name), + || { + if stuck { + return None; + } + if startup_failed { + let reason = match scope.child().map(|child| child.exit_reason()) { + Some(None) if !startup_timed_out => "unknown", + Some(reason) => startup_reason(reason.as_ref()), + None => startup_reason(None), + }; + trace::emit(core.trace, || Event::new("StartupFailed", c, reason)); + } + Some(Event::new("CleanupStuck", c, "")) + }, + ); + view.mark_stuck(); + scope.joined().await; + trace::atomic(|| { + trace::tree(core.trace, |id| { + TreeEvent::new("TJoined", id).child(c, generation) + }); + core.cohort(|| Event::new("CJoined", c, "").count(generation as usize)); + }); + break; + } + trace::atomic(|| { + trace::tree(core.trace, |id| { + TreeEvent::new("TJoined", id).child(c, generation) + }); + core.cohort(|| Event::new("CJoined", c, "").count(generation as usize)); + }); + if exit.is_none() { + exit = scope.child().and_then(|child| child.exit_reason()); + } + if !stuck { + trace::atomic(|| { + if startup_failed { + trace::emit(core.trace, || { + Event::new("StartupFailed", c, startup_reason(exit.as_ref())) + }); + } + trace::emit(core.trace, || Event::new("CleanupDone", c, "")); + }); + } + let cleanup_failure = scope.child().and_then(|child| child.cleanup_failure()); + view.change(|v| { + v.snapshot.last_exit = exit.clone(); + if cleanup_failure.is_some() { + v.snapshot.last_error = cleanup_failure.clone(); + } + if v.snapshot.actor_id.is_none() { + v.snapshot.actor_id = scope.child().map(|child| child.id()); + } + v.scope = None; + }); + // A stable generation resets its failure streak even when another + // child's recovery supersedes this controller during cleanup. + if scope.was_stable(core.config.backoff.stable_after) { + failures = 0; + } + if core.gate.is_open() && view.recovery(last_recovery).is_some() { + continue; + } + // S1: a child's first start that fails (any cause) aborts the + // supervisor's start whatever the restart type, as OTP's + // `start_children` returns `failed_to_start_child`. + let first_start = ready_at.is_none() && !*view.initial.borrow(); + // A generation that never became ready after the child's initial + // readiness is a failed restart attempt: retried within the budget + // whatever its exit reason, as OTP retries a failed restart + // (`try_again_restart`), unless Temporary. + let failed_restart = ready_at.is_none() && !first_start; + let restart = if first_start { + false + } else if startup_timed_out || failed_restart { + spec.restart != RestartMode::Temporary + } else { + spec.restart + .restarts(exit.as_ref(), cleanup_failure.is_some()) + }; + // S1: a ready child's exit during the initial start is recorded and + // processed once the tree commits ready, as OTP handles the queued + // 'EXIT' after `init` returns. If the start aborts instead, the + // exit is part of the rollback. Only the decision waits: cleanup + // already ran within its deadline. + let deferred = ready_at.is_some() + && trace::atomic(|| { + let defer = core.gate.is_open() && !*core.ready.borrow(); + if defer { + trace::emit(core.trace, || Event::new("DeferExit", c, "")); + } + defer + }); + if deferred { + let _ = select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(system::wait_true(core.ready.subscribe())), + ) + .await; + trace::emit(core.trace, || Event::new("Undefer", c, "")); + } + // Gate check, decision and any resulting failure happen in one step, + // so a concurrent stop is never followed by a failure of this child. + let again = trace::atomic(|| { + if !core.gate.is_open() { + if !stuck { + trace::emit(core.trace, || Event::new("Decide", c, "closed")); + } + return false; + } + if spec.group.is_closed() { + core.fail_record_if_open_as( + child_failure( + &spec, + &view, + if first_start { + FailureKind::FirstStart + } else { + FailureKind::Other + }, + format!("execution group closed for {}", spec.name), + ), + || Some(Event::new("Unmodelled", c, "GroupClosed")), + ); + return false; + } + if restart { + trace::emit(core.trace, || Event::new("Decide", c, "restart")); + return true; + } + if ready_at.is_some() { + trace::emit(core.trace, || Event::new("Decide", c, "stop")); + } else { + let (kind, message) = if first_start { + let snapshot = view.snapshot(); + let cause = snapshot + .last_error + .clone() + .or_else(|| exit.as_ref().map(|reason| format!("exited {reason}"))) + .unwrap_or_else(|| "unknown".into()); + ( + FailureKind::FirstStart, + format!("failed to start child {}: {cause}", spec.name), + ) + } else { + ( + FailureKind::Other, + format!("non-restartable child failed startup: {}", spec.name), + ) + }; + core.fail_record_if_open_as(child_failure(&spec, &view, kind, message), || { + Some(Event::new("Decide", c, "fail")) + }); + } + false + }); + if !again { + if core.config.strategy == SupervisorStrategy::OneForOne || !core.gate.is_open() { + break; + } + let failed = system::lock(&core.failure).is_some(); + if trace::atomic(|| { + let finished = view.finish_unless_recovering(last_recovery, failed); + if finished { + trace::emit(core.trace, || Event::new("Finish", c, "")); + core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); + } + finished + }) { + core.log_failure_once(); + return; + } + continue; + } + // A failed replacement inside a recovery round requests a new round + // like any other failure, so OneForAll stops and rebuilds the whole + // cohort again (and RestForOne the failed child and its later + // siblings), as OTP's `try_again_restart` re-runs the strategy. + if core.config.strategy != SupervisorStrategy::OneForOne && *core.ready.borrow() { + core.request_recovery(view.index, last_recovery); + if matches!( + select( + Box::pin(core.admission_closed.cancelled()), + Box::pin(view.next_recovery(last_recovery)) + ) + .await, + Either::Left(_) + ) { + break; + } + } + } + // Record Finish and publish completion together, so the supervisor's + // deadline check sees both or neither. + trace::atomic(|| { + trace::emit(core.trace, || Event::new("Finish", c, "")); + view.finish(system::lock(&core.failure).is_some()); + core.cohort(|| Event::new("CRetire", c, "").round(last_recovery)); + }); + core.log_failure_once(); +} + +pub(super) fn child_failure( + spec: &ChildSpec, + view: &View, + kind: FailureKind, + message: String, +) -> Failure { + let mut snapshot = view.snapshot(); + snapshot.name = spec.name.clone(); + snapshot_failure(snapshot, kind, message) +} + +pub(super) fn snapshot_failure( + snapshot: ChildSnapshot, + kind: FailureKind, + message: String, +) -> Failure { + Failure::new( + kind, + message, + Some(FailureOrigin { + child: snapshot.name, + generation: snapshot.generation, + reason: snapshot.last_exit, + last_error: snapshot.last_error, + }), + ) +} diff --git a/concurrency/src/supervision/state.rs b/concurrency/src/supervision/state.rs new file mode 100644 index 00000000..f5ecad4d --- /dev/null +++ b/concurrency/src/supervision/state.rs @@ -0,0 +1,469 @@ +//! The supervisor core: restart budget, failure records, the shared `Core` state and its restart, shutdown and cleanup bookkeeping. +use super::*; + +pub(super) struct RetryBudget { + pub(super) attempts: VecDeque, + pub(super) reserved: usize, + pub(super) max: usize, + pub(super) window: Duration, +} +impl RetryBudget { + pub(super) fn available(&self, now: Instant) -> bool { + self.attempts + .iter() + .filter(|&&time| now.saturating_duration_since(time) < self.window) + .count() + .saturating_add(self.reserved) + < self.max + } + pub(super) fn take(&mut self, now: Instant) -> bool { + admit_in_window( + &mut self.attempts, + self.max.saturating_sub(self.reserved), + self.window, + now, + |now, t| now.saturating_duration_since(t), + ) + } +} + +/// Rolling-window rate limit: forget stamps at least `window` old, then admit +/// `now` only while fewer than `max` remain. `since(now, t)` is the elapsed +/// time from `t` to `now`, generic so the window can be checked without clocks. +pub(crate) fn admit_in_window( + attempts: &mut VecDeque, + max: usize, + window: Duration, + now: T, + since: impl Fn(T, T) -> Duration, +) -> bool { + while attempts.front().is_some_and(|t| since(now, *t) >= window) { + attempts.pop_front(); + } + if attempts.len() >= max { + return false; + } + attempts.push_back(now); + true +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum FailureKind { + Other, + /// A child's first start failed (S1). + FirstStart, + Cleanup, + Coordinator, + RestartBudget, +} +#[derive(Debug, Clone)] +pub(super) struct FailureOrigin { + pub(super) child: String, + pub(super) generation: u64, + pub(super) reason: Option, + pub(super) last_error: Option, +} +#[derive(Debug, Clone)] +pub(super) struct Failure { + pub(super) kind: FailureKind, + pub(super) message: String, + pub(super) origin: Option, + pub(super) logged: bool, +} +impl Failure { + pub(super) fn new(kind: FailureKind, message: String, origin: Option) -> Self { + Self { + kind, + message, + origin, + logged: false, + } + } +} +impl std::ops::Deref for Failure { + type Target = str; + fn deref(&self) -> &str { + &self.message + } +} + +pub(super) struct Core { + pub(super) recovery: Mutex, + pub(super) recovery_changed: watch::Sender, + pub(super) gate: Arc, + pub(super) cancel: CancellationToken, + pub(super) admission_closed: CancellationToken, + pub(super) done: watch::Sender, + pub(super) ready: watch::Sender, + pub(super) failure: Mutex>, + pub(super) failed: Arc, + pub(super) cleanup_failure: Mutex>, + /// Every recorded cleanup failure, for the rollback list of a failed + /// start (S2). + pub(super) cleanup_failures: Mutex>, + pub(super) children: Mutex>>, + pub(super) budget: Mutex, + pub(super) config: SupervisorConfig, + pub(super) trace: TraceId, + #[cfg(test)] + pub(super) initial_ready_pause: Mutex)>>, + #[cfg(test)] + pub(super) retry_admission_pause: Mutex)>>, + #[cfg(feature = "lifecycle-trace")] + pub(super) deadline_recorded: std::sync::atomic::AtomicBool, +} +impl Core { + pub(super) fn close_admission(&self) { + trace::atomic(|| { + if self.gate.is_open() { + self.cohort(|| Event::sup("CClose")); + trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); + } + #[cfg(feature = "lifecycle-trace")] + if self.gate.is_open() && !trace::close_suppressed() { + trace::emit(self.trace, || Event::sup("Close")); + } + self.gate.close(); + }); + self.admission_closed.cancel(); + let children = system::lock(&self.children).clone(); + for child in children { + child.close_admission(); + } + } + pub(super) fn close(&self) { + trace::atomic(|| { + self.close_admission(); + if !self.cancel.is_cancelled() { + trace::tree(self.trace, |id| TreeEvent::new("TCancel", id)); + self.cancel.cancel(); + } + }); + } + pub(super) fn cancel_unready_startup(&self) { + trace::atomic(|| { + let cancel = self.gate.close_unless( + || *self.ready.borrow(), + || { + self.cohort(|| Event::sup("CClose")); + trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); + #[cfg(feature = "lifecycle-trace")] + if !trace::close_suppressed() { + trace::emit(self.trace, || Event::sup("Close")); + } + }, + ); + if cancel { + self.close(); + } + }); + } + pub(super) fn stop_remaining(&self) { + trace::atomic(|| { + self.close(); + #[cfg(feature = "lifecycle-trace")] + if !*self.done.borrow() + && !self + .deadline_recorded + .swap(true, std::sync::atomic::Ordering::Relaxed) + { + trace::emit(self.trace, || Event::sup("ShutdownDeadline")); + } + }); + let children = system::lock(&self.children).clone(); + for child in children.iter().rev() { + child.stop(); + } + } + #[cfg(test)] + pub(super) fn fail(&self, error: String) { + self.fail_kind(FailureKind::Other, error); + } + pub(super) fn fail_kind(&self, kind: FailureKind, error: String) { + self.fail_record(Failure::new(kind, error, None)); + } + pub(super) fn fail_record(&self, record: Failure) { + let mut failure = system::lock(&self.failure); + if failure.is_none() { + *failure = Some(record); + } + self.failed + .store(true, std::sync::atomic::Ordering::Release); + drop(failure); + self.close(); + } + #[inline(always)] + pub(super) fn cohort(&self, make: impl FnOnce() -> Event) { + #[cfg(feature = "lifecycle-trace")] + if self.config.strategy != SupervisorStrategy::OneForOne { + trace::emit(self.trace, make); + } + #[cfg(not(feature = "lifecycle-trace"))] + let _ = make; + } + pub(super) fn record_cleanup_failure(&self, error: &str) { + system::lock(&self.cleanup_failures).push(error.to_string()); + let mut failure = system::lock(&self.cleanup_failure); + if failure.is_none() { + *failure = Some(error.to_string()); + } + } + pub(super) fn fail_cleanup(&self, error: String) { + self.record_cleanup_failure(&error); + self.fail_kind(FailureKind::Cleanup, error); + } + pub(super) fn fail_cleanup_for( + &self, + snapshot: ChildSnapshot, + error: String, + event: impl FnOnce() -> Option, + ) { + self.record_cleanup_failure(&error); + let record = snapshot_failure(snapshot, FailureKind::Cleanup, error); + trace::failing(|| { + trace::emit_some(self.trace, event); + self.fail_record(record); + }); + } + pub(super) fn request_recovery(&self, index: usize, after: u64) { + trace::atomic(|| { + self.gate.if_open(|| { + let mut queue = system::lock(&self.recovery); + let accepted = queue.request(index, after); + self.cohort(|| { + Event::new("CRequest", index + 1, trace::flag(accepted)).round(after) + }); + if accepted { + self.recovery_changed + .send_modify(|n| *n = n.wrapping_add(1)); + } + }) + }); + } + /// `fail` as part of the model action `event` (whose `Fail` closes the + /// gate), so the gate flip is not traced as a separate `Close`. + pub(super) fn fail_as_kind( + &self, + kind: FailureKind, + error: String, + event: impl FnOnce() -> Option, + ) { + trace::failing(|| { + trace::emit_some(self.trace, event); + self.fail_kind(kind, error); + }); + } + pub(super) fn log_failure_once(&self) { + let failure = { + let mut slot = system::lock(&self.failure); + let Some(failure) = slot.as_mut() else { + return; + }; + if failure.logged { + return; + } + failure.logged = true; + failure.clone() + }; + let (child, generation, reason, last_error) = failure + .origin + .as_ref() + .map(|origin| { + ( + origin.child.as_str(), + origin.generation, + origin.reason.as_ref(), + origin.last_error.as_deref(), + ) + }) + .unwrap_or(("", 0, None, None)); + let event = if failure.kind == FailureKind::RestartBudget { + "restart_budget_exhausted" + } else { + "supervision_give_up" + }; + if let Err(payload) = std::panic::catch_unwind(AssertUnwindSafe(|| { + tracing::warn!(event, child, generation, ?reason, ?last_error, + kind = ?failure.kind, error = %failure.message, "supervisor failed"); + })) { + crate::mailbox::dispose_panic(payload); + } + } + /// A concurrent ordinary stop wins over a later child failure. The gate + /// lock, rather than optional trace instrumentation, commits this decision. + #[cfg(test)] + pub(super) fn fail_if_open_as( + &self, + error: String, + event: impl FnOnce() -> Option, + ) -> bool { + self.fail_record_if_open_as(Failure::new(FailureKind::Other, error, None), event) + } + pub(super) fn fail_record_if_open_as( + &self, + record: Failure, + event: impl FnOnce() -> Option, + ) -> bool { + trace::failing(|| { + let committed = self.gate.close_if_open(|| { + trace::emit_some(self.trace, event); + self.cohort(|| Event::sup("CClose")); + trace::tree(self.trace, |id| TreeEvent::new("TClose", id)); + let mut failure = system::lock(&self.failure); + if failure.is_none() { + *failure = Some(record); + } + self.failed + .store(true, std::sync::atomic::Ordering::Release); + }); + if committed { + self.close(); + } + committed + }) + } + pub(super) fn reserve_retry(&self, child: usize) -> Option> { + trace::atomic(|| { + self.gate + .if_open(|| { + let mut budget = system::lock(&self.budget); + let now = Instant::now(); + // Pending backoffs reserve capacity, so siblings cannot all sleep + // against the same final permit. Only actual grants enter the window. + while budget + .attempts + .front() + .is_some_and(|time| now.saturating_duration_since(*time) >= budget.window) + { + budget.attempts.pop_front(); + } + if !budget.available(now) { + return None; + } + budget.reserved += 1; + trace::emit(self.trace, || { + Event::new("RetryReserve", child, "") + .count(budget.attempts.len() + budget.reserved) + }); + Some(RetryReservation { + core: self, + child, + held: true, + }) + }) + .flatten() + }) + } + /// One OneForAll/RestForOne recovery round is one restart action (OTP's + /// `add_restart` runs once per restart, before the cohort is rebuilt): + /// take its grant now. The round's controllers then rebuild without a + /// grant of their own. Pending per-child reservations (retries during + /// the initial start) keep their capacity. Returns whether it was taken. + pub(super) fn charge_round(&self) -> bool { + let mut budget = system::lock(&self.budget); + budget.take(Instant::now()) + } + /// Caller holds the admission gate and slot lock; charge atomically with + /// generation admission, so placement failure or supersession refunds it. + pub(super) fn retry_under_gate(&self, reservation: &mut RetryReservation<'_>) -> bool { + let mut budget = system::lock(&self.budget); + budget.reserved -= 1; + reservation.held = false; + let taken = budget.take(Instant::now()); + debug_assert!(taken, "reserved retry lost its capacity"); + if taken { + trace::emit(self.trace, || { + Event::new("BackoffDone", reservation.child, "retry") + .count(budget.attempts.len() + budget.reserved) + }); + } + taken + } + #[cfg(test)] + pub(super) fn retry(&self, reservation: &mut RetryReservation<'_>) -> bool { + self.gate + .if_open(|| self.retry_under_gate(reservation)) + .unwrap_or(false) + } +} +pub(super) struct RetryReservation<'a> { + pub(super) core: &'a Core, + pub(super) child: usize, + pub(super) held: bool, +} +impl Drop for RetryReservation<'_> { + fn drop(&mut self) { + if self.held { + trace::atomic(|| { + let mut budget = system::lock(&self.core.budget); + budget.reserved -= 1; + trace::emit(self.core.trace, || { + Event::new("RetryRelease", self.child, "") + .count(budget.attempts.len() + budget.reserved) + }); + }); + } + } +} +impl ShutdownUnit for Core { + fn close_admission(&self) { + Core::close_admission(self); + } + fn stop(&self) { + Core::close(self); + } + fn stop_remaining(&self) { + Core::stop_remaining(self); + } + fn joined(&self) -> BoxFuture<'_, ()> { + Box::pin(async move { + let _ = system::wait_true(self.done.subscribe()).await; + }) + } + fn is_done(&self) -> bool { + *self.done.borrow() + } + fn failure(&self) -> Option { + system::lock(&self.failure) + .as_ref() + .map(|f| f.message.clone()) + } +} + +pub(super) trait Runner: Send { + fn run(self: Box, core: Arc) -> BoxFuture<'static, ()>; +} +pub(super) struct TypedRunner { + pub(super) spec: ChildSpec, + pub(super) view: Arc>, +} +impl Runner for TypedRunner { + fn run(self: Box, core: Arc) -> BoxFuture<'static, ()> { + let view = self.view.clone(); + Box::pin(async move { + if let Err(payload) = AssertUnwindSafe(run_child(self.spec, self.view, core.clone())) + .catch_unwind() + .await + { + let message = format!( + "child controller panicked: {}", + crate::error::panic_message(&*payload) + ); + crate::mailbox::dispose_panic(payload); + core.fail_cleanup_for(view.snapshot(), message, || { + Some(Event::new( + "Unmodelled", + view.trace_index, + "ControllerPanicked", + )) + }); + let scope = system::lock(&view.data).scope.clone(); + if let Some(scope) = scope { + scope.stop(ExitReason::Shutdown); + scope.joined().await; + } + view.finish(true); + } + }) + } +} diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index f99ecbbf..34b6c48a 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -35,6 +35,14 @@ pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); mod jobs; pub use jobs::{BlockingJobPermit, JobCtx, JobPermit}; +mod envelope; +use envelope::*; +mod actor_ref; +mod context; +mod lifecycle; +#[cfg(test)] +use lifecycle::run_actor; + // --------------------------------------------------------------------------- // Backend // --------------------------------------------------------------------------- @@ -152,552 +160,6 @@ pub trait Handler: Actor { ) -> impl Future + Send; } -// --------------------------------------------------------------------------- -// Envelope (type-erasure on the actor side) -// --------------------------------------------------------------------------- - -trait Envelope: Send { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "other" - } - #[cfg(any(feature = "diagnostics", spawned_sim))] - fn message_type(&self) -> &'static str { - std::any::type_name::() - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - None - } - #[cfg(feature = "metrics")] - fn reject(self: Box) {} - - /// The owner reached its death point with this envelope undispatched: - /// answer its request now (the final reason), before any payload of the - /// mailbox is destroyed. No user code runs. - fn discharge(&mut self, _replies: Option<&Registry>) {} - - /// Dispatch once: the turn's future, and the request's caller if any. - /// The envelope keeps the request's slot for the whole turn and lends - /// it to the handler's `PendingReply` (no reference count change per - /// message); the run loop answers the lent caller at the death point if - /// the runtime destroys the turn before it finishes. - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a>; -} - -/// A dispatched turn: None means the guarded dispatch already completed; -/// otherwise its future and caller stay lent by the envelope for the turn. -type Dispatch<'a> = ( - Option + Send + 'a>>>, - Option<&'a dyn Caller>, -); - -/// Dispatch `envelope`, catching a panic while the handler builds its -/// future (a manual `Handler` may panic before it returns one): a handler -/// panic like any other. -fn dispatch<'a, A: Actor>( - envelope: &'a mut (dyn Envelope + Send), - actor: &'a mut A, - ctx: &'a Context, -) -> Result, Box> { - let mut parts = Some((envelope, actor)); - std::panic::catch_unwind(AssertUnwindSafe(move || { - let (envelope, actor) = parts.take().expect("dispatched once"); - crate::lifecycle_trace::with_actor(ctx.shared.id.raw(), || envelope.handle(actor, ctx)) - })) -} - -/// Take an envelope's payload at its one dispatch. -fn dispatched(payload: &mut Option) -> T { - payload.take().expect("an envelope is dispatched once") -} - -/// Answer every request queued in a dead owner's mailbox, destroying -/// nothing: the drained envelopes come back, to be destroyed once the death -/// point's wake-ups ran. Admission is closed first, so nothing is queued -/// after. -fn answer_mailbox( - rx: &mut mpsc::Receiver + Send>>, - replies: Option<&Registry>, -) -> Vec + Send>> { - let mut queued = rx.drain(); - for envelope in &mut queued { - envelope.discharge(replies); - } - queued -} - -/// A never-polled actor's startup payload, dropped with its runtime. -type StartupPayload = ( - A, - Context, - mpsc::Receiver + Send>>, - StopToken, -); -fn discharge_startup(payload: &mut StartupPayload) -> Vec> { - let replies = payload.1.shared.replies.clone(); - erase(answer_mailbox(&mut payload.2, Some(&replies))) -} -fn erase(items: Vec) -> Vec> { - items - .into_iter() - .map(|item| Box::new(item) as Box) - .collect() -} - -struct MessageEnvelope { - /// The request's slot: held until dispatch (dropped undispatched, it is - /// disposed of as a `SlotHandle`), then lent to the turn. - slot: Option>>, - // Some until dispatch; taking the message is also the dispatch marker. - msg: Option, - #[cfg(feature = "metrics")] - observation: Option, - /// The recorded actor that sent it (0: outside code or a request); its - /// mailbox item kind is then the model's `msg`. - #[cfg(feature = "lifecycle-trace")] - from: u64, -} - -/// An undispatched request (mailbox disposal without the registry, a -/// refused admission) is answered as its `SlotHandle` would be. -impl Drop for MessageEnvelope { - #[inline] - fn drop(&mut self) { - if self.msg.is_some() && self.slot.is_some() { - drop(SlotHandle::of(self.slot.take())); - } - } -} - -impl Envelope for MessageEnvelope -where - A: Actor + Handler, - M: Message, -{ - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - if self.from != 0 { - "msg" - } else { - crate::lifecycle_trace::life_kind::() - } - } - #[cfg(any(feature = "diagnostics", spawned_sim))] - fn message_type(&self) -> &'static str { - std::any::type_name::() - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - self.observation.take() - } - #[cfg(feature = "metrics")] - fn reject(mut self: Box) { - if let Some(observation) = self.observation.take() { - observation.reject(); - } - } - fn discharge(&mut self, replies: Option<&Registry>) { - if self.msg.is_some() { - let mut held = SlotHandle::of(self.slot.take()); - held.dispose(replies); - } - } - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let Self { slot, msg, .. } = self; - let msg = dispatched(msg); - let slot: &'a Option>> = slot; - let reply = PendingReply::lent(slot.as_ref(), &ctx.shared.replies); - let caller = slot.as_ref().map(|slot| slot as &dyn Caller); - match >::__spawned_handle_immediate(actor, msg, reply, ctx) { - None => (None, caller), - Some((msg, reply)) => (Some(Box::pin(actor.handle(msg, reply, ctx))), caller), - } - } -} - -/// A sent message (no caller): no reply slot to hold, lend or dispose of, -/// so its dispatch builds only a detached `PendingReply`. -struct SendEnvelope { - msg: Option, - #[cfg(feature = "metrics")] - observation: Option, - #[cfg(feature = "lifecycle-trace")] - from: u64, -} - -impl Envelope for SendEnvelope -where - A: Actor + Handler, - M: Message, -{ - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - if self.from != 0 { - "msg" - } else { - crate::lifecycle_trace::life_kind::() - } - } - #[cfg(any(feature = "diagnostics", spawned_sim))] - fn message_type(&self) -> &'static str { - std::any::type_name::() - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - self.observation.take() - } - #[cfg(feature = "metrics")] - fn reject(mut self: Box) { - if let Some(observation) = self.observation.take() { - observation.reject(); - } - } - #[inline] - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let msg = dispatched(&mut self.msg); - match >::__spawned_handle_immediate( - actor, - msg, - PendingReply::detached(), - ctx, - ) { - None => (None, None), - Some((msg, reply)) => (Some(Box::pin(actor.handle(msg, reply, ctx))), None), - } - } -} - -impl> Envelope for DownEnvelope { - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "down" - } - #[cfg(any(feature = "diagnostics", spawned_sim))] - fn message_type(&self) -> &'static str { - std::any::type_name::() - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - self.observation.take() - } - #[cfg(feature = "metrics")] - fn reject(mut self: Box) { - if let Some(observation) = self.observation.take() { - observation.reject(); - } - } - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let down = dispatched(&mut self.down); - let turn = Box::pin(async move { - fail_point!("monitor.down_before_dispatch", async); - if crate::monitor::fence::take_queued(&ctx.shared.monitors, &down.monitor_ref) { - let reply = PendingReply::new(None, &ctx.shared.replies); - actor.handle(down, reply, ctx).await; - } - }); - (Some(turn), None) - } -} - -/// Admit a message. Past the death point (the registry is sealed, or the -/// mailbox is gone) it reports `NoProc`; before it, including while the -/// actor exits, it is queued, and a request is answered at the death point -/// if it is not handled. -#[inline] -fn enqueue( - sender: &mpsc::Sender + Send>>, - _mailbox: &crate::instrumentation::Mailbox, - replies: &Registry, - msg: M, - slot: SlotHandle, -) -> Result<(), ActorError> -where - A: Actor + Handler, - M: Message, -{ - let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); - let mut slot = slot; - let envelope: Box + Send> = Box::new(MessageEnvelope { - msg: Some(msg), - slot: slot.take(), - #[cfg(feature = "metrics")] - observation: Some(_mailbox.enqueued()), - #[cfg(feature = "lifecycle-trace")] - from, - }); - admit::(sender, replies, envelope, from) -} - -/// Admit a sent message (no caller): no reply slot at all. -fn enqueue_send( - sender: &mpsc::Sender + Send>>, - _mailbox: &crate::instrumentation::Mailbox, - replies: &Registry, - msg: M, -) -> Result<(), ActorError> -where - A: Actor + Handler, - M: Message, -{ - let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), true); - let envelope: Box + Send> = Box::new(SendEnvelope { - msg: Some(msg), - #[cfg(feature = "metrics")] - observation: Some(_mailbox.enqueued()), - #[cfg(feature = "lifecycle-trace")] - from, - }); - admit::(sender, replies, envelope, from) -} - -/// A monitor's `Down`, enqueued by the framework (the waiter at the target's -/// publication, or `monitor` past the target's death point). Its dispatch -/// runs `Handler` only while the monitor's table entry is still -/// queued: a `demonitor` that returned first flushed it (OTP -/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary -/// message. -struct DownEnvelope { - down: Option, - #[cfg(feature = "metrics")] - observation: Option, -} - -fn enqueue_down>( - sender: &mpsc::Sender + Send>>, - _mailbox: &crate::instrumentation::Mailbox, - replies: &Registry, - down: Down, -) -> Result<(), ActorError> { - let envelope: Box + Send> = Box::new(DownEnvelope { - down: Some(down), - #[cfg(feature = "metrics")] - observation: Some(_mailbox.enqueued()), - }); - admit::(sender, replies, envelope, 0) -} - -/// Admission of a built envelope (`M` names its kind for the trace). -fn admit( - sender: &mpsc::Sender + Send>>, - replies: &Registry, - envelope: Box + Send>, - _from: u64, -) -> Result<(), ActorError> { - // Admission and its trace event are one step of the lifecycle trace. - let owner = replies.owner().raw(); - crate::lifecycle_trace::life_section(owner, || { - if replies.refuses_admission() { - #[cfg(feature = "metrics")] - envelope.reject(); - crate::lifecycle_trace::life_send::(owner, false, _from); - return Err(ActorError::NoProc); - } - match sender.send(envelope) { - Ok(()) => { - #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - crate::lifecycle_trace::life_send::(owner, true, _from); - Ok(()) - } - Err(_error) => { - #[cfg(feature = "metrics")] - { - _error.0.reject(); - } - crate::lifecycle_trace::life_send::(owner, false, _from); - Err(ActorError::NoProc) - } - } - }) -} - -/// A closure run in one of the actor's turns (`spawned_test::tasks::in_actor`). -type InTurn = Box) -> R + Send>; -struct InTurnEnvelope { - slot: SlotHandle, - lent: Option>>, - f: Option>, -} - -impl Envelope for InTurnEnvelope { - #[cfg(feature = "diagnostics")] - fn message_type(&self) -> &'static str { - "InTurn" - } - fn discharge(&mut self, replies: Option<&Registry>) { - self.slot.dispose(replies); - drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); - } - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let Self { slot, lent, f } = self; - let f = dispatched(f); - *lent = slot.take(); - let lent: &'a Option>> = lent; - // Inside the future, so a panicking closure is a handler panic. - let turn = Box::pin(async move { - let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); - let value = f(actor, ctx); - let _ = reply.reply(value); - }); - (Some(turn), lent.as_ref().map(|slot| slot as &dyn Caller)) - } -} - -// Private lifecycle signal, excluded from application message/handler metrics. -// Nested supervisor adapters have no public mailbox traffic to bury this signal. -struct FailureEnvelope(Option); -impl Envelope for FailureEnvelope { - #[cfg(feature = "diagnostics")] - fn message_type(&self) -> &'static str { - "Exit" - } - - fn handle<'a>(&'a mut self, _: &'a mut A, _: &'a Context) -> Dispatch<'a> { - let failure = dispatched(&mut self.0); - ( - Some(Box::pin(async move { - panic!("nested supervisor failed: {failure}"); - })), - None, - ) - } -} - -/// System envelope that dispatches an `Exit` notification to the actor's -/// `exit_received` callback. Unlike `MessageEnvelope`, this does not require -/// `Handler` on the actor — every actor can receive `Exit` via the -/// trait's default-no-op `exit_received` callback. -struct ExitEnvelope { - exit: Option, -} - -impl Envelope for ExitEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "exit" - } - #[cfg(feature = "diagnostics")] - fn message_type(&self) -> &'static str { - "Exit" - } - - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let exit = dispatched(&mut self.exit); - ( - Some(Box::pin(async move { - actor.exit_received(exit, ctx).await; - })), - None, - ) - } -} - -/// A parent exit (E3): when dispatched, exit begins with the parent's reason -/// and `stopped()` runs. `exit_received` never sees it. -struct ParentExitEnvelope { - exit: Option, -} - -impl Envelope for ParentExitEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "pexit" - } - #[cfg(feature = "diagnostics")] - fn message_type(&self) -> &'static str { - "ParentExit" - } - - fn handle<'a>(&'a mut self, _actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let exit = dispatched(&mut self.exit); - ( - Some(Box::pin(async move { ctx.parent_exit(exit.reason) })), - None, - ) - } -} - -/// An exit signal that arrived while `started()` ran, in its mailbox place -/// (phase 0). If the actor trapped exits when `started()` returned it is an -/// `Exit` (or parent exit); otherwise it was resolved then and is inert. -struct HeldEnvelope { - exit: Option, - parent: bool, -} - -impl Envelope for HeldEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "held" - } - #[cfg(feature = "diagnostics")] - fn message_type(&self) -> &'static str { - "Exit" - } - - fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { - let exit = dispatched(&mut self.exit); - let parent = self.parent; - let turn = Box::pin(async move { - if !link::lock_request(&ctx.shared.linked_reason).held_trapped() { - return; - } - if parent { - ctx.parent_exit(exit.reason); - } else { - actor.exit_received(exit, ctx).await; - } - }); - (Some(turn), None) - } -} - -/// Build the signal target of a new actor instance. -fn signal_target( - id: ActorId, - parent: Option, - sender: &mpsc::Sender + Send>>, - trap: TrapExitFlag, - request: LinkedExitReason, - token: StopToken, -) -> Arc { - let exit_tx = sender.clone(); - let parent_tx = sender.clone(); - let held_tx = sender.clone(); - let send_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit: Some(exit) }); - exit_tx.send(envelope).map_err(|_| ActorError::NoProc) - }); - let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = - Box::new(ParentExitEnvelope { exit: Some(exit) }); - parent_tx.send(envelope).map_err(|_| ActorError::NoProc) - }); - let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { - let envelope: Box + Send> = Box::new(HeldEnvelope { - exit: Some(exit), - parent, - }); - held_tx.send(envelope).map_err(|_| ActorError::NoProc) - }); - Arc::new(SignalTarget { - id, - parent, - trap, - request, - wake: Arc::new(move || token.cancel()), - send_exit, - send_parent_exit, - send_held, - }) -} - // --------------------------------------------------------------------------- // Context // --------------------------------------------------------------------------- @@ -753,4277 +215,270 @@ impl Debug for Context { } } -impl Context { - /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). - pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { - Self { - shared: actor_ref.shared.clone(), - } - } - - /// Waiting messages, including in-flight enqueue reservations, excluding - /// the active handler and internal exit/shutdown control envelopes. - /// Concurrent observations are approximate; no registry or actor labels. - #[cfg(feature = "metrics")] - pub fn mailbox_depth(&self) -> u64 { - self.shared.mailbox.depth() - } - - /// Deferred-reply credits in use by this instance: live deferred - /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). - /// Experimental in 0.8 (for tests and diagnostics; the `metrics` - /// feature exports the same count): it may change before 1.0. - pub fn deferred_outstanding(&self) -> usize { - self.shared.replies.outstanding() - } - - /// The actor's unique identity. - pub fn id(&self) -> ActorId { - self.shared.id - } - - /// Self-exit: the current handler finishes, then `stopped()` is called - /// and the actor exits `Normal`. - pub fn stop(&self) { - self.stop_with_reason(ExitReason::Normal); - } - - /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without - /// unwinding. The current handler finishes and `stopped()` runs. The first - /// requested self-exit or stop wins; until the death point a kill - /// replaces it with `Killed`, an untrapped exit signal with its reason, and - /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays - /// `Kill` and propagates over links as `Kill` (trappable). - pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(self.shared.id, &self.shared.linked_reason, reason, || { - self.shared.cancellation_token.cancel() - }); - } - - /// A parent exit being dispatched: normal termination with its reason - /// (part of the model's `Dispatch`, so not recorded as a stop). - fn parent_exit(&self, reason: ExitReason) { - link::request_stop(&self.shared.linked_reason, reason, || { - self.shared.cancellation_token.cancel() - }); - } - - /// Send an exit signal from this actor to `target` (OTP - /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the - /// delivery rules. From the target's `start_linked` parent the signal is a - /// parent exit (a trapping target ends normally, through `stopped()`). - /// Works until this actor's death point, including from `stopped()`; - /// inert after it. A signal to this actor's own handle is a signal, not a - /// self-exit: a non-trapping actor ignores `Normal`. - pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { - // Held until delivered: the death point waits for it, so a signal - // sent before death precedes the death notices (and none is sent - // after it). - let Some(_sending) = self.shared.replies.signal_permit() else { - return; - }; - if link::lock_request(&self.shared.linked_reason).is_dead() { - return; - } - fail_point!("actor.exit_before_deliver"); - target.exit_from(self.shared.id, reason, false); - } - - /// Send a fire-and-forget message to this actor. - pub fn send(&self, msg: M) -> Result<(), ActorError> - where - A: Handler, - M: Message, - { - enqueue_send( - &self.shared.sender, - &self.shared.mailbox, - &self.shared.replies, - msg, - ) - } +// --------------------------------------------------------------------------- +// Receiver trait (object-safe) + Recipient alias +// --------------------------------------------------------------------------- - /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get - /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from - /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait - /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) - /// uses). The handler sees it through `PendingReply::deadline`. - pub fn send_request( +/// Object-safe trait for sending a single message type to an actor. +/// +/// Implemented automatically by `ActorRef` and `Context` for any +/// message type that `A` handles. Sealed: it cannot be implemented outside +/// spawned (a method can then be added without a breaking change). +pub trait Receiver: Send + Sync + crate::__private::Sealed { + fn send(&self, msg: M) -> Result<(), ActorError>; + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its + /// deadline is fixed now (the slot carries it). On a [`Recipient`], + /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). + fn send_request( &self, msg: M, - deadline: impl Into, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let deadline = deadline.into(); - self.send_request_with(msg, deadline.ticks(), true) - } - - /// `send_request`; `raw` puts a deadline in the owner's heap now (the - /// receiver may be awaited directly). The `request` paths arm their own - /// timer and pass `false`. - #[inline] - pub(crate) fn send_request_with( + deadline: Deadline, + ) -> Result, ActorError>; + /// `send_request` for the `request` helper, which arms its own timer: + /// the deadline does not go to the owner's heap. + #[doc(hidden)] + fn send_request_timed( &self, msg: M, - deadline: u64, - raw: bool, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); - enqueue( - &self.shared.sender, - &self.shared.mailbox, - &self.shared.replies, - msg, - slot, - )?; - Ok(rx) + deadline: Deadline, + ) -> Result, ActorError> { + self.send_request(msg, deadline) } +} - /// Send a request and wait for the reply (default 5s timeout). - /// - /// Uses the same request state machine as `request_with_timeout`, with - /// a zero-sized fixed budget instead of capturing a Duration. - pub fn request( - &self, - msg: M, - ) -> impl std::future::Future> + Send + '_ - where - A: Handler, - M: Message, - { - self.request_with_budget(msg, crate::reply::DefaultRequestBudget) - } +/// Type-erased reference for sending a single message type. +/// +/// Obtained via `actor_ref.recipient::()` or `ctx.recipient::()`. +pub type Recipient = Arc>; - /// Send a request and wait for the reply with a custom timeout. - pub fn request_with_timeout( - &self, - msg: M, - duration: Duration, - ) -> impl Future> + Send + '_ - where - A: Handler, - M: Message, - { - self.request_with_budget(msg, duration) - } +/// Send a request through a type-erased `Receiver` with a custom timeout. +pub async fn request( + recipient: &dyn Receiver, + msg: M, + timeout_duration: Duration, +) -> Result { + let observation = crate::instrumentation::RequestObservation::new(); + // The deadline is computed before enqueueing (send time, as OTP's call). + // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). + let (deadline, timer) = crate::reply::request_timer(timeout_duration); + let rx = match recipient.send_request_timed(msg, Deadline::of_ticks(deadline)) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } + }; + let result = rx.into_timed_future_with(timer).await; + observation.finish(&result); + result +} - async fn request_with_budget(&self, msg: M, budget: B) -> Result - where - A: Handler, - M: Message, - B: crate::reply::RequestBudget, - { - let duration = budget.into_duration(); - let observation = crate::instrumentation::RequestObservation::new(); - // Bind the receiver before awaiting it: awaiting inside a `match` on - // the `Result` keeps the scrutinee's storage alive beside the awaited - // receiver and doubles this future's size. - let (deadline, timer) = crate::reply::request_timer(duration); - let rx = match self.send_request_with(msg, deadline, false) { - Ok(rx) => rx, - Err(error) => { - let result = Err(error); - observation.finish(&result); - return result; - } - }; - let result = rx.into_timed_future_with(timer).await; - observation.finish(&result); - result - } +// --------------------------------------------------------------------------- +// ActorRef +// --------------------------------------------------------------------------- + +/// External handle to a running actor. Cloneable, `Send + Sync`. +/// +/// Use this to send messages, make requests, or wait for the actor to stop. +/// To stop the actor, send an explicit shutdown message through your protocol, +/// or call [`Context::stop`] from within a handler. +pub struct ActorRef { + shared: Arc>, +} - /// Get a type-erased `Recipient` for sending a single message type - /// to this actor. - pub fn recipient(&self) -> Recipient - where - A: Handler, - M: Message, - { - Arc::new(self.clone()) +impl Debug for ActorRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ActorRef").finish_non_exhaustive() } +} - /// Get an `ActorRef` from this context. - pub fn actor_ref(&self) -> ActorRef { - ActorRef { +impl Clone for ActorRef { + fn clone(&self) -> Self { + Self { shared: self.shared.clone(), } } +} - /// Set up a unidirectional monitor on another actor. - /// - /// Returns a [`MonitorRef`] that can be used to cancel the monitor via - /// [`Context::demonitor`]. When the monitored actor's exit is published, - /// a [`Down`] with its final reason is delivered via `Handler`. - /// - /// If the target is past its death point, exactly one - /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`; use - /// [`ChildHandle::exit_reason`] for the real reason). - /// - /// Multiple independent monitors are allowed on the same target — each - /// call returns a distinct `MonitorRef`. - /// - /// Monitors are unidirectional: the monitored actor is unaware of the - /// monitor and unaffected by it. - /// Registration after owner shutdown starts (including in `stopped()`) - /// returns an inert reference and creates no waiter or notification. - pub fn monitor(&self, target: &ChildHandle) -> MonitorRef - where - A: Handler, - { - fail_point!("monitor.before_install"); - let monitor_ref = MonitorRef::next(); - let active = spawned_rt::tasks::CancellationToken::new(); - - let observer = self.shared.id.raw(); - let registered = crate::lifecycle_trace::life_section(observer, || { - let event = |x| { - crate::lifecycle_trace::LifeEvent::new("Monitor", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x(x) - }; - crate::monitor::fence::register( - &self.shared.monitors, - monitor_ref, - active.clone(), - // The token check makes registration inert once shutdown - // starts. The closed table serializes registration with exit - // cleanup, even on paths that unwind before cancelling the - // owner token. - || self.shared.cancellation_token.is_cancelled(), - // Linearised against the target's death point (E4): past it - // the only notice is NoProc, delivered under the observer's - // lock. - || target.is_past_death(), - || { - crate::lifecycle_trace::life_emit(observer, || event("late")); - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x("NoProc") - .k("down") - .o("late") - }); - let _ = enqueue_down( - &self.shared.sender, - &self.shared.mailbox, - &self.shared.replies, - Down { - monitor_ref, - actor: target.id(), - reason: ExitReason::NoProc, - }, - ); - }, - || crate::lifecycle_trace::life_emit(observer, || event("active")), - ) == crate::monitor::fence::Registration::Active - }); - if !registered { - return monitor_ref; - } - - let target = target.clone(); - let actor_ref = self.actor_ref(); - let monitors = self.shared.monitors.clone(); - let registration = crate::monitor::MonitorRegistration(monitors.clone(), monitor_ref); - - self.spawn_framework(async move { - let _registration = registration; - let reason = match futures::future::select( - Box::pin(active.cancelled()), - Box::pin(target.wait_exit_async()), - ) - .await - { - futures::future::Either::Left(_) => return, - futures::future::Either::Right((reason, _)) => reason, - }; - // Receiver-side fence (E6): the entry is removed and the Down - // enqueued in one critical section, so once `demonitor` returns - // no Down of this monitor is enqueued. - crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::deliver(&monitors, &monitor_ref, || { - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x(crate::lifecycle_trace::life_reason(&reason)) - .k("down") - .o("death") - }); - let _ = enqueue_down( - &actor_ref.shared.sender, - &actor_ref.shared.mailbox, - &actor_ref.shared.replies, - Down { - monitor_ref, - actor: target.id(), - reason, - }, - ); - }); - }); - }); - - monitor_ref - } - - /// Cancel a previously-set monitor. - /// - /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it - /// returns, no `Down` of this monitor is enqueued or handled; one already - /// in the mailbox is skipped. Demonitoring an unknown, delivered or - /// already cancelled monitor does nothing. - pub fn demonitor(&self, monitor_ref: MonitorRef) { - let observer = self.shared.id.raw(); - let active = crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::demonitor(&self.shared.monitors, &monitor_ref, || { - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) - .m(monitor_ref.raw()) - }); - }) - }); - // Waking the waiter must happen after releasing the table lock. - if let Some(active) = active { - active.cancel(); - } - } - - /// Set up a bidirectional link with another actor (OTP `link/1`). - /// - /// When either side's exit is published, the other receives a link - /// notice with the final reason: a non-trapping peer ignores `Normal` and - /// otherwise exits with that reason without running `stopped()`; a - /// trapping peer gets an `Exit` via [`Actor::exit_received`]. - /// - /// If the target is past its death point, this actor receives a `NoProc` - /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits - /// `NoProc` (decision 1). Use [`try_link`](Self::try_link) to get - /// `Err(NoProc)` instead. An existing link, or a pending notice from that - /// peer, makes this a no-op. Inert once this actor's exit began (in - /// `stopped()`, from a helper after the stop, or past its death point): - /// no record and no notice, as `monitor`. - /// - /// [`Actor::exit_received`]: Actor::exit_received - pub fn link(&self, target: &ChildHandle) { - link::register_link( - &self.shared.signal, - &self.shared.links, - target.signal_target(), - target.links(), - false, - ); - } - - /// Link as [`link`](Self::link), but report a target past its death - /// point as `Err(NoProc)` and keep no record, whether or not this actor - /// traps exits. - /// A caller whose exit already began does nothing and returns `Ok(())`; - /// an exit racing this call can linearize after the link attempt. - pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { - match link::register_link( - &self.shared.signal, - &self.shared.links, - target.signal_target(), - target.links(), - true, - ) { - LinkOutcome::NoProc => Err(NoProc), - LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), - } - } - - /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has - /// no further effect on this actor, including a notice already taken at - /// the peer's death; an `Exit` already queued stays. - pub fn unlink(&self, target: &ChildHandle) { - link::unregister_link( - self.shared.id, - &self.shared.links, - target.id(), - target.links(), - ); - } - - /// Whether this actor's exit has begun or a signal that will end it is - /// already pending, for a trapping actor that wants to refuse late work. - /// - /// `true` once: - /// - exit has begun: `ctx.stop()`, an administrative stop - /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit - /// signal or link notice, a panic; it stays `true` in `stopped()`; - /// - a parent exit is queued for this trapping actor (E3: the - /// supervisor's or system's shutdown, or a signal from its - /// `start_linked` parent). The parent exit is dispatched after the - /// messages queued before it, and handlers of those messages see - /// `true`. - /// - /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent - /// `exit` is not a pending exit: `exit_received` decides. Signals that - /// arrive while `started()` runs count once `started()` returns, when - /// they are resolved. Without trapping, a pending signal has already - /// begun the exit, so this is also `true`. - /// - /// Advisory: a signal can arrive right after it returns `false`. It - /// takes the actor's exit-request lock; nothing is added per message. - pub fn exit_pending(&self) -> bool { - link::lock_request(&self.shared.linked_reason).exit_pending() - } - - /// Control how exit signals (link notices and `exit` signals) are - /// handled; read when each signal arrives (OTP `trap_exit`). - /// - /// - `false` (default): `Normal` is ignored; any other reason ends this - /// actor after its current handler, without `stopped()`. - /// - `true`: the signal is queued as an `Exit` for - /// [`Actor::exit_received`]. - /// - /// Only a kill is untrappable. A parent's exit (the `start_linked` parent, - /// the supervisor or the system) ends a trapping actor normally, through - /// `stopped()`, and never reaches `exit_received`. Set it first thing in - /// `started()`: signals that arrive while `started()` runs are resolved - /// when it returns. Handlers are never preempted. - pub fn trap_exit(&self, enabled: bool) { - let id = self.shared.id.raw(); - crate::lifecycle_trace::life_section(id, || { - // Recorded only when it changes (the model's SetTrap). - if self.shared.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { - crate::lifecycle_trace::life_emit(id, || { - crate::lifecycle_trace::LifeEvent::new("Trap", id) - .x(crate::lifecycle_trace::flag(enabled)) - }); - } - }); - } - - /// Type-erased handle to this actor, without an intermediate `ActorRef`. - pub(crate) fn child_handle(&self) -> ChildHandle { - self.shared.child_handle() - } - - /// Spawn a lifecycle-owned helper on this actor's assigned execution group. - /// Construct runtime-bound work inside the async block. Closure of the scope - /// cancels the helper and waits for its future to drop before cleanup ends. - /// Returns [`crate::system::SystemError::UnmanagedActor`] for actors started - /// with `.start()`; use `.start_on(&group)` to establish cleanup ownership. - pub fn spawn_helper( - &self, - future: F, - ) -> Result, crate::system::SystemError> - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - self.shared - .managed - .as_ref() - .ok_or(crate::system::SystemError::UnmanagedActor)? - .spawn_helper(future) - } - - /// Register blocking work until the native closure actually finishes. - /// Cancellation cannot preempt it. Raw nested spawn calls are not implicitly - /// owned; use this method rather than detaching a blocking task inside a - /// cancellable async helper when recovery must await its resource cleanup. - /// Requires `.start_on(&group)`, like [`Self::spawn_helper`]. - pub fn spawn_blocking_helper( - &self, - work: F, - ) -> Result, crate::system::SystemError> - where - F: FnOnce() -> T + Send + 'static, - T: Send + 'static, - { - self.shared - .managed - .as_ref() - .ok_or(crate::system::SystemError::UnmanagedActor)? - .spawn_blocking_helper(work) - } - - pub fn execution_group(&self) -> Option { - self.shared.managed.as_ref().map(|s| s.group()) - } - - /// Register this managed actor until its owned cleanup completes. Removal - /// compares identity and registration generation, preserving replacements. - pub fn register_name( - &self, - name: &str, - ) -> Result<(), crate::registry::ManagedRegistrationError> { - self.shared - .managed - .as_ref() - .ok_or(crate::registry::ManagedRegistrationError::OwnerClosed)? - .register(name, self.shared.id, self.actor_ref()) - } - - pub(crate) fn managed_scope(&self) -> Option<&Arc> { - self.shared.managed.as_ref() - } - pub(crate) fn spawn_framework(&self, future: F) -> JoinHandle<()> - where - F: Future + Send + 'static, - { - match &self.shared.managed { - Some(scope) => scope.spawn_framework(future), - None => rt::spawn(future), - } - } - - pub(crate) fn cancellation_token(&self) -> StopToken { - self.shared.cancellation_token.clone() - } -} - -impl crate::__private::Sealed for Context {} -impl crate::__private::Sealed for ActorRef {} - -// Bridge: Context implements Receiver for any M that A handles -impl Receiver for Context -where - A: Actor + Handler, - M: Message, -{ - fn send(&self, msg: M) -> Result<(), ActorError> { - Context::send(self, msg) - } - - fn send_request( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - Context::send_request(self, msg, deadline) - } - - fn send_request_timed( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - Context::send_request_with(self, msg, deadline.ticks(), false) - } -} - -// --------------------------------------------------------------------------- -// Receiver trait (object-safe) + Recipient alias -// --------------------------------------------------------------------------- - -/// Object-safe trait for sending a single message type to an actor. -/// -/// Implemented automatically by `ActorRef` and `Context` for any -/// message type that `A` handles. Sealed: it cannot be implemented outside -/// spawned (a method can then be added without a breaking change). -pub trait Receiver: Send + Sync + crate::__private::Sealed { - fn send(&self, msg: M) -> Result<(), ActorError>; - /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its - /// deadline is fixed now (the slot carries it). On a [`Recipient`], - /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). - fn send_request( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError>; - /// `send_request` for the `request` helper, which arms its own timer: - /// the deadline does not go to the owner's heap. - #[doc(hidden)] - fn send_request_timed( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - self.send_request(msg, deadline) - } -} - -/// Type-erased reference for sending a single message type. -/// -/// Obtained via `actor_ref.recipient::()` or `ctx.recipient::()`. -pub type Recipient = Arc>; - -/// Send a request through a type-erased `Receiver` with a custom timeout. -pub async fn request( - recipient: &dyn Receiver, - msg: M, - timeout_duration: Duration, -) -> Result { - let observation = crate::instrumentation::RequestObservation::new(); - // The deadline is computed before enqueueing (send time, as OTP's call). - // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). - let (deadline, timer) = crate::reply::request_timer(timeout_duration); - let rx = match recipient.send_request_timed(msg, Deadline::of_ticks(deadline)) { - Ok(rx) => rx, - Err(error) => { - let result = Err(error); - observation.finish(&result); - return result; - } - }; - let result = rx.into_timed_future_with(timer).await; - observation.finish(&result); - result -} - -// --------------------------------------------------------------------------- -// ActorRef -// --------------------------------------------------------------------------- - -/// External handle to a running actor. Cloneable, `Send + Sync`. -/// -/// Use this to send messages, make requests, or wait for the actor to stop. -/// To stop the actor, send an explicit shutdown message through your protocol, -/// or call [`Context::stop`] from within a handler. -pub struct ActorRef { - shared: Arc>, -} - -impl Debug for ActorRef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ActorRef").finish_non_exhaustive() - } -} - -impl Clone for ActorRef { - fn clone(&self) -> Self { - Self { - shared: self.shared.clone(), - } - } -} - -impl ActorRef { - /// Clone this handle with a timeout override for generated protocol calls. - /// Per-call override wins over method and protocol defaults. No message is - /// sent until a protocol method is called; timeout does not cancel its work. - pub fn with_request_timeout(&self, timeout: Duration) -> crate::response::RequestOptions { - crate::response::RequestOptions::new(self.clone(), timeout) - } - - /// Await this managed instance's initial readiness barrier. Success does not - /// guarantee it remains alive afterward; ordinary request errors still apply. - pub async fn ready(&self) -> Result<(), crate::system::SystemError> { - self.shared - .managed - .as_ref() - .ok_or(crate::system::SystemError::UnmanagedActor)? - .wait_ready() - .await - } - pub(crate) fn signal_failure(&self, error: String) { - let _ = self - .shared - .sender - .send(Box::new(FailureEnvelope(Some(error)))); - } - /// Send a fire-and-forget message to the actor. - pub fn send(&self, msg: M) -> Result<(), ActorError> - where - A: Handler, - M: Message, - { - enqueue_send( - &self.shared.sender, - &self.shared.mailbox, - &self.shared.replies, - msg, - ) - } - - /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get - /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from - /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait - /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) - /// uses). The handler sees it through `PendingReply::deadline`. - pub fn send_request( - &self, - msg: M, - deadline: impl Into, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let deadline = deadline.into(); - self.send_request_with(msg, deadline.ticks(), true) - } - - /// `send_request`; `raw` puts a deadline in the owner's heap now (the - /// receiver may be awaited directly). The `request` paths arm their own - /// timer and pass `false`. - #[inline] - pub(crate) fn send_request_with( - &self, - msg: M, - deadline: u64, - raw: bool, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); - enqueue( - &self.shared.sender, - &self.shared.mailbox, - &self.shared.replies, - msg, - slot, - )?; - Ok(rx) - } - - /// Send a request and wait for the reply (default 5s timeout). - /// - /// Uses the same request state machine as `request_with_timeout`, with - /// a zero-sized fixed budget instead of capturing a Duration. - pub fn request( - &self, - msg: M, - ) -> impl std::future::Future> + Send + '_ - where - A: Handler, - M: Message, - { - self.request_with_budget(msg, crate::reply::DefaultRequestBudget) - } - - /// Run `f` in one of the actor's turns, with its state and `Context`, - /// and receive its result. For tests: `spawned_test::tasks::in_actor` (through - /// `__private`). - pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> - where - R: Send + 'static, - F: FnOnce(&mut A, &Context) -> R + Send + 'static, - { - let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.shared.replies), false); - let envelope: Box + Send> = Box::new(InTurnEnvelope { - f: Some(Box::new(f)), - slot, - lent: None, - }); - // Past the death point the actor is gone (E4), as for `enqueue`. - if self.shared.replies.refuses_admission() { - return Err(ActorError::NoProc); - } - self.shared - .sender - .send(envelope) - .map_err(|_| ActorError::NoProc)?; - Ok(rx) - } - - /// Send a request and wait for the reply with a custom timeout. - pub fn request_with_timeout( - &self, - msg: M, - duration: Duration, - ) -> impl Future> + Send + '_ - where - A: Handler, - M: Message, - { - self.request_with_budget(msg, duration) - } - - async fn request_with_budget(&self, msg: M, budget: B) -> Result - where - A: Handler, - M: Message, - B: crate::reply::RequestBudget, - { - let duration = budget.into_duration(); - let observation = crate::instrumentation::RequestObservation::new(); - // Bind the receiver before awaiting it: awaiting inside a `match` on - // the `Result` keeps the scrutinee's storage alive beside the awaited - // receiver and doubles this future's size. - let (deadline, timer) = crate::reply::request_timer(duration); - let rx = match self.send_request_with(msg, deadline, false) { - Ok(rx) => rx, - Err(error) => { - let result = Err(error); - observation.finish(&result); - return result; - } - }; - let result = rx.into_timed_future_with(timer).await; - observation.finish(&result); - result - } - - /// Get a type-erased `Recipient` for this actor. - pub fn recipient(&self) -> Recipient - where - A: Handler, - M: Message, - { - Arc::new(self.clone()) - } - - /// Get a `Context` from this ref, for timer setup or stream listeners - /// from outside the actor. - pub fn context(&self) -> Context { - Context::from_ref(self) - } - - /// Wait until the actor has fully stopped (including `stopped()` callback). - /// - /// Returns when cleanup completes: the actor died and its cancelled jobs - /// were joined (bounded by the cleanup deadline; jobs still running then - /// are reported, not awaited). A `stopped()` or state destructor that - /// never returns keeps it waiting; bound it with a timeout, or use - /// [`wait_exit`](Self::wait_exit) to observe death alone. - pub async fn join(&self) { - let _ = self.wait_exit().await; - let mut cleaned = self.shared.cleaned.clone(); - loop { - if *cleaned.borrow_and_update() { - return; - } - if cleaned.changed().await.is_err() { - return; - } - } - } - - /// Poll the exit reason. Returns `None` until the exit is published. - /// A completion channel closed without a reason reports - /// [`ExitReason::Killed`], matching the wait methods (for example, after - /// runtime shutdown). - pub fn exit_reason(&self) -> Option { - crate::child_handle::poll_exit_reason(&self.shared.completion_rx) - } - - /// Wait until the actor's exit is published and return its final reason. - pub async fn wait_exit(&self) -> ExitReason { - crate::child_handle::wait_loop(self.shared.completion_rx.clone()).await - } - - /// Waiting messages, including in-flight enqueue reservations, excluding - /// the active handler and internal exit/shutdown control envelopes. - /// Concurrent observations are approximate; no registry or actor labels. - #[cfg(feature = "metrics")] - pub fn mailbox_depth(&self) -> u64 { - self.shared.mailbox.depth() - } - - /// The actor's unique identity. - pub fn id(&self) -> ActorId { - self.shared.id - } - - /// Get a type-erased `ChildHandle` for this actor. - pub fn child_handle(&self) -> ChildHandle { - self.shared.child_handle() - } - - /// The shared state, for tests proving it is freed once every handle is gone. - #[cfg(test)] - fn shared_weak(&self) -> std::sync::Weak> { - Arc::downgrade(&self.shared) - } -} - -impl ActorShared { - fn child_handle(&self) -> ChildHandle { - ChildHandle::new( - self.signal.clone(), - Completion::Tasks(self.completion_rx.clone()), - self.links.clone(), - ) - .with_mailbox_failures(self.sender.failures()) - } -} - -impl From> for ChildHandle { - fn from(actor_ref: ActorRef) -> Self { - actor_ref.shared.child_handle() - } -} - -// Bridge: ActorRef implements Receiver for any M that A handles -impl Receiver for ActorRef -where - A: Actor + Handler, - M: Message, -{ - fn send(&self, msg: M) -> Result<(), ActorError> { - ActorRef::send(self, msg) - } - - fn send_request( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - ActorRef::send_request(self, msg, deadline) - } - - fn send_request_timed( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - ActorRef::send_request_with(self, msg, deadline.ticks(), false) - } -} - -// --------------------------------------------------------------------------- -// Actor startup + main loop -// --------------------------------------------------------------------------- - -impl ActorRef { - fn spawn(actor: A, backend: Backend, parent: Option) -> Self { - Self::spawn_inner(actor, backend, parent, None, None) - .expect("unmanaged actor dispatch failed") - } - - pub(crate) fn spawn_managed( - actor: A, - scope: Arc, - lease: crate::system::Lease, - ) -> Result { - if !scope.can_start() { - return Err(crate::system::SystemError::Closed); - } - Self::spawn_inner(actor, Backend::Async, None, Some(scope), Some(lease)) - } - - fn spawn_inner( - actor: A, - backend: Backend, - parent: Option, - managed: Option>, - lease: Option, - ) -> Result { - let (tx, rx) = mpsc::channel:: + Send>>(); - let cancellation_token = managed - .as_ref() - .map(|s| s.token.clone()) - .unwrap_or_default(); - let (completion_tx, completion_rx) = watch::channel(None); - let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); - let trap_exit = new_trap_exit_flag(); - let links = new_link_table(); - let linked_reason = managed - .as_ref() - .map(|s| s.cause.clone()) - .unwrap_or_else(new_linked_exit_reason); - - let id = ActorId::next(); - // Phase 0: signals that arrive before started() returns are held. - link::lock_request(&linked_reason).begin_starting(); - let signal = signal_target::( - id, - parent.as_ref().map(ChildHandle::id), - &tx, - trap_exit.clone(), - linked_reason.clone(), - cancellation_token.clone(), - ); - let (cleaned_tx, cleaned_rx) = watch::channel(false); - // A supervised child's `ChildSpec` may override the actor's bounds. - let (max_outstanding, max_native) = managed - .as_ref() - .map(|scope| scope.bounds()) - .unwrap_or_default() - .resolve(A::MAX_OUTSTANDING, A::MAX_NATIVE_JOBS); - let actor_ref = ActorRef { - shared: Arc::new(ActorShared { - id, - replies: Registry::new(id, max_outstanding), - jobs: Arc::new(crate::reply::jobs::JobSet::new(id, max_native)), - cleaned: cleaned_rx, - sender: tx, - cancellation_token: cancellation_token.clone(), - completion_rx, - monitors, - trap_exit, - links: links.clone(), - linked_reason: linked_reason.clone(), - signal, - mailbox: crate::instrumentation::Mailbox::default(), - managed: managed.clone(), - }), - }; - let ctx = actor_ref.context(); - - if let Some(scope) = &managed { - scope.attach::(actor_ref.child_handle()); - scope.set_jobs(Arc::downgrade(&actor_ref.shared.jobs)); - } - - let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); - if let Some(parent) = parent { - ctx.link(&parent); - } - // Recorded from here: the start_linked link is the model's initial one. - crate::lifecycle_trace::life_spawn(id.raw(), parent_id); - - let cleanup_token = cancellation_token.clone(); - let cleanup = crate::actor_cleanup::ActorCleanup::new( - actor_ref.shared.id, - links, - linked_reason, - actor_ref.shared.monitors.clone(), - completion_tx, - move || cleanup_token.cancel(), - ) - .with_mailbox_failures(actor_ref.shared.sender.failures()) - .with_replies(actor_ref.shared.replies.clone()) - .with_jobs(actor_ref.shared.jobs.clone()); - let startup = crate::actor_cleanup::ActorStartup::new( - (actor, ctx, rx, cancellation_token), - cleanup, - discharge_startup::, - ); - let job_owner = actor_ref.shared.clone(); - let cleaned_tx = cleaned_tx; - // Destruction-safe: if the runtime drops this future before it - // completes, `ActorStartup` (never polled) or `ExitOnDrop` (polled) - // still reach D before destroying anything, so no caller waits for an - // instance that can never answer. - let inner_future = async move { - let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); - run_actor(actor, ctx, rx, cancellation_token, &mut cleanup).await; - let reason = cleanup.die(); - if let Some(scope) = &managed { - scope.exited(reason); - } - // Publication (wait_exit, monitors, link notices) after the - // state drop, before the job join. - drop(cleanup); - // Job cleanup (deferred-replies §5 step 4), after death: join the - // cancelled jobs, bounded by the cleanup deadline; a job still - // running then does not undo death and is reported unfinished. - if job_owner.jobs.running() > 0 { - let timeout = job_owner - .managed - .as_ref() - .map_or(DEFAULT_STOP_DEADLINE, |scope| scope.cleanup_timeout()); - let remaining = job_owner.replies.cleanup_remaining(timeout); - let unfinished = job_owner.jobs.join(remaining).await; - crate::reply::jobs::report_unfinished(job_owner.id, unfinished); - } - drop(job_owner); - let _ = cleaned_tx.send(true); - }; - - if let Some(lease) = lease { - let scope = actor_ref - .shared - .managed - .as_ref() - .expect("managed lease without scope"); - scope.dispatch_actor(crate::system::Tracked { - future: inner_future, - lease, - })?; - return Ok(actor_ref); - } - - match backend { - Backend::Async => { - #[cfg(debug_assertions)] - let inner_future = warn_on_block::WarnOnBlocking::new(inner_future); - let _handle = rt::spawn(inner_future); - } - Backend::Blocking => { - let _handle = rt::spawn_blocking(move || rt::block_on(inner_future)); - } - Backend::Thread => { - let _handle = threads::spawn(move || threads::block_on(inner_future)); - } - } - - Ok(actor_ref) - } -} - -/// A pending user future of the lifecycle: `started()`, a turn (handler, -/// `exit_received`, a job's `done`) or `stopped()`. If the runtime drops the -/// lifecycle while it is pending (G1), its `Drop` reaches the death point -/// first: the turn's own caller, every deferred token and every queued -/// request are answered and woken before this future and its captures (the -/// message, the handler's locals) are destroyed. A destructor panic of the -/// future is a cleanup failure. -struct DeathFirst<'a, A: Actor, C: Fn()> { - inner: Option + Send + 'a>>>, - rx: &'a mut mpsc::Receiver + Send>>, - cleanup: &'a mut crate::actor_cleanup::ActorCleanup, - caller: Option<&'a dyn Caller>, - /// The run loop already polled `inner` once in this task poll (it was - /// pending): the next poll returns `Pending` without polling it again. - polled: bool, -} -impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { - fn new( - inner: Pin + Send + 'a>>, - rx: &'a mut mpsc::Receiver + Send>>, - cleanup: &'a mut crate::actor_cleanup::ActorCleanup, - caller: Option<&'a dyn Caller>, - ) -> Self { - Self { - inner: Some(inner), - rx, - cleanup, - caller, - polled: false, - } - } - - /// A turn the run loop polled once and found pending. - fn pending( - inner: Pin + Send + 'a>>, - rx: &'a mut mpsc::Receiver + Send>>, - cleanup: &'a mut crate::actor_cleanup::ActorCleanup, - caller: Option<&'a dyn Caller>, - ) -> Self { - let mut turn = Self::new(inner, rx, cleanup, caller); - turn.polled = true; - turn - } -} - -/// A boxed turn of the lifecycle. -type TurnFuture<'a> = Pin + Send + 'a>>; - -/// Poll `turn` once, catching its panic. -#[inline] -fn poll_caught( - turn: &mut TurnFuture<'_>, - cx: &mut std::task::Context<'_>, -) -> Result, Panic> { - std::panic::catch_unwind(AssertUnwindSafe(|| turn.as_mut().poll(cx))) -} - -/// Drop a turn that panicked; a panic of its destructor is disposed of. -fn drop_panicked(turn: Option>) { - if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(turn))) { - crate::mailbox::dispose_panic(secondary); - } -} -/// A caught panic payload. -type Panic = Box; - -impl Future for DeathFirst<'_, A, C> { - /// The future's panic, caught: the turn's (or hook's) panic, not a - /// runtime drop. The future is gone before it is returned. - type Output = Result<(), Panic>; - fn poll( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - let this = self.get_mut(); - if std::mem::take(&mut this.polled) { - // Polled (and its waker registered) in this same task poll. - return std::task::Poll::Pending; - } - let inner = this.inner.as_mut().expect("polled after completion"); - match poll_caught(inner, cx) { - Ok(std::task::Poll::Pending) => std::task::Poll::Pending, - Ok(std::task::Poll::Ready(())) => { - this.inner = None; - std::task::Poll::Ready(Ok(())) - } - Err(panic) => { - drop_panicked(this.inner.take()); - std::task::Poll::Ready(Err(panic)) - } - } - } -} -impl Drop for DeathFirst<'_, A, C> { - fn drop(&mut self) { - let Some(inner) = self.inner.take() else { - return; - }; - // Destroyed while pending: the runtime dropped the actor. - self.cleanup.die_dropped(); - if let Some(caller) = self.caller.take() { - self.cleanup.answer_turn(caller.retain()); - } - let queued = answer_mailbox(self.rx, self.cleanup.replies()); - self.cleanup.discharged(); - self.rx.discard_all(queued); - if let Err(panic) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { - let message = panic_message(&*panic); - crate::mailbox::dispose_panic(panic); - self.cleanup - .cleanup_failed(format!("dropped turn destructor: {message}")); - } - } -} - -/// The actor's state and mailbox while its lifecycle runs. They are -/// destroyed only after the death point D: normally at the end of -/// [`run_actor`]; if the lifecycle future is dropped mid-way (its runtime was -/// destroyed), here, which reaches D first (`Killed` unless a reason was -/// requested), so tokens and queued requests answer `ActorStopped` before -/// any destructor runs. -struct ExitOnDrop<'c, A: Actor, C: Fn()> { - actor: Option, - rx: mpsc::Receiver + Send>>, - cleanup: &'c mut crate::actor_cleanup::ActorCleanup, -} -impl ExitOnDrop<'_, A, C> { - /// D, then the queued requests, then destruction (mailbox, then state). - fn finish(&mut self, dropped: bool) { - let Some(actor) = self.actor.take() else { - return; - }; - if dropped { - self.cleanup.die_dropped(); - } else { - self.cleanup.die(); - } - let queued = answer_mailbox(&mut self.rx, self.cleanup.replies()); - // Every caller is answered: wake them, then destroy (queued - // payloads, then state). A destructor panic is a cleanup failure, - // never the reason. - self.cleanup.discharged(); - self.rx.discard_all(queued); - if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { - self.cleanup.cleanup_failed(failure); - } - } -} -impl Drop for ExitOnDrop<'_, A, C> { - fn drop(&mut self) { - self.finish(true); - } -} - -async fn run_actor( - actor: A, - ctx: Context, - rx: mpsc::Receiver + Send>>, - cancellation_token: StopToken, - cleanup: &mut crate::actor_cleanup::ActorCleanup, -) { - let request = ctx.shared.linked_reason.clone(); - let mut owned = ExitOnDrop { - actor: Some(actor), - rx, - cleanup, - }; - if let Err(payload) = AssertUnwindSafe(run_actor_lifecycle( - owned.actor.as_mut().expect("the state lives until death"), - ctx, - &mut owned.rx, - &mut *owned.cleanup, - cancellation_token, - )) - .catch_unwind() - .await - { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - link::lock_request(&request).panic(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))); - } - // The death point D, only now that `stopped()` returned (or was - // skipped): the reason becomes final, the link table closes, job - // admission and the registry close (every open token is answered), then - // the mailbox refuses admission and its queued requests are answered, - // all before any destructor. Early returns (startup panic, closed scope) - // and an unwound lifecycle also come here, so jobs never outlive their - // owner. - owned.finish(false); -} - -async fn run_actor_lifecycle( - actor: &mut A, - ctx: Context, - rx: &mut mpsc::Receiver + Send>>, - cleanup: &mut crate::actor_cleanup::ActorCleanup, - cancellation_token: StopToken, -) { - let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); - let request = ctx.shared.linked_reason.clone(); - let trace_id = ctx.shared.id.raw(); - if let Some(scope) = &ctx.shared.managed { - if !scope.can_start() { - scope.stop(ExitReason::Shutdown); - return; - } - } - let start_result = AssertUnwindSafe(async { - fail_point!("actor.before_started", async); - DeathFirst::new( - crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.started(&ctx))), - &mut *rx, - &mut *cleanup, - None, - ) - .await - .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) - }) - .catch_unwind() - .await; - if let Err(panic) = start_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in started() callback: {msg}"); - }); - // No stopped() after a failed start (L5). - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); - }); - cancellation_token.cancel(); - return; - } - // Phase 0 ends: resolve signals held during started() with the trap - // flag read now. - if link::lock_request(&request).end_starting(ctx.shared.trap_exit.load(Ordering::Acquire)) { - cancellation_token.cancel(); - } - - fail_point!("actor.after_started_before_ready", async); - if let Some(scope) = &ctx.shared.managed { - if !scope.mark_ready() { - scope.stop(ExitReason::Shutdown); - } - } - - // One cancellation future for the whole loop: polled only when the - // mailbox has nothing, so a busy actor registers no waiter per message. - let mut cancelled = pin!(cancellation_token.cancelled()); - // Check stop after receiving and before dispatch: a message published - // after stop must not slip through an earlier snapshot. A rejected receive - // stays in the mailbox wrapper until the death point answers its caller. - // The fast check is one atomic load; only parked loops register a waiter. - loop { - let msg = { - let recv = pin!(future::poll_fn(|cx| { - // Stop and closed-mailbox both take the normal lifecycle exit, - // just like the cancellation arm below. Check after the - // mailbox acquire and retain a rejected item for shutdown. - // The receive, check and trace event are one step - // of the lifecycle trace: a recorded stop is seen here. - crate::lifecycle_trace::life_section(trace_id, || { - #[cfg(all(test, not(feature = "lifecycle-trace"), not(spawned_sim)))] - stop_receive_probe::before_receive(); - let received = rx.poll_recv(cx); - let result = cancellation_token.after_receive(received, |item| { - rx.hold_for_shutdown(item); - }); - #[cfg(feature = "lifecycle-trace")] - if let std::task::Poll::Ready(Some(envelope)) = &result { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) - .k(envelope.trace_kind()) - }); - } - result - }) - })); - let mut recv = recv; - future::poll_fn(|cx| match recv.as_mut().poll(cx) { - std::task::Poll::Pending => match cancelled.as_mut().poll(cx) { - std::task::Poll::Ready(()) => std::task::Poll::Ready(None), - std::task::Poll::Pending => std::task::Poll::Pending, - }, - ready => ready, - }) - .await - }; - match msg { - Some(envelope) => { - // Simulator turn budget (rank 8b); compiled only under the cfg. - #[cfg(spawned_sim)] - let _sim_turn = rt::sim::turn(std::any::type_name::(), envelope.message_type()); - #[cfg(feature = "diagnostics")] - let _turn = ctx - .shared - .managed - .as_ref() - .map(|scope| scope.begin_turn(envelope.message_type())); - #[cfg(feature = "metrics")] - let mut envelope = envelope; - - #[cfg(feature = "metrics")] - let observation = envelope - .observation() - .map(|observation| observation.begin()); - ctx.shared.replies.begin_turn(); - // The handler's future is built inside the guarded future: a - // manual `Handler` may panic before it returns one, and that - // is a handler panic like any other (threads mode catches the - // same call). The envelope outlives the turn and lends it the - // request's slot. - let mut envelope = envelope; - let result = match dispatch(&mut *envelope, &mut *actor, &ctx) { - // Dispatch completed the turn synchronously. The caller - // hold is only needed if an unfinished future is dropped; - // turn accounting and observation bracket either branch. - Ok((None, _)) => Ok(()), - Ok((Some(turn), caller)) => { - let mut turn = - Some(crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn)); - // Most turns finish in their first poll, which runs - // here, with no await point before it returns: only - // a turn that stays pending is guarded by - // `DeathFirst` against the runtime dropping it. - let first = future::poll_fn(|cx| { - std::task::Poll::Ready(poll_caught( - turn.as_mut().expect("polled once"), - cx, - )) - }) - .await; - match first { - Ok(std::task::Poll::Ready(())) => { - drop(turn); - Ok(()) - } - Ok(std::task::Poll::Pending) => { - let turn = turn.take().expect("pending turn"); - DeathFirst::pending(turn, &mut *rx, &mut *cleanup, caller).await - } - Err(panic) => { - drop_panicked(turn.take()); - Err(panic) - } - } - } - Err(panic) => Err(panic), - }; - ctx.shared.replies.end_turn(result.is_err()); - #[cfg(feature = "diagnostics")] - drop(_turn); - #[cfg(feature = "metrics")] - if let Some(observation) = observation { - observation.finish(result.is_err()); - } - if let Err(panic) = result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in message handler: {msg}"); - }); - // A panic begins exit, or replaces the requested reason - // unless a kill was received. - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); - }); - break; - } - } - None => break, - } - } - - if let Some(scope) = &ctx.shared.managed { - scope.end_running(); - } - cancellation_token.cancel(); - ctx.shared.replies.exiting(); - // Payload drops are held until every caller is answered (DropFailures). - ctx.shared.sender.failures().hold(); - ctx.shared.jobs.cancel_all(); - // stopped() is skipped when a kill, an untrapped exit signal or link - // notice began the exit (decision 2); one that already began finishes. - let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { - let mut req = link::lock_request(&request); - let run = req.begin_hook(); - let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); - drop(req); - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { - "run" - } else { - "skip" - }) - }); - (run, skipped) - }); - if let Some((cause, reason)) = skipped { - crate::actor_cleanup::log_stopped_skipped( - ctx.shared.id, - std::any::type_name::(), - cause, - reason.as_ref(), - ); - } - if !run_hook { - return; - } - let stop_result = AssertUnwindSafe(async { - fail_point!("actor.before_stopped", async); - DeathFirst::new( - crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.stopped(&ctx))), - &mut *rx, - &mut *cleanup, - None, - ) - .await - .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) - }) - .catch_unwind() - .await; - if let Err(panic) = stop_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); - }); - } -} - -/// A context and its cleanup without spawning, for run-loop unit tests. -#[cfg(test)] -fn test_context( - tx: &mpsc::Sender + Send>>, - token: &StopToken, -) -> (Context, crate::actor_cleanup::ActorCleanup) { - let id = ActorId::next(); - let (completion_tx, completion_rx) = watch::channel(None); - let trap = new_trap_exit_flag(); - let links = new_link_table(); - let request = new_linked_exit_reason(); - let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); - let signal = signal_target::(id, None, tx, trap.clone(), request.clone(), token.clone()); - let ctx = Context { - shared: Arc::new(ActorShared { - id, - sender: tx.clone(), - cancellation_token: token.clone(), - completion_rx, - monitors: monitors.clone(), - trap_exit: trap, - links: links.clone(), - linked_reason: request.clone(), - signal, - mailbox: crate::instrumentation::Mailbox::default(), - managed: None, - replies: Registry::new(id, DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(id, 1)), - cleaned: watch::channel(false).1, - }), - }; - let noop: fn() = || {}; - let cleanup = - crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop) - .with_replies(ctx.shared.replies.clone()); - (ctx, cleanup) -} - -// --------------------------------------------------------------------------- -// Actor::start -// --------------------------------------------------------------------------- - -/// Extension trait for starting an actor. Automatically implemented for all [`Actor`] types. -pub trait ActorStart: Actor { - /// Start the actor with the default backend ([`Backend::Async`]). - fn start(self) -> ActorRef { - self.start_with_backend(Backend::default()) - } - - /// Admit this actor to an owned parked async execution group. - fn start_on( - self, - group: &crate::system::ExecutionGroup, - ) -> Result, crate::system::SystemError> { - group.start(self) - } - - /// Start the actor with a specific [`Backend`]. - fn start_with_backend(self, backend: Backend) -> ActorRef { - ActorRef::spawn(self, backend, None) - } - - /// Start linked to `parent_ctx`'s actor, which becomes this instance's - /// parent (E3). The link is registered before `started()`. The parent's - /// exit ends a trapping child normally (its `stopped()` runs; it never - /// reaches `exit_received`), a non-trapping child as any link notice. A - /// parent already past its death point gives the child a `NoProc` parent - /// exit. - /// - /// The child is always an unmanaged `Backend::Async` actor, even when - /// the parent is managed: it is outside the parent's execution group and - /// its ordered shutdown (`spawn_helper` on it returns `UnmanagedActor`). - /// The unified start API (planned for 0.9) will inherit the placement. - fn start_linked(self, parent_ctx: &Context

) -> ActorRef { - ActorRef::spawn(self, Backend::Async, Some(parent_ctx.child_handle())) - } -} - -impl ActorStart for A {} - -// --------------------------------------------------------------------------- -// send_message_on (utility) -// --------------------------------------------------------------------------- - -/// Send a message to an actor when a future completes. -/// -/// Spawns a task that races the future against the actor's cancellation token. -/// If the actor stops before the future completes, the message is not sent. -pub fn send_message_on(ctx: Context, future: U, msg: M) -> JoinHandle<()> -where - A: Actor + Handler, - M: Message, - U: Future + Send + 'static, - ::Output: Send, -{ - let cancellation_token = ctx.cancellation_token(); - let spawn_ctx = ctx.clone(); - let join_handle = spawn_ctx.spawn_framework(async move { - let is_cancelled = pin!(cancellation_token.cancelled()); - let signal = pin!(future); - match future::select(is_cancelled, signal).await { - future::Either::Left(_) => tracing::debug!("Actor stopped"), - future::Either::Right(_) => { - if let Err(e) = ctx.send(msg) { - tracing::error!("Failed to send message: {e:?}") - } - } - } - }); - join_handle -} - -// --------------------------------------------------------------------------- -// WarnOnBlocking (debug only) -// --------------------------------------------------------------------------- - -#[cfg(debug_assertions)] -mod warn_on_block { - use super::*; - use std::time::Instant; - use tracing::warn; - - pin_project_lite::pin_project! { - pub struct WarnOnBlocking{ - #[pin] - inner: F - } - } - - impl WarnOnBlocking { - pub fn new(inner: F) -> Self { - Self { inner } - } - } - - impl Future for WarnOnBlocking { - type Output = F::Output; - - fn poll( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll { - let type_id = std::any::type_name::(); - let task_id = rt::task_id(); - let this = self.project(); - let now = Instant::now(); - let res = this.inner.poll(cx); - let elapsed = now.elapsed(); - if elapsed > Duration::from_millis(10) { - warn!(task = ?task_id, future = ?type_id, elapsed = ?elapsed, "Blocking operation detected"); - } - res - } - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use crate::message::Message; - use std::{ - sync::{atomic, Arc}, - thread, - time::Duration, - }; - - // --- Counter actor for basic tests --- - - struct Counter { - count: u64, - } - - struct GetCount; - impl Message for GetCount { - type Result = u64; - } - - struct Increment; - impl Message for Increment { - type Result = u64; - } - - struct StopCounter; - impl Message for StopCounter { - type Result = u64; - } - - impl Actor for Counter {} - - impl Handler for Counter { - async fn handle( - &mut self, - _msg: GetCount, - reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - reply.reply(self.count); - } - } - - impl Handler for Counter { - async fn handle( - &mut self, - _msg: Increment, - reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - let __reply_value: u64 = { - self.count += 1; - self.count - }; - reply.reply(__reply_value); - } - } - - impl Handler for Counter { - async fn handle( - &mut self, - _msg: StopCounter, - reply: crate::reply::PendingReply<'_, u64>, - ctx: &Context, - ) { - let __reply_value: u64 = { - ctx.stop(); - self.count - }; - reply.reply(__reply_value); - } - } - - #[test] - pub fn backend_default_is_async() { - assert_eq!(Backend::default(), Backend::Async); - } - - #[test] - #[allow(clippy::clone_on_copy)] - pub fn backend_enum_is_copy_and_clone() { - let backend = Backend::Async; - let copied = backend; - let cloned = backend.clone(); - assert_eq!(backend, copied); - assert_eq!(backend, cloned); - } - - #[test] - pub fn backend_enum_debug_format() { - assert_eq!(format!("{:?}", Backend::Async), "Async"); - assert_eq!(format!("{:?}", Backend::Blocking), "Blocking"); - assert_eq!(format!("{:?}", Backend::Thread), "Thread"); - } - - #[test] - pub fn backend_enum_equality() { - assert_eq!(Backend::Async, Backend::Async); - assert_eq!(Backend::Blocking, Backend::Blocking); - assert_eq!(Backend::Thread, Backend::Thread); - assert_ne!(Backend::Async, Backend::Blocking); - assert_ne!(Backend::Async, Backend::Thread); - assert_ne!(Backend::Blocking, Backend::Thread); - } - - #[test] - pub fn backend_async_handles_send_and_request() { - crate::test_wait::bounded( - concat!(module_path!(), "::backend_async_handles_send_and_request"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); - - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); - - // fire-and-forget send; the request below shares its mailbox, so - // it is handled after the send without any extra wait. - counter.send(Increment).unwrap(); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); - - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); - }, - ); - } - - #[test] - pub fn backend_blocking_handles_send_and_request() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::backend_blocking_handles_send_and_request" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); - - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); - - counter.send(Increment).unwrap(); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); - - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); - }, - ); - } - - #[test] - pub fn backend_thread_handles_send_and_request() { - crate::test_wait::bounded( - concat!(module_path!(), "::backend_thread_handles_send_and_request"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); - - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); - - counter.send(Increment).unwrap(); - - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); - - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); - }, - ); - } - - #[test] - pub fn multiple_backends_concurrent() { - crate::test_wait::bounded( - concat!(module_path!(), "::multiple_backends_concurrent"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let async_counter = Counter { count: 0 }.start(); - let blocking_counter = - Counter { count: 100 }.start_with_backend(Backend::Blocking); - let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); - - async_counter.request(Increment).await.unwrap(); - blocking_counter.request(Increment).await.unwrap(); - thread_counter.request(Increment).await.unwrap(); - - let async_val = async_counter.request(GetCount).await.unwrap(); - let blocking_val = blocking_counter.request(GetCount).await.unwrap(); - let thread_val = thread_counter.request(GetCount).await.unwrap(); - - assert_eq!(async_val, 1); - assert_eq!(blocking_val, 101); - assert_eq!(thread_val, 201); - - async_counter.request(StopCounter).await.unwrap(); - blocking_counter.request(StopCounter).await.unwrap(); - thread_counter.request(StopCounter).await.unwrap(); - }); - }, - ); - } - - #[test] - pub fn request_timeout() { - crate::test_wait::bounded(concat!(module_path!(), "::request_timeout"), move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct SlowActor; - struct SlowOp; - impl Message for SlowOp { - type Result = (); - } - impl Actor for SlowActor {} - impl Handler for SlowActor { - async fn handle( - &mut self, - _msg: SlowOp, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - // Never replies, so the timeout is the only possible outcome - // however late the runtime polls it. - std::future::pending::<()>().await; - reply.reply(()); - } - } - - let actor = SlowActor.start(); - let result = actor - .request_with_timeout(SlowOp, Duration::from_millis(50)) - .await; - assert!(matches!(result, Err(ActorError::RequestTimeout))); - }); - }); - } - - #[test] - pub fn recipient_type_erasure() { - crate::test_wait::bounded( - concat!(module_path!(), "::recipient_type_erasure"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 42 }.start(); - let recipient: Recipient = counter.recipient(); - - let rx = recipient - .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - .unwrap(); - let result = rx.await.unwrap(); - assert_eq!(result, 42); - - // Also test request helper - let result = request(&*recipient, GetCount, Duration::from_secs(5)) - .await - .unwrap(); - assert_eq!(result, 42); - }); - }, - ); - } - - // --- SlowShutdownActor for join tests --- - - struct SlowShutdownActor; - - struct StopSlow; - impl Message for StopSlow { - type Result = (); - } - - impl Actor for SlowShutdownActor { - async fn stopped(&mut self, _ctx: &Context) { - thread::sleep(Duration::from_millis(500)); - } - } - - impl Handler for SlowShutdownActor { - async fn handle( - &mut self, - _msg: StopSlow, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - /// Blocks its thread in a callback until the test releases it. - struct HeldThread { - entered: Arc, - release: std::sync::mpsc::Receiver<()>, - released: Arc, - } - - impl HeldThread { - fn new() -> (Self, std::sync::mpsc::Sender<()>) { - let (tx, release) = std::sync::mpsc::channel(); - let actor = HeldThread { - entered: Arc::default(), - release, - released: Arc::default(), - }; - (actor, tx) - } - - fn hold(&mut self) { - self.entered.store(true, atomic::Ordering::SeqCst); - let released = self.release.recv_timeout(crate::test_wait::WAIT).is_ok(); - self.released.store(released, atomic::Ordering::SeqCst); - } - } - - impl Actor for HeldThread { - async fn stopped(&mut self, _ctx: &Context) { - // Only the join test sends StopSlow; DoBlock already held. - if !self.entered.load(atomic::Ordering::SeqCst) { - self.hold(); - } - } - } - - impl Handler for HeldThread { - async fn handle( - &mut self, - _msg: StopSlow, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - impl Handler for HeldThread { - async fn handle( - &mut self, - _msg: DoBlock, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - self.hold(); - ctx.stop(); - reply.reply(()); - } - } - - #[test] - pub fn thread_backend_join_does_not_block_runtime() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::thread_backend_join_does_not_block_runtime" - ), - move || { - // stopped() blocks its OS thread until a task on this current-thread - // runtime releases it. If join() blocked the runtime, that task could - // never run and stopped() would give up after the timeout instead. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - runtime.block_on(async move { - let (held, release) = HeldThread::new(); - let released = held.released.clone(); - let actor = held.start_with_backend(Backend::Thread); - actor.send(StopSlow).unwrap(); - let _releaser = rt::spawn(async move { - let _ = release.send(()); - }); - - actor.join().await; - - assert!( - released.load(atomic::Ordering::SeqCst), - "join() blocked the runtime: the releasing task never ran" - ); - }); - }, - ); - } - - #[test] - pub fn multiple_join_callers_all_notified() { - crate::test_wait::bounded( - concat!(module_path!(), "::multiple_join_callers_all_notified"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = SlowShutdownActor.start(); - let actor_clone1 = actor.clone(); - let actor_clone2 = actor.clone(); - - let join1 = rt::spawn(async move { - actor_clone1.join().await; - 1u32 - }); - let join2 = rt::spawn(async move { - actor_clone2.join().await; - 2u32 - }); - - rt::sleep(Duration::from_millis(10)).await; - - actor.send(StopSlow).unwrap(); - - let (r1, r2) = tokio::join!(join1, join2); - assert_eq!(r1.unwrap(), 1); - assert_eq!(r2.unwrap(), 2); - - actor.join().await; - }); - }, - ); - } - - // --- Badly behaved actors for blocking tests --- - - struct BadlyBehavedTask; - - struct DoBlock; - impl Message for DoBlock { - type Result = (); - } - - impl Actor for BadlyBehavedTask {} - - impl Handler for BadlyBehavedTask { - async fn handle( - &mut self, - _msg: DoBlock, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - rt::sleep(Duration::from_millis(20)).await; - thread::sleep(Duration::from_secs(2)); - ctx.stop(); - reply.reply(()); - } - } - - struct IncrementWell; - impl Message for IncrementWell { - type Result = (); - } - - struct WellBehavedTask { - pub count: u64, - } - - impl Actor for WellBehavedTask {} - - impl Handler for WellBehavedTask { - async fn handle( - &mut self, - _msg: GetCount, - reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - reply.reply(self.count); - } - } - - impl Handler for WellBehavedTask { - async fn handle( - &mut self, - _msg: StopCounter, - reply: crate::reply::PendingReply<'_, u64>, - ctx: &Context, - ) { - let __reply_value: u64 = { - ctx.stop(); - self.count - }; - reply.reply(__reply_value); - } - } - - impl Handler for WellBehavedTask { - async fn handle( - &mut self, - _msg: IncrementWell, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - self.count += 1; - use crate::tasks::send_after; - send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); - reply.reply(()); - } - } - - #[test] - pub fn badly_behaved_thread_non_blocking() { - crate::test_wait::bounded( - concat!(module_path!(), "::badly_behaved_thread_non_blocking"), - move || { - // An async actor that blocks its thread starves everything else on - // it. With one runtime thread this is deterministic; on a multi-thread - // runtime the well-behaved actor may run on another worker. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async move { - let badboy = BadlyBehavedTask.start(); - badboy.send(DoBlock).unwrap(); - let goodboy = WellBehavedTask { count: 0 }.start(); - goodboy.send(IncrementWell).unwrap(); - rt::sleep(Duration::from_secs(1)).await; - let count = goodboy.request(GetCount).await.unwrap(); - // The blocking handler held the only thread for 2 s of the 1 s - // window, so the 100 ms chain could not get far. - assert!(count < 10, "blocked runtime still made progress: {count}"); - goodboy.request(StopCounter).await.unwrap(); - }); - }, - ); - } - - /// An async actor on a single-threaded runtime keeps making progress while - /// an actor on `backend` is blocked inside a handler. With only one runtime - /// thread, a blocked handler on that thread would stop all progress. - fn blocked_handler_does_not_stall_runtime(backend: Backend) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async move { - let (held, release) = HeldThread::new(); - let entered = held.entered.clone(); - let badboy = held.start_with_backend(backend); - badboy.send(DoBlock).unwrap(); - let entered = crate::test_wait::wait_for_async( - || async { entered.load(atomic::Ordering::SeqCst) }, - |entered| *entered, - ) - .await; - assert!(entered, "blocking handler never started"); - - let goodboy = WellBehavedTask { count: 0 }.start(); - goodboy.send(IncrementWell).unwrap(); - let count = crate::test_wait::wait_for_async( - || goodboy.request(GetCount), - |count| matches!(count, Ok(n) if *n >= 5), - ) - .await - .unwrap(); - assert!(count >= 5, "runtime stalled behind a blocked handler"); - - release.send(()).unwrap(); - badboy.join().await; - goodboy.request(StopCounter).await.unwrap(); - }); - } - - #[test] - pub fn badly_behaved_thread() { - blocked_handler_does_not_stall_runtime(Backend::Blocking); - } - - #[test] - pub fn backend_thread_isolates_blocking_work() { - blocked_handler_does_not_stall_runtime(Backend::Thread); - } - - // --- Panic recovery tests --- - - #[test] - pub fn panic_in_started_stops_actor() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_started_stops_actor"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicOnStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - async fn handle( - &mut self, - _msg: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } - - let actor = PanicOnStart.start(); - actor.join().await; - let result = actor.send(Ping); - assert!(result.is_err()); - }); - }, - ); - } - - #[test] - pub fn panic_in_handler_stops_actor() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_handler_stops_actor"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnMsg; - struct Explode; - impl Message for Explode { - type Result = (); - } - struct Check; - impl Message for Check { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - async fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); - } - } - } - impl Handler for PanicOnMsg { - async fn handle( - &mut self, - _msg: Check, - reply: crate::reply::PendingReply<'_, u32>, - _ctx: &Context, - ) { - reply.reply(42); - } - } - - let actor = PanicOnMsg.start(); - actor.send(Explode).unwrap(); - actor.join().await; - let result = actor.request(Check).await; - assert!(result.is_err()); - }); - }, - ); - } - - /// Review P1 (Codex 9): a manual `Handler` may be a plain `fn` that - /// panics before it returns its future. That is a handler panic like - /// any other: the turn ends, `stopped()` runs, and the caller and - /// `wait_exit` see the handler-panic reason. The future used to be built - /// outside the turn's `catch_unwind`, so the panic unwound the whole - /// lifecycle (no `stopped()`, an "actor lifecycle unwound" reason). - #[test] - pub fn panic_building_the_handler_future_is_a_handler_panic() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::panic_building_the_handler_future_is_a_handler_panic" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct Eager(Arc); - struct Explode; - impl Message for Explode { - type Result = (); - } - impl Actor for Eager { - async fn stopped(&mut self, _ctx: &Context) { - self.0.store(true, Ordering::SeqCst); - } - } - impl Handler for Eager { - #[allow(unreachable_code, clippy::diverging_sub_expression)] - fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) -> impl Future + Send { - panic!("boom building the handler future"); - async {} - } - } - - let stopped = Arc::new(atomic::AtomicBool::new(false)); - let actor = Eager(stopped.clone()).start(); - let reply = actor.request(Explode).await; - let reason = actor.wait_exit().await; - let ExitReason::Panic(message) = &reason else { - panic!("unexpected reason {reason:?}"); - }; - assert!( - message.starts_with("panic in handler: boom building"), - "not a handler panic: {message}" - ); - assert!( - matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), - "caller saw {reply:?}" - ); - assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); - }); - }, - ); - } - - #[test] - pub fn panic_in_stopped_still_completes() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_stopped_still_completes"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStop; - struct StopMe; - impl Message for StopMe { - type Result = (); - } - impl Actor for PanicOnStop { - async fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - async fn handle( - &mut self, - _msg: StopMe, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - let actor = PanicOnStop.start(); - actor.send(StopMe).unwrap(); - actor.join().await; - }); - }, - ); - } - - #[test] - pub fn send_message_on_delivers() { - crate::test_wait::bounded( - concat!(module_path!(), "::send_message_on_delivers"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); - let count = crate::test_wait::wait_for_async( - || counter.request(GetCount), - |count| matches!(count, Ok(1)), - ) - .await - .unwrap(); - assert_eq!(count, 1); - }); - }, - ); - } - - #[test] - pub fn send_message_on_cancelled() { - crate::test_wait::bounded( - concat!(module_path!(), "::send_message_on_cancelled"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); - // Stop actor before the future resolves - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 0, "message should not have been delivered"); - counter.join().await; - }); - }, - ); - } - - // --- ExitReason tests --- - - #[test] - pub fn exit_reason_normal_on_clean_stop() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).await.unwrap(); - let reason = actor.wait_exit().await; - assert!(matches!(reason, ExitReason::Normal)); - }); - }, - ); - } - - #[test] - pub fn exit_reason_panic_in_started() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_panic_in_started"), - move || { - struct PanicStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStart { - async fn handle( - &mut self, - _msg: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = PanicStart.start(); - let reason = actor.wait_exit().await; - assert!( - matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) - ); - }); - }, - ); - } - - #[test] - pub fn exit_reason_panic_in_handler() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_panic_in_handler"), - move || { - struct PanicHandler; - struct Explode; - impl Message for Explode { - type Result = (); - } - impl Actor for PanicHandler {} - impl Handler for PanicHandler { - async fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); - } - } - } - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = PanicHandler.start(); - let _ = actor.send(Explode); - let reason = actor.wait_exit().await; - assert!( - matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) - ); - }); - }, - ); - } - - #[test] - pub fn exit_reason_poll_returns_none_while_running() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::exit_reason_poll_returns_none_while_running" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).await.unwrap(); - actor.join().await; - assert!(actor.exit_reason().is_some()); - }); - }, - ); - } - - // --- Monitor tests --- - - struct GetDowns; - impl Message for GetDowns { - type Result = Vec; - } - - /// Actor that exposes `monitor`/`demonitor` via messages, so tests can - /// drive it from outside. Records all received Down messages. - struct Watcher { - downs: Arc>>, - } - - struct StartMonitor(crate::ChildHandle); - impl Message for StartMonitor { - type Result = crate::monitor::MonitorRef; - } - struct CallDemonitor(crate::monitor::MonitorRef); - impl Message for CallDemonitor { - type Result = (); - } - /// Monitor, then demonitor in the same turn. - struct MonitorThenDemonitor(crate::ChildHandle); - impl Message for MonitorThenDemonitor { - type Result = (); - } - - impl Actor for Watcher {} - - impl Handler for Watcher { - async fn handle( - &mut self, - msg: StartMonitor, - reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, - ctx: &Context, - ) { - reply.reply(ctx.monitor(&msg.0)); - } - } - - impl Handler for Watcher { - async fn handle( - &mut self, - msg: MonitorThenDemonitor, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - let monitor = ctx.monitor(&msg.0); - ctx.demonitor(monitor); - reply.reply(()); - } - } - - impl Handler for Watcher { - async fn handle( - &mut self, - msg: CallDemonitor, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.demonitor(msg.0); - reply.reply(()); - } - } - - impl Handler for Watcher { - async fn handle( - &mut self, - msg: crate::monitor::Down, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - self.downs.lock().unwrap().push(msg); - reply.reply(()); - } - } - - impl Handler for Watcher { - async fn handle( - &mut self, - _msg: GetDowns, - reply: crate::reply::PendingReply<'_, Vec>, - _ctx: &Context, - ) { - reply.reply(self.downs.lock().unwrap().clone()); - } - } - - #[test] - pub fn monitor_running_actor_delivers_down_on_exit() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitor_running_actor_delivers_down_on_exit" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - - // Stop the target — Down should be delivered - target.request(StopCounter).await.unwrap(); - target.join().await; - - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); - }); - }, - ); - } - - /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox - /// (here the `NoProc` a monitor of a dead target queues at once) is - /// never handled once `demonitor` returned. - #[test] - pub fn demonitor_flushes_a_queued_down() { - crate::test_wait::bounded( - concat!(module_path!(), "::demonitor_flushes_a_queued_down"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).await.unwrap(); - target.join().await; - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - watcher - .request(MonitorThenDemonitor(target.child_handle())) - .await - .unwrap(); - // FIFO: the queued Down is dispatched before this request. - let downs = watcher.request(GetDowns).await.unwrap(); - assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); - }); - }, - ); - } - - #[test] - pub fn monitor_already_dead_actor_delivers_down_immediately() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitor_already_dead_actor_delivers_down_immediately" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).await.unwrap(); - target.join().await; - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - }); - }, - ); - } - - #[test] - pub fn demonitor_before_target_dies_suppresses_down() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::demonitor_before_target_dies_suppresses_down" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); - - // Now stop the target - target.request(StopCounter).await.unwrap(); - target.join().await; - rt::sleep(Duration::from_millis(50)).await; - - let downs = watcher.request(GetDowns).await.unwrap(); - assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); - }); - }, - ); - } - - #[test] - pub fn multiple_monitors_each_get_own_ref_and_down() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::multiple_monitors_each_get_own_ref_and_down" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .await - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); - }); - }, - ); - } - - #[test] - pub fn monitor_table_is_cleaned_up_after_target_dies() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitor_table_is_cleaned_up_after_target_dies" - ), - move || { - // Watcher should remove its entry from the monitor table after the - // target dies, so the table doesn't accumulate stale entries. - struct Inspect; - impl Message for Inspect { - type Result = usize; - } - impl Handler for Watcher { - async fn handle( - &mut self, - _msg: Inspect, - reply: crate::reply::PendingReply<'_, usize>, - ctx: &Context, - ) { - reply.reply( - ctx.shared - .monitors - .lock() - .unwrap() - .as_ref() - .map_or(0, HashMap::len), - ); - } - } - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher - .request(StartMonitor(target.child_handle())) - .await - .unwrap(); - assert_eq!(watcher.request(Inspect).await.unwrap(), 1); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let entries = crate::test_wait::wait_for_async( - || watcher.request(Inspect), - |entries| !matches!(entries, Ok(n) if *n > 0), - ) - .await; - assert_eq!( - entries.unwrap(), - 0, - "monitor table should be empty after target died" - ); - }); - }, - ); - } - - #[test] - pub fn monitor_observes_panic_reason() { - crate::test_wait::bounded( - concat!(module_path!(), "::monitor_observes_panic_reason"), - move || { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - async fn handle( - &mut self, - _msg: PanicMsg, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic"); - } - } - } - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = PanicMe.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let _ = target.send(PanicMsg); - - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); - }); - }, - ); - } - - #[test] - pub fn monitoring_actor_stops_before_target_does_not_panic() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitoring_actor_stops_before_target_does_not_panic" - ), - move || { - // Regression: if the monitoring actor stops while the target is still - // alive, the watcher must not crash when the target eventually dies. - // The watcher's send() will fail silently (mailbox closed), and the - // watcher exits cleanly. - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - - // Stop the monitoring actor first - let watcher_handle = watcher.child_handle(); - watcher_handle.stop(); - watcher_handle.wait_exit_async().await; - - // Now stop the target — the watcher should clean up without panicking - target.request(StopCounter).await.unwrap(); - target.join().await; - // Give the orphaned watcher task time to attempt delivery - rt::sleep(Duration::from_millis(50)).await; - // If we got here without panicking, the test passes. - }); - }, - ); - } - - // --- Link tests --- - - /// Trapping actor that records `Exit` notifications. - struct TrapActor { - exits: Arc>>, - trap: bool, - } - - struct GetExits; - impl Message for GetExits { - type Result = Vec; - } - struct LinkTo(crate::ChildHandle); - impl Message for LinkTo { - type Result = (); - } - - impl Actor for TrapActor { - async fn started(&mut self, ctx: &Context) { - ctx.trap_exit(self.trap); - } - async fn exit_received(&mut self, exit: Exit, _ctx: &Context) { - self.exits.lock().unwrap().push(exit); - } - } - - impl Handler for TrapActor { - async fn handle( - &mut self, - msg: LinkTo, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.link(&msg.0); - reply.reply(()); - } - } - - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: GetExits, - reply: crate::reply::PendingReply<'_, Vec>, - _ctx: &Context, - ) { - reply.reply(self.exits.lock().unwrap().clone()); - } - } - - fn make_trapper(trap: bool) -> ActorRef { - TrapActor { - exits: Arc::new(Mutex::new(Vec::new())), - trap, - } - .start() - } - - #[test] - pub fn link_propagates_panic_to_non_trapping_peer() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_propagates_panic_to_non_trapping_peer" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Two non-trapping actors. When A panics, B should be terminated. - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).await.unwrap(); - - // Make A panic by dropping its only ActorRef while it's mid-handler... easier: - // make A panic via a panic message - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); - } - } - } - - let _ = a.send(Boom); - // Both should die - let reason_a = a.wait_exit().await; - let reason_b = b.wait_exit().await; - assert!(matches!(reason_a, ExitReason::Panic(_))); - // B should propagate A's reason (transitive propagation) - assert!(matches!(reason_b, ExitReason::Panic(_))); - }); - }, - ); - } - - #[test] - pub fn link_delivers_exit_to_trapping_peer() { - crate::test_wait::bounded( - concat!(module_path!(), "::link_delivers_exit_to_trapping_peer"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = make_trapper(false); - let b = make_trapper(true); // B traps - b.request(LinkTo(a.child_handle())).await.unwrap(); - - struct Boom2; - impl Message for Boom2 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom2, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom2"); - } - } - } - - let _ = a.send(Boom2); - a.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - - // B should still be alive and have received an Exit - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); - - // Clean up - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); - }, - ); - } - - #[test] - pub fn link_normal_exit_not_propagated_to_non_trapping_peer() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_normal_exit_not_propagated_to_non_trapping_peer" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(false); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // A stops cleanly - a.request(StopCounter).await.unwrap(); - a.join().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should still be alive - assert!(b.exit_reason().is_none()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); - }, - ); - } - - #[test] - pub fn link_to_already_dead_actor_delivers_noproc() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_to_already_dead_actor_delivers_noproc" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Start and panic a target - struct Boom3; - impl Message for Boom3 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom3, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom3"); - } - } - } - let target = make_trapper(false); - let _ = target.send(Boom3); - target.wait_exit().await; - - // Now link from a trapping observer: the target is past its - // death point, so the Exit carries NoProc (E4), not its reason. - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); +/// A context and its cleanup without spawning, for run-loop unit tests. +#[cfg(test)] +fn test_context( + tx: &mpsc::Sender + Send>>, + token: &StopToken, +) -> (Context, crate::actor_cleanup::ActorCleanup) { + let id = ActorId::next(); + let (completion_tx, completion_rx) = watch::channel(None); + let trap = new_trap_exit_flag(); + let links = new_link_table(); + let request = new_linked_exit_reason(); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); + let signal = signal_target::(id, None, tx, trap.clone(), request.clone(), token.clone()); + let ctx = Context { + shared: Arc::new(ActorShared { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion_rx, + monitors: monitors.clone(), + trap_exit: trap, + links: links.clone(), + linked_reason: request.clone(), + signal, + mailbox: crate::instrumentation::Mailbox::default(), + managed: None, + replies: Registry::new(id, DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, 1)), + cleaned: watch::channel(false).1, + }), + }; + let noop: fn() = || {}; + let cleanup = + crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop) + .with_replies(ctx.shared.replies.clone()); + (ctx, cleanup) +} - let exits = crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].reason, ExitReason::NoProc); - assert!(matches!(target.exit_reason(), Some(ExitReason::Panic(_)))); +// --------------------------------------------------------------------------- +// Actor::start +// --------------------------------------------------------------------------- - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); - }, - ); +/// Extension trait for starting an actor. Automatically implemented for all [`Actor`] types. +pub trait ActorStart: Actor { + /// Start the actor with the default backend ([`Backend::Async`]). + fn start(self) -> ActorRef { + self.start_with_backend(Backend::default()) } - #[test] - pub fn unlink_prevents_signal_delivery() { - crate::test_wait::bounded( - concat!(module_path!(), "::unlink_prevents_signal_delivery"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // Now unlink - struct UnlinkFrom(crate::ChildHandle); - impl Message for UnlinkFrom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - msg: UnlinkFrom, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.unlink(&msg.0); - reply.reply(()); - } - } - b.request(UnlinkFrom(a.child_handle())).await.unwrap(); - - // A panics - struct Boom4; - impl Message for Boom4 { - type Result = u64; - } - impl Handler for Counter { - async fn handle( - &mut self, - _msg: Boom4, - _reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - panic!("boom4"); - } - } - // Counter's Boom4 returns u64; just use send with no expectation of reply - let _ = a.send(Boom4); - a.wait_exit().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should NOT have received an Exit - let exits = b.request(GetExits).await.unwrap(); - assert!(exits.is_empty()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); - }, - ); + /// Admit this actor to an owned parked async execution group. + fn start_on( + self, + group: &crate::system::ExecutionGroup, + ) -> Result, crate::system::SystemError> { + group.start(self) } - #[test] - pub fn duplicate_link_is_idempotent() { - crate::test_wait::bounded( - concat!(module_path!(), "::duplicate_link_is_idempotent"), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate - - struct Boom5; - impl Message for Boom5 { - type Result = u64; - } - impl Handler for Counter { - async fn handle( - &mut self, - _msg: Boom5, - _reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - panic!("boom5"); - } - } - let _ = a.send(Boom5); - a.wait_exit().await; - crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; - - // Should receive only ONE Exit, not two - let exits = b.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); - }, - ); + /// Start the actor with a specific [`Backend`]. + fn start_with_backend(self, backend: Backend) -> ActorRef { + ActorRef::spawn(self, backend, None) } - #[test] - pub fn chain_propagation_through_non_trapping_middle() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::chain_propagation_through_non_trapping_middle" - ), - move || { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // A linked to B linked to C. A panics → B dies (not trapping) → - // B's death propagates to C. - let a = make_trapper(false); - let b = make_trapper(false); - let c = make_trapper(true); // C traps - - a.request(LinkTo(b.child_handle())).await.unwrap(); - c.request(LinkTo(b.child_handle())).await.unwrap(); - - struct Boom6; - impl Message for Boom6 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom6, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom6"); - } - } - } - let _ = a.send(Boom6); - a.wait_exit().await; - b.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || c.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); - - let ch = c.child_handle(); - ch.stop(); - ch.wait_exit_async().await; - }); - }, - ); + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. A + /// parent already past its death point gives the child a `NoProc` parent + /// exit. + /// + /// The child is always an unmanaged `Backend::Async` actor, even when + /// the parent is managed: it is outside the parent's execution group and + /// its ordered shutdown (`spawn_helper` on it returns `UnmanagedActor`). + /// The unified start API (planned for 0.9) will inherit the placement. + fn start_linked(self, parent_ctx: &Context

) -> ActorRef { + ActorRef::spawn(self, Backend::Async, Some(parent_ctx.child_handle())) } +} - #[test] - pub fn link_to_already_dead_delivers_exactly_once_to_trapping_peer() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_to_already_dead_delivers_exactly_once_to_trapping_peer" - ), - move || { - // Regression: previously, `ctx.link()` would deliver a duplicate Exit - // to trapping actors when the target died concurrently — once from - // propagate_exit (since register_link inserted self in target's table - // before target's drain ran) and once from the fallback exit_reason() - // check. - struct Boom7; - impl Message for Boom7 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom7, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom7"); - } - } - } - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = make_trapper(false); - // Kick off the panic - let _ = target.send(Boom7); - // Wait for target to fully die so exit_reason() is Some - target.wait_exit().await; - - // Now link from a trapping observer — should receive Exit EXACTLY ONCE - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); - - crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; - let exits = observer.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); +impl ActorStart for A {} - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); - }, - ); - } +// --------------------------------------------------------------------------- +// send_message_on (utility) +// --------------------------------------------------------------------------- - #[test] - pub fn start_linked_links_atomically() { - crate::test_wait::bounded( - concat!(module_path!(), "::start_linked_links_atomically"), - move || { - // Parent + child via start_linked. When parent stops via panic, - // child (not trapping) should die too. - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - async fn handle( - &mut self, - _msg: PanicParent, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("parent boom"); - } - } +/// Send a message to an actor when a future completes. +/// +/// Spawns a task that races the future against the actor's cancellation token. +/// If the actor stops before the future completes, the message is not sent. +pub fn send_message_on(ctx: Context, future: U, msg: M) -> JoinHandle<()> +where + A: Actor + Handler, + M: Message, + U: Future + Send + 'static, + ::Output: Send, +{ + let cancellation_token = ctx.cancellation_token(); + let spawn_ctx = ctx.clone(); + let join_handle = spawn_ctx.spawn_framework(async move { + let is_cancelled = pin!(cancellation_token.cancelled()); + let signal = pin!(future); + match future::select(is_cancelled, signal).await { + future::Either::Left(_) => tracing::debug!("Actor stopped"), + future::Either::Right(_) => { + if let Err(e) = ctx.send(msg) { + tracing::error!("Failed to send message: {e:?}") } - - struct Child; - impl Actor for Child {} - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); - - // Parent panics — child should die too - let _ = parent.send(PanicParent); - parent.wait_exit().await; - // Child should also die (link propagation) - let reason = child.wait_exit().await; - // Reason should be the parent's panic - assert!(matches!(reason, ExitReason::Panic(_))); - }); - }, - ); - } -} - -#[cfg(test)] -mod monitor_cleanup_regressions { - use super::*; - struct Idle; - impl Actor for Idle {} - impl Handler for Idle { - async fn handle( - &mut self, - _: Down, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - reply.reply(()); - } - } - #[test] - fn closed_monitor_table_rejects_late_registration() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::closed_monitor_table_rejects_late_registration" - ), - move || { - let rt = rt::Runtime::new().unwrap(); - let _entered = rt.enter(); - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.shared.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.shared.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - rt.block_on(observer.join()); - rt.block_on(target.join()); - }, - ); - } - fn check_cleanup(stop_owner: bool) { - let rt = rt::Runtime::new().unwrap(); - let _entered = rt.enter(); - let target = Idle.start(); - let observer = Idle.start(); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - let monitor = observer.context().monitor(&child); - drop(child); - if stop_owner { - observer.child_handle().stop(); - rt.block_on(observer.join()); - drop(observer); - } else { - observer.context().demonitor(monitor); - // Only the waiter now owns this particular ChildHandle allocation. - let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); - observer.child_handle().stop(); - rt.block_on(observer.join()); - drop(observer); - target.child_handle().stop(); - rt.block_on(target.join()); - assert!(released, "demonitor retained its waiter until target exit"); - return; + } } - let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); - target.child_handle().stop(); - rt.block_on(target.join()); - assert!(released, "stopped observer retained its monitor waiter"); - } - #[test] - fn demonitor_releases_waiter() { - check_cleanup(false); - } - #[test] - fn stopped_observer_releases_waiter() { - check_cleanup(true); - } + }); + join_handle } -#[cfg(test)] -mod early_stop_regression { +// --------------------------------------------------------------------------- +// WarnOnBlocking (debug only) +// --------------------------------------------------------------------------- + +#[cfg(debug_assertions)] +mod warn_on_block { use super::*; - struct EarlyStop; - impl Actor for EarlyStop { - async fn started(&mut self, ctx: &Context) { - ctx.stop(); - } - async fn stopped(&mut self, _: &Context) { - panic!("cleanup failed"); + use std::time::Instant; + use tracing::warn; + + pin_project_lite::pin_project! { + pub struct WarnOnBlocking{ + #[pin] + inner: F } } - #[test] - fn panic_during_early_stop_is_reported() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_during_early_stop_is_reported"), - move || { - let rt = rt::Runtime::new().unwrap(); - let _enter = rt.enter(); - let actor = EarlyStop.start(); - assert!( - matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) - ); - }, - ); - } -} -#[cfg(test)] -mod idle_stop_regression { - use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - struct Counter(Arc); - impl Actor for Counter {} - struct Increment; - impl Message for Increment { - type Result = (); - } - impl Handler for Counter { - async fn handle( - &mut self, - _: Increment, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - self.0.fetch_add(1, Ordering::SeqCst); - reply.reply(()); + impl WarnOnBlocking { + pub fn new(inner: F) -> Self { + Self { inner } } } - #[test] - fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::stop_while_idle_does_not_dispatch_a_queued_request_on_resume" - ), - move || { - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = StopToken::new(); - let (ctx, mut cleanup) = test_context(&tx, &token); - let count = Arc::new(AtomicUsize::new(0)); - let mut actor = Box::pin(run_actor( - Counter(count.clone()), - ctx, - rx, - token.clone(), - &mut cleanup, - )); - // Start and park the actual actor loop, without allowing another - // executor task to run it between admission and stop below. - assert!(futures::poll!(actor.as_mut()).is_pending()); - let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); - tx.send(Box::new(MessageEnvelope { - msg: Some(Increment), - slot: { - let mut slot = slot; - slot.take() - }, - #[cfg(feature = "metrics")] - observation: None, - #[cfg(feature = "lifecycle-trace")] - from: 0, - }) as Box + Send>) - .unwrap(); - token.cancel(); - actor.await; - assert_eq!(cleanup.die(), ExitReason::Normal); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "idle actor dispatched queued work after stop" - ); - assert!(response.await.is_err(), "queued request unexpectedly ran"); - }); - }, - ); - } -} -/// Context and request APIs that the behavioural tests above do not reach. -#[cfg(test)] -mod api_coverage { - use super::*; - use crate::test_wait::WAIT; + impl Future for WarnOnBlocking { + type Output = F::Output; - struct Echo; - struct Ask(u32); - impl Message for Ask { - type Result = u32; - } - /// Holds the actor inside a handler until released, then stops it, so a - /// request queued behind it is admitted but never answered. - struct StopAfter(rt::oneshot::Receiver<()>); - impl Message for StopAfter { - type Result = (); - } - struct OwnRecipient; - impl Message for OwnRecipient { - type Result = (Recipient, ActorId); - } - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Actor for Echo {} - impl Handler for Echo { - async fn handle( - &mut self, - msg: Ask, - reply: crate::reply::PendingReply<'_, u32>, - _: &Context, - ) { - reply.reply(msg.0); - } - } - impl Handler for Echo { - async fn handle( - &mut self, - msg: StopAfter, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - let _ = msg.0.await; - ctx.stop(); - reply.reply(()); - } - } - impl Handler for Echo { - async fn handle( - &mut self, - _: OwnRecipient, - reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, - ctx: &Context, - ) { - reply.reply((ctx.recipient::(), ctx.id())); - } - } - impl Handler for Echo { - async fn handle( - &mut self, - _: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - { - panic!("boom"); + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let type_id = std::any::type_name::(); + let task_id = rt::task_id(); + let this = self.project(); + let now = Instant::now(); + let res = this.inner.poll(cx); + let elapsed = now.elapsed(); + if elapsed > Duration::from_millis(10) { + warn!(task = ?task_id, future = ?type_id, elapsed = ?elapsed, "Blocking operation detected"); } + res } } - - #[test] - fn context_recipient_reaches_its_actor() { - crate::test_wait::bounded( - concat!(module_path!(), "::context_recipient_reaches_its_actor"), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join().await; - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); - }); - }, - ); - } - - #[test] - fn requests_to_a_stopped_actor_fail_without_waiting() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::requests_to_a_stopped_actor_fail_without_waiting" - ), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join().await; - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT).await, - Err(ActorError::NoProc) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT).await, - Err(ActorError::NoProc) - )); - }); - }, - ); - } - - #[test] - fn admitted_requests_fail_when_the_actor_stops_before_replying() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::admitted_requests_fail_when_the_actor_stops_before_replying" - ), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); - let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); - assert!(futures::poll!(direct.as_mut()).is_pending()); - assert!(futures::poll!(erased.as_mut()).is_pending()); - release.send(()).unwrap(); - assert!(matches!( - direct.await, - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - assert!(matches!( - erased.await, - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - }); - }, - ); - } - - #[test] - fn erased_request_times_out() { - crate::test_wait::bounded( - concat!(module_path!(), "::erased_request_times_out"), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - assert!(matches!( - request(&*recipient, Ask(1), Duration::from_millis(1)).await, - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join().await; - }); - }, - ); - } - - #[test] - fn default_exit_received_keeps_a_trapping_actor_alive() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::default_exit_received_keeps_a_trapping_actor_alive" - ), - move || { - struct Trapper; - impl Actor for Trapper { - async fn started(&mut self, ctx: &Context) { - ctx.trap_exit(true); - } - } - impl Handler for Trapper { - async fn handle( - &mut self, - msg: Ask, - reply: crate::reply::PendingReply<'_, u32>, - _: &Context, - ) { - reply.reply(msg.0); - } - } - rt::Runtime::new().unwrap().block_on(async { - let trapper = Trapper.start(); - assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); - }); - }, - ); - } - - #[test] - fn debug_output_names_the_handle() { - crate::test_wait::bounded( - concat!(module_path!(), "::debug_output_names_the_handle"), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join().await; - }); - }, - ); - } } -#[cfg(test)] -mod shared_state_release { - //! `ActorRef` and `Context` share one `Arc`. Once the actor has - //! stopped and every handle is dropped, that block must be freed: nothing - //! the runtime keeps (links, monitors, child handles, the run task) may hold it. - use super::*; - use std::sync::Weak; +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- - async fn freed(weak: &Weak>) -> bool { - for _ in 0..500 { - if weak.upgrade().is_none() { - return true; - } - rt::sleep(Duration::from_millis(10)).await; - } - false - } +#[cfg(test)] +mod tests; - struct Plain; - impl Actor for Plain {} - impl Handler for Plain { - async fn handle( - &mut self, - _: Down, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - reply.reply(()); - } - } +#[cfg(test)] +mod monitor_cleanup_regressions; - /// Keeps a clone of its own `Context` in its state. - struct Holder(Option>); - impl Actor for Holder { - async fn started(&mut self, ctx: &Context) { - self.0 = Some(ctx.clone()); - } - } +#[cfg(test)] +mod early_stop_regression; - #[test] - fn freed_after_stop_and_last_handle_dropped() { - crate::test_wait::bounded( - concat!(module_path!(), "::freed_after_stop_and_last_handle_dropped"), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Plain.start(); - let weak = actor.shared_weak(); - let extra = (actor.clone(), actor.context(), actor.child_handle()); - actor.context().stop(); - actor.join().await; - drop(extra); - drop(actor); - assert!(freed(&weak).await, "ActorShared outlived every handle"); - }); - }, - ); - } +#[cfg(test)] +mod idle_stop_regression; - #[test] - fn freed_when_context_was_stored_in_actor_state() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::freed_when_context_was_stored_in_actor_state" - ), - move || { - rt::Runtime::new().unwrap().block_on(async { - let actor = Holder(None).start(); - let weak = actor.shared_weak(); - actor.context().stop(); - actor.join().await; - drop(actor); - assert!(freed(&weak).await, "Context kept in actor state leaked"); - }); - }, - ); - } +/// Context and request APIs that the behavioural tests above do not reach. +#[cfg(test)] +mod api_coverage; - #[test] - fn freed_with_links_and_monitors() { - crate::test_wait::bounded( - concat!(module_path!(), "::freed_with_links_and_monitors"), - move || { - rt::Runtime::new().unwrap().block_on(async { - let a = Plain.start(); - let b = Plain.start(); - let watcher = Plain.start(); - a.context().trap_exit(true); - a.context().link(&b.child_handle()); - let _monitor = watcher.context().monitor(&a.child_handle()); - watcher.context().monitor(&b.child_handle()); - let child = Plain.start_linked(&a.context()); - let weaks = [ - a.shared_weak(), - b.shared_weak(), - watcher.shared_weak(), - child.shared_weak(), - ]; - for actor in [&b, &a, &watcher] { - actor.context().stop(); - actor.join().await; - } - child.context().stop(); - child.join().await; - drop((a, b, watcher, child)); - for weak in &weaks { - assert!(freed(weak).await, "linked or monitored actor leaked"); - } - }); - }, - ); - } -} +#[cfg(test)] +mod shared_state_release; #[cfg(all(test, not(feature = "lifecycle-trace"), not(spawned_sim)))] -mod stop_receive_probe { - use super::*; - use std::{cell::RefCell, rc::Rc, sync::atomic::AtomicUsize}; - thread_local! { - static INTERLEAVE: RefCell>> = const { RefCell::new(None) }; - } - pub(super) fn before_receive() { - let action = INTERLEAVE.with(|hook| hook.borrow_mut().take()); - if let Some(action) = action { - action(); - } - } - struct Count(Arc); - impl Actor for Count {} - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Handler for Count { - async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { - self.0.fetch_add(1, Ordering::SeqCst); - reply.reply(()); - } - } - #[test] - fn stop_between_check_and_receive_discards_post_stop_request() { - crate::test_wait::bounded( - "stop_between_check_and_receive_discards_post_stop_request", - || { - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = StopToken::new(); - let (ctx, mut cleanup) = test_context::(&tx, &token); - let hits = Arc::new(AtomicUsize::new(0)); - let response = Rc::new(RefCell::new(None)); - let stop_ctx = ctx.clone(); - let saved = response.clone(); - INTERLEAVE.with(|hook| { - *hook.borrow_mut() = Some(Box::new(move || { - stop_ctx.stop(); - *saved.borrow_mut() = - Some(stop_ctx.send_request(Ping, Deadline::Never).unwrap()); - })) - }); - run_actor(Count(hits.clone()), ctx, rx, token, &mut cleanup).await; - assert_eq!(cleanup.die(), ExitReason::Normal); - let response = response - .borrow_mut() - .take() - .expect("receive probe was reached"); - let reply = response.await; - assert!( - matches!(reply, Err(ActorError::ActorStopped(ExitReason::Normal))), - "post-stop request was handled: {reply:?}" - ); - assert_eq!(hits.load(Ordering::SeqCst), 0); - }); - }, - ); - } - - struct CheckDrop { - owner: Arc, - response: Arc>>>, - checked: Arc, - } - impl Message for CheckDrop { - type Result = (); - } - impl Drop for CheckDrop { - fn drop(&mut self) { - let reply = self - .response - .lock() - .unwrap() - .as_mut() - .and_then(ReplyReceiver::try_recv); - let after_death = match reply { - Some(Err(ActorError::ActorStopped(ref reason))) => { - self.owner.final_reason() == Some(reason) - } - _ => false, - }; - self.checked.store(after_death, Ordering::SeqCst); - } - } - struct CleanupPause { - hits: Arc, - park: bool, - } - impl Actor for CleanupPause { - async fn stopped(&mut self, _: &Context) { - if self.park { - std::future::pending::<()>().await; - } - } - } - impl Handler for CleanupPause { - async fn handle( - &mut self, - _msg: CheckDrop, - reply: PendingReply<'_, ()>, - _: &Context, - ) { - self.hits.fetch_add(1, Ordering::SeqCst); - reply.reply(()); - } - } - #[test] - fn receive_rejected_by_stop_keeps_its_caller_until_the_death_point() { - crate::test_wait::bounded( - "receive_rejected_by_stop_keeps_its_caller_until_the_death_point", - || { - for (park, requested) in [ - (false, ExitReason::Normal), - (true, ExitReason::Normal), - (false, ExitReason::Shutdown), - (true, ExitReason::Shutdown), - (false, ExitReason::Panic("requested".into())), - (true, ExitReason::Panic("requested".into())), - (false, ExitReason::Kill), - (true, ExitReason::Kill), - ] { - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = StopToken::new(); - let (ctx, mut cleanup) = test_context::(&tx, &token); - let hits = Arc::new(AtomicUsize::new(0)); - let checked = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let response = Arc::new(std::sync::Mutex::new(None)); - let stop_ctx = ctx.clone(); - let probe = CheckDrop { - owner: ctx.shared.replies.clone(), - response: response.clone(), - checked: checked.clone(), - }; - let stop_reason = requested.clone(); - INTERLEAVE.with(|hook| { - *hook.borrow_mut() = Some(Box::new(move || { - stop_ctx.stop_with_reason(stop_reason); - let reply = stop_ctx.send_request(probe, Deadline::Never).unwrap(); - *response.lock().unwrap() = Some(reply); - })) - }); - let mut actor = Box::pin(run_actor( - CleanupPause { - hits: hits.clone(), - park, - }, - ctx, - rx, - token, - &mut cleanup, - )); - let expected = if park { - assert!(futures::poll!(actor.as_mut()).is_pending()); - assert!( - !checked.load(Ordering::SeqCst), - "payload was dropped before stopped() finished" - ); - drop(actor); - if requested == ExitReason::Normal { - ExitReason::Killed - } else { - requested - } - } else { - actor.await; - requested - }; - assert_eq!(cleanup.die(), expected); - assert_eq!(hits.load(Ordering::SeqCst), 0); - assert!( - checked.load(Ordering::SeqCst), - "held caller was not answered before payload destruction" - ); - }); - } - }, - ); - } -} +mod stop_receive_probe; diff --git a/concurrency/src/tasks/actor/actor_ref.rs b/concurrency/src/tasks/actor/actor_ref.rs new file mode 100644 index 00000000..c32dc8d3 --- /dev/null +++ b/concurrency/src/tasks/actor/actor_ref.rs @@ -0,0 +1,282 @@ +//! `ActorRef`: the external handle (sends, requests, links, monitors, join and exit observation). +use super::*; + +impl ActorRef { + /// Clone this handle with a timeout override for generated protocol calls. + /// Per-call override wins over method and protocol defaults. No message is + /// sent until a protocol method is called; timeout does not cancel its work. + pub fn with_request_timeout(&self, timeout: Duration) -> crate::response::RequestOptions { + crate::response::RequestOptions::new(self.clone(), timeout) + } + + /// Await this managed instance's initial readiness barrier. Success does not + /// guarantee it remains alive afterward; ordinary request errors still apply. + pub async fn ready(&self) -> Result<(), crate::system::SystemError> { + self.shared + .managed + .as_ref() + .ok_or(crate::system::SystemError::UnmanagedActor)? + .wait_ready() + .await + } + pub(crate) fn signal_failure(&self, error: String) { + let _ = self + .shared + .sender + .send(Box::new(FailureEnvelope(Some(error)))); + } + /// Send a fire-and-forget message to the actor. + pub fn send(&self, msg: M) -> Result<(), ActorError> + where + A: Handler, + M: Message, + { + enqueue_send( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + #[inline] + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; + Ok(rx) + } + + /// Send a request and wait for the reply (default 5s timeout). + /// + /// Uses the same request state machine as `request_with_timeout`, with + /// a zero-sized fixed budget instead of capturing a Duration. + pub fn request( + &self, + msg: M, + ) -> impl std::future::Future> + Send + '_ + where + A: Handler, + M: Message, + { + self.request_with_budget(msg, crate::reply::DefaultRequestBudget) + } + + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: `spawned_test::tasks::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.shared.replies), false); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Some(Box::new(f)), + slot, + lent: None, + }); + // Past the death point the actor is gone (E4), as for `enqueue`. + if self.shared.replies.refuses_admission() { + return Err(ActorError::NoProc); + } + self.shared + .sender + .send(envelope) + .map_err(|_| ActorError::NoProc)?; + Ok(rx) + } + + /// Send a request and wait for the reply with a custom timeout. + pub fn request_with_timeout( + &self, + msg: M, + duration: Duration, + ) -> impl Future> + Send + '_ + where + A: Handler, + M: Message, + { + self.request_with_budget(msg, duration) + } + + pub(super) async fn request_with_budget( + &self, + msg: M, + budget: B, + ) -> Result + where + A: Handler, + M: Message, + B: crate::reply::RequestBudget, + { + let duration = budget.into_duration(); + let observation = crate::instrumentation::RequestObservation::new(); + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let (deadline, timer) = crate::reply::request_timer(duration); + let rx = match self.send_request_with(msg, deadline, false) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } + }; + let result = rx.into_timed_future_with(timer).await; + observation.finish(&result); + result + } + + /// Get a type-erased `Recipient` for this actor. + pub fn recipient(&self) -> Recipient + where + A: Handler, + M: Message, + { + Arc::new(self.clone()) + } + + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. + pub fn context(&self) -> Context { + Context::from_ref(self) + } + + /// Wait until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline; jobs still running then + /// are reported, not awaited). A `stopped()` or state destructor that + /// never returns keeps it waiting; bound it with a timeout, or use + /// [`wait_exit`](Self::wait_exit) to observe death alone. + pub async fn join(&self) { + let _ = self.wait_exit().await; + let mut cleaned = self.shared.cleaned.clone(); + loop { + if *cleaned.borrow_and_update() { + return; + } + if cleaned.changed().await.is_err() { + return; + } + } + } + + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods (for example, after + /// runtime shutdown). + pub fn exit_reason(&self) -> Option { + crate::child_handle::poll_exit_reason(&self.shared.completion_rx) + } + + /// Wait until the actor's exit is published and return its final reason. + pub async fn wait_exit(&self) -> ExitReason { + crate::child_handle::wait_loop(self.shared.completion_rx.clone()).await + } + + /// Waiting messages, including in-flight enqueue reservations, excluding + /// the active handler and internal exit/shutdown control envelopes. + /// Concurrent observations are approximate; no registry or actor labels. + #[cfg(feature = "metrics")] + pub fn mailbox_depth(&self) -> u64 { + self.shared.mailbox.depth() + } + + /// The actor's unique identity. + pub fn id(&self) -> ActorId { + self.shared.id + } + + /// Get a type-erased `ChildHandle` for this actor. + pub fn child_handle(&self) -> ChildHandle { + self.shared.child_handle() + } + + /// The shared state, for tests proving it is freed once every handle is gone. + #[cfg(test)] + pub(super) fn shared_weak(&self) -> std::sync::Weak> { + Arc::downgrade(&self.shared) + } +} + +impl ActorShared { + pub(super) fn child_handle(&self) -> ChildHandle { + ChildHandle::new( + self.signal.clone(), + Completion::Tasks(self.completion_rx.clone()), + self.links.clone(), + ) + .with_mailbox_failures(self.sender.failures()) + } +} + +impl From> for ChildHandle { + fn from(actor_ref: ActorRef) -> Self { + actor_ref.shared.child_handle() + } +} + +// Bridge: ActorRef implements Receiver for any M that A handles +impl Receiver for ActorRef +where + A: Actor + Handler, + M: Message, +{ + fn send(&self, msg: M) -> Result<(), ActorError> { + ActorRef::send(self, msg) + } + + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline.ticks(), false) + } +} diff --git a/concurrency/src/tasks/actor/api_coverage.rs b/concurrency/src/tasks/actor/api_coverage.rs new file mode 100644 index 00000000..ba23654d --- /dev/null +++ b/concurrency/src/tasks/actor/api_coverage.rs @@ -0,0 +1,225 @@ +use super::*; +use crate::test_wait::WAIT; + +struct Echo; +struct Ask(u32); +impl Message for Ask { + type Result = u32; +} +/// Holds the actor inside a handler until released, then stops it, so a +/// request queued behind it is admitted but never answered. +struct StopAfter(rt::oneshot::Receiver<()>); +impl Message for StopAfter { + type Result = (); +} +struct OwnRecipient; +impl Message for OwnRecipient { + type Result = (Recipient, ActorId); +} +struct Boom; +impl Message for Boom { + type Result = (); +} +impl Actor for Echo {} +impl Handler for Echo { + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); + } +} +impl Handler for Echo { + async fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let _ = msg.0.await; + ctx.stop(); + reply.reply(()); + } +} +impl Handler for Echo { + async fn handle( + &mut self, + _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, + ctx: &Context, + ) { + reply.reply((ctx.recipient::(), ctx.id())); + } +} +impl Handler for Echo { + async fn handle( + &mut self, + _: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("boom"); + } + } +} + +#[test] +fn context_recipient_reaches_its_actor() { + crate::test_wait::bounded( + concat!(module_path!(), "::context_recipient_reaches_its_actor"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }); + }, + ); +} + +#[test] +fn requests_to_a_stopped_actor_fail_without_waiting() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + }); + }, + ); +} + +#[test] +fn admitted_requests_fail_when_the_actor_stops_before_replying() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_actor_stops_before_replying" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); + let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); + assert!(futures::poll!(direct.as_mut()).is_pending()); + assert!(futures::poll!(erased.as_mut()).is_pending()); + release.send(()).unwrap(); + assert!(matches!( + direct.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + assert!(matches!( + erased.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + }); + }, + ); +} + +#[test] +fn erased_request_times_out() { + crate::test_wait::bounded( + concat!(module_path!(), "::erased_request_times_out"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + assert!(matches!( + request(&*recipient, Ask(1), Duration::from_millis(1)).await, + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join().await; + }); + }, + ); +} + +#[test] +fn default_exit_received_keeps_a_trapping_actor_alive() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + struct Trapper; + impl Actor for Trapper { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + } + impl Handler for Trapper { + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); + } + } + rt::Runtime::new().unwrap().block_on(async { + let trapper = Trapper.start(); + assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); + }); + }, + ); +} + +#[test] +fn debug_output_names_the_handle() { + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join().await; + }); + }, + ); +} diff --git a/concurrency/src/tasks/actor/context.rs b/concurrency/src/tasks/actor/context.rs new file mode 100644 index 00000000..ebaad305 --- /dev/null +++ b/concurrency/src/tasks/actor/context.rs @@ -0,0 +1,568 @@ +//! `Context`: the handler-side handle (sends, requests, timers, links, monitors, stop and exit). +use super::*; + +impl Context { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { + Self { + shared: actor_ref.shared.clone(), + } + } + + /// Waiting messages, including in-flight enqueue reservations, excluding + /// the active handler and internal exit/shutdown control envelopes. + /// Concurrent observations are approximate; no registry or actor labels. + #[cfg(feature = "metrics")] + pub fn mailbox_depth(&self) -> u64 { + self.shared.mailbox.depth() + } + + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. + pub fn deferred_outstanding(&self) -> usize { + self.shared.replies.outstanding() + } + + /// The actor's unique identity. + pub fn id(&self) -> ActorId { + self.shared.id + } + + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. + pub fn stop(&self) { + self.stop_with_reason(ExitReason::Normal); + } + + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). + pub fn stop_with_reason(&self, reason: ExitReason) { + link::stop_with_reason(self.shared.id, &self.shared.linked_reason, reason, || { + self.shared.cancellation_token.cancel() + }); + } + + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + pub(super) fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.shared.linked_reason, reason, || { + self.shared.cancellation_token.cancel() + }); + } + + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. A signal to this actor's own handle is a signal, not a + /// self-exit: a non-trapping actor ignores `Normal`. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.shared.replies.signal_permit() else { + return; + }; + if link::lock_request(&self.shared.linked_reason).is_dead() { + return; + } + fail_point!("actor.exit_before_deliver"); + target.exit_from(self.shared.id, reason, false); + } + + /// Send a fire-and-forget message to this actor. + pub fn send(&self, msg: M) -> Result<(), ActorError> + where + A: Handler, + M: Message, + { + enqueue_send( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + #[inline] + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; + Ok(rx) + } + + /// Send a request and wait for the reply (default 5s timeout). + /// + /// Uses the same request state machine as `request_with_timeout`, with + /// a zero-sized fixed budget instead of capturing a Duration. + pub fn request( + &self, + msg: M, + ) -> impl std::future::Future> + Send + '_ + where + A: Handler, + M: Message, + { + self.request_with_budget(msg, crate::reply::DefaultRequestBudget) + } + + /// Send a request and wait for the reply with a custom timeout. + pub fn request_with_timeout( + &self, + msg: M, + duration: Duration, + ) -> impl Future> + Send + '_ + where + A: Handler, + M: Message, + { + self.request_with_budget(msg, duration) + } + + pub(super) async fn request_with_budget( + &self, + msg: M, + budget: B, + ) -> Result + where + A: Handler, + M: Message, + B: crate::reply::RequestBudget, + { + let duration = budget.into_duration(); + let observation = crate::instrumentation::RequestObservation::new(); + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let (deadline, timer) = crate::reply::request_timer(duration); + let rx = match self.send_request_with(msg, deadline, false) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } + }; + let result = rx.into_timed_future_with(timer).await; + observation.finish(&result); + result + } + + /// Get a type-erased `Recipient` for sending a single message type + /// to this actor. + pub fn recipient(&self) -> Recipient + where + A: Handler, + M: Message, + { + Arc::new(self.clone()) + } + + /// Get an `ActorRef` from this context. + pub fn actor_ref(&self) -> ActorRef { + ActorRef { + shared: self.shared.clone(), + } + } + + /// Set up a unidirectional monitor on another actor. + /// + /// Returns a [`MonitorRef`] that can be used to cancel the monitor via + /// [`Context::demonitor`]. When the monitored actor's exit is published, + /// a [`Down`] with its final reason is delivered via `Handler`. + /// + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`; use + /// [`ChildHandle::exit_reason`] for the real reason). + /// + /// Multiple independent monitors are allowed on the same target — each + /// call returns a distinct `MonitorRef`. + /// + /// Monitors are unidirectional: the monitored actor is unaware of the + /// monitor and unaffected by it. + /// Registration after owner shutdown starts (including in `stopped()`) + /// returns an inert reference and creates no waiter or notification. + pub fn monitor(&self, target: &ChildHandle) -> MonitorRef + where + A: Handler, + { + fail_point!("monitor.before_install"); + let monitor_ref = MonitorRef::next(); + let active = spawned_rt::tasks::CancellationToken::new(); + + let observer = self.shared.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) + }; + crate::monitor::fence::register( + &self.shared.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.shared.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = enqueue_down( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + Down { + monitor_ref, + actor: target.id(), + reason: ExitReason::NoProc, + }, + ); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active + }); + if !registered { + return monitor_ref; + } + + let target = target.clone(); + let actor_ref = self.actor_ref(); + let monitors = self.shared.monitors.clone(); + let registration = crate::monitor::MonitorRegistration(monitors.clone(), monitor_ref); + + self.spawn_framework(async move { + let _registration = registration; + let reason = match futures::future::select( + Box::pin(active.cancelled()), + Box::pin(target.wait_exit_async()), + ) + .await + { + futures::future::Either::Left(_) => return, + futures::future::Either::Right((reason, _)) => reason, + }; + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = enqueue_down( + &actor_ref.shared.sender, + &actor_ref.shared.mailbox, + &actor_ref.shared.replies, + Down { + monitor_ref, + actor: target.id(), + reason, + }, + ); + }); + }); + }); + + monitor_ref + } + + /// Cancel a previously-set monitor. + /// + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. + pub fn demonitor(&self, monitor_ref: MonitorRef) { + let observer = self.shared.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::demonitor(&self.shared.monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + }) + }); + // Waking the waiter must happen after releasing the table lock. + if let Some(active) = active { + active.cancel(); + } + } + + /// Set up a bidirectional link with another actor (OTP `link/1`). + /// + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. + /// + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc` (decision 1). Use [`try_link`](Self::try_link) to get + /// `Err(NoProc)` instead. An existing link, or a pending notice from that + /// peer, makes this a no-op. Inert once this actor's exit began (in + /// `stopped()`, from a helper after the stop, or past its death point): + /// no record and no notice, as `monitor`. + /// + /// [`Actor::exit_received`]: Actor::exit_received + pub fn link(&self, target: &ChildHandle) { + link::register_link( + &self.shared.signal, + &self.shared.links, + target.signal_target(), + target.links(), + false, + ); + } + + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record, whether or not this actor + /// traps exits. + /// A caller whose exit already began does nothing and returns `Ok(())`; + /// an exit racing this call can linearize after the link attempt. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.shared.signal, + &self.shared.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor, including a notice already taken at + /// the peer's death; an `Exit` already queued stays. + pub fn unlink(&self, target: &ChildHandle) { + link::unregister_link( + self.shared.id, + &self.shared.links, + target.id(), + target.links(), + ); + } + + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. + /// + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. + /// + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.shared.linked_reason).exit_pending() + } + + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). + /// + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for + /// [`Actor::exit_received`]. + /// + /// Only a kill is untrappable. A parent's exit (the `start_linked` parent, + /// the supervisor or the system) ends a trapping actor normally, through + /// `stopped()`, and never reaches `exit_received`. Set it first thing in + /// `started()`: signals that arrive while `started()` runs are resolved + /// when it returns. Handlers are never preempted. + pub fn trap_exit(&self, enabled: bool) { + let id = self.shared.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.shared.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); + } + + /// Type-erased handle to this actor, without an intermediate `ActorRef`. + pub(crate) fn child_handle(&self) -> ChildHandle { + self.shared.child_handle() + } + + /// Spawn a lifecycle-owned helper on this actor's assigned execution group. + /// Construct runtime-bound work inside the async block. Closure of the scope + /// cancels the helper and waits for its future to drop before cleanup ends. + /// Returns [`crate::system::SystemError::UnmanagedActor`] for actors started + /// with `.start()`; use `.start_on(&group)` to establish cleanup ownership. + pub fn spawn_helper( + &self, + future: F, + ) -> Result, crate::system::SystemError> + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + self.shared + .managed + .as_ref() + .ok_or(crate::system::SystemError::UnmanagedActor)? + .spawn_helper(future) + } + + /// Register blocking work until the native closure actually finishes. + /// Cancellation cannot preempt it. Raw nested spawn calls are not implicitly + /// owned; use this method rather than detaching a blocking task inside a + /// cancellable async helper when recovery must await its resource cleanup. + /// Requires `.start_on(&group)`, like [`Self::spawn_helper`]. + pub fn spawn_blocking_helper( + &self, + work: F, + ) -> Result, crate::system::SystemError> + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + self.shared + .managed + .as_ref() + .ok_or(crate::system::SystemError::UnmanagedActor)? + .spawn_blocking_helper(work) + } + + pub fn execution_group(&self) -> Option { + self.shared.managed.as_ref().map(|s| s.group()) + } + + /// Register this managed actor until its owned cleanup completes. Removal + /// compares identity and registration generation, preserving replacements. + pub fn register_name( + &self, + name: &str, + ) -> Result<(), crate::registry::ManagedRegistrationError> { + self.shared + .managed + .as_ref() + .ok_or(crate::registry::ManagedRegistrationError::OwnerClosed)? + .register(name, self.shared.id, self.actor_ref()) + } + + pub(crate) fn managed_scope(&self) -> Option<&Arc> { + self.shared.managed.as_ref() + } + pub(crate) fn spawn_framework(&self, future: F) -> JoinHandle<()> + where + F: Future + Send + 'static, + { + match &self.shared.managed { + Some(scope) => scope.spawn_framework(future), + None => rt::spawn(future), + } + } + + pub(crate) fn cancellation_token(&self) -> StopToken { + self.shared.cancellation_token.clone() + } +} + +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + +// Bridge: Context implements Receiver for any M that A handles +impl Receiver for Context +where + A: Actor + Handler, + M: Message, +{ + fn send(&self, msg: M) -> Result<(), ActorError> { + Context::send(self, msg) + } + + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline.ticks(), false) + } +} diff --git a/concurrency/src/tasks/actor/early_stop_regression.rs b/concurrency/src/tasks/actor/early_stop_regression.rs new file mode 100644 index 00000000..ef7add53 --- /dev/null +++ b/concurrency/src/tasks/actor/early_stop_regression.rs @@ -0,0 +1,24 @@ +use super::*; +struct EarlyStop; +impl Actor for EarlyStop { + async fn started(&mut self, ctx: &Context) { + ctx.stop(); + } + async fn stopped(&mut self, _: &Context) { + panic!("cleanup failed"); + } +} +#[test] +fn panic_during_early_stop_is_reported() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let rt = rt::Runtime::new().unwrap(); + let _enter = rt.enter(); + let actor = EarlyStop.start(); + assert!( + matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, + ); +} diff --git a/concurrency/src/tasks/actor/envelope.rs b/concurrency/src/tasks/actor/envelope.rs new file mode 100644 index 00000000..28122808 --- /dev/null +++ b/concurrency/src/tasks/actor/envelope.rs @@ -0,0 +1,548 @@ +//! Mailbox envelopes: type erasure of messages, requests, `Down`, exit signals and in-turn closures on the actor side. +use super::*; + +// --------------------------------------------------------------------------- +// Envelope (type-erasure on the actor side) +// --------------------------------------------------------------------------- + +pub(super) trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + None + } + #[cfg(feature = "metrics")] + fn reject(self: Box) {} + + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self, _replies: Option<&Registry>) {} + + /// Dispatch once: the turn's future, and the request's caller if any. + /// The envelope keeps the request's slot for the whole turn and lends + /// it to the handler's `PendingReply` (no reference count change per + /// message); the run loop answers the lent caller at the death point if + /// the runtime destroys the turn before it finishes. + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a>; +} + +/// A dispatched turn: None means the guarded dispatch already completed; +/// otherwise its future and caller stay lent by the envelope for the turn. +pub(super) type Dispatch<'a> = ( + Option + Send + 'a>>>, + Option<&'a dyn Caller>, +); + +/// Dispatch `envelope`, catching a panic while the handler builds its +/// future (a manual `Handler` may panic before it returns one): a handler +/// panic like any other. +pub(super) fn dispatch<'a, A: Actor>( + envelope: &'a mut (dyn Envelope + Send), + actor: &'a mut A, + ctx: &'a Context, +) -> Result, Box> { + let mut parts = Some((envelope, actor)); + std::panic::catch_unwind(AssertUnwindSafe(move || { + let (envelope, actor) = parts.take().expect("dispatched once"); + crate::lifecycle_trace::with_actor(ctx.shared.id.raw(), || envelope.handle(actor, ctx)) + })) +} + +/// Take an envelope's payload at its one dispatch. +pub(super) fn dispatched(payload: &mut Option) -> T { + payload.take().expect("an envelope is dispatched once") +} + +/// Answer every request queued in a dead owner's mailbox, destroying +/// nothing: the drained envelopes come back, to be destroyed once the death +/// point's wake-ups ran. Admission is closed first, so nothing is queued +/// after. +pub(super) fn answer_mailbox( + rx: &mut mpsc::Receiver + Send>>, + replies: Option<&Registry>, +) -> Vec + Send>> { + let mut queued = rx.drain(); + for envelope in &mut queued { + envelope.discharge(replies); + } + queued +} + +/// A never-polled actor's startup payload, dropped with its runtime. +pub(super) type StartupPayload = ( + A, + Context, + mpsc::Receiver + Send>>, + StopToken, +); +pub(super) fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + let replies = payload.1.shared.replies.clone(); + erase(answer_mailbox(&mut payload.2, Some(&replies))) +} +pub(super) fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() +} + +pub(super) struct MessageEnvelope { + /// The request's slot: held until dispatch (dropped undispatched, it is + /// disposed of as a `SlotHandle`), then lent to the turn. + pub(super) slot: Option>>, + // Some until dispatch; taking the message is also the dispatch marker. + pub(super) msg: Option, + #[cfg(feature = "metrics")] + pub(super) observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + pub(super) from: u64, +} + +/// An undispatched request (mailbox disposal without the registry, a +/// refused admission) is answered as its `SlotHandle` would be. +impl Drop for MessageEnvelope { + #[inline] + fn drop(&mut self) { + if self.msg.is_some() && self.slot.is_some() { + drop(SlotHandle::of(self.slot.take())); + } + } +} + +impl Envelope for MessageEnvelope +where + A: Actor + Handler, + M: Message, +{ + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn discharge(&mut self, replies: Option<&Registry>) { + if self.msg.is_some() { + let mut held = SlotHandle::of(self.slot.take()); + held.dispose(replies); + } + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { slot, msg, .. } = self; + let msg = dispatched(msg); + let slot: &'a Option>> = slot; + let reply = PendingReply::lent(slot.as_ref(), &ctx.shared.replies); + let caller = slot.as_ref().map(|slot| slot as &dyn Caller); + match >::__spawned_handle_immediate(actor, msg, reply, ctx) { + None => (None, caller), + Some((msg, reply)) => (Some(Box::pin(actor.handle(msg, reply, ctx))), caller), + } + } +} + +/// A sent message (no caller): no reply slot to hold, lend or dispose of, +/// so its dispatch builds only a detached `PendingReply`. +pub(super) struct SendEnvelope { + pub(super) msg: Option, + #[cfg(feature = "metrics")] + pub(super) observation: Option, + #[cfg(feature = "lifecycle-trace")] + pub(super) from: u64, +} + +impl Envelope for SendEnvelope +where + A: Actor + Handler, + M: Message, +{ + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + #[inline] + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let msg = dispatched(&mut self.msg); + match >::__spawned_handle_immediate( + actor, + msg, + PendingReply::detached(), + ctx, + ) { + None => (None, None), + Some((msg, reply)) => (Some(Box::pin(actor.handle(msg, reply, ctx))), None), + } + } +} + +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let down = dispatched(&mut self.down); + let turn = Box::pin(async move { + fail_point!("monitor.down_before_dispatch", async); + if crate::monitor::fence::take_queued(&ctx.shared.monitors, &down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.shared.replies); + actor.handle(down, reply, ctx).await; + } + }); + (Some(turn), None) + } +} + +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. +#[inline] +pub(super) fn enqueue( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + msg: M, + slot: SlotHandle, +) -> Result<(), ActorError> +where + A: Actor + Handler, + M: Message, +{ + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); + let mut slot = slot; + let envelope: Box + Send> = Box::new(MessageEnvelope { + msg: Some(msg), + slot: slot.take(), + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// Admit a sent message (no caller): no reply slot at all. +pub(super) fn enqueue_send( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + msg: M, +) -> Result<(), ActorError> +where + A: Actor + Handler, + M: Message, +{ + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), true); + let envelope: Box + Send> = Box::new(SendEnvelope { + msg: Some(msg), + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +pub(super) struct DownEnvelope { + pub(super) down: Option, + #[cfg(feature = "metrics")] + pub(super) observation: Option, +} + +pub(super) fn enqueue_down>( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down: Some(down), + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + }); + admit::(sender, replies, envelope, 0) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +pub(super) fn admit( + sender: &mpsc::Sender + Send>>, + replies: &Registry, + envelope: Box + Send>, + _from: u64, +) -> Result<(), ActorError> { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { + #[cfg(feature = "metrics")] + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false, _from); + return Err(ActorError::NoProc); + } + match sender.send(envelope) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true, _from); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + _error.0.reject(); + } + crate::lifecycle_trace::life_send::(owner, false, _from); + Err(ActorError::NoProc) + } + } + }) +} + +/// A closure run in one of the actor's turns (`spawned_test::tasks::in_actor`). +pub(super) type InTurn = Box) -> R + Send>; +pub(super) struct InTurnEnvelope { + pub(super) slot: SlotHandle, + pub(super) lent: Option>>, + pub(super) f: Option>, +} + +impl Envelope for InTurnEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "InTurn" + } + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { slot, lent, f } = self; + let f = dispatched(f); + *lent = slot.take(); + let lent: &'a Option>> = lent; + // Inside the future, so a panicking closure is a handler panic. + let turn = Box::pin(async move { + let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); + }); + (Some(turn), lent.as_ref().map(|slot| slot as &dyn Caller)) + } +} + +// Private lifecycle signal, excluded from application message/handler metrics. +// Nested supervisor adapters have no public mailbox traffic to bury this signal. +pub(super) struct FailureEnvelope(pub(super) Option); +impl Envelope for FailureEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "Exit" + } + + fn handle<'a>(&'a mut self, _: &'a mut A, _: &'a Context) -> Dispatch<'a> { + let failure = dispatched(&mut self.0); + ( + Some(Box::pin(async move { + panic!("nested supervisor failed: {failure}"); + })), + None, + ) + } +} + +/// System envelope that dispatches an `Exit` notification to the actor's +/// `exit_received` callback. Unlike `MessageEnvelope`, this does not require +/// `Handler` on the actor — every actor can receive `Exit` via the +/// trait's default-no-op `exit_received` callback. +pub(super) struct ExitEnvelope { + pub(super) exit: Option, +} + +impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "Exit" + } + + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + ( + Some(Box::pin(async move { + actor.exit_received(exit, ctx).await; + })), + None, + ) + } +} + +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +pub(super) struct ParentExitEnvelope { + pub(super) exit: Option, +} + +impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "ParentExit" + } + + fn handle<'a>(&'a mut self, _actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + ( + Some(Box::pin(async move { ctx.parent_exit(exit.reason) })), + None, + ) + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +pub(super) struct HeldEnvelope { + pub(super) exit: Option, + pub(super) parent: bool, +} + +impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "Exit" + } + + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + let parent = self.parent; + let turn = Box::pin(async move { + if !link::lock_request(&ctx.shared.linked_reason).held_trapped() { + return; + } + if parent { + ctx.parent_exit(exit.reason); + } else { + actor.exit_received(exit, ctx).await; + } + }); + (Some(turn), None) + } +} + +/// Build the signal target of a new actor instance. +pub(super) fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender + Send>>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: StopToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit: Some(exit) }); + exit_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = + Box::new(ParentExitEnvelope { exit: Some(exit) }); + parent_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { + exit: Some(exit), + parent, + }); + held_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} diff --git a/concurrency/src/tasks/actor/idle_stop_regression.rs b/concurrency/src/tasks/actor/idle_stop_regression.rs new file mode 100644 index 00000000..81956895 --- /dev/null +++ b/concurrency/src/tasks/actor/idle_stop_regression.rs @@ -0,0 +1,68 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; +struct Counter(Arc); +impl Actor for Counter {} +struct Increment; +impl Message for Increment { + type Result = (); +} +impl Handler for Counter { + async fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } +} +#[test] +fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_idle_does_not_dispatch_a_queued_request_on_resume" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = StopToken::new(); + let (ctx, mut cleanup) = test_context(&tx, &token); + let count = Arc::new(AtomicUsize::new(0)); + let mut actor = Box::pin(run_actor( + Counter(count.clone()), + ctx, + rx, + token.clone(), + &mut cleanup, + )); + // Start and park the actual actor loop, without allowing another + // executor task to run it between admission and stop below. + assert!(futures::poll!(actor.as_mut()).is_pending()); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); + tx.send(Box::new(MessageEnvelope { + msg: Some(Increment), + slot: { + let mut slot = slot; + slot.take() + }, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }) as Box + Send>) + .unwrap(); + token.cancel(); + actor.await; + assert_eq!(cleanup.die(), ExitReason::Normal); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "idle actor dispatched queued work after stop" + ); + assert!(response.await.is_err(), "queued request unexpectedly ran"); + }); + }, + ); +} diff --git a/concurrency/src/tasks/actor/lifecycle.rs b/concurrency/src/tasks/actor/lifecycle.rs new file mode 100644 index 00000000..a88489ac --- /dev/null +++ b/concurrency/src/tasks/actor/lifecycle.rs @@ -0,0 +1,619 @@ +//! Actor startup and the main loop: spawn on a backend, `started()`, the dispatch loop, `stopped()` and the death point. +use super::*; + +// --------------------------------------------------------------------------- +// Actor startup + main loop +// --------------------------------------------------------------------------- + +impl ActorRef { + pub(super) fn spawn(actor: A, backend: Backend, parent: Option) -> Self { + Self::spawn_inner(actor, backend, parent, None, None) + .expect("unmanaged actor dispatch failed") + } + + pub(crate) fn spawn_managed( + actor: A, + scope: Arc, + lease: crate::system::Lease, + ) -> Result { + if !scope.can_start() { + return Err(crate::system::SystemError::Closed); + } + Self::spawn_inner(actor, Backend::Async, None, Some(scope), Some(lease)) + } + + pub(super) fn spawn_inner( + actor: A, + backend: Backend, + parent: Option, + managed: Option>, + lease: Option, + ) -> Result { + let (tx, rx) = mpsc::channel:: + Send>>(); + let cancellation_token = managed + .as_ref() + .map(|s| s.token.clone()) + .unwrap_or_default(); + let (completion_tx, completion_rx) = watch::channel(None); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); + let trap_exit = new_trap_exit_flag(); + let links = new_link_table(); + let linked_reason = managed + .as_ref() + .map(|s| s.cause.clone()) + .unwrap_or_else(new_linked_exit_reason); + + let id = ActorId::next(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); + let (cleaned_tx, cleaned_rx) = watch::channel(false); + // A supervised child's `ChildSpec` may override the actor's bounds. + let (max_outstanding, max_native) = managed + .as_ref() + .map(|scope| scope.bounds()) + .unwrap_or_default() + .resolve(A::MAX_OUTSTANDING, A::MAX_NATIVE_JOBS); + let actor_ref = ActorRef { + shared: Arc::new(ActorShared { + id, + replies: Registry::new(id, max_outstanding), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, max_native)), + cleaned: cleaned_rx, + sender: tx, + cancellation_token: cancellation_token.clone(), + completion_rx, + monitors, + trap_exit, + links: links.clone(), + linked_reason: linked_reason.clone(), + signal, + mailbox: crate::instrumentation::Mailbox::default(), + managed: managed.clone(), + }), + }; + let ctx = actor_ref.context(); + + if let Some(scope) = &managed { + scope.attach::(actor_ref.child_handle()); + scope.set_jobs(Arc::downgrade(&actor_ref.shared.jobs)); + } + + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); + if let Some(parent) = parent { + ctx.link(&parent); + } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); + + let cleanup_token = cancellation_token.clone(); + let cleanup = crate::actor_cleanup::ActorCleanup::new( + actor_ref.shared.id, + links, + linked_reason, + actor_ref.shared.monitors.clone(), + completion_tx, + move || cleanup_token.cancel(), + ) + .with_mailbox_failures(actor_ref.shared.sender.failures()) + .with_replies(actor_ref.shared.replies.clone()) + .with_jobs(actor_ref.shared.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); + let job_owner = actor_ref.shared.clone(); + let cleaned_tx = cleaned_tx; + // Destruction-safe: if the runtime drops this future before it + // completes, `ActorStartup` (never polled) or `ExitOnDrop` (polled) + // still reach D before destroying anything, so no caller waits for an + // instance that can never answer. + let inner_future = async move { + let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); + run_actor(actor, ctx, rx, cancellation_token, &mut cleanup).await; + let reason = cleanup.die(); + if let Some(scope) = &managed { + scope.exited(reason); + } + // Publication (wait_exit, monitors, link notices) after the + // state drop, before the job join. + drop(cleanup); + // Job cleanup (deferred-replies §5 step 4), after death: join the + // cancelled jobs, bounded by the cleanup deadline; a job still + // running then does not undo death and is reported unfinished. + if job_owner.jobs.running() > 0 { + let timeout = job_owner + .managed + .as_ref() + .map_or(DEFAULT_STOP_DEADLINE, |scope| scope.cleanup_timeout()); + let remaining = job_owner.replies.cleanup_remaining(timeout); + let unfinished = job_owner.jobs.join(remaining).await; + crate::reply::jobs::report_unfinished(job_owner.id, unfinished); + } + drop(job_owner); + let _ = cleaned_tx.send(true); + }; + + if let Some(lease) = lease { + let scope = actor_ref + .shared + .managed + .as_ref() + .expect("managed lease without scope"); + scope.dispatch_actor(crate::system::Tracked { + future: inner_future, + lease, + })?; + return Ok(actor_ref); + } + + match backend { + Backend::Async => { + #[cfg(debug_assertions)] + let inner_future = warn_on_block::WarnOnBlocking::new(inner_future); + let _handle = rt::spawn(inner_future); + } + Backend::Blocking => { + let _handle = rt::spawn_blocking(move || rt::block_on(inner_future)); + } + Backend::Thread => { + let _handle = threads::spawn(move || threads::block_on(inner_future)); + } + } + + Ok(actor_ref) + } +} + +/// A pending user future of the lifecycle: `started()`, a turn (handler, +/// `exit_received`, a job's `done`) or `stopped()`. If the runtime drops the +/// lifecycle while it is pending (G1), its `Drop` reaches the death point +/// first: the turn's own caller, every deferred token and every queued +/// request are answered and woken before this future and its captures (the +/// message, the handler's locals) are destroyed. A destructor panic of the +/// future is a cleanup failure. +pub(super) struct DeathFirst<'a, A: Actor, C: Fn()> { + pub(super) inner: Option + Send + 'a>>>, + pub(super) rx: &'a mut mpsc::Receiver + Send>>, + pub(super) cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + pub(super) caller: Option<&'a dyn Caller>, + /// The run loop already polled `inner` once in this task poll (it was + /// pending): the next poll returns `Pending` without polling it again. + pub(super) polled: bool, +} +impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { + pub(super) fn new( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + ) -> Self { + Self { + inner: Some(inner), + rx, + cleanup, + caller, + polled: false, + } + } + + /// A turn the run loop polled once and found pending. + pub(super) fn pending( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + ) -> Self { + let mut turn = Self::new(inner, rx, cleanup, caller); + turn.polled = true; + turn + } +} + +/// A boxed turn of the lifecycle. +pub(super) type TurnFuture<'a> = Pin + Send + 'a>>; + +/// Poll `turn` once, catching its panic. +#[inline] +pub(super) fn poll_caught( + turn: &mut TurnFuture<'_>, + cx: &mut std::task::Context<'_>, +) -> Result, Panic> { + std::panic::catch_unwind(AssertUnwindSafe(|| turn.as_mut().poll(cx))) +} + +/// Drop a turn that panicked; a panic of its destructor is disposed of. +pub(super) fn drop_panicked(turn: Option>) { + if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(turn))) { + crate::mailbox::dispose_panic(secondary); + } +} +/// A caught panic payload. +pub(super) type Panic = Box; + +impl Future for DeathFirst<'_, A, C> { + /// The future's panic, caught: the turn's (or hook's) panic, not a + /// runtime drop. The future is gone before it is returned. + type Output = Result<(), Panic>; + fn poll( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let this = self.get_mut(); + if std::mem::take(&mut this.polled) { + // Polled (and its waker registered) in this same task poll. + return std::task::Poll::Pending; + } + let inner = this.inner.as_mut().expect("polled after completion"); + match poll_caught(inner, cx) { + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Ok(std::task::Poll::Ready(())) => { + this.inner = None; + std::task::Poll::Ready(Ok(())) + } + Err(panic) => { + drop_panicked(this.inner.take()); + std::task::Poll::Ready(Err(panic)) + } + } + } +} +impl Drop for DeathFirst<'_, A, C> { + fn drop(&mut self) { + let Some(inner) = self.inner.take() else { + return; + }; + // Destroyed while pending: the runtime dropped the actor. + self.cleanup.die_dropped(); + if let Some(caller) = self.caller.take() { + self.cleanup.answer_turn(caller.retain()); + } + let queued = answer_mailbox(self.rx, self.cleanup.replies()); + self.cleanup.discharged(); + self.rx.discard_all(queued); + if let Err(panic) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { + let message = panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + self.cleanup + .cleanup_failed(format!("dropped turn destructor: {message}")); + } + } +} + +/// The actor's state and mailbox while its lifecycle runs. They are +/// destroyed only after the death point D: normally at the end of +/// [`run_actor`]; if the lifecycle future is dropped mid-way (its runtime was +/// destroyed), here, which reaches D first (`Killed` unless a reason was +/// requested), so tokens and queued requests answer `ActorStopped` before +/// any destructor runs. +pub(super) struct ExitOnDrop<'c, A: Actor, C: Fn()> { + pub(super) actor: Option, + pub(super) rx: mpsc::Receiver + Send>>, + pub(super) cleanup: &'c mut crate::actor_cleanup::ActorCleanup, +} +impl ExitOnDrop<'_, A, C> { + /// D, then the queued requests, then destruction (mailbox, then state). + pub(super) fn finish(&mut self, dropped: bool) { + let Some(actor) = self.actor.take() else { + return; + }; + if dropped { + self.cleanup.die_dropped(); + } else { + self.cleanup.die(); + } + let queued = answer_mailbox(&mut self.rx, self.cleanup.replies()); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). A destructor panic is a cleanup failure, + // never the reason. + self.cleanup.discharged(); + self.rx.discard_all(queued); + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + self.cleanup.cleanup_failed(failure); + } + } +} +impl Drop for ExitOnDrop<'_, A, C> { + fn drop(&mut self) { + self.finish(true); + } +} + +pub(super) async fn run_actor( + actor: A, + ctx: Context, + rx: mpsc::Receiver + Send>>, + cancellation_token: StopToken, + cleanup: &mut crate::actor_cleanup::ActorCleanup, +) { + let request = ctx.shared.linked_reason.clone(); + let mut owned = ExitOnDrop { + actor: Some(actor), + rx, + cleanup, + }; + if let Err(payload) = AssertUnwindSafe(run_actor_lifecycle( + owned.actor.as_mut().expect("the state lives until death"), + ctx, + &mut owned.rx, + &mut *owned.cleanup, + cancellation_token, + )) + .catch_unwind() + .await + { + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + } + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is answered), then + // the mailbox refuses admission and its queued requests are answered, + // all before any destructor. Early returns (startup panic, closed scope) + // and an unwound lifecycle also come here, so jobs never outlive their + // owner. + owned.finish(false); +} + +pub(super) async fn run_actor_lifecycle( + actor: &mut A, + ctx: Context, + rx: &mut mpsc::Receiver + Send>>, + cleanup: &mut crate::actor_cleanup::ActorCleanup, + cancellation_token: StopToken, +) { + let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); + let request = ctx.shared.linked_reason.clone(); + let trace_id = ctx.shared.id.raw(); + if let Some(scope) = &ctx.shared.managed { + if !scope.can_start() { + scope.stop(ExitReason::Shutdown); + return; + } + } + let start_result = AssertUnwindSafe(async { + fail_point!("actor.before_started", async); + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.started(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }) + .catch_unwind() + .await; + if let Err(panic) = start_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in started() callback: {msg}"); + }); + // No stopped() after a failed start (L5). + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); + cancellation_token.cancel(); + return; + } + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.shared.trap_exit.load(Ordering::Acquire)) { + cancellation_token.cancel(); + } + + fail_point!("actor.after_started_before_ready", async); + if let Some(scope) = &ctx.shared.managed { + if !scope.mark_ready() { + scope.stop(ExitReason::Shutdown); + } + } + + // One cancellation future for the whole loop: polled only when the + // mailbox has nothing, so a busy actor registers no waiter per message. + let mut cancelled = pin!(cancellation_token.cancelled()); + // Check stop after receiving and before dispatch: a message published + // after stop must not slip through an earlier snapshot. A rejected receive + // stays in the mailbox wrapper until the death point answers its caller. + // The fast check is one atomic load; only parked loops register a waiter. + loop { + let msg = { + let recv = pin!(future::poll_fn(|cx| { + // Stop and closed-mailbox both take the normal lifecycle exit, + // just like the cancellation arm below. Check after the + // mailbox acquire and retain a rejected item for shutdown. + // The receive, check and trace event are one step + // of the lifecycle trace: a recorded stop is seen here. + crate::lifecycle_trace::life_section(trace_id, || { + #[cfg(all(test, not(feature = "lifecycle-trace"), not(spawned_sim)))] + stop_receive_probe::before_receive(); + let received = rx.poll_recv(cx); + let result = cancellation_token.after_receive(received, |item| { + rx.hold_for_shutdown(item); + }); + #[cfg(feature = "lifecycle-trace")] + if let std::task::Poll::Ready(Some(envelope)) = &result { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + } + result + }) + })); + let mut recv = recv; + future::poll_fn(|cx| match recv.as_mut().poll(cx) { + std::task::Poll::Pending => match cancelled.as_mut().poll(cx) { + std::task::Poll::Ready(()) => std::task::Poll::Ready(None), + std::task::Poll::Pending => std::task::Poll::Pending, + }, + ready => ready, + }) + .await + }; + match msg { + Some(envelope) => { + // Simulator turn budget (rank 8b); compiled only under the cfg. + #[cfg(spawned_sim)] + let _sim_turn = rt::sim::turn(std::any::type_name::(), envelope.message_type()); + #[cfg(feature = "diagnostics")] + let _turn = ctx + .shared + .managed + .as_ref() + .map(|scope| scope.begin_turn(envelope.message_type())); + #[cfg(feature = "metrics")] + let mut envelope = envelope; + + #[cfg(feature = "metrics")] + let observation = envelope + .observation() + .map(|observation| observation.begin()); + ctx.shared.replies.begin_turn(); + // The handler's future is built inside the guarded future: a + // manual `Handler` may panic before it returns one, and that + // is a handler panic like any other (threads mode catches the + // same call). The envelope outlives the turn and lends it the + // request's slot. + let mut envelope = envelope; + let result = match dispatch(&mut *envelope, &mut *actor, &ctx) { + // Dispatch completed the turn synchronously. The caller + // hold is only needed if an unfinished future is dropped; + // turn accounting and observation bracket either branch. + Ok((None, _)) => Ok(()), + Ok((Some(turn), caller)) => { + let mut turn = + Some(crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn)); + // Most turns finish in their first poll, which runs + // here, with no await point before it returns: only + // a turn that stays pending is guarded by + // `DeathFirst` against the runtime dropping it. + let first = future::poll_fn(|cx| { + std::task::Poll::Ready(poll_caught( + turn.as_mut().expect("polled once"), + cx, + )) + }) + .await; + match first { + Ok(std::task::Poll::Ready(())) => { + drop(turn); + Ok(()) + } + Ok(std::task::Poll::Pending) => { + let turn = turn.take().expect("pending turn"); + DeathFirst::pending(turn, &mut *rx, &mut *cleanup, caller).await + } + Err(panic) => { + drop_panicked(turn.take()); + Err(panic) + } + } + } + Err(panic) => Err(panic), + }; + ctx.shared.replies.end_turn(result.is_err()); + #[cfg(feature = "diagnostics")] + drop(_turn); + #[cfg(feature = "metrics")] + if let Some(observation) = observation { + observation.finish(result.is_err()); + } + if let Err(panic) = result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in message handler: {msg}"); + }); + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); + break; + } + } + None => break, + } + } + + if let Some(scope) = &ctx.shared.managed { + scope.end_running(); + } + cancellation_token.cancel(); + ctx.shared.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.shared.sender.failures().hold(); + ctx.shared.jobs.cancel_all(); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) + }); + (run, skipped) + }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.shared.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } + if !run_hook { + return; + } + let stop_result = AssertUnwindSafe(async { + fail_point!("actor.before_stopped", async); + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.stopped(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }) + .catch_unwind() + .await; + if let Err(panic) = stop_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in stopped() callback: {msg}"); + }); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); + } +} diff --git a/concurrency/src/tasks/actor/monitor_cleanup_regressions.rs b/concurrency/src/tasks/actor/monitor_cleanup_regressions.rs new file mode 100644 index 00000000..27bbc6eb --- /dev/null +++ b/concurrency/src/tasks/actor/monitor_cleanup_regressions.rs @@ -0,0 +1,78 @@ +use super::*; +struct Idle; +impl Actor for Idle {} +impl Handler for Idle { + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +#[test] +fn closed_monitor_table_rejects_late_registration() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let rt = rt::Runtime::new().unwrap(); + let _entered = rt.enter(); + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.shared.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.shared.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + rt.block_on(observer.join()); + rt.block_on(target.join()); + }, + ); +} +fn check_cleanup(stop_owner: bool) { + let rt = rt::Runtime::new().unwrap(); + let _entered = rt.enter(); + let target = Idle.start(); + let observer = Idle.start(); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + let monitor = observer.context().monitor(&child); + drop(child); + if stop_owner { + observer.child_handle().stop(); + rt.block_on(observer.join()); + drop(observer); + } else { + observer.context().demonitor(monitor); + // Only the waiter now owns this particular ChildHandle allocation. + let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); + observer.child_handle().stop(); + rt.block_on(observer.join()); + drop(observer); + target.child_handle().stop(); + rt.block_on(target.join()); + assert!(released, "demonitor retained its waiter until target exit"); + return; + } + let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); + target.child_handle().stop(); + rt.block_on(target.join()); + assert!(released, "stopped observer retained its monitor waiter"); +} +#[test] +fn demonitor_releases_waiter() { + check_cleanup(false); +} +#[test] +fn stopped_observer_releases_waiter() { + check_cleanup(true); +} diff --git a/concurrency/src/tasks/actor/shared_state_release.rs b/concurrency/src/tasks/actor/shared_state_release.rs new file mode 100644 index 00000000..224fa339 --- /dev/null +++ b/concurrency/src/tasks/actor/shared_state_release.rs @@ -0,0 +1,110 @@ +//! `ActorRef` and `Context` share one `Arc`. Once the actor has +//! stopped and every handle is dropped, that block must be freed: nothing +//! the runtime keeps (links, monitors, child handles, the run task) may hold it. +use super::*; +use std::sync::Weak; + +async fn freed(weak: &Weak>) -> bool { + for _ in 0..500 { + if weak.upgrade().is_none() { + return true; + } + rt::sleep(Duration::from_millis(10)).await; + } + false +} + +struct Plain; +impl Actor for Plain {} +impl Handler for Plain { + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} + +/// Keeps a clone of its own `Context` in its state. +struct Holder(Option>); +impl Actor for Holder { + async fn started(&mut self, ctx: &Context) { + self.0 = Some(ctx.clone()); + } +} + +#[test] +fn freed_after_stop_and_last_handle_dropped() { + crate::test_wait::bounded( + concat!(module_path!(), "::freed_after_stop_and_last_handle_dropped"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Plain.start(); + let weak = actor.shared_weak(); + let extra = (actor.clone(), actor.context(), actor.child_handle()); + actor.context().stop(); + actor.join().await; + drop(extra); + drop(actor); + assert!(freed(&weak).await, "ActorShared outlived every handle"); + }); + }, + ); +} + +#[test] +fn freed_when_context_was_stored_in_actor_state() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::freed_when_context_was_stored_in_actor_state" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Holder(None).start(); + let weak = actor.shared_weak(); + actor.context().stop(); + actor.join().await; + drop(actor); + assert!(freed(&weak).await, "Context kept in actor state leaked"); + }); + }, + ); +} + +#[test] +fn freed_with_links_and_monitors() { + crate::test_wait::bounded( + concat!(module_path!(), "::freed_with_links_and_monitors"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let a = Plain.start(); + let b = Plain.start(); + let watcher = Plain.start(); + a.context().trap_exit(true); + a.context().link(&b.child_handle()); + let _monitor = watcher.context().monitor(&a.child_handle()); + watcher.context().monitor(&b.child_handle()); + let child = Plain.start_linked(&a.context()); + let weaks = [ + a.shared_weak(), + b.shared_weak(), + watcher.shared_weak(), + child.shared_weak(), + ]; + for actor in [&b, &a, &watcher] { + actor.context().stop(); + actor.join().await; + } + child.context().stop(); + child.join().await; + drop((a, b, watcher, child)); + for weak in &weaks { + assert!(freed(weak).await, "linked or monitored actor leaked"); + } + }); + }, + ); +} diff --git a/concurrency/src/tasks/actor/stop_receive_probe.rs b/concurrency/src/tasks/actor/stop_receive_probe.rs new file mode 100644 index 00000000..6155de04 --- /dev/null +++ b/concurrency/src/tasks/actor/stop_receive_probe.rs @@ -0,0 +1,175 @@ +use super::*; +use std::{cell::RefCell, rc::Rc, sync::atomic::AtomicUsize}; +thread_local! { + static INTERLEAVE: RefCell>> = const { RefCell::new(None) }; +} +pub(super) fn before_receive() { + let action = INTERLEAVE.with(|hook| hook.borrow_mut().take()); + if let Some(action) = action { + action(); + } +} +struct Count(Arc); +impl Actor for Count {} +struct Ping; +impl Message for Ping { + type Result = (); +} +impl Handler for Count { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } +} +#[test] +fn stop_between_check_and_receive_discards_post_stop_request() { + crate::test_wait::bounded( + "stop_between_check_and_receive_discards_post_stop_request", + || { + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = StopToken::new(); + let (ctx, mut cleanup) = test_context::(&tx, &token); + let hits = Arc::new(AtomicUsize::new(0)); + let response = Rc::new(RefCell::new(None)); + let stop_ctx = ctx.clone(); + let saved = response.clone(); + INTERLEAVE.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move || { + stop_ctx.stop(); + *saved.borrow_mut() = + Some(stop_ctx.send_request(Ping, Deadline::Never).unwrap()); + })) + }); + run_actor(Count(hits.clone()), ctx, rx, token, &mut cleanup).await; + assert_eq!(cleanup.die(), ExitReason::Normal); + let response = response + .borrow_mut() + .take() + .expect("receive probe was reached"); + let reply = response.await; + assert!( + matches!(reply, Err(ActorError::ActorStopped(ExitReason::Normal))), + "post-stop request was handled: {reply:?}" + ); + assert_eq!(hits.load(Ordering::SeqCst), 0); + }); + }, + ); +} + +struct CheckDrop { + owner: Arc, + response: Arc>>>, + checked: Arc, +} +impl Message for CheckDrop { + type Result = (); +} +impl Drop for CheckDrop { + fn drop(&mut self) { + let reply = self + .response + .lock() + .unwrap() + .as_mut() + .and_then(ReplyReceiver::try_recv); + let after_death = match reply { + Some(Err(ActorError::ActorStopped(ref reason))) => { + self.owner.final_reason() == Some(reason) + } + _ => false, + }; + self.checked.store(after_death, Ordering::SeqCst); + } +} +struct CleanupPause { + hits: Arc, + park: bool, +} +impl Actor for CleanupPause { + async fn stopped(&mut self, _: &Context) { + if self.park { + std::future::pending::<()>().await; + } + } +} +impl Handler for CleanupPause { + async fn handle(&mut self, _msg: CheckDrop, reply: PendingReply<'_, ()>, _: &Context) { + self.hits.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } +} +#[test] +fn receive_rejected_by_stop_keeps_its_caller_until_the_death_point() { + crate::test_wait::bounded( + "receive_rejected_by_stop_keeps_its_caller_until_the_death_point", + || { + for (park, requested) in [ + (false, ExitReason::Normal), + (true, ExitReason::Normal), + (false, ExitReason::Shutdown), + (true, ExitReason::Shutdown), + (false, ExitReason::Panic("requested".into())), + (true, ExitReason::Panic("requested".into())), + (false, ExitReason::Kill), + (true, ExitReason::Kill), + ] { + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = StopToken::new(); + let (ctx, mut cleanup) = test_context::(&tx, &token); + let hits = Arc::new(AtomicUsize::new(0)); + let checked = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let response = Arc::new(std::sync::Mutex::new(None)); + let stop_ctx = ctx.clone(); + let probe = CheckDrop { + owner: ctx.shared.replies.clone(), + response: response.clone(), + checked: checked.clone(), + }; + let stop_reason = requested.clone(); + INTERLEAVE.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move || { + stop_ctx.stop_with_reason(stop_reason); + let reply = stop_ctx.send_request(probe, Deadline::Never).unwrap(); + *response.lock().unwrap() = Some(reply); + })) + }); + let mut actor = Box::pin(run_actor( + CleanupPause { + hits: hits.clone(), + park, + }, + ctx, + rx, + token, + &mut cleanup, + )); + let expected = if park { + assert!(futures::poll!(actor.as_mut()).is_pending()); + assert!( + !checked.load(Ordering::SeqCst), + "payload was dropped before stopped() finished" + ); + drop(actor); + if requested == ExitReason::Normal { + ExitReason::Killed + } else { + requested + } + } else { + actor.await; + requested + }; + assert_eq!(cleanup.die(), expected); + assert_eq!(hits.load(Ordering::SeqCst), 0); + assert!( + checked.load(Ordering::SeqCst), + "held caller was not answered before payload destruction" + ); + }); + } + }, + ); +} diff --git a/concurrency/src/tasks/actor/tests.rs b/concurrency/src/tasks/actor/tests.rs new file mode 100644 index 00000000..ac6e18db --- /dev/null +++ b/concurrency/src/tasks/actor/tests.rs @@ -0,0 +1,1859 @@ +use super::*; +use crate::message::Message; +use std::{ + sync::{atomic, Arc}, + thread, + time::Duration, +}; + +// --- Counter actor for basic tests --- + +struct Counter { + count: u64, +} + +struct GetCount; +impl Message for GetCount { + type Result = u64; +} + +struct Increment; +impl Message for Increment { + type Result = u64; +} + +struct StopCounter; +impl Message for StopCounter { + type Result = u64; +} + +impl Actor for Counter {} + +impl Handler for Counter { + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); + } +} + +impl Handler for Counter { + async fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { + self.count += 1; + self.count + }; + reply.reply(__reply_value); + } +} + +impl Handler for Counter { + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); + } +} + +#[test] +pub fn backend_default_is_async() { + assert_eq!(Backend::default(), Backend::Async); +} + +#[test] +#[allow(clippy::clone_on_copy)] +pub fn backend_enum_is_copy_and_clone() { + let backend = Backend::Async; + let copied = backend; + let cloned = backend.clone(); + assert_eq!(backend, copied); + assert_eq!(backend, cloned); +} + +#[test] +pub fn backend_enum_debug_format() { + assert_eq!(format!("{:?}", Backend::Async), "Async"); + assert_eq!(format!("{:?}", Backend::Blocking), "Blocking"); + assert_eq!(format!("{:?}", Backend::Thread), "Thread"); +} + +#[test] +pub fn backend_enum_equality() { + assert_eq!(Backend::Async, Backend::Async); + assert_eq!(Backend::Blocking, Backend::Blocking); + assert_eq!(Backend::Thread, Backend::Thread); + assert_ne!(Backend::Async, Backend::Blocking); + assert_ne!(Backend::Async, Backend::Thread); + assert_ne!(Backend::Blocking, Backend::Thread); +} + +#[test] +pub fn backend_async_handles_send_and_request() { + crate::test_wait::bounded( + concat!(module_path!(), "::backend_async_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); + + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); + + // fire-and-forget send; the request below shares its mailbox, so + // it is handled after the send without any extra wait. + counter.send(Increment).unwrap(); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); + + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); +} + +#[test] +pub fn backend_blocking_handles_send_and_request() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::backend_blocking_handles_send_and_request" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); + + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); + + counter.send(Increment).unwrap(); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); + + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); +} + +#[test] +pub fn backend_thread_handles_send_and_request() { + crate::test_wait::bounded( + concat!(module_path!(), "::backend_thread_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); + + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); + + counter.send(Increment).unwrap(); + + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); + + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); +} + +#[test] +pub fn multiple_backends_concurrent() { + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_backends_concurrent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let async_counter = Counter { count: 0 }.start(); + let blocking_counter = Counter { count: 100 }.start_with_backend(Backend::Blocking); + let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); + + async_counter.request(Increment).await.unwrap(); + blocking_counter.request(Increment).await.unwrap(); + thread_counter.request(Increment).await.unwrap(); + + let async_val = async_counter.request(GetCount).await.unwrap(); + let blocking_val = blocking_counter.request(GetCount).await.unwrap(); + let thread_val = thread_counter.request(GetCount).await.unwrap(); + + assert_eq!(async_val, 1); + assert_eq!(blocking_val, 101); + assert_eq!(thread_val, 201); + + async_counter.request(StopCounter).await.unwrap(); + blocking_counter.request(StopCounter).await.unwrap(); + thread_counter.request(StopCounter).await.unwrap(); + }); + }, + ); +} + +#[test] +pub fn request_timeout() { + crate::test_wait::bounded(concat!(module_path!(), "::request_timeout"), move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct SlowActor; + struct SlowOp; + impl Message for SlowOp { + type Result = (); + } + impl Actor for SlowActor {} + impl Handler for SlowActor { + async fn handle( + &mut self, + _msg: SlowOp, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + // Never replies, so the timeout is the only possible outcome + // however late the runtime polls it. + std::future::pending::<()>().await; + reply.reply(()); + } + } + + let actor = SlowActor.start(); + let result = actor + .request_with_timeout(SlowOp, Duration::from_millis(50)) + .await; + assert!(matches!(result, Err(ActorError::RequestTimeout))); + }); + }); +} + +#[test] +pub fn recipient_type_erasure() { + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 42 }.start(); + let recipient: Recipient = counter.recipient(); + + let rx = recipient + .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + .unwrap(); + let result = rx.await.unwrap(); + assert_eq!(result, 42); + + // Also test request helper + let result = request(&*recipient, GetCount, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(result, 42); + }); + }, + ); +} + +// --- SlowShutdownActor for join tests --- + +struct SlowShutdownActor; + +struct StopSlow; +impl Message for StopSlow { + type Result = (); +} + +impl Actor for SlowShutdownActor { + async fn stopped(&mut self, _ctx: &Context) { + thread::sleep(Duration::from_millis(500)); + } +} + +impl Handler for SlowShutdownActor { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } +} + +/// Blocks its thread in a callback until the test releases it. +struct HeldThread { + entered: Arc, + release: std::sync::mpsc::Receiver<()>, + released: Arc, +} + +impl HeldThread { + fn new() -> (Self, std::sync::mpsc::Sender<()>) { + let (tx, release) = std::sync::mpsc::channel(); + let actor = HeldThread { + entered: Arc::default(), + release, + released: Arc::default(), + }; + (actor, tx) + } + + fn hold(&mut self) { + self.entered.store(true, atomic::Ordering::SeqCst); + let released = self.release.recv_timeout(crate::test_wait::WAIT).is_ok(); + self.released.store(released, atomic::Ordering::SeqCst); + } +} + +impl Actor for HeldThread { + async fn stopped(&mut self, _ctx: &Context) { + // Only the join test sends StopSlow; DoBlock already held. + if !self.entered.load(atomic::Ordering::SeqCst) { + self.hold(); + } + } +} + +impl Handler for HeldThread { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } +} + +impl Handler for HeldThread { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.hold(); + ctx.stop(); + reply.reply(()); + } +} + +#[test] +pub fn thread_backend_join_does_not_block_runtime() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::thread_backend_join_does_not_block_runtime" + ), + move || { + // stopped() blocks its OS thread until a task on this current-thread + // runtime releases it. If join() blocked the runtime, that task could + // never run and stopped() would give up after the timeout instead. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async move { + let (held, release) = HeldThread::new(); + let released = held.released.clone(); + let actor = held.start_with_backend(Backend::Thread); + actor.send(StopSlow).unwrap(); + let _releaser = rt::spawn(async move { + let _ = release.send(()); + }); + + actor.join().await; + + assert!( + released.load(atomic::Ordering::SeqCst), + "join() blocked the runtime: the releasing task never ran" + ); + }); + }, + ); +} + +#[test] +pub fn multiple_join_callers_all_notified() { + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_join_callers_all_notified"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = SlowShutdownActor.start(); + let actor_clone1 = actor.clone(); + let actor_clone2 = actor.clone(); + + let join1 = rt::spawn(async move { + actor_clone1.join().await; + 1u32 + }); + let join2 = rt::spawn(async move { + actor_clone2.join().await; + 2u32 + }); + + rt::sleep(Duration::from_millis(10)).await; + + actor.send(StopSlow).unwrap(); + + let (r1, r2) = tokio::join!(join1, join2); + assert_eq!(r1.unwrap(), 1); + assert_eq!(r2.unwrap(), 2); + + actor.join().await; + }); + }, + ); +} + +// --- Badly behaved actors for blocking tests --- + +struct BadlyBehavedTask; + +struct DoBlock; +impl Message for DoBlock { + type Result = (); +} + +impl Actor for BadlyBehavedTask {} + +impl Handler for BadlyBehavedTask { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + rt::sleep(Duration::from_millis(20)).await; + thread::sleep(Duration::from_secs(2)); + ctx.stop(); + reply.reply(()); + } +} + +struct IncrementWell; +impl Message for IncrementWell { + type Result = (); +} + +struct WellBehavedTask { + pub count: u64, +} + +impl Actor for WellBehavedTask {} + +impl Handler for WellBehavedTask { + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); + } +} + +impl Handler for WellBehavedTask { + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); + } +} + +impl Handler for WellBehavedTask { + async fn handle( + &mut self, + _msg: IncrementWell, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.count += 1; + use crate::tasks::send_after; + send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); + reply.reply(()); + } +} + +#[test] +pub fn badly_behaved_thread_non_blocking() { + crate::test_wait::bounded( + concat!(module_path!(), "::badly_behaved_thread_non_blocking"), + move || { + // An async actor that blocks its thread starves everything else on + // it. With one runtime thread this is deterministic; on a multi-thread + // runtime the well-behaved actor may run on another worker. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let badboy = BadlyBehavedTask.start(); + badboy.send(DoBlock).unwrap(); + let goodboy = WellBehavedTask { count: 0 }.start(); + goodboy.send(IncrementWell).unwrap(); + rt::sleep(Duration::from_secs(1)).await; + let count = goodboy.request(GetCount).await.unwrap(); + // The blocking handler held the only thread for 2 s of the 1 s + // window, so the 100 ms chain could not get far. + assert!(count < 10, "blocked runtime still made progress: {count}"); + goodboy.request(StopCounter).await.unwrap(); + }); + }, + ); +} + +/// An async actor on a single-threaded runtime keeps making progress while +/// an actor on `backend` is blocked inside a handler. With only one runtime +/// thread, a blocked handler on that thread would stop all progress. +fn blocked_handler_does_not_stall_runtime(backend: Backend) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let (held, release) = HeldThread::new(); + let entered = held.entered.clone(); + let badboy = held.start_with_backend(backend); + badboy.send(DoBlock).unwrap(); + let entered = crate::test_wait::wait_for_async( + || async { entered.load(atomic::Ordering::SeqCst) }, + |entered| *entered, + ) + .await; + assert!(entered, "blocking handler never started"); + + let goodboy = WellBehavedTask { count: 0 }.start(); + goodboy.send(IncrementWell).unwrap(); + let count = crate::test_wait::wait_for_async( + || goodboy.request(GetCount), + |count| matches!(count, Ok(n) if *n >= 5), + ) + .await + .unwrap(); + assert!(count >= 5, "runtime stalled behind a blocked handler"); + + release.send(()).unwrap(); + badboy.join().await; + goodboy.request(StopCounter).await.unwrap(); + }); +} + +#[test] +pub fn badly_behaved_thread() { + blocked_handler_does_not_stall_runtime(Backend::Blocking); +} + +#[test] +pub fn backend_thread_isolates_blocking_work() { + blocked_handler_does_not_stall_runtime(Backend::Thread); +} + +// --- Panic recovery tests --- + +#[test] +pub fn panic_in_started_stops_actor() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicOnStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } + + let actor = PanicOnStart.start(); + actor.join().await; + let result = actor.send(Ping); + assert!(result.is_err()); + }); + }, + ); +} + +#[test] +pub fn panic_in_handler_stops_actor() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnMsg; + struct Explode; + impl Message for Explode { + type Result = (); + } + struct Check; + impl Message for Check { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Check, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } + } + + let actor = PanicOnMsg.start(); + actor.send(Explode).unwrap(); + actor.join().await; + let result = actor.request(Check).await; + assert!(result.is_err()); + }); + }, + ); +} + +/// Review P1 (Codex 9): a manual `Handler` may be a plain `fn` that +/// panics before it returns its future. That is a handler panic like +/// any other: the turn ends, `stopped()` runs, and the caller and +/// `wait_exit` see the handler-panic reason. The future used to be built +/// outside the turn's `catch_unwind`, so the panic unwound the whole +/// lifecycle (no `stopped()`, an "actor lifecycle unwound" reason). +#[test] +pub fn panic_building_the_handler_future_is_a_handler_panic() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::panic_building_the_handler_future_is_a_handler_panic" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct Eager(Arc); + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for Eager { + async fn stopped(&mut self, _ctx: &Context) { + self.0.store(true, Ordering::SeqCst); + } + } + impl Handler for Eager { + #[allow(unreachable_code, clippy::diverging_sub_expression)] + fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) -> impl Future + Send { + panic!("boom building the handler future"); + async {} + } + } + + let stopped = Arc::new(atomic::AtomicBool::new(false)); + let actor = Eager(stopped.clone()).start(); + let reply = actor.request(Explode).await; + let reason = actor.wait_exit().await; + let ExitReason::Panic(message) = &reason else { + panic!("unexpected reason {reason:?}"); + }; + assert!( + message.starts_with("panic in handler: boom building"), + "not a handler panic: {message}" + ); + assert!( + matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), + "caller saw {reply:?}" + ); + assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); + }); + }, + ); +} + +#[test] +pub fn panic_in_stopped_still_completes() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStop; + struct StopMe; + impl Message for StopMe { + type Result = (); + } + impl Actor for PanicOnStop { + async fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + async fn handle( + &mut self, + _msg: StopMe, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } + + let actor = PanicOnStop.start(); + actor.send(StopMe).unwrap(); + actor.join().await; + }); + }, + ); +} + +#[test] +pub fn send_message_on_delivers() { + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); + let count = crate::test_wait::wait_for_async( + || counter.request(GetCount), + |count| matches!(count, Ok(1)), + ) + .await + .unwrap(); + assert_eq!(count, 1); + }); + }, + ); +} + +#[test] +pub fn send_message_on_cancelled() { + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_cancelled"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); + // Stop actor before the future resolves + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 0, "message should not have been delivered"); + counter.join().await; + }); + }, + ); +} + +// --- ExitReason tests --- + +#[test] +pub fn exit_reason_normal_on_clean_stop() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).await.unwrap(); + let reason = actor.wait_exit().await; + assert!(matches!(reason, ExitReason::Normal)); + }); + }, + ); +} + +#[test] +pub fn exit_reason_panic_in_started() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = PanicStart.start(); + let reason = actor.wait_exit().await; + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) + ); + }); + }, + ); +} + +#[test] +pub fn exit_reason_panic_in_handler() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandler; + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for PanicHandler {} + impl Handler for PanicHandler { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = PanicHandler.start(); + let _ = actor.send(Explode); + let reason = actor.wait_exit().await; + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) + ); + }); + }, + ); +} + +#[test] +pub fn exit_reason_poll_returns_none_while_running() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).await.unwrap(); + actor.join().await; + assert!(actor.exit_reason().is_some()); + }); + }, + ); +} + +// --- Monitor tests --- + +struct GetDowns; +impl Message for GetDowns { + type Result = Vec; +} + +/// Actor that exposes `monitor`/`demonitor` via messages, so tests can +/// drive it from outside. Records all received Down messages. +struct Watcher { + downs: Arc>>, +} + +struct StartMonitor(crate::ChildHandle); +impl Message for StartMonitor { + type Result = crate::monitor::MonitorRef; +} +struct CallDemonitor(crate::monitor::MonitorRef); +impl Message for CallDemonitor { + type Result = (); +} +/// Monitor, then demonitor in the same turn. +struct MonitorThenDemonitor(crate::ChildHandle); +impl Message for MonitorThenDemonitor { + type Result = (); +} + +impl Actor for Watcher {} + +impl Handler for Watcher { + async fn handle( + &mut self, + msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); + } +} + +impl Handler for Watcher { + async fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); + } +} + +impl Handler for Watcher { + async fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.demonitor(msg.0); + reply.reply(()); + } +} + +impl Handler for Watcher { + async fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + self.downs.lock().unwrap().push(msg); + reply.reply(()); + } +} + +impl Handler for Watcher { + async fn handle( + &mut self, + _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.downs.lock().unwrap().clone()); + } +} + +#[test] +pub fn monitor_running_actor_delivers_down_on_exit() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + + // Stop the target — Down should be delivered + target.request(StopCounter).await.unwrap(); + target.join().await; + + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }); + }, + ); +} + +/// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox +/// (here the `NoProc` a monitor of a dead target queues at once) is +/// never handled once `demonitor` returned. +#[test] +pub fn demonitor_flushes_a_queued_down() { + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .await + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }); + }, + ); +} + +#[test] +pub fn monitor_already_dead_actor_delivers_down_immediately() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + }); + }, + ); +} + +#[test] +pub fn demonitor_before_target_dies_suppresses_down() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); + + // Now stop the target + target.request(StopCounter).await.unwrap(); + target.join().await; + rt::sleep(Duration::from_millis(50)).await; + + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); + }); + }, + ); +} + +#[test] +pub fn multiple_monitors_each_get_own_ref_and_down() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .await + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }); + }, + ); +} + +#[test] +pub fn monitor_table_is_cleaned_up_after_target_dies() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_table_is_cleaned_up_after_target_dies" + ), + move || { + // Watcher should remove its entry from the monitor table after the + // target dies, so the table doesn't accumulate stale entries. + struct Inspect; + impl Message for Inspect { + type Result = usize; + } + impl Handler for Watcher { + async fn handle( + &mut self, + _msg: Inspect, + reply: crate::reply::PendingReply<'_, usize>, + ctx: &Context, + ) { + reply.reply( + ctx.shared + .monitors + .lock() + .unwrap() + .as_ref() + .map_or(0, HashMap::len), + ); + } + } + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher + .request(StartMonitor(target.child_handle())) + .await + .unwrap(); + assert_eq!(watcher.request(Inspect).await.unwrap(), 1); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let entries = crate::test_wait::wait_for_async( + || watcher.request(Inspect), + |entries| !matches!(entries, Ok(n) if *n > 0), + ) + .await; + assert_eq!( + entries.unwrap(), + 0, + "monitor table should be empty after target died" + ); + }); + }, + ); +} + +#[test] +pub fn monitor_observes_panic_reason() { + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + async fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } + } + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = PanicMe.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = target.send(PanicMsg); + + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }); + }, + ); +} + +#[test] +pub fn monitoring_actor_stops_before_target_does_not_panic() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitoring_actor_stops_before_target_does_not_panic" + ), + move || { + // Regression: if the monitoring actor stops while the target is still + // alive, the watcher must not crash when the target eventually dies. + // The watcher's send() will fail silently (mailbox closed), and the + // watcher exits cleanly. + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + + // Stop the monitoring actor first + let watcher_handle = watcher.child_handle(); + watcher_handle.stop(); + watcher_handle.wait_exit_async().await; + + // Now stop the target — the watcher should clean up without panicking + target.request(StopCounter).await.unwrap(); + target.join().await; + // Give the orphaned watcher task time to attempt delivery + rt::sleep(Duration::from_millis(50)).await; + // If we got here without panicking, the test passes. + }); + }, + ); +} + +// --- Link tests --- + +/// Trapping actor that records `Exit` notifications. +struct TrapActor { + exits: Arc>>, + trap: bool, +} + +struct GetExits; +impl Message for GetExits { + type Result = Vec; +} +struct LinkTo(crate::ChildHandle); +impl Message for LinkTo { + type Result = (); +} + +impl Actor for TrapActor { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(self.trap); + } + async fn exit_received(&mut self, exit: Exit, _ctx: &Context) { + self.exits.lock().unwrap().push(exit); + } +} + +impl Handler for TrapActor { + async fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.link(&msg.0); + reply.reply(()); + } +} + +impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.exits.lock().unwrap().clone()); + } +} + +fn make_trapper(trap: bool) -> ActorRef { + TrapActor { + exits: Arc::new(Mutex::new(Vec::new())), + trap, + } + .start() +} + +#[test] +pub fn link_propagates_panic_to_non_trapping_peer() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Two non-trapping actors. When A panics, B should be terminated. + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).await.unwrap(); + + // Make A panic by dropping its only ActorRef while it's mid-handler... easier: + // make A panic via a panic message + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } + + let _ = a.send(Boom); + // Both should die + let reason_a = a.wait_exit().await; + let reason_b = b.wait_exit().await; + assert!(matches!(reason_a, ExitReason::Panic(_))); + // B should propagate A's reason (transitive propagation) + assert!(matches!(reason_b, ExitReason::Panic(_))); + }); + }, + ); +} + +#[test] +pub fn link_delivers_exit_to_trapping_peer() { + crate::test_wait::bounded( + concat!(module_path!(), "::link_delivers_exit_to_trapping_peer"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = make_trapper(false); + let b = make_trapper(true); // B traps + b.request(LinkTo(a.child_handle())).await.unwrap(); + + struct Boom2; + impl Message for Boom2 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom2"); + } + } + } + + let _ = a.send(Boom2); + a.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + + // B should still be alive and have received an Exit + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + + // Clean up + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn link_normal_exit_not_propagated_to_non_trapping_peer() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_normal_exit_not_propagated_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(false); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // A stops cleanly + a.request(StopCounter).await.unwrap(); + a.join().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should still be alive + assert!(b.exit_reason().is_none()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn link_to_already_dead_actor_delivers_noproc() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_actor_delivers_noproc" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Start and panic a target + struct Boom3; + impl Message for Boom3 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom3, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom3"); + } + } + } + let target = make_trapper(false); + let _ = target.send(Boom3); + target.wait_exit().await; + + // Now link from a trapping observer: the target is past its + // death point, so the Exit carries NoProc (E4), not its reason. + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + let exits = crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].reason, ExitReason::NoProc); + assert!(matches!(target.exit_reason(), Some(ExitReason::Panic(_)))); + + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn unlink_prevents_signal_delivery() { + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // Now unlink + struct UnlinkFrom(crate::ChildHandle); + impl Message for UnlinkFrom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + msg: UnlinkFrom, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.unlink(&msg.0); + reply.reply(()); + } + } + b.request(UnlinkFrom(a.child_handle())).await.unwrap(); + + // A panics + struct Boom4; + impl Message for Boom4 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom4, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom4"); + } + } + // Counter's Boom4 returns u64; just use send with no expectation of reply + let _ = a.send(Boom4); + a.wait_exit().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should NOT have received an Exit + let exits = b.request(GetExits).await.unwrap(); + assert!(exits.is_empty()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn duplicate_link_is_idempotent() { + crate::test_wait::bounded( + concat!(module_path!(), "::duplicate_link_is_idempotent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate + + struct Boom5; + impl Message for Boom5 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom5, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom5"); + } + } + let _ = a.send(Boom5); + a.wait_exit().await; + crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; + + // Should receive only ONE Exit, not two + let exits = b.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn chain_propagation_through_non_trapping_middle() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::chain_propagation_through_non_trapping_middle" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // A linked to B linked to C. A panics → B dies (not trapping) → + // B's death propagates to C. + let a = make_trapper(false); + let b = make_trapper(false); + let c = make_trapper(true); // C traps + + a.request(LinkTo(b.child_handle())).await.unwrap(); + c.request(LinkTo(b.child_handle())).await.unwrap(); + + struct Boom6; + impl Message for Boom6 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom6, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom6"); + } + } + } + let _ = a.send(Boom6); + a.wait_exit().await; + b.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || c.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); + + let ch = c.child_handle(); + ch.stop(); + ch.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn link_to_already_dead_delivers_exactly_once_to_trapping_peer() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_delivers_exactly_once_to_trapping_peer" + ), + move || { + // Regression: previously, `ctx.link()` would deliver a duplicate Exit + // to trapping actors when the target died concurrently — once from + // propagate_exit (since register_link inserted self in target's table + // before target's drain ran) and once from the fallback exit_reason() + // check. + struct Boom7; + impl Message for Boom7 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom7, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom7"); + } + } + } + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = make_trapper(false); + // Kick off the panic + let _ = target.send(Boom7); + // Wait for target to fully die so exit_reason() is Some + target.wait_exit().await; + + // Now link from a trapping observer — should receive Exit EXACTLY ONCE + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; + let exits = observer.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); + + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); +} + +#[test] +pub fn start_linked_links_atomically() { + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_links_atomically"), + move || { + // Parent + child via start_linked. When parent stops via panic, + // child (not trapping) should die too. + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + async fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } + } + + struct Child; + impl Actor for Child {} + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); + + // Parent panics — child should die too + let _ = parent.send(PanicParent); + parent.wait_exit().await; + // Child should also die (link propagation) + let reason = child.wait_exit().await; + // Reason should be the parent's panic + assert!(matches!(reason, ExitReason::Panic(_))); + }); + }, + ); +} diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 358eb24d..19a3fa6e 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -31,6 +31,14 @@ pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); mod jobs; pub use jobs::{BlockingJobPermit, JobCtx}; +mod envelope; +use envelope::*; +mod actor_ref; +mod context; +mod lifecycle; +#[cfg(test)] +use lifecycle::run_actor_receiving; + // --------------------------------------------------------------------------- // Actor trait // --------------------------------------------------------------------------- @@ -81,376 +89,6 @@ pub trait Handler: Actor { fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, ctx: &Context); } -// --------------------------------------------------------------------------- -// Envelope (type-erasure) -// --------------------------------------------------------------------------- - -trait Envelope: Send { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "other" - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - None - } - #[cfg(feature = "metrics")] - fn reject(self: Box) {} - - /// The owner reached its death point with this envelope undispatched: - /// answer its request now (the final reason), before any payload of the - /// mailbox is destroyed. No user code runs. - fn discharge(&mut self, _replies: Option<&Registry>) {} - - fn handle(self: Box, actor: &mut A, ctx: &Context); -} - -/// Answer every request queued in a dead owner's mailbox (and `extra`, an -/// envelope received after the stop), destroying nothing: the items come -/// back, to be destroyed once the death point's wake-ups ran. Admission is -/// closed first (waiting for in-flight sends), so nothing is queued after. -fn answer_mailbox( - rx: &mut mpsc::Receiver>, - extra: Option + Send>>, - replies: Option<&Registry>, -) -> Vec> { - let mut queued = rx.drain(); - queued.extend(extra.map(MailboxItem::Message)); - for item in &mut queued { - if let MailboxItem::Message(envelope) = item { - envelope.discharge(replies); - } - } - queued -} - -/// A never-polled actor's startup payload (its thread was never started). -type StartupPayload = ( - A, - Context, - mpsc::Receiver>, - CancellationToken, -); -fn discharge_startup(payload: &mut StartupPayload) -> Vec> { - let replies = payload.1.replies.clone(); - erase(answer_mailbox(&mut payload.2, None, Some(&replies))) -} -fn erase(items: Vec) -> Vec> { - items - .into_iter() - .map(|item| Box::new(item) as Box) - .collect() -} - -struct MessageEnvelope { - // Before the message: a plain drop also answers the request first. - slot: SlotHandle, - msg: M, - #[cfg(feature = "metrics")] - observation: Option, - /// The recorded actor that sent it (0: outside code or a request); its - /// mailbox item kind is then the model's `msg`. - #[cfg(feature = "lifecycle-trace")] - from: u64, -} - -impl Envelope for MessageEnvelope -where - A: Actor + Handler, - M: Message, -{ - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - if self.from != 0 { - "msg" - } else { - crate::lifecycle_trace::life_kind::() - } - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - self.observation.take() - } - #[cfg(feature = "metrics")] - fn reject(mut self: Box) { - if let Some(observation) = self.observation.take() { - observation.reject(); - } - } - fn discharge(&mut self, replies: Option<&Registry>) { - self.slot.dispose(replies); - drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); - } - fn handle(self: Box, actor: &mut A, ctx: &Context) { - let mut this = *self; - let reply = PendingReply::new(this.slot.take(), &ctx.replies); - actor.handle(this.msg, reply, ctx); - } -} - -impl> Envelope for DownEnvelope { - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "down" - } - #[cfg(feature = "metrics")] - fn observation(&mut self) -> Option { - self.observation.take() - } - #[cfg(feature = "metrics")] - fn reject(mut self: Box) { - if let Some(observation) = self.observation.take() { - observation.reject(); - } - } - fn handle(self: Box, actor: &mut A, ctx: &Context) { - fail_point!("monitor.down_before_dispatch"); - if crate::monitor::fence::take_queued(&ctx.monitors, &self.down.monitor_ref) { - let reply = PendingReply::new(None, &ctx.replies); - actor.handle(self.down, reply, ctx); - } - } -} - -#[cfg(all(spawned_sim, feature = "async"))] -fn assert_native_clock_context() { - assert!( - spawned_rt::tasks::Handle::try_current_or_ambient().is_err(), - "spawned_sim: native actors cannot run in or receive messages from a simulated world; use tasks actors for simulation, or run native actors outside the simulation" - ); -} - -/// Admit a message. Past the death point (the registry is sealed, or the -/// mailbox is gone) it reports `NoProc`; before it, including while the -/// actor exits, it is queued, and a request is answered at the death point -/// if it is not handled. -fn enqueue( - sender: &mpsc::Sender>, - _mailbox: &crate::instrumentation::Mailbox, - replies: &Registry, - msg: M, - slot: SlotHandle, -) -> Result<(), ActorError> -where - A: Actor + Handler, - M: Message, -{ - #[cfg(all(spawned_sim, feature = "async"))] - assert_native_clock_context(); - let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); - let envelope: Box + Send> = Box::new(MessageEnvelope { - msg, - slot, - #[cfg(feature = "metrics")] - observation: Some(_mailbox.enqueued()), - #[cfg(feature = "lifecycle-trace")] - from, - }); - admit::(sender, replies, envelope, from) -} - -/// A monitor's `Down`, enqueued by the framework (the waiter at the target's -/// publication, or `monitor` past the target's death point). Its dispatch -/// runs `Handler` only while the monitor's table entry is still -/// queued: a `demonitor` that returned first flushed it (OTP -/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary -/// message. -struct DownEnvelope { - down: Down, - #[cfg(feature = "metrics")] - observation: Option, -} - -fn enqueue_down>( - sender: &mpsc::Sender>, - _mailbox: &crate::instrumentation::Mailbox, - replies: &Registry, - down: Down, -) -> Result<(), ActorError> { - let envelope: Box + Send> = Box::new(DownEnvelope { - down, - #[cfg(feature = "metrics")] - observation: Some(_mailbox.enqueued()), - }); - admit::(sender, replies, envelope, 0) -} - -/// Admission of a built envelope (`M` names its kind for the trace). -fn admit( - sender: &mpsc::Sender>, - replies: &Registry, - envelope: Box + Send>, - _from: u64, -) -> Result<(), ActorError> { - // Admission and its trace event are one step of the lifecycle trace. - let owner = replies.owner().raw(); - crate::lifecycle_trace::life_section(owner, || { - if replies.refuses_admission() { - #[cfg(feature = "metrics")] - envelope.reject(); - crate::lifecycle_trace::life_send::(owner, false, _from); - return Err(ActorError::NoProc); - } - match sender.send(MailboxItem::Message(envelope)) { - Ok(()) => { - #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - crate::lifecycle_trace::life_send::(owner, true, _from); - Ok(()) - } - Err(_error) => { - #[cfg(feature = "metrics")] - { - if let MailboxItem::Message(envelope) = _error.0 { - envelope.reject(); - } - } - crate::lifecycle_trace::life_send::(owner, false, _from); - Err(ActorError::NoProc) - } - } - }) -} - -/// A closure run in one of the actor's turns (`spawned_test::threads::in_actor`). -type InTurn = Box) -> R + Send>; -struct InTurnEnvelope { - slot: SlotHandle, - f: InTurn, -} - -impl Envelope for InTurnEnvelope { - fn discharge(&mut self, replies: Option<&Registry>) { - self.slot.dispose(replies); - drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); - } - fn handle(self: Box, actor: &mut A, ctx: &Context) { - let InTurnEnvelope { f, mut slot } = *self; - let reply = PendingReply::new(slot.take(), &ctx.replies); - let value = f(actor, ctx); - let _ = reply.reply(value); - } -} - -/// System envelope that dispatches an `Exit` notification to the actor's -/// `exit_received` callback. Does not require `Handler` on the actor. -struct ExitEnvelope { - exit: Exit, -} - -impl Envelope for ExitEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "exit" - } - fn handle(self: Box, actor: &mut A, ctx: &Context) { - actor.exit_received(self.exit, ctx); - } -} - -/// A parent exit (E3): when dispatched, exit begins with the parent's reason -/// and `stopped()` runs. `exit_received` never sees it. -struct ParentExitEnvelope { - exit: Exit, -} - -impl Envelope for ParentExitEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "pexit" - } - fn handle(self: Box, _actor: &mut A, ctx: &Context) { - ctx.parent_exit(self.exit.reason); - } -} - -/// An exit signal that arrived while `started()` ran, in its mailbox place -/// (phase 0). If the actor trapped exits when `started()` returned it is an -/// `Exit` (or parent exit); otherwise it was resolved then and is inert. -struct HeldEnvelope { - exit: Exit, - parent: bool, -} - -impl Envelope for HeldEnvelope { - /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. - #[cfg(feature = "lifecycle-trace")] - fn trace_kind(&self) -> &'static str { - "held" - } - fn handle(self: Box, actor: &mut A, ctx: &Context) { - if !link::lock_request(&ctx.linked_reason).held_trapped() { - return; - } - if self.parent { - ctx.parent_exit(self.exit.reason); - } else { - actor.exit_received(self.exit, ctx); - } - } -} - -/// Build the signal target of a new actor instance. -fn signal_target( - id: ActorId, - parent: Option, - sender: &mpsc::Sender>, - trap: TrapExitFlag, - request: LinkedExitReason, - token: CancellationToken, -) -> Arc { - let exit_tx = sender.clone(); - let parent_tx = sender.clone(); - let held_tx = sender.clone(); - let send_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - exit_tx - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::NoProc) - }); - let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); - parent_tx - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::NoProc) - }); - let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { - let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); - held_tx - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::NoProc) - }); - Arc::new(SignalTarget { - id, - parent, - trap, - request, - wake: Arc::new(move || token.cancel()), - send_exit, - send_parent_exit, - send_held, - }) -} - -// A private wakeup is distinct from user envelopes and never invokes a handler. -enum MailboxItem { - Message(Box + Send>), - Shutdown, -} - -fn register_stop_wakeup(token: &CancellationToken, sender: mpsc::Sender>) { - // One callback covers Context, ChildHandle, linked exits and framework cleanup. - // CancellationToken drains this FnOnce callback exactly once, including when - // multiple callers race to cancel. Repeated stop calls cannot grow the queue. - token.on_cancel(Box::new(move || { - let _ = sender.send(MailboxItem::Shutdown); - })); -} - // --------------------------------------------------------------------------- // Context // --------------------------------------------------------------------------- @@ -501,484 +139,6 @@ impl Debug for Context { } } -impl Context { - /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). - pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { - Self { - id: actor_ref.id, - sender: actor_ref.sender.clone(), - cancellation_token: actor_ref.cancellation_token.clone(), - completion: actor_ref.completion.clone(), - monitors: actor_ref.monitors.clone(), - trap_exit: actor_ref.trap_exit.clone(), - links: actor_ref.links.clone(), - linked_reason: actor_ref.linked_reason.clone(), - signal: actor_ref.signal.clone(), - mailbox: actor_ref.mailbox.clone(), - replies: actor_ref.replies.clone(), - jobs: actor_ref.jobs.clone(), - cleaned: actor_ref.cleaned.clone(), - } - } - - /// Waiting messages, including in-flight enqueue reservations, excluding - /// the active handler and internal exit/shutdown control envelopes. - /// Concurrent observations are approximate; no registry or actor labels. - #[cfg(feature = "metrics")] - pub fn mailbox_depth(&self) -> u64 { - self.mailbox.depth() - } - - /// Deferred-reply credits in use by this instance: live deferred - /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). - /// Experimental in 0.8 (for tests and diagnostics; the `metrics` - /// feature exports the same count): it may change before 1.0. - pub fn deferred_outstanding(&self) -> usize { - self.replies.outstanding() - } - - /// The actor's unique identity. - pub fn id(&self) -> ActorId { - self.id - } - - /// Self-exit: the current handler finishes, then `stopped()` is called - /// and the actor exits `Normal`. - pub fn stop(&self) { - self.stop_with_reason(ExitReason::Normal); - } - - /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without - /// unwinding. The current handler finishes and `stopped()` runs. The first - /// requested self-exit or stop wins; until the death point a kill - /// replaces it with `Killed`, an untrapped exit signal with its reason, and - /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays - /// `Kill` and propagates over links as `Kill` (trappable). - pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(self.id, &self.linked_reason, reason, || { - self.cancellation_token.cancel() - }); - } - - /// A parent exit being dispatched: normal termination with its reason - /// (part of the model's `Dispatch`, so not recorded as a stop). - fn parent_exit(&self, reason: ExitReason) { - link::request_stop(&self.linked_reason, reason, || { - self.cancellation_token.cancel() - }); - } - - /// Send an exit signal from this actor to `target` (OTP - /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the - /// delivery rules. From the target's `start_linked` parent the signal is a - /// parent exit (a trapping target ends normally, through `stopped()`). - /// Works until this actor's death point, including from `stopped()`; - /// inert after it. - pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { - // Held until delivered: the death point waits for it, so a signal - // sent before death precedes the death notices (and none is sent - // after it). - let Some(_sending) = self.replies.signal_permit() else { - return; - }; - if link::lock_request(&self.linked_reason).is_dead() { - return; - } - fail_point!("actor.exit_before_deliver"); - target.exit_from(self.id, reason, false); - } - - /// Send a fire-and-forget message to this actor. - pub fn send(&self, msg: M) -> Result<(), ActorError> - where - A: Handler, - M: Message, - { - enqueue( - &self.sender, - &self.mailbox, - &self.replies, - msg, - SlotHandle::detached(), - ) - } - - /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get - /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from - /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait - /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) - /// uses). The handler sees it through `PendingReply::deadline`. - pub fn send_request( - &self, - msg: M, - deadline: impl Into, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let deadline = deadline.into(); - self.send_request_with(msg, deadline.ticks(), true) - } - - /// `send_request`; `raw` puts a deadline in the owner's heap now (the - /// receiver may be awaited directly). The `request` paths arm their own - /// timer and pass `false`. - pub(crate) fn send_request_with( - &self, - msg: M, - deadline: u64, - raw: bool, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); - enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; - Ok(rx) - } - - /// Send a request and block until the reply arrives (default 5s timeout). - pub fn request(&self, msg: M) -> Result - where - A: Handler, - M: Message, - { - self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) - } - - /// Send a request and block until the reply arrives, with a custom timeout. - pub fn request_with_timeout( - &self, - msg: M, - duration: Duration, - ) -> Result - where - A: Handler, - M: Message, - { - let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) - { - Err(error) => Err(error), - Ok(rx) => rx.recv(), - }; - observation.finish(&result); - result - } - - /// Get a type-erased `Recipient` for sending a single message type - /// to this actor. - pub fn recipient(&self) -> Recipient - where - A: Handler, - M: Message, - { - Arc::new(self.clone()) - } - - /// Get an `ActorRef` from this context. - pub fn actor_ref(&self) -> ActorRef { - ActorRef { - id: self.id, - sender: self.sender.clone(), - cancellation_token: self.cancellation_token.clone(), - completion: self.completion.clone(), - monitors: self.monitors.clone(), - trap_exit: self.trap_exit.clone(), - links: self.links.clone(), - linked_reason: self.linked_reason.clone(), - signal: self.signal.clone(), - mailbox: self.mailbox.clone(), - replies: self.replies.clone(), - jobs: self.jobs.clone(), - cleaned: self.cleaned.clone(), - } - } - - /// Set up a unidirectional monitor on another actor. - /// - /// Returns a [`MonitorRef`] that can be used to cancel the monitor via - /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] - /// message is delivered to this actor's mailbox via `Handler`. - /// - /// If the target is past its death point, exactly one - /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`). - /// - /// Multiple independent monitors are allowed on the same target — each - /// call returns a distinct `MonitorRef`. - /// - /// Monitors are unidirectional: the monitored actor is unaware of the - /// monitor and unaffected by it. - /// Registration after owner shutdown starts (including in `stopped()`) - /// returns an inert reference and creates no waiter or notification. - /// - /// **Resource cost (threads mode):** each active monitor occupies one OS - /// thread until the target exits, the monitor is cancelled, or its owner - /// stops, waiting on the target's completion signal. For supervisors with many long-lived - /// children, consider using tasks mode instead. - pub fn monitor(&self, target: &ChildHandle) -> MonitorRef - where - A: Handler, - { - fail_point!("monitor.before_install"); - let monitor_ref = MonitorRef::next(); - let active = spawned_rt::sync::CancellationToken::new(); - - let observer = self.id.raw(); - let registered = crate::lifecycle_trace::life_section(observer, || { - let event = |x| { - crate::lifecycle_trace::LifeEvent::new("Monitor", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x(x) - }; - crate::monitor::fence::register( - &self.monitors, - monitor_ref, - active.clone(), - // The token check makes registration inert once shutdown - // starts. The closed table serializes registration with exit - // cleanup, even on paths that unwind before cancelling the - // owner token. - || self.cancellation_token.is_cancelled(), - // Linearised against the target's death point (E4): past it - // the only notice is NoProc, delivered under the observer's - // lock. - || target.is_past_death(), - || { - crate::lifecycle_trace::life_emit(observer, || event("late")); - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x("NoProc") - .k("down") - .o("late") - }); - let _ = enqueue_down( - &self.sender, - &self.mailbox, - &self.replies, - Down { - monitor_ref, - actor: target.id(), - reason: ExitReason::NoProc, - }, - ); - }, - || crate::lifecycle_trace::life_emit(observer, || event("active")), - ) == crate::monitor::fence::Registration::Active - }); - if !registered { - return monitor_ref; - } - - let target = target.clone(); - let actor_ref = self.actor_ref(); - let monitors = self.monitors.clone(); - let registration = crate::monitor::MonitorRegistration(monitors.clone(), monitor_ref); - - rt::spawn(move || { - let _registration = registration; - let reason = futures::executor::block_on(async { - match futures::future::select( - Box::pin(active.cancelled()), - Box::pin(target.wait_exit_async()), - ) - .await - { - futures::future::Either::Left(_) => None, - futures::future::Either::Right((reason, _)) => Some(reason), - } - }); - let Some(reason) = reason else { return }; - // Receiver-side fence (E6): the entry is removed and the Down - // enqueued in one critical section, so once `demonitor` returns - // no Down of this monitor is enqueued. - crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::deliver(&monitors, &monitor_ref, || { - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x(crate::lifecycle_trace::life_reason(&reason)) - .k("down") - .o("death") - }); - let _ = enqueue_down( - &actor_ref.sender, - &actor_ref.mailbox, - &actor_ref.replies, - Down { - monitor_ref, - actor: target.id(), - reason, - }, - ); - }); - }); - }); - - monitor_ref - } - - /// Cancel a previously-set monitor. - /// - /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it - /// returns, no `Down` of this monitor is enqueued or handled; one already - /// in the mailbox is skipped. Demonitoring an unknown, delivered or - /// already cancelled monitor does nothing. - pub fn demonitor(&self, monitor_ref: MonitorRef) { - let observer = self.id.raw(); - let active = crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::demonitor(&self.monitors, &monitor_ref, || { - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) - .m(monitor_ref.raw()) - }); - }) - }); - // Waking the waiter must happen after releasing the table lock. - if let Some(active) = active { - active.cancel(); - } - } - - /// Set up a bidirectional link with another actor (OTP `link/1`). - /// - /// When either side's exit is published, the other receives a link - /// notice with the final reason: a non-trapping peer ignores `Normal` and - /// otherwise exits with that reason without running `stopped()`; a - /// trapping peer gets an `Exit` via [`Actor::exit_received`]. - /// - /// If the target is past its death point, this actor receives a `NoProc` - /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits - /// `NoProc`. Use [`try_link`](Self::try_link) to get `Err(NoProc)` - /// instead. Inert once this actor's exit began (in `stopped()`, from a - /// helper after the stop, or past its death point): no record and no - /// notice, as `monitor`. An existing link, or a pending notice from that peer, makes - /// this a no-op. - /// - /// [`Actor::exit_received`]: Actor::exit_received - pub fn link(&self, target: &ChildHandle) { - link::register_link( - &self.signal, - &self.links, - target.signal_target(), - target.links(), - false, - ); - } - - /// Link as [`link`](Self::link), but report a target past its death - /// point as `Err(NoProc)` and keep no record. - /// A caller whose exit already began does nothing and returns `Ok(())`; - /// an exit racing this call can linearize after the link attempt. - pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { - match link::register_link( - &self.signal, - &self.links, - target.signal_target(), - target.links(), - true, - ) { - LinkOutcome::NoProc => Err(NoProc), - LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), - } - } - - /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has - /// no further effect on this actor; an `Exit` already queued stays. - pub fn unlink(&self, target: &ChildHandle) { - link::unregister_link(self.id, &self.links, target.id(), target.links()); - } - - /// Whether this actor's exit has begun or a signal that will end it is - /// already pending, for a trapping actor that wants to refuse late work. - /// - /// `true` once: - /// - exit has begun: `ctx.stop()`, an administrative stop - /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit - /// signal or link notice, a panic; it stays `true` in `stopped()`; - /// - a parent exit is queued for this trapping actor (E3: the - /// supervisor's or system's shutdown, or a signal from its - /// `start_linked` parent). The parent exit is dispatched after the - /// messages queued before it, and handlers of those messages see - /// `true`. - /// - /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent - /// `exit` is not a pending exit: `exit_received` decides. Signals that - /// arrive while `started()` runs count once `started()` returns, when - /// they are resolved. Without trapping, a pending signal has already - /// begun the exit, so this is also `true`. - /// - /// Advisory: a signal can arrive right after it returns `false`. It - /// takes the actor's exit-request lock; nothing is added per message. - pub fn exit_pending(&self) -> bool { - link::lock_request(&self.linked_reason).exit_pending() - } - - /// Control how exit signals (link notices and `exit` signals) are - /// handled; read when each signal arrives (OTP `trap_exit`). - /// - /// - `false` (default): `Normal` is ignored; any other reason ends this - /// actor after its current handler, without `stopped()`. - /// - `true`: the signal is queued as an `Exit` for - /// [`Actor::exit_received`]. - /// - /// Only a kill is untrappable. A `start_linked` parent's exit ends a - /// trapping actor normally, through `stopped()`. Signals that arrive while - /// `started()` runs are resolved when it returns. - pub fn trap_exit(&self, enabled: bool) { - let id = self.id.raw(); - crate::lifecycle_trace::life_section(id, || { - // Recorded only when it changes (the model's SetTrap). - if self.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { - crate::lifecycle_trace::life_emit(id, || { - crate::lifecycle_trace::LifeEvent::new("Trap", id) - .x(crate::lifecycle_trace::flag(enabled)) - }); - } - }); - } - - pub(crate) fn cancellation_token(&self) -> CancellationToken { - self.cancellation_token.clone() - } -} - -impl crate::__private::Sealed for Context {} -impl crate::__private::Sealed for ActorRef {} - -// Bridge: Context implements Receiver for any M that A handles -impl Receiver for Context -where - A: Actor + Handler, - M: Message, -{ - fn send(&self, msg: M) -> Result<(), ActorError> { - Context::send(self, msg) - } - - fn send_request( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - Context::send_request(self, msg, deadline) - } - - fn send_request_timed( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - Context::send_request_with(self, msg, deadline.ticks(), false) - } -} - // --------------------------------------------------------------------------- // Receiver trait (object-safe) + Recipient alias // --------------------------------------------------------------------------- @@ -1079,2260 +239,68 @@ impl Clone for ActorRef { } } -impl ActorRef { - /// Clone this handle with a timeout override for generated protocol calls. - /// Per-call override wins over method and protocol defaults. No message is - /// sent until a protocol method is called; timeout does not cancel its work. - pub fn with_request_timeout(&self, timeout: Duration) -> crate::response::RequestOptions { - crate::response::RequestOptions::new(self.clone(), timeout) - } +// --------------------------------------------------------------------------- +// Actor::start +// --------------------------------------------------------------------------- - /// Send a fire-and-forget message to the actor. - pub fn send(&self, msg: M) -> Result<(), ActorError> - where - A: Handler, - M: Message, - { - enqueue( - &self.sender, - &self.mailbox, - &self.replies, - msg, - SlotHandle::detached(), - ) +/// Extension trait for starting an actor. Automatically implemented for all [`Actor`] types. +pub trait ActorStart: Actor { + /// Start the actor on a dedicated OS thread. + fn start(self) -> ActorRef { + ActorRef::spawn(self, None) } - /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get - /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from - /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait - /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) - /// uses). The handler sees it through `PendingReply::deadline`. - pub fn send_request( - &self, - msg: M, - deadline: impl Into, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let deadline = deadline.into(); - self.send_request_with(msg, deadline.ticks(), true) + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. + fn start_linked(self, parent_ctx: &Context

) -> ActorRef { + ActorRef::spawn(self, Some(parent_ctx.actor_ref().child_handle())) } +} - /// `send_request`; `raw` puts a deadline in the owner's heap now (the - /// receiver may be awaited directly). The `request` paths arm their own - /// timer and pass `false`. - pub(crate) fn send_request_with( - &self, - msg: M, - deadline: u64, - raw: bool, - ) -> Result, ActorError> - where - A: Handler, - M: Message, - { - let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); - enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; - Ok(rx) - } +impl ActorStart for A {} - /// Send a request and block until the reply arrives (default 5s timeout). - pub fn request(&self, msg: M) -> Result - where - A: Handler, - M: Message, - { - self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) - } +// --------------------------------------------------------------------------- +// send_message_on (utility) +// --------------------------------------------------------------------------- - /// Run `f` in one of the actor's turns, with its state and `Context`, - /// and receive its result. For tests: `spawned_test::threads::in_actor` (through - /// `__private`). - pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> - where - R: Send + 'static, - F: FnOnce(&mut A, &Context) -> R + Send + 'static, - { - let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.replies), false); - let envelope: Box + Send> = Box::new(InTurnEnvelope { - f: Box::new(f), - slot, - }); - // Past the death point the actor is gone (E4), as for `enqueue`. - if self.replies.refuses_admission() { - return Err(ActorError::NoProc); +/// Send a message to an actor when a blocking closure completes. +/// +/// Spawns a thread that runs `f()`, then sends `msg` to the actor. +/// If the actor stops before `f()` returns, the message is not sent. +pub fn send_message_on(ctx: Context, f: F, msg: M) -> rt::JoinHandle<()> +where + A: Actor + Handler, + M: Message, + F: FnOnce() + Send + 'static, +{ + let cancellation_token = ctx.cancellation_token(); + rt::spawn(move || { + f(); + if !cancellation_token.is_cancelled() { + if let Err(e) = ctx.send(msg) { + tracing::error!("Failed to send message: {e:?}") + } } - self.sender - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::NoProc)?; - Ok(rx) - } - - /// Send a request and block until the reply arrives, with a custom timeout. - pub fn request_with_timeout( - &self, - msg: M, - duration: Duration, - ) -> Result - where - A: Handler, - M: Message, - { - let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) - { - Err(error) => Err(error), - Ok(rx) => rx.recv(), - }; - observation.finish(&result); - result - } - - /// Get a type-erased `Recipient` for this actor. - pub fn recipient(&self) -> Recipient - where - A: Handler, - M: Message, - { - Arc::new(self.clone()) - } - - /// Get a `Context` from this ref, for timer setup or stream listeners - /// from outside the actor. - pub fn context(&self) -> Context { - Context::from_ref(self) - } - - /// Block until the actor has fully stopped (including `stopped()` callback). - /// - /// Returns when cleanup completes: the actor died and its cancelled jobs - /// were joined (bounded by the cleanup deadline). Use - /// [`wait_exit`](Self::wait_exit) to observe death alone. - pub fn join(&self) { - let _ = self.wait_exit(); - self.cleaned.wait(); - } - - /// Poll the exit reason. Returns `None` until the exit is published. - /// A completion channel closed without a reason reports - /// [`ExitReason::Killed`], matching the wait methods. - pub fn exit_reason(&self) -> Option { - crate::child_handle::poll_exit_reason(&self.completion) - } - - /// Block until the actor stops and return the exit reason. - pub fn wait_exit(&self) -> ExitReason { - crate::child_handle::wait_for_thread_exit_blocking(self.completion.clone()) - } - - /// Waiting messages, including in-flight enqueue reservations, excluding - /// the active handler and internal exit/shutdown control envelopes. - /// Concurrent observations are approximate; no registry or actor labels. - #[cfg(feature = "metrics")] - pub fn mailbox_depth(&self) -> u64 { - self.mailbox.depth() - } - - /// The actor's unique identity. - pub fn id(&self) -> ActorId { - self.id - } - - /// Get a type-erased `ChildHandle` for this actor. - pub fn child_handle(&self) -> ChildHandle { - ChildHandle::from(self.clone()) - } -} - -impl From> for ChildHandle { - fn from(actor_ref: ActorRef) -> Self { - let failures = actor_ref.sender.failures(); - ChildHandle::new( - actor_ref.signal, - Completion::Threads(actor_ref.completion), - actor_ref.links, - ) - .with_mailbox_failures(failures) - } -} - -// Bridge: ActorRef implements Receiver for any M that A handles -impl Receiver for ActorRef -where - A: Actor + Handler, - M: Message, -{ - fn send(&self, msg: M) -> Result<(), ActorError> { - ActorRef::send(self, msg) - } - - fn send_request( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - ActorRef::send_request(self, msg, deadline) - } - - fn send_request_timed( - &self, - msg: M, - deadline: Deadline, - ) -> Result, ActorError> { - ActorRef::send_request_with(self, msg, deadline.ticks(), false) - } -} - -// --------------------------------------------------------------------------- -// Actor startup + main loop -// --------------------------------------------------------------------------- - -impl ActorRef { - fn spawn(actor: A, parent: Option) -> Self { - #[cfg(all(spawned_sim, feature = "async"))] - assert_native_clock_context(); - let (tx, rx) = mpsc::channel::>(); - let cancellation_token = CancellationToken::new(); - register_stop_wakeup(&cancellation_token, tx.clone()); - let (completion_tx, completion) = spawned_rt::sync::watch::channel(None); - let id = ActorId::next(); - let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); - let trap_exit = new_trap_exit_flag(); - let links = new_link_table(); - let linked_reason = new_linked_exit_reason(); - // Phase 0: signals that arrive before started() returns are held. - link::lock_request(&linked_reason).begin_starting(); - let signal = signal_target::( - id, - parent.as_ref().map(ChildHandle::id), - &tx, - trap_exit.clone(), - linked_reason.clone(), - cancellation_token.clone(), - ); - - let actor_ref = ActorRef { - id, - sender: tx.clone(), - cancellation_token: cancellation_token.clone(), - completion: completion.clone(), - monitors: monitors.clone(), - trap_exit: trap_exit.clone(), - links: links.clone(), - linked_reason: linked_reason.clone(), - signal: signal.clone(), - mailbox: crate::instrumentation::Mailbox::default(), - replies: Registry::new_native(id, A::MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new( - id, - A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), - )), - cleaned: Arc::default(), - }; - - let ctx = Context { - id, - sender: tx, - cancellation_token: cancellation_token.clone(), - completion: actor_ref.completion.clone(), - monitors, - trap_exit, - links: links.clone(), - linked_reason: linked_reason.clone(), - signal, - mailbox: actor_ref.mailbox.clone(), - replies: actor_ref.replies.clone(), - jobs: actor_ref.jobs.clone(), - cleaned: actor_ref.cleaned.clone(), - }; - - let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); - if let Some(parent) = parent { - actor_ref.context().link(&parent); - } - // Recorded from here: the start_linked link is the model's initial one. - crate::lifecycle_trace::life_spawn(id.raw(), parent_id); - - let cleanup_token = cancellation_token.clone(); - let cleanup = crate::actor_cleanup::ActorCleanup::new( - id, - links, - linked_reason, - actor_ref.monitors.clone(), - completion_tx, - move || cleanup_token.cancel(), - ) - .with_mailbox_failures(actor_ref.sender.failures()) - .with_replies(actor_ref.replies.clone()) - .with_jobs(actor_ref.jobs.clone()); - let startup = crate::actor_cleanup::ActorStartup::new( - (actor, ctx, rx, cancellation_token), - cleanup, - discharge_startup::, - ); - let replies = actor_ref.replies.clone(); - let jobs = actor_ref.jobs.clone(); - let cleaned = crate::reply::jobs::CleanedGuard(actor_ref.cleaned.clone()); - let _thread_handle = rt::spawn(move || { - let _cleaned = cleaned; - let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); - let request = ctx.linked_reason.clone(); - let discarded = match catch_unwind(AssertUnwindSafe(|| { - run_actor(&mut actor, ctx, &mut rx, cancellation_token) - })) { - Ok(discarded) => discarded, - Err(payload) => { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - link::lock_request(&request).panic(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))); - None - } - }; - // The death point D, only now that `stopped()` returned (or was - // skipped): the reason becomes final, the link table closes, job - // admission and the registry close (every open token is - // answered), then the mailbox refuses admission and its queued - // requests (and one received after the stop) are answered, all - // before any destructor. - cleanup.die(); - let queued = answer_mailbox(&mut rx, discarded, Some(&replies)); - // Every caller is answered: wake them, then destroy (queued - // payloads, then state). - cleanup.discharged(); - rx.discard_all(queued); - drop(rx); - // Destruction: a destructor panic is a cleanup failure only. - if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { - cleanup.cleanup_failed(failure); - } - // Publication (wait_exit, monitors, link notices) before the job join. - drop(cleanup); - // Job cleanup (§5 step 4), after death: join the cancelled jobs - // that signal completion, bounded by the cleanup deadline. - if jobs.running() > 0 { - let remaining = replies.cleanup_remaining(DEFAULT_STOP_DEADLINE); - let unfinished = jobs.join_blocking(std::time::Instant::now() + remaining); - crate::reply::jobs::report_unfinished(jobs.owner(), unfinished); - } - }); - - actor_ref - } -} - -/// Returns an envelope received after the stop (undispatched; answered at -/// the death point). -fn run_actor( - actor: &mut A, - ctx: Context, - rx: &mut mpsc::Receiver>, - cancellation_token: CancellationToken, -) -> Option + Send>> { - run_actor_receiving( - actor, - ctx, - move || { - rx.recv() - .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) - }, - cancellation_token, - ) -} - -fn run_actor_receiving( - actor: &mut A, - ctx: Context, - mut receive: impl FnMut() -> Result, std::sync::mpsc::RecvTimeoutError>, - cancellation_token: CancellationToken, -) -> Option + Send>> { - let mut discarded = None; - let request = ctx.linked_reason.clone(); - let trace_id = ctx.id.raw(); - let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); - let start_result = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_started"); - crate::lifecycle_trace::with_actor(trace_id, || actor.started(&ctx)); - })); - if let Err(panic) = start_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in started() callback: {msg}"); - }); - // No stopped() after a failed start (L5). - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); - }); - cancellation_token.cancel(); - return None; - } - // Phase 0 ends: resolve signals held during started() with the trap - // flag read now. - if link::lock_request(&request).end_starting(ctx.trap_exit.load(Ordering::Acquire)) { - cancellation_token.cancel(); - } - - while !cancellation_token.is_cancelled() { - let msg = match receive() { - Ok(MailboxItem::Message(msg)) => Some(msg), - Ok(MailboxItem::Shutdown) => None, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => None, - }; - match msg { - Some(envelope) => { - #[cfg(feature = "metrics")] - let mut envelope = envelope; - - // A stop observed after receive wins over dispatch, whether - // the channel waited or was buffered. Defer the message drop. - // The check and the trace event are one lifecycle-trace step. - let stopped = crate::lifecycle_trace::life_section(trace_id, || { - if cancellation_token.is_cancelled() { - return true; - } - #[cfg(feature = "lifecycle-trace")] - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) - .k(envelope.trace_kind()) - }); - false - }); - if stopped { - discarded = Some(envelope); - break; - } - #[cfg(feature = "metrics")] - let observation = envelope - .observation() - .map(|observation| observation.begin()); - ctx.replies.begin_turn(); - let result = catch_unwind(AssertUnwindSafe(|| { - crate::lifecycle_trace::with_actor(trace_id, || envelope.handle(actor, &ctx)); - })); - ctx.replies.end_turn(result.is_err()); - #[cfg(feature = "metrics")] - if let Some(observation) = observation { - observation.finish(result.is_err()); - } - if let Err(panic) = result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in message handler: {msg}"); - }); - // A panic begins exit, or replaces the requested reason - // unless a kill was received. - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); - }); - break; - } - } - None => break, - } - } - - cancellation_token.cancel(); - ctx.replies.exiting(); - // Payload drops are held until every caller is answered (DropFailures). - ctx.sender.failures().hold(); - ctx.jobs.cancel_all(); - // stopped() is skipped when a kill, an untrapped exit signal or link - // notice began the exit (decision 2); one that already began finishes. - let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { - let mut req = link::lock_request(&request); - let run = req.begin_hook(); - let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); - drop(req); - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { - "run" - } else { - "skip" - }) - }); - (run, skipped) - }); - if let Some((cause, reason)) = skipped { - crate::actor_cleanup::log_stopped_skipped( - ctx.id, - std::any::type_name::(), - cause, - reason.as_ref(), - ); - } - if run_hook { - let stop_result = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_stopped"); - crate::lifecycle_trace::with_actor(trace_id, || actor.stopped(&ctx)); - })); - if let Err(panic) = stop_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - crate::lifecycle_trace::life_section(trace_id, || { - crate::lifecycle_trace::life_emit(trace_id, || { - crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") - }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); - }); - } - } - - // Message destructors and reply cancellation may re-enter monitor state. - // Preserve stopped() -> monitor cleanup -> undelivered-message drop (the - // envelope received after the stop is answered at the death point, then - // dropped with the mailbox). - drop(_monitor_cleanup); - discarded -} - -// --------------------------------------------------------------------------- -// Actor::start -// --------------------------------------------------------------------------- - -/// Extension trait for starting an actor. Automatically implemented for all [`Actor`] types. -pub trait ActorStart: Actor { - /// Start the actor on a dedicated OS thread. - fn start(self) -> ActorRef { - ActorRef::spawn(self, None) - } - - /// Start linked to `parent_ctx`'s actor, which becomes this instance's - /// parent (E3). The link is registered before `started()`. The parent's - /// exit ends a trapping child normally (its `stopped()` runs; it never - /// reaches `exit_received`), a non-trapping child as any link notice. - fn start_linked(self, parent_ctx: &Context

) -> ActorRef { - ActorRef::spawn(self, Some(parent_ctx.actor_ref().child_handle())) - } -} - -impl ActorStart for A {} - -// --------------------------------------------------------------------------- -// send_message_on (utility) -// --------------------------------------------------------------------------- - -/// Send a message to an actor when a blocking closure completes. -/// -/// Spawns a thread that runs `f()`, then sends `msg` to the actor. -/// If the actor stops before `f()` returns, the message is not sent. -pub fn send_message_on(ctx: Context, f: F, msg: M) -> rt::JoinHandle<()> -where - A: Actor + Handler, - M: Message, - F: FnOnce() + Send + 'static, -{ - let cancellation_token = ctx.cancellation_token(); - rt::spawn(move || { - f(); - if !cancellation_token.is_cancelled() { - if let Err(e) = ctx.send(msg) { - tracing::error!("Failed to send message: {e:?}") - } - } - }) -} + }) +} #[cfg(test)] -mod tests { - use super::*; - use crate::message::Message; - use std::thread; - - struct Counter { - count: u64, - } - - struct GetCount; - impl Message for GetCount { - type Result = u64; - } - - struct Increment; - impl Message for Increment { - type Result = u64; - } - - struct StopCounter; - impl Message for StopCounter { - type Result = u64; - } - - impl Actor for Counter {} - - impl Handler for Counter { - fn handle( - &mut self, - _msg: GetCount, - reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - reply.reply(self.count); - } - } - - impl Handler for Counter { - fn handle( - &mut self, - _msg: Increment, - reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - let __reply_value: u64 = { - self.count += 1; - self.count - }; - reply.reply(__reply_value); - } - } - - impl Handler for Counter { - fn handle( - &mut self, - _msg: StopCounter, - reply: crate::reply::PendingReply<'_, u64>, - ctx: &Context, - ) { - let __reply_value: u64 = { - ctx.stop(); - self.count - }; - reply.reply(__reply_value); - } - } - - #[test] - fn basic_send_and_request() { - crate::test_wait::bounded( - concat!(module_path!(), "::basic_send_and_request"), - move || { - let actor = Counter { count: 0 }.start(); - assert_eq!(actor.request(GetCount).unwrap(), 0); - assert_eq!(actor.request(Increment).unwrap(), 1); - // Same mailbox: the request is handled after the send. - actor.send(Increment).unwrap(); - assert_eq!(actor.request(GetCount).unwrap(), 2); - actor.request(StopCounter).unwrap(); - }, - ); - } - - #[test] - fn join_waits_for_completion() { - crate::test_wait::bounded( - concat!(module_path!(), "::join_waits_for_completion"), - move || { - struct SlowStop; - struct StopSlow; - impl Message for StopSlow { - type Result = (); - } - impl Actor for SlowStop { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(300)); - } - } - impl Handler for SlowStop { - fn handle( - &mut self, - _msg: StopSlow, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - let actor = SlowStop.start(); - actor.send(StopSlow).unwrap(); - actor.join(); - // If join() returned, stopped() has completed - }, - ); - } - - #[test] - fn join_multiple_callers() { - crate::test_wait::bounded( - concat!(module_path!(), "::join_multiple_callers"), - move || { - struct SlowStop2; - struct StopSlow2; - impl Message for StopSlow2 { - type Result = (); - } - impl Actor for SlowStop2 { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(200)); - } - } - impl Handler for SlowStop2 { - fn handle( - &mut self, - _msg: StopSlow2, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - let actor = SlowStop2.start(); - let a1 = actor.clone(); - let a2 = actor.clone(); - let t1 = thread::spawn(move || { - a1.join(); - 1u32 - }); - let t2 = thread::spawn(move || { - a2.join(); - 2u32 - }); - actor.send(StopSlow2).unwrap(); - assert_eq!(t1.join().unwrap(), 1); - assert_eq!(t2.join().unwrap(), 2); - }, - ); - } - - #[test] - fn panic_in_started_stops_actor() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_started_stops_actor"), - move || { - struct PanicOnStart; - struct PingThread; - impl Message for PingThread { - type Result = (); - } - impl Actor for PanicOnStart { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - fn handle( - &mut self, - _msg: PingThread, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } - - let actor = PanicOnStart.start(); - actor.join(); - let result = actor.send(PingThread); - assert!(result.is_err()); - }, - ); - } - - #[test] - fn panic_in_handler_stops_actor() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_handler_stops_actor"), - move || { - struct PanicOnMsg; - struct ExplodeThread; - impl Message for ExplodeThread { - type Result = (); - } - struct CheckThread; - impl Message for CheckThread { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - fn handle( - &mut self, - _msg: ExplodeThread, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); - } - } - } - impl Handler for PanicOnMsg { - fn handle( - &mut self, - _msg: CheckThread, - reply: crate::reply::PendingReply<'_, u32>, - _ctx: &Context, - ) { - reply.reply(42); - } - } - - let actor = PanicOnMsg.start(); - actor.send(ExplodeThread).unwrap(); - actor.join(); - let result = actor.request(CheckThread); - assert!(result.is_err()); - }, - ); - } - - #[test] - fn panic_in_stopped_still_completes() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_in_stopped_still_completes"), - move || { - struct PanicOnStop; - struct StopMeThread; - impl Message for StopMeThread { - type Result = (); - } - impl Actor for PanicOnStop { - fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - fn handle( - &mut self, - _msg: StopMeThread, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - let actor = PanicOnStop.start(); - actor.send(StopMeThread).unwrap(); - actor.join(); - }, - ); - } - - #[test] - fn recipient_type_erasure() { - crate::test_wait::bounded( - concat!(module_path!(), "::recipient_type_erasure"), - move || { - let actor = Counter { count: 42 }.start(); - let recipient: Recipient = actor.recipient(); - let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); - assert_eq!(result, 42); - }, - ); - } - - #[test] - fn send_message_on_delivers() { - crate::test_wait::bounded( - concat!(module_path!(), "::send_message_on_delivers"), - move || { - let actor = Counter { count: 0 }.start(); - let ctx = actor.context(); - send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); - let count = - crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))) - .unwrap(); - assert_eq!(count, 1); - }, - ); - } - - // --- ExitReason tests --- - - #[test] - fn exit_reason_normal_on_clean_stop() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), - move || { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).unwrap(); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Normal)); - }, - ); - } - - #[test] - fn exit_reason_panic_in_started() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_panic_in_started"), - move || { - struct PanicStartThread; - struct PingThread2; - impl Message for PingThread2 { - type Result = (); - } - impl Actor for PanicStartThread { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStartThread { - fn handle( - &mut self, - _msg: PingThread2, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } - - let actor = PanicStartThread.start(); - let reason = actor.wait_exit(); - assert!( - matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) - ); - }, - ); - } - - #[test] - fn exit_reason_panic_in_handler() { - crate::test_wait::bounded( - concat!(module_path!(), "::exit_reason_panic_in_handler"), - move || { - struct PanicHandlerThread; - struct ExplodeThread2; - impl Message for ExplodeThread2 { - type Result = (); - } - impl Actor for PanicHandlerThread {} - impl Handler for PanicHandlerThread { - fn handle( - &mut self, - _msg: ExplodeThread2, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); - } - } - } - - let actor = PanicHandlerThread.start(); - let _ = actor.send(ExplodeThread2); - let reason = actor.wait_exit(); - assert!( - matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) - ); - }, - ); - } - - #[test] - fn exit_reason_poll_returns_none_while_running() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::exit_reason_poll_returns_none_while_running" - ), - move || { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).unwrap(); - actor.join(); - assert!(actor.exit_reason().is_some()); - }, - ); - } - - // --- Monitor tests --- - - struct GetDowns; - impl Message for GetDowns { - type Result = Vec; - } - - struct Watcher { - downs: Arc>>, - } - - struct StartMonitor(crate::ChildHandle); - impl Message for StartMonitor { - type Result = crate::monitor::MonitorRef; - } - struct CallDemonitor(crate::monitor::MonitorRef); - impl Message for CallDemonitor { - type Result = (); - } - /// Monitor, then demonitor in the same turn. - struct MonitorThenDemonitor(crate::ChildHandle); - impl Message for MonitorThenDemonitor { - type Result = (); - } - - impl Actor for Watcher {} - - impl Handler for Watcher { - fn handle( - &mut self, - msg: StartMonitor, - reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, - ctx: &Context, - ) { - reply.reply(ctx.monitor(&msg.0)); - } - } - - impl Handler for Watcher { - fn handle( - &mut self, - msg: MonitorThenDemonitor, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - let monitor = ctx.monitor(&msg.0); - ctx.demonitor(monitor); - reply.reply(()); - } - } - - impl Handler for Watcher { - fn handle( - &mut self, - msg: CallDemonitor, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.demonitor(msg.0); - reply.reply(()); - } - } - - impl Handler for Watcher { - fn handle( - &mut self, - msg: crate::monitor::Down, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - self.downs.lock().unwrap().push(msg); - reply.reply(()); - } - } - - impl Handler for Watcher { - fn handle( - &mut self, - _msg: GetDowns, - reply: crate::reply::PendingReply<'_, Vec>, - _ctx: &Context, - ) { - reply.reply(self.downs.lock().unwrap().clone()); - } - } - - fn make_watcher() -> ActorRef { - Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start() - } - - #[test] - fn monitor_running_actor_delivers_down_on_exit() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitor_running_actor_delivers_down_on_exit" - ), - move || { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); - - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); - }, - ); - } - - /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox - /// (here the `NoProc` a monitor of a dead target queues at once) is - /// never handled once `demonitor` returned. - #[test] - fn demonitor_flushes_a_queued_down() { - crate::test_wait::bounded( - concat!(module_path!(), "::demonitor_flushes_a_queued_down"), - move || { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).unwrap(); - target.join(); - let watcher = make_watcher(); - watcher - .request(MonitorThenDemonitor(target.child_handle())) - .unwrap(); - // FIFO: the queued Down is dispatched before this request. - let downs = watcher.request(GetDowns).unwrap(); - assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); - }, - ); - } - - #[test] - fn monitor_already_dead_actor_delivers_down_immediately() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::monitor_already_dead_actor_delivers_down_immediately" - ), - move || { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).unwrap(); - target.join(); - let target_handle = target.child_handle(); - - let watcher = make_watcher(); - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - }, - ); - } - - #[test] - fn demonitor_before_target_dies_suppresses_down() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::demonitor_before_target_dies_suppresses_down" - ), - move || { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); - watcher.request(CallDemonitor(monitor_ref)).unwrap(); - - target.request(StopCounter).unwrap(); - target.join(); - rt::sleep(Duration::from_millis(150)); - - let downs = watcher.request(GetDowns).unwrap(); - assert!(downs.is_empty()); - }, - ); - } - - #[test] - fn multiple_monitors_each_get_own_ref_and_down() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::multiple_monitors_each_get_own_ref_and_down" - ), - move || { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); - }, - ); - } - - #[test] - fn monitor_observes_panic_reason() { - crate::test_wait::bounded( - concat!(module_path!(), "::monitor_observes_panic_reason"), - move || { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - fn handle( - &mut self, - _msg: PanicMsg, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic"); - } - } - } - - let target = PanicMe.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let _ = target.send(PanicMsg); - - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); - }, - ); - } - - // --- Link tests --- - - struct TrapActor { - exits: Arc>>, - trap: bool, - } - - struct GetExits; - impl Message for GetExits { - type Result = Vec; - } - struct LinkTo(crate::ChildHandle); - impl Message for LinkTo { - type Result = (); - } - - impl Actor for TrapActor { - fn started(&mut self, ctx: &Context) { - ctx.trap_exit(self.trap); - } - fn exit_received(&mut self, exit: Exit, _ctx: &Context) { - self.exits.lock().unwrap().push(exit); - } - } - - impl Handler for TrapActor { - fn handle( - &mut self, - msg: LinkTo, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.link(&msg.0); - reply.reply(()); - } - } - - impl Handler for TrapActor { - fn handle( - &mut self, - _msg: GetExits, - reply: crate::reply::PendingReply<'_, Vec>, - _ctx: &Context, - ) { - reply.reply(self.exits.lock().unwrap().clone()); - } - } - - fn make_trapper(trap: bool) -> ActorRef { - TrapActor { - exits: Arc::new(Mutex::new(Vec::new())), - trap, - } - .start() - } - - #[test] - fn link_propagates_panic_to_non_trapping_peer_threads() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_propagates_panic_to_non_trapping_peer_threads" - ), - move || { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); - } - } - } - - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).unwrap(); - - let _ = a.send(Boom); - let reason_a = a.wait_exit(); - let reason_b = b.wait_exit(); - assert!(matches!(reason_a, ExitReason::Panic(_))); - assert!(matches!(reason_b, ExitReason::Panic(_))); - }, - ); - } - - #[test] - fn link_delivers_exit_to_trapping_peer_threads() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::link_delivers_exit_to_trapping_peer_threads" - ), - move || { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); - } - } - } - - let a = make_trapper(false); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).unwrap(); - - let _ = a.send(Boom); - a.wait_exit(); - let exits = crate::test_wait::wait_for( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .unwrap(); - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_blocking(); - }, - ); - } - - #[test] - fn start_linked_threads() { - crate::test_wait::bounded( - concat!(module_path!(), "::start_linked_threads"), - move || { - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - fn handle( - &mut self, - _msg: PanicParent, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("parent boom"); - } - } - } - - struct Child; - impl Actor for Child {} - - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); - - let _ = parent.send(PanicParent); - parent.wait_exit(); - let reason = child.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(_))); - }, - ); - } -} +mod tests; #[cfg(test)] -mod monitor_cleanup_regressions { - use super::*; - struct Idle; - impl Actor for Idle {} - impl Handler for Idle { - fn handle( - &mut self, - _: Down, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - reply.reply(()); - } - } - #[test] - fn closed_monitor_table_rejects_late_registration() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::closed_monitor_table_rejects_late_registration" - ), - move || { - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - observer.join(); - target.join(); - }, - ); - } - fn check_cleanup(stop_owner: bool) { - let target = Idle.start(); - let observer = Idle.start(); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - let monitor = observer.context().monitor(&child); - drop(child); - if stop_owner { - observer.child_handle().stop(); - observer.join(); - drop(observer); - } else { - observer.context().demonitor(monitor); - // Only the waiter now owns this particular ChildHandle allocation. - let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); - observer.child_handle().stop(); - observer.join(); - drop(observer); - target.child_handle().stop(); - target.join(); - assert!(released, "demonitor retained its waiter until target exit"); - return; - } - let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); - target.child_handle().stop(); - target.join(); - assert!(released, "stopped observer retained its monitor waiter"); - } - #[test] - fn demonitor_releases_waiter() { - check_cleanup(false); - } - #[test] - fn stopped_observer_releases_waiter() { - check_cleanup(true); - } -} +mod monitor_cleanup_regressions; #[cfg(test)] -mod early_stop_regression { - use super::*; - struct EarlyStop; - impl Actor for EarlyStop { - fn started(&mut self, ctx: &Context) { - ctx.stop(); - } - fn stopped(&mut self, _: &Context) { - panic!("cleanup failed"); - } - } - #[test] - fn panic_during_early_stop_is_reported() { - crate::test_wait::bounded( - concat!(module_path!(), "::panic_during_early_stop_is_reported"), - move || { - let actor = EarlyStop.start(); - assert!( - matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) - ); - }, - ); - } -} +mod early_stop_regression; #[cfg(test)] -mod idle_stop_regression { - use super::*; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Barrier, - }; - struct Counter(Arc, Arc>>); - impl Actor for Counter { - fn stopped(&mut self, _: &Context) { - self.1.lock().unwrap().push("stopped"); - } - } - impl Drop for Counter { - fn drop(&mut self) { - self.1.lock().unwrap().push("actor-dropped"); - } - } - struct Increment(Arc>>, MonitorTable); - impl Drop for Increment { - fn drop(&mut self) { - assert!( - self.1.lock().unwrap().is_none(), - "message dropped before monitor cleanup" - ); - self.0.lock().unwrap().push("message-dropped"); - } - } - impl Message for Increment { - type Result = (); - } - impl Handler for Counter { - fn handle( - &mut self, - _: Increment, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - self.0.fetch_add(1, Ordering::SeqCst); - reply.reply(()); - } - } - #[test] - fn stop_while_receiving_does_not_dispatch_a_later_request() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::stop_while_receiving_does_not_dispatch_a_later_request" - ), - move || { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (_, completion_rx) = spawned_rt::sync::watch::channel(None); - let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); - let events = Arc::new(Mutex::new(Vec::new())); - let (id, trap, request) = ( - ActorId::next(), - new_trap_exit_flag(), - new_linked_exit_reason(), - ); - let ctx = Context { - id, - sender: tx.clone(), - cancellation_token: token.clone(), - completion: completion_rx, - monitors: monitors.clone(), - trap_exit: trap.clone(), - links: new_link_table(), - linked_reason: request.clone(), - signal: signal_target::<_>( - id, - None, - &tx, - trap.clone(), - request.clone(), - token.clone(), - ), - mailbox: crate::instrumentation::Mailbox::default(), - replies: Registry::new_native(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: Arc::default(), - }; - let count = Arc::new(AtomicUsize::new(0)); - let seen = count.clone(); - let entered = Arc::new(Barrier::new(2)); - let release = Arc::new(Barrier::new(2)); - let (e, r) = (entered.clone(), release.clone()); - let cancel = token.clone(); - let trace = events.clone(); - let actor = std::thread::spawn(move || { - let mut first = true; - let mut counter = Counter(seen, trace); - // The envelope received after the stop comes back undispatched; - // the death point answers it and destroys it before the state. - let discarded = run_actor_receiving( - &mut counter, - ctx, - move || { - // Pause the real receive boundary after the started/cancel - // checks; the standard channel still carries the envelope. - if first { - first = false; - e.wait(); - r.wait(); - } - rx.recv_timeout(Duration::from_secs(5)) - }, - cancel, - ); - drop(discarded); - drop(counter); - }); - entered.wait(); - let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); - tx.send(MailboxItem::Message(Box::new(MessageEnvelope { - msg: Increment(events.clone(), monitors), - slot, - #[cfg(feature = "metrics")] - observation: None, - #[cfg(feature = "lifecycle-trace")] - from: 0, - }))) - .unwrap(); - token.cancel(); - release.wait(); - actor.join().unwrap(); - assert_eq!( - link::lock_request(&request).reason(), - None, - "a plain stop request" - ); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "thread actor dispatched work after idle stop" - ); - assert!(response.recv().is_err()); - assert_eq!( - *events.lock().unwrap(), - ["stopped", "message-dropped", "actor-dropped"] - ); - }, - ); - } -} +mod idle_stop_regression; #[cfg(test)] -mod stop_wakeup_regression { - use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - - struct Idle(mpsc::Sender<()>); - impl Actor for Idle { - fn stopped(&mut self, _: &Context) { - let _ = self.0.send(()); - } - } - struct Noop; - impl Message for Noop { - type Result = (); - } - impl Handler for Idle { - fn handle( - &mut self, - _: Noop, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - reply.reply(()); - } - } - - #[test] - fn concurrent_cancellation_enqueues_exactly_one_shutdown() { - for already_cancelled in [false, true] { - let (tx, rx) = mpsc::channel::>(); - let token = CancellationToken::new(); - if already_cancelled { - token.cancel(); - } - register_stop_wakeup(&token, tx.clone()); - std::thread::scope(|scope| { - for _ in 0..16 { - let token = &token; - scope.spawn(move || { - for _ in 0..256 { - token.cancel(); - } - }); - } - }); - assert!(matches!(rx.try_recv(), Ok(MailboxItem::Shutdown))); - // Keep tx alive: Empty proves no duplicate, rather than disconnection. - assert!(matches!( - rx.try_recv(), - Err(std::sync::mpsc::TryRecvError::Empty) - )); - } - } - - #[test] - fn cancellation_wakes_the_real_blocking_receive_boundary() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::cancellation_wakes_the_real_blocking_receive_boundary" - ), - move || { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - register_stop_wakeup(&token, tx.clone()); - let (_, completion) = spawned_rt::sync::watch::channel(None); - let (id, trap, request) = ( - ActorId::next(), - new_trap_exit_flag(), - new_linked_exit_reason(), - ); - let ctx = Context { - id, - sender: tx.clone(), - cancellation_token: token.clone(), - completion, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: trap.clone(), - links: new_link_table(), - linked_reason: request.clone(), - signal: signal_target::<_>( - id, - None, - &tx, - trap.clone(), - request.clone(), - token.clone(), - ), - mailbox: crate::instrumentation::Mailbox::default(), - replies: Registry::new_native(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: Arc::default(), - }; - let (entered_tx, entered) = mpsc::channel(); - let (stopped_tx, stopped) = mpsc::channel(); - let cancel = token.clone(); - let actor = std::thread::spawn(move || { - run_actor_receiving( - &mut Idle(stopped_tx), - ctx, - move || { - let _ = entered_tx.send(()); - rx.recv() - .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) - }, - cancel, - ) - }); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - token.cancel(); - stopped - .recv_timeout(Duration::from_secs(2)) - .expect("cancel did not wake recv"); - actor.join().unwrap(); - assert_eq!( - link::lock_request(&request).reason(), - None, - "a plain stop request" - ); - }, - ); - } - - struct Crash; - impl Message for Crash { - type Result = (); - } - struct Failing; - impl Actor for Failing {} - impl Handler for Failing { - fn handle( - &mut self, - _: Crash, - _reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - { - panic!("linked wakeup regression"); - } - } - } - - #[test] - fn every_stop_route_wakes_an_idle_thread() { - crate::test_wait::bounded( - concat!(module_path!(), "::every_stop_route_wakes_an_idle_thread"), - move || { - for route in 0..4 { - let (stopped_tx, stopped) = mpsc::channel(); - let actor = Idle(stopped_tx).start(); - // A reply ensures startup is complete before the cancellation race. - actor.request(Noop).unwrap(); - let failing = if route == 3 { - let peer = Failing.start(); - actor.context().link(&peer.child_handle()); - Some(peer) - } else { - None - }; - match route { - 0 => actor.context().stop(), - 1 => actor.child_handle().stop(), - 2 => actor.cancellation_token.cancel(), - _ => failing.as_ref().unwrap().send(Crash).unwrap(), - } - if route == 3 { - // An untrapped link notice ends the idle actor without - // stopped() (decision 2): the wakeup is observed as its exit. - let reason = actor.child_handle().wait_exit_blocking(); - assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); - assert!(stopped.try_recv().is_err(), "stopped() ran"); - failing.unwrap().join(); - continue; - } - let result = stopped.recv_timeout(Duration::from_secs(2)); - // A failed test must not hang cleanup on a broken wakeup callback. - if result.is_err() { - actor.child_handle().stop(); - let _ = actor.send(Noop); - } - assert!( - result.is_ok(), - "stop route {route} failed to wake the mailbox" - ); - let reason = actor.wait_exit(); - if route == 3 { - assert!(matches!(reason, ExitReason::Panic(_))); - failing.unwrap().join(); - } else { - assert_eq!(reason, ExitReason::Normal); - } - } - }, - ); - } - - struct Busy { - entered: mpsc::Sender<()>, - release: mpsc::Receiver<()>, - handled: Arc, - } - impl Actor for Busy {} - struct Work; - impl Message for Work { - type Result = (); - } - impl Handler for Busy { - fn handle( - &mut self, - _: Work, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - self.handled.fetch_add(1, Ordering::SeqCst); - let _ = self.entered.send(()); - self.release.recv_timeout(Duration::from_secs(5)).unwrap(); - reply.reply(()); - } - } - - #[test] - fn repeated_stop_during_a_handler_discards_queued_requests() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::repeated_stop_during_a_handler_discards_queued_requests" - ), - move || { - let (entered_tx, entered) = mpsc::channel(); - let (release, release_rx) = mpsc::channel(); - let handled = Arc::new(AtomicUsize::new(0)); - let actor = Busy { - entered: entered_tx, - release: release_rx, - handled: handled.clone(), - } - .start(); - let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - let replies: Vec<_> = (0..256) - .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) - .collect(); - std::thread::scope(|scope| { - for _ in 0..16 { - let child = actor.child_handle(); - scope.spawn(move || { - for _ in 0..256 { - child.stop(); - } - }); - } - }); - release.send(()).unwrap(); - first.recv().unwrap(); - for reply in replies { - assert!(matches!( - reply.recv(), - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - } - assert_eq!(actor.wait_exit(), ExitReason::Normal); - assert_eq!(handled.load(Ordering::SeqCst), 1); - }, - ); - } -} +mod stop_wakeup_regression; /// Context and request APIs that the behavioural tests above do not reach. #[cfg(test)] -mod api_coverage { - use super::*; - use crate::test_wait::WAIT; - use std::sync::mpsc as std_mpsc; - - struct Echo; - struct Ask(u32); - impl Message for Ask { - type Result = u32; - } - /// Holds the actor inside a handler until released, then stops it, so a - /// request queued behind it is admitted but never answered. - struct StopAfter(std_mpsc::Receiver<()>); - impl Message for StopAfter { - type Result = (); - } - struct OwnRecipient; - impl Message for OwnRecipient { - type Result = (Recipient, ActorId); - } - struct Boom; - impl Message for Boom { - type Result = (); - } - struct Unlink(ChildHandle); - impl Message for Unlink { - type Result = (); - } - impl Actor for Echo {} - impl Handler for Echo { - fn handle( - &mut self, - msg: Ask, - reply: crate::reply::PendingReply<'_, u32>, - _: &Context, - ) { - reply.reply(msg.0); - } - } - impl Handler for Echo { - fn handle( - &mut self, - msg: StopAfter, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - let _ = msg.0.recv_timeout(WAIT); - ctx.stop(); - reply.reply(()); - } - } - impl Handler for Echo { - fn handle( - &mut self, - _: OwnRecipient, - reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, - ctx: &Context, - ) { - reply.reply((ctx.recipient::(), ctx.id())); - } - } - impl Handler for Echo { - fn handle( - &mut self, - _: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - { - panic!("boom"); - } - } - } - - #[test] - fn context_recipient_and_identity_reach_their_actor() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::context_recipient_and_identity_reach_their_actor" - ), - move || { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join(); - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); - }, - ); - } - - #[test] - fn requests_to_a_stopped_actor_fail_without_waiting() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::requests_to_a_stopped_actor_fail_without_waiting" - ), - move || { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join(); - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT), - Err(ActorError::NoProc) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT), - Err(ActorError::NoProc) - )); - }, - ); - } - - #[test] - fn admitted_requests_fail_when_the_handler_unwinds_before_replying() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::admitted_requests_fail_when_the_handler_unwinds_before_replying" - ), - move || { - let actor = Echo.start(); - assert!(matches!( - actor.request_with_timeout(Boom, WAIT), - Err(ActorError::ActorStopped(ExitReason::Panic(_))) - )); - let other = Echo.start(); - let recipient = other.recipient::(); - assert!(matches!( - request(&*recipient, Boom, WAIT), - Err(ActorError::ActorStopped(ExitReason::Panic(_))) - )); - }, - ); - } - - #[test] - fn requests_time_out_while_the_actor_is_busy() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::requests_time_out_while_the_actor_is_busy" - ), - move || { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = std_mpsc::channel(); - actor.send(StopAfter(held)).unwrap(); - let short = Duration::from_millis(1); - assert!(matches!( - actor.request_with_timeout(Ask(1), short), - Err(ActorError::RequestTimeout) - )); - assert!(matches!( - request(&*recipient, Ask(1), short), - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join(); - }, - ); - } - - struct Trapper { - trap: bool, - } - impl Actor for Trapper { - fn started(&mut self, ctx: &Context) { - ctx.trap_exit(self.trap); - } - } - impl Handler for Trapper { - fn handle( - &mut self, - msg: Ask, - reply: crate::reply::PendingReply<'_, u32>, - _: &Context, - ) { - reply.reply(msg.0); - } - } - impl Handler for Trapper { - fn handle( - &mut self, - msg: Unlink, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.unlink(&msg.0); - reply.reply(()); - } - } - - #[test] - fn default_exit_received_keeps_a_trapping_actor_alive() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::default_exit_received_keeps_a_trapping_actor_alive" - ), - move || { - let trapper = Trapper { trap: true }.start(); - assert!(matches!(trapper.request(Ask(0)), Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)), Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); - }, - ); - } - - #[test] - fn unlink_prevents_signal_delivery() { - crate::test_wait::bounded( - concat!(module_path!(), "::unlink_prevents_signal_delivery"), - move || { - // Not trapping: a linked peer's panic would take this actor down. - let trapper = Trapper { trap: false }.start(); - let peer = Echo.start(); - let peer_handle = peer.child_handle(); - trapper.context().link(&peer_handle); - trapper.request(Unlink(peer_handle.clone())).unwrap(); - // Unlinking yourself or an already-unlinked peer is a no-op. - trapper.request(Unlink(trapper.child_handle())).unwrap(); - trapper.request(Unlink(peer_handle)).unwrap(); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - assert!(matches!(trapper.request(Ask(1)), Ok(1))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - trapper.join(); - }, - ); - } - - #[test] - fn opposite_links_between_the_same_pair_do_not_deadlock() { - crate::test_wait::bounded( - concat!( - module_path!(), - "::opposite_links_between_the_same_pair_do_not_deadlock" - ), - move || { - // Each link/unlink locks both link tables. Both directions must take - // the locks in the same global order, or these two loops deadlock. - let a = Trapper { trap: true }.start(); - let b = Trapper { trap: true }.start(); - let (done, finished) = std_mpsc::channel(); - let workers: Vec<_> = [ - (a.context(), b.child_handle()), - (b.context(), a.child_handle()), - ] - .into_iter() - .map(|(own, peer)| { - let done = done.clone(); - std::thread::spawn(move || { - for _ in 0..2_000 { - own.link(&peer); - own.unlink(&peer); - } - done.send(()).unwrap(); - }) - }) - .collect(); - for _ in 0..2 { - finished - .recv_timeout(WAIT) - .expect("opposite link directions deadlocked"); - } - for worker in workers { - worker.join().unwrap(); - } - for actor in [a, b] { - actor - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - actor.join(); - } - }, - ); - } - - #[test] - fn debug_output_names_the_handle() { - crate::test_wait::bounded( - concat!(module_path!(), "::debug_output_names_the_handle"), - move || { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join(); - }, - ); - } -} +mod api_coverage; diff --git a/concurrency/src/threads/actor/actor_ref.rs b/concurrency/src/threads/actor/actor_ref.rs new file mode 100644 index 00000000..ff6ac15f --- /dev/null +++ b/concurrency/src/threads/actor/actor_ref.rs @@ -0,0 +1,208 @@ +//! `ActorRef`: the external handle (sends, requests, links, monitors, join and exit observation). +use super::*; + +impl ActorRef { + /// Clone this handle with a timeout override for generated protocol calls. + /// Per-call override wins over method and protocol defaults. No message is + /// sent until a protocol method is called; timeout does not cancel its work. + pub fn with_request_timeout(&self, timeout: Duration) -> crate::response::RequestOptions { + crate::response::RequestOptions::new(self.clone(), timeout) + } + + /// Send a fire-and-forget message to the actor. + pub fn send(&self, msg: M) -> Result<(), ActorError> + where + A: Handler, + M: Message, + { + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; + Ok(rx) + } + + /// Send a request and block until the reply arrives (default 5s timeout). + pub fn request(&self, msg: M) -> Result + where + A: Handler, + M: Message, + { + self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) + } + + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: `spawned_test::threads::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.replies), false); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Box::new(f), + slot, + }); + // Past the death point the actor is gone (E4), as for `enqueue`. + if self.replies.refuses_admission() { + return Err(ActorError::NoProc); + } + self.sender + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc)?; + Ok(rx) + } + + /// Send a request and block until the reply arrives, with a custom timeout. + pub fn request_with_timeout( + &self, + msg: M, + duration: Duration, + ) -> Result + where + A: Handler, + M: Message, + { + let observation = crate::instrumentation::RequestObservation::new(); + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { + Err(error) => Err(error), + Ok(rx) => rx.recv(), + }; + observation.finish(&result); + result + } + + /// Get a type-erased `Recipient` for this actor. + pub fn recipient(&self) -> Recipient + where + A: Handler, + M: Message, + { + Arc::new(self.clone()) + } + + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. + pub fn context(&self) -> Context { + Context::from_ref(self) + } + + /// Block until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline). Use + /// [`wait_exit`](Self::wait_exit) to observe death alone. + pub fn join(&self) { + let _ = self.wait_exit(); + self.cleaned.wait(); + } + + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods. + pub fn exit_reason(&self) -> Option { + crate::child_handle::poll_exit_reason(&self.completion) + } + + /// Block until the actor stops and return the exit reason. + pub fn wait_exit(&self) -> ExitReason { + crate::child_handle::wait_for_thread_exit_blocking(self.completion.clone()) + } + + /// Waiting messages, including in-flight enqueue reservations, excluding + /// the active handler and internal exit/shutdown control envelopes. + /// Concurrent observations are approximate; no registry or actor labels. + #[cfg(feature = "metrics")] + pub fn mailbox_depth(&self) -> u64 { + self.mailbox.depth() + } + + /// The actor's unique identity. + pub fn id(&self) -> ActorId { + self.id + } + + /// Get a type-erased `ChildHandle` for this actor. + pub fn child_handle(&self) -> ChildHandle { + ChildHandle::from(self.clone()) + } +} + +impl From> for ChildHandle { + fn from(actor_ref: ActorRef) -> Self { + let failures = actor_ref.sender.failures(); + ChildHandle::new( + actor_ref.signal, + Completion::Threads(actor_ref.completion), + actor_ref.links, + ) + .with_mailbox_failures(failures) + } +} + +// Bridge: ActorRef implements Receiver for any M that A handles +impl Receiver for ActorRef +where + A: Actor + Handler, + M: Message, +{ + fn send(&self, msg: M) -> Result<(), ActorError> { + ActorRef::send(self, msg) + } + + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline.ticks(), false) + } +} diff --git a/concurrency/src/threads/actor/api_coverage.rs b/concurrency/src/threads/actor/api_coverage.rs new file mode 100644 index 00000000..fcd94996 --- /dev/null +++ b/concurrency/src/threads/actor/api_coverage.rs @@ -0,0 +1,295 @@ +use super::*; +use crate::test_wait::WAIT; +use std::sync::mpsc as std_mpsc; + +struct Echo; +struct Ask(u32); +impl Message for Ask { + type Result = u32; +} +/// Holds the actor inside a handler until released, then stops it, so a +/// request queued behind it is admitted but never answered. +struct StopAfter(std_mpsc::Receiver<()>); +impl Message for StopAfter { + type Result = (); +} +struct OwnRecipient; +impl Message for OwnRecipient { + type Result = (Recipient, ActorId); +} +struct Boom; +impl Message for Boom { + type Result = (); +} +struct Unlink(ChildHandle); +impl Message for Unlink { + type Result = (); +} +impl Actor for Echo {} +impl Handler for Echo { + fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} +impl Handler for Echo { + fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let _ = msg.0.recv_timeout(WAIT); + ctx.stop(); + reply.reply(()); + } +} +impl Handler for Echo { + fn handle( + &mut self, + _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, + ctx: &Context, + ) { + reply.reply((ctx.recipient::(), ctx.id())); + } +} +impl Handler for Echo { + fn handle(&mut self, _: Boom, _reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { + panic!("boom"); + } + } +} + +#[test] +fn context_recipient_and_identity_reach_their_actor() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::context_recipient_and_identity_reach_their_actor" + ), + move || { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }, + ); +} + +#[test] +fn requests_to_a_stopped_actor_fail_without_waiting() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT), + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT), + Err(ActorError::NoProc) + )); + }, + ); +} + +#[test] +fn admitted_requests_fail_when_the_handler_unwinds_before_replying() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_handler_unwinds_before_replying" + ), + move || { + let actor = Echo.start(); + assert!(matches!( + actor.request_with_timeout(Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + let other = Echo.start(); + let recipient = other.recipient::(); + assert!(matches!( + request(&*recipient, Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + }, + ); +} + +#[test] +fn requests_time_out_while_the_actor_is_busy() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_time_out_while_the_actor_is_busy" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = std_mpsc::channel(); + actor.send(StopAfter(held)).unwrap(); + let short = Duration::from_millis(1); + assert!(matches!( + actor.request_with_timeout(Ask(1), short), + Err(ActorError::RequestTimeout) + )); + assert!(matches!( + request(&*recipient, Ask(1), short), + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join(); + }, + ); +} + +struct Trapper { + trap: bool, +} +impl Actor for Trapper { + fn started(&mut self, ctx: &Context) { + ctx.trap_exit(self.trap); + } +} +impl Handler for Trapper { + fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} +impl Handler for Trapper { + fn handle( + &mut self, + msg: Unlink, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.unlink(&msg.0); + reply.reply(()); + } +} + +#[test] +fn default_exit_received_keeps_a_trapping_actor_alive() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + let trapper = Trapper { trap: true }.start(); + assert!(matches!(trapper.request(Ask(0)), Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)), Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); + }, + ); +} + +#[test] +fn unlink_prevents_signal_delivery() { + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + // Not trapping: a linked peer's panic would take this actor down. + let trapper = Trapper { trap: false }.start(); + let peer = Echo.start(); + let peer_handle = peer.child_handle(); + trapper.context().link(&peer_handle); + trapper.request(Unlink(peer_handle.clone())).unwrap(); + // Unlinking yourself or an already-unlinked peer is a no-op. + trapper.request(Unlink(trapper.child_handle())).unwrap(); + trapper.request(Unlink(peer_handle)).unwrap(); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + assert!(matches!(trapper.request(Ask(1)), Ok(1))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + trapper.join(); + }, + ); +} + +#[test] +fn opposite_links_between_the_same_pair_do_not_deadlock() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::opposite_links_between_the_same_pair_do_not_deadlock" + ), + move || { + // Each link/unlink locks both link tables. Both directions must take + // the locks in the same global order, or these two loops deadlock. + let a = Trapper { trap: true }.start(); + let b = Trapper { trap: true }.start(); + let (done, finished) = std_mpsc::channel(); + let workers: Vec<_> = [ + (a.context(), b.child_handle()), + (b.context(), a.child_handle()), + ] + .into_iter() + .map(|(own, peer)| { + let done = done.clone(); + std::thread::spawn(move || { + for _ in 0..2_000 { + own.link(&peer); + own.unlink(&peer); + } + done.send(()).unwrap(); + }) + }) + .collect(); + for _ in 0..2 { + finished + .recv_timeout(WAIT) + .expect("opposite link directions deadlocked"); + } + for worker in workers { + worker.join().unwrap(); + } + for actor in [a, b] { + actor + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + actor.join(); + } + }, + ); +} + +#[test] +fn debug_output_names_the_handle() { + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join(); + }, + ); +} diff --git a/concurrency/src/threads/actor/context.rs b/concurrency/src/threads/actor/context.rs new file mode 100644 index 00000000..80ad748e --- /dev/null +++ b/concurrency/src/threads/actor/context.rs @@ -0,0 +1,480 @@ +//! `Context`: the handler-side handle (sends, requests, timers, links, monitors, stop and exit). +use super::*; + +impl Context { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { + Self { + id: actor_ref.id, + sender: actor_ref.sender.clone(), + cancellation_token: actor_ref.cancellation_token.clone(), + completion: actor_ref.completion.clone(), + monitors: actor_ref.monitors.clone(), + trap_exit: actor_ref.trap_exit.clone(), + links: actor_ref.links.clone(), + linked_reason: actor_ref.linked_reason.clone(), + signal: actor_ref.signal.clone(), + mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), + } + } + + /// Waiting messages, including in-flight enqueue reservations, excluding + /// the active handler and internal exit/shutdown control envelopes. + /// Concurrent observations are approximate; no registry or actor labels. + #[cfg(feature = "metrics")] + pub fn mailbox_depth(&self) -> u64 { + self.mailbox.depth() + } + + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. + pub fn deferred_outstanding(&self) -> usize { + self.replies.outstanding() + } + + /// The actor's unique identity. + pub fn id(&self) -> ActorId { + self.id + } + + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. + pub fn stop(&self) { + self.stop_with_reason(ExitReason::Normal); + } + + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). + pub fn stop_with_reason(&self, reason: ExitReason) { + link::stop_with_reason(self.id, &self.linked_reason, reason, || { + self.cancellation_token.cancel() + }); + } + + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + pub(super) fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.linked_reason, reason, || { + self.cancellation_token.cancel() + }); + } + + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.replies.signal_permit() else { + return; + }; + if link::lock_request(&self.linked_reason).is_dead() { + return; + } + fail_point!("actor.exit_before_deliver"); + target.exit_from(self.id, reason, false); + } + + /// Send a fire-and-forget message to this actor. + pub fn send(&self, msg: M) -> Result<(), ActorError> + where + A: Handler, + M: Message, + { + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; + Ok(rx) + } + + /// Send a request and block until the reply arrives (default 5s timeout). + pub fn request(&self, msg: M) -> Result + where + A: Handler, + M: Message, + { + self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) + } + + /// Send a request and block until the reply arrives, with a custom timeout. + pub fn request_with_timeout( + &self, + msg: M, + duration: Duration, + ) -> Result + where + A: Handler, + M: Message, + { + let observation = crate::instrumentation::RequestObservation::new(); + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { + Err(error) => Err(error), + Ok(rx) => rx.recv(), + }; + observation.finish(&result); + result + } + + /// Get a type-erased `Recipient` for sending a single message type + /// to this actor. + pub fn recipient(&self) -> Recipient + where + A: Handler, + M: Message, + { + Arc::new(self.clone()) + } + + /// Get an `ActorRef` from this context. + pub fn actor_ref(&self) -> ActorRef { + ActorRef { + id: self.id, + sender: self.sender.clone(), + cancellation_token: self.cancellation_token.clone(), + completion: self.completion.clone(), + monitors: self.monitors.clone(), + trap_exit: self.trap_exit.clone(), + links: self.links.clone(), + linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), + mailbox: self.mailbox.clone(), + replies: self.replies.clone(), + jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), + } + } + + /// Set up a unidirectional monitor on another actor. + /// + /// Returns a [`MonitorRef`] that can be used to cancel the monitor via + /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] + /// message is delivered to this actor's mailbox via `Handler`. + /// + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`). + /// + /// Multiple independent monitors are allowed on the same target — each + /// call returns a distinct `MonitorRef`. + /// + /// Monitors are unidirectional: the monitored actor is unaware of the + /// monitor and unaffected by it. + /// Registration after owner shutdown starts (including in `stopped()`) + /// returns an inert reference and creates no waiter or notification. + /// + /// **Resource cost (threads mode):** each active monitor occupies one OS + /// thread until the target exits, the monitor is cancelled, or its owner + /// stops, waiting on the target's completion signal. For supervisors with many long-lived + /// children, consider using tasks mode instead. + pub fn monitor(&self, target: &ChildHandle) -> MonitorRef + where + A: Handler, + { + fail_point!("monitor.before_install"); + let monitor_ref = MonitorRef::next(); + let active = spawned_rt::sync::CancellationToken::new(); + + let observer = self.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) + }; + crate::monitor::fence::register( + &self.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = enqueue_down( + &self.sender, + &self.mailbox, + &self.replies, + Down { + monitor_ref, + actor: target.id(), + reason: ExitReason::NoProc, + }, + ); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active + }); + if !registered { + return monitor_ref; + } + + let target = target.clone(); + let actor_ref = self.actor_ref(); + let monitors = self.monitors.clone(); + let registration = crate::monitor::MonitorRegistration(monitors.clone(), monitor_ref); + + rt::spawn(move || { + let _registration = registration; + let reason = futures::executor::block_on(async { + match futures::future::select( + Box::pin(active.cancelled()), + Box::pin(target.wait_exit_async()), + ) + .await + { + futures::future::Either::Left(_) => None, + futures::future::Either::Right((reason, _)) => Some(reason), + } + }); + let Some(reason) = reason else { return }; + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = enqueue_down( + &actor_ref.sender, + &actor_ref.mailbox, + &actor_ref.replies, + Down { + monitor_ref, + actor: target.id(), + reason, + }, + ); + }); + }); + }); + + monitor_ref + } + + /// Cancel a previously-set monitor. + /// + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. + pub fn demonitor(&self, monitor_ref: MonitorRef) { + let observer = self.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::demonitor(&self.monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + }) + }); + // Waking the waiter must happen after releasing the table lock. + if let Some(active) = active { + active.cancel(); + } + } + + /// Set up a bidirectional link with another actor (OTP `link/1`). + /// + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. + /// + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc`. Use [`try_link`](Self::try_link) to get `Err(NoProc)` + /// instead. Inert once this actor's exit began (in `stopped()`, from a + /// helper after the stop, or past its death point): no record and no + /// notice, as `monitor`. An existing link, or a pending notice from that peer, makes + /// this a no-op. + /// + /// [`Actor::exit_received`]: Actor::exit_received + pub fn link(&self, target: &ChildHandle) { + link::register_link( + &self.signal, + &self.links, + target.signal_target(), + target.links(), + false, + ); + } + + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record. + /// A caller whose exit already began does nothing and returns `Ok(())`; + /// an exit racing this call can linearize after the link attempt. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.signal, + &self.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor; an `Exit` already queued stays. + pub fn unlink(&self, target: &ChildHandle) { + link::unregister_link(self.id, &self.links, target.id(), target.links()); + } + + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. + /// + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. + /// + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.linked_reason).exit_pending() + } + + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). + /// + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for + /// [`Actor::exit_received`]. + /// + /// Only a kill is untrappable. A `start_linked` parent's exit ends a + /// trapping actor normally, through `stopped()`. Signals that arrive while + /// `started()` runs are resolved when it returns. + pub fn trap_exit(&self, enabled: bool) { + let id = self.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + self.cancellation_token.clone() + } +} + +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + +// Bridge: Context implements Receiver for any M that A handles +impl Receiver for Context +where + A: Actor + Handler, + M: Message, +{ + fn send(&self, msg: M) -> Result<(), ActorError> { + Context::send(self, msg) + } + + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline.ticks(), false) + } +} diff --git a/concurrency/src/threads/actor/early_stop_regression.rs b/concurrency/src/threads/actor/early_stop_regression.rs new file mode 100644 index 00000000..a66e19e0 --- /dev/null +++ b/concurrency/src/threads/actor/early_stop_regression.rs @@ -0,0 +1,22 @@ +use super::*; +struct EarlyStop; +impl Actor for EarlyStop { + fn started(&mut self, ctx: &Context) { + ctx.stop(); + } + fn stopped(&mut self, _: &Context) { + panic!("cleanup failed"); + } +} +#[test] +fn panic_during_early_stop_is_reported() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let actor = EarlyStop.start(); + assert!( + matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, + ); +} diff --git a/concurrency/src/threads/actor/envelope.rs b/concurrency/src/threads/actor/envelope.rs new file mode 100644 index 00000000..de991546 --- /dev/null +++ b/concurrency/src/threads/actor/envelope.rs @@ -0,0 +1,375 @@ +//! Mailbox envelopes: type erasure of messages, requests, `Down`, exit signals, in-turn closures and the stop wakeup on the actor side. +use super::*; + +// --------------------------------------------------------------------------- +// Envelope (type-erasure) +// --------------------------------------------------------------------------- + +pub(super) trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + None + } + #[cfg(feature = "metrics")] + fn reject(self: Box) {} + + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self, _replies: Option<&Registry>) {} + + fn handle(self: Box, actor: &mut A, ctx: &Context); +} + +/// Answer every request queued in a dead owner's mailbox (and `extra`, an +/// envelope received after the stop), destroying nothing: the items come +/// back, to be destroyed once the death point's wake-ups ran. Admission is +/// closed first (waiting for in-flight sends), so nothing is queued after. +pub(super) fn answer_mailbox( + rx: &mut mpsc::Receiver>, + extra: Option + Send>>, + replies: Option<&Registry>, +) -> Vec> { + let mut queued = rx.drain(); + queued.extend(extra.map(MailboxItem::Message)); + for item in &mut queued { + if let MailboxItem::Message(envelope) = item { + envelope.discharge(replies); + } + } + queued +} + +/// A never-polled actor's startup payload (its thread was never started). +pub(super) type StartupPayload = ( + A, + Context, + mpsc::Receiver>, + CancellationToken, +); +pub(super) fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + let replies = payload.1.replies.clone(); + erase(answer_mailbox(&mut payload.2, None, Some(&replies))) +} +pub(super) fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() +} + +pub(super) struct MessageEnvelope { + // Before the message: a plain drop also answers the request first. + pub(super) slot: SlotHandle, + pub(super) msg: M, + #[cfg(feature = "metrics")] + pub(super) observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + pub(super) from: u64, +} + +impl Envelope for MessageEnvelope +where + A: Actor + Handler, + M: Message, +{ + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let mut this = *self; + let reply = PendingReply::new(this.slot.take(), &ctx.replies); + actor.handle(this.msg, reply, ctx); + } +} + +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + fail_point!("monitor.down_before_dispatch"); + if crate::monitor::fence::take_queued(&ctx.monitors, &self.down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.replies); + actor.handle(self.down, reply, ctx); + } + } +} + +#[cfg(all(spawned_sim, feature = "async"))] +pub(super) fn assert_native_clock_context() { + assert!( + spawned_rt::tasks::Handle::try_current_or_ambient().is_err(), + "spawned_sim: native actors cannot run in or receive messages from a simulated world; use tasks actors for simulation, or run native actors outside the simulation" + ); +} + +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. +pub(super) fn enqueue( + sender: &mpsc::Sender>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + msg: M, + slot: SlotHandle, +) -> Result<(), ActorError> +where + A: Actor + Handler, + M: Message, +{ + #[cfg(all(spawned_sim, feature = "async"))] + assert_native_clock_context(); + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); + let envelope: Box + Send> = Box::new(MessageEnvelope { + msg, + slot, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +pub(super) struct DownEnvelope { + pub(super) down: Down, + #[cfg(feature = "metrics")] + pub(super) observation: Option, +} + +pub(super) fn enqueue_down>( + sender: &mpsc::Sender>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + }); + admit::(sender, replies, envelope, 0) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +pub(super) fn admit( + sender: &mpsc::Sender>, + replies: &Registry, + envelope: Box + Send>, + _from: u64, +) -> Result<(), ActorError> { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { + #[cfg(feature = "metrics")] + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false, _from); + return Err(ActorError::NoProc); + } + match sender.send(MailboxItem::Message(envelope)) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true, _from); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + if let MailboxItem::Message(envelope) = _error.0 { + envelope.reject(); + } + } + crate::lifecycle_trace::life_send::(owner, false, _from); + Err(ActorError::NoProc) + } + } + }) +} + +/// A closure run in one of the actor's turns (`spawned_test::threads::in_actor`). +pub(super) type InTurn = Box) -> R + Send>; +pub(super) struct InTurnEnvelope { + pub(super) slot: SlotHandle, + pub(super) f: InTurn, +} + +impl Envelope for InTurnEnvelope { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let InTurnEnvelope { f, mut slot } = *self; + let reply = PendingReply::new(slot.take(), &ctx.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); + } +} + +/// System envelope that dispatches an `Exit` notification to the actor's +/// `exit_received` callback. Does not require `Handler` on the actor. +pub(super) struct ExitEnvelope { + pub(super) exit: Exit, +} + +impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + actor.exit_received(self.exit, ctx); + } +} + +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +pub(super) struct ParentExitEnvelope { + pub(super) exit: Exit, +} + +impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } + fn handle(self: Box, _actor: &mut A, ctx: &Context) { + ctx.parent_exit(self.exit.reason); + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +pub(super) struct HeldEnvelope { + pub(super) exit: Exit, + pub(super) parent: bool, +} + +impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + if !link::lock_request(&ctx.linked_reason).held_trapped() { + return; + } + if self.parent { + ctx.parent_exit(self.exit.reason); + } else { + actor.exit_received(self.exit, ctx); + } + } +} + +/// Build the signal target of a new actor instance. +pub(super) fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: CancellationToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); + exit_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); + parent_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); + held_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} + +// A private wakeup is distinct from user envelopes and never invokes a handler. +pub(super) enum MailboxItem { + Message(Box + Send>), + Shutdown, +} + +pub(super) fn register_stop_wakeup( + token: &CancellationToken, + sender: mpsc::Sender>, +) { + // One callback covers Context, ChildHandle, linked exits and framework cleanup. + // CancellationToken drains this FnOnce callback exactly once, including when + // multiple callers race to cancel. Repeated stop calls cannot grow the queue. + token.on_cancel(Box::new(move || { + let _ = sender.send(MailboxItem::Shutdown); + })); +} diff --git a/concurrency/src/threads/actor/idle_stop_regression.rs b/concurrency/src/threads/actor/idle_stop_regression.rs new file mode 100644 index 00000000..66c7bea5 --- /dev/null +++ b/concurrency/src/threads/actor/idle_stop_regression.rs @@ -0,0 +1,142 @@ +use super::*; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Barrier, +}; +struct Counter(Arc, Arc>>); +impl Actor for Counter { + fn stopped(&mut self, _: &Context) { + self.1.lock().unwrap().push("stopped"); + } +} +impl Drop for Counter { + fn drop(&mut self) { + self.1.lock().unwrap().push("actor-dropped"); + } +} +struct Increment(Arc>>, MonitorTable); +impl Drop for Increment { + fn drop(&mut self) { + assert!( + self.1.lock().unwrap().is_none(), + "message dropped before monitor cleanup" + ); + self.0.lock().unwrap().push("message-dropped"); + } +} +impl Message for Increment { + type Result = (); +} +impl Handler for Counter { + fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } +} +#[test] +fn stop_while_receiving_does_not_dispatch_a_later_request() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_receiving_does_not_dispatch_a_later_request" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (_, completion_rx) = spawned_rt::sync::watch::channel(None); + let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); + let events = Arc::new(Mutex::new(Vec::new())); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion: completion_rx, + monitors: monitors.clone(), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new_native(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let count = Arc::new(AtomicUsize::new(0)); + let seen = count.clone(); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let (e, r) = (entered.clone(), release.clone()); + let cancel = token.clone(); + let trace = events.clone(); + let actor = std::thread::spawn(move || { + let mut first = true; + let mut counter = Counter(seen, trace); + // The envelope received after the stop comes back undispatched; + // the death point answers it and destroys it before the state. + let discarded = run_actor_receiving( + &mut counter, + ctx, + move || { + // Pause the real receive boundary after the started/cancel + // checks; the standard channel still carries the envelope. + if first { + first = false; + e.wait(); + r.wait(); + } + rx.recv_timeout(Duration::from_secs(5)) + }, + cancel, + ); + drop(discarded); + drop(counter); + }); + entered.wait(); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); + tx.send(MailboxItem::Message(Box::new(MessageEnvelope { + msg: Increment(events.clone(), monitors), + slot, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }))) + .unwrap(); + token.cancel(); + release.wait(); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "thread actor dispatched work after idle stop" + ); + assert!(response.recv().is_err()); + assert_eq!( + *events.lock().unwrap(), + ["stopped", "message-dropped", "actor-dropped"] + ); + }, + ); +} diff --git a/concurrency/src/threads/actor/lifecycle.rs b/concurrency/src/threads/actor/lifecycle.rs new file mode 100644 index 00000000..bbe86a11 --- /dev/null +++ b/concurrency/src/threads/actor/lifecycle.rs @@ -0,0 +1,317 @@ +//! Actor startup and the main loop on a dedicated OS thread: `started()`, the dispatch loop, `stopped()` and the death point. +use super::*; + +// --------------------------------------------------------------------------- +// Actor startup + main loop +// --------------------------------------------------------------------------- + +impl ActorRef { + pub(super) fn spawn(actor: A, parent: Option) -> Self { + #[cfg(all(spawned_sim, feature = "async"))] + assert_native_clock_context(); + let (tx, rx) = mpsc::channel::>(); + let cancellation_token = CancellationToken::new(); + register_stop_wakeup(&cancellation_token, tx.clone()); + let (completion_tx, completion) = spawned_rt::sync::watch::channel(None); + let id = ActorId::next(); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); + let trap_exit = new_trap_exit_flag(); + let links = new_link_table(); + let linked_reason = new_linked_exit_reason(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); + + let actor_ref = ActorRef { + id, + sender: tx.clone(), + cancellation_token: cancellation_token.clone(), + completion: completion.clone(), + monitors: monitors.clone(), + trap_exit: trap_exit.clone(), + links: links.clone(), + linked_reason: linked_reason.clone(), + signal: signal.clone(), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new_native(id, A::MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new( + id, + A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), + )), + cleaned: Arc::default(), + }; + + let ctx = Context { + id, + sender: tx, + cancellation_token: cancellation_token.clone(), + completion: actor_ref.completion.clone(), + monitors, + trap_exit, + links: links.clone(), + linked_reason: linked_reason.clone(), + signal, + mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), + }; + + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); + if let Some(parent) = parent { + actor_ref.context().link(&parent); + } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); + + let cleanup_token = cancellation_token.clone(); + let cleanup = crate::actor_cleanup::ActorCleanup::new( + id, + links, + linked_reason, + actor_ref.monitors.clone(), + completion_tx, + move || cleanup_token.cancel(), + ) + .with_mailbox_failures(actor_ref.sender.failures()) + .with_replies(actor_ref.replies.clone()) + .with_jobs(actor_ref.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); + let replies = actor_ref.replies.clone(); + let jobs = actor_ref.jobs.clone(); + let cleaned = crate::reply::jobs::CleanedGuard(actor_ref.cleaned.clone()); + let _thread_handle = rt::spawn(move || { + let _cleaned = cleaned; + let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); + let request = ctx.linked_reason.clone(); + let discarded = match catch_unwind(AssertUnwindSafe(|| { + run_actor(&mut actor, ctx, &mut rx, cancellation_token) + })) { + Ok(discarded) => discarded, + Err(payload) => { + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + None + } + }; + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is + // answered), then the mailbox refuses admission and its queued + // requests (and one received after the stop) are answered, all + // before any destructor. + cleanup.die(); + let queued = answer_mailbox(&mut rx, discarded, Some(&replies)); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). + cleanup.discharged(); + rx.discard_all(queued); + drop(rx); + // Destruction: a destructor panic is a cleanup failure only. + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + cleanup.cleanup_failed(failure); + } + // Publication (wait_exit, monitors, link notices) before the job join. + drop(cleanup); + // Job cleanup (§5 step 4), after death: join the cancelled jobs + // that signal completion, bounded by the cleanup deadline. + if jobs.running() > 0 { + let remaining = replies.cleanup_remaining(DEFAULT_STOP_DEADLINE); + let unfinished = jobs.join_blocking(std::time::Instant::now() + remaining); + crate::reply::jobs::report_unfinished(jobs.owner(), unfinished); + } + }); + + actor_ref + } +} + +/// Returns an envelope received after the stop (undispatched; answered at +/// the death point). +pub(super) fn run_actor( + actor: &mut A, + ctx: Context, + rx: &mut mpsc::Receiver>, + cancellation_token: CancellationToken, +) -> Option + Send>> { + run_actor_receiving( + actor, + ctx, + move || { + rx.recv() + .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) + }, + cancellation_token, + ) +} + +pub(super) fn run_actor_receiving( + actor: &mut A, + ctx: Context, + mut receive: impl FnMut() -> Result, std::sync::mpsc::RecvTimeoutError>, + cancellation_token: CancellationToken, +) -> Option + Send>> { + let mut discarded = None; + let request = ctx.linked_reason.clone(); + let trace_id = ctx.id.raw(); + let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); + let start_result = catch_unwind(AssertUnwindSafe(|| { + fail_point!("actor.before_started"); + crate::lifecycle_trace::with_actor(trace_id, || actor.started(&ctx)); + })); + if let Err(panic) = start_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in started() callback: {msg}"); + }); + // No stopped() after a failed start (L5). + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); + cancellation_token.cancel(); + return None; + } + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.trap_exit.load(Ordering::Acquire)) { + cancellation_token.cancel(); + } + + while !cancellation_token.is_cancelled() { + let msg = match receive() { + Ok(MailboxItem::Message(msg)) => Some(msg), + Ok(MailboxItem::Shutdown) => None, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => None, + }; + match msg { + Some(envelope) => { + #[cfg(feature = "metrics")] + let mut envelope = envelope; + + // A stop observed after receive wins over dispatch, whether + // the channel waited or was buffered. Defer the message drop. + // The check and the trace event are one lifecycle-trace step. + let stopped = crate::lifecycle_trace::life_section(trace_id, || { + if cancellation_token.is_cancelled() { + return true; + } + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + false + }); + if stopped { + discarded = Some(envelope); + break; + } + #[cfg(feature = "metrics")] + let observation = envelope + .observation() + .map(|observation| observation.begin()); + ctx.replies.begin_turn(); + let result = catch_unwind(AssertUnwindSafe(|| { + crate::lifecycle_trace::with_actor(trace_id, || envelope.handle(actor, &ctx)); + })); + ctx.replies.end_turn(result.is_err()); + #[cfg(feature = "metrics")] + if let Some(observation) = observation { + observation.finish(result.is_err()); + } + if let Err(panic) = result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in message handler: {msg}"); + }); + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); + break; + } + } + None => break, + } + } + + cancellation_token.cancel(); + ctx.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.sender.failures().hold(); + ctx.jobs.cancel_all(); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) + }); + (run, skipped) + }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } + if run_hook { + let stop_result = catch_unwind(AssertUnwindSafe(|| { + fail_point!("actor.before_stopped"); + crate::lifecycle_trace::with_actor(trace_id, || actor.stopped(&ctx)); + })); + if let Err(panic) = stop_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in stopped() callback: {msg}"); + }); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); + } + } + + // Message destructors and reply cancellation may re-enter monitor state. + // Preserve stopped() -> monitor cleanup -> undelivered-message drop (the + // envelope received after the stop is answered at the death point, then + // dropped with the mailbox). + drop(_monitor_cleanup); + discarded +} diff --git a/concurrency/src/threads/actor/monitor_cleanup_regressions.rs b/concurrency/src/threads/actor/monitor_cleanup_regressions.rs new file mode 100644 index 00000000..649f3e93 --- /dev/null +++ b/concurrency/src/threads/actor/monitor_cleanup_regressions.rs @@ -0,0 +1,69 @@ +use super::*; +struct Idle; +impl Actor for Idle {} +impl Handler for Idle { + fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} +#[test] +fn closed_monitor_table_rejects_late_registration() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + observer.join(); + target.join(); + }, + ); +} +fn check_cleanup(stop_owner: bool) { + let target = Idle.start(); + let observer = Idle.start(); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + let monitor = observer.context().monitor(&child); + drop(child); + if stop_owner { + observer.child_handle().stop(); + observer.join(); + drop(observer); + } else { + observer.context().demonitor(monitor); + // Only the waiter now owns this particular ChildHandle allocation. + let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); + observer.child_handle().stop(); + observer.join(); + drop(observer); + target.child_handle().stop(); + target.join(); + assert!(released, "demonitor retained its waiter until target exit"); + return; + } + let released = crate::test_wait::wait_for(|| waiter.strong_count() == 0, |r| *r); + target.child_handle().stop(); + target.join(); + assert!(released, "stopped observer retained its monitor waiter"); +} +#[test] +fn demonitor_releases_waiter() { + check_cleanup(false); +} +#[test] +fn stopped_observer_releases_waiter() { + check_cleanup(true); +} diff --git a/concurrency/src/threads/actor/stop_wakeup_regression.rs b/concurrency/src/threads/actor/stop_wakeup_regression.rs new file mode 100644 index 00000000..c4c456ad --- /dev/null +++ b/concurrency/src/threads/actor/stop_wakeup_regression.rs @@ -0,0 +1,248 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct Idle(mpsc::Sender<()>); +impl Actor for Idle { + fn stopped(&mut self, _: &Context) { + let _ = self.0.send(()); + } +} +struct Noop; +impl Message for Noop { + type Result = (); +} +impl Handler for Idle { + fn handle(&mut self, _: Noop, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +#[test] +fn concurrent_cancellation_enqueues_exactly_one_shutdown() { + for already_cancelled in [false, true] { + let (tx, rx) = mpsc::channel::>(); + let token = CancellationToken::new(); + if already_cancelled { + token.cancel(); + } + register_stop_wakeup(&token, tx.clone()); + std::thread::scope(|scope| { + for _ in 0..16 { + let token = &token; + scope.spawn(move || { + for _ in 0..256 { + token.cancel(); + } + }); + } + }); + assert!(matches!(rx.try_recv(), Ok(MailboxItem::Shutdown))); + // Keep tx alive: Empty proves no duplicate, rather than disconnection. + assert!(matches!( + rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + } +} + +#[test] +fn cancellation_wakes_the_real_blocking_receive_boundary() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::cancellation_wakes_the_real_blocking_receive_boundary" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + register_stop_wakeup(&token, tx.clone()); + let (_, completion) = spawned_rt::sync::watch::channel(None); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion, + monitors: Arc::new(Mutex::new(Some(HashMap::new()))), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new_native(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let (entered_tx, entered) = mpsc::channel(); + let (stopped_tx, stopped) = mpsc::channel(); + let cancel = token.clone(); + let actor = std::thread::spawn(move || { + run_actor_receiving( + &mut Idle(stopped_tx), + ctx, + move || { + let _ = entered_tx.send(()); + rx.recv() + .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) + }, + cancel, + ) + }); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + token.cancel(); + stopped + .recv_timeout(Duration::from_secs(2)) + .expect("cancel did not wake recv"); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + }, + ); +} + +struct Crash; +impl Message for Crash { + type Result = (); +} +struct Failing; +impl Actor for Failing {} +impl Handler for Failing { + fn handle(&mut self, _: Crash, _reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { + panic!("linked wakeup regression"); + } + } +} + +#[test] +fn every_stop_route_wakes_an_idle_thread() { + crate::test_wait::bounded( + concat!(module_path!(), "::every_stop_route_wakes_an_idle_thread"), + move || { + for route in 0..4 { + let (stopped_tx, stopped) = mpsc::channel(); + let actor = Idle(stopped_tx).start(); + // A reply ensures startup is complete before the cancellation race. + actor.request(Noop).unwrap(); + let failing = if route == 3 { + let peer = Failing.start(); + actor.context().link(&peer.child_handle()); + Some(peer) + } else { + None + }; + match route { + 0 => actor.context().stop(), + 1 => actor.child_handle().stop(), + 2 => actor.cancellation_token.cancel(), + _ => failing.as_ref().unwrap().send(Crash).unwrap(), + } + if route == 3 { + // An untrapped link notice ends the idle actor without + // stopped() (decision 2): the wakeup is observed as its exit. + let reason = actor.child_handle().wait_exit_blocking(); + assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); + assert!(stopped.try_recv().is_err(), "stopped() ran"); + failing.unwrap().join(); + continue; + } + let result = stopped.recv_timeout(Duration::from_secs(2)); + // A failed test must not hang cleanup on a broken wakeup callback. + if result.is_err() { + actor.child_handle().stop(); + let _ = actor.send(Noop); + } + assert!( + result.is_ok(), + "stop route {route} failed to wake the mailbox" + ); + let reason = actor.wait_exit(); + if route == 3 { + assert!(matches!(reason, ExitReason::Panic(_))); + failing.unwrap().join(); + } else { + assert_eq!(reason, ExitReason::Normal); + } + } + }, + ); +} + +struct Busy { + entered: mpsc::Sender<()>, + release: mpsc::Receiver<()>, + handled: Arc, +} +impl Actor for Busy {} +struct Work; +impl Message for Work { + type Result = (); +} +impl Handler for Busy { + fn handle(&mut self, _: Work, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + self.handled.fetch_add(1, Ordering::SeqCst); + let _ = self.entered.send(()); + self.release.recv_timeout(Duration::from_secs(5)).unwrap(); + reply.reply(()); + } +} + +#[test] +fn repeated_stop_during_a_handler_discards_queued_requests() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::repeated_stop_during_a_handler_discards_queued_requests" + ), + move || { + let (entered_tx, entered) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let handled = Arc::new(AtomicUsize::new(0)); + let actor = Busy { + entered: entered_tx, + release: release_rx, + handled: handled.clone(), + } + .start(); + let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + let replies: Vec<_> = (0..256) + .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) + .collect(); + std::thread::scope(|scope| { + for _ in 0..16 { + let child = actor.child_handle(); + scope.spawn(move || { + for _ in 0..256 { + child.stop(); + } + }); + } + }); + release.send(()).unwrap(); + first.recv().unwrap(); + for reply in replies { + assert!(matches!( + reply.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + } + assert_eq!(actor.wait_exit(), ExitReason::Normal); + assert_eq!(handled.load(Ordering::SeqCst), 1); + }, + ); +} diff --git a/concurrency/src/threads/actor/tests.rs b/concurrency/src/threads/actor/tests.rs new file mode 100644 index 00000000..0dbc1f31 --- /dev/null +++ b/concurrency/src/threads/actor/tests.rs @@ -0,0 +1,854 @@ +use super::*; +use crate::message::Message; +use std::thread; + +struct Counter { + count: u64, +} + +struct GetCount; +impl Message for GetCount { + type Result = u64; +} + +struct Increment; +impl Message for Increment { + type Result = u64; +} + +struct StopCounter; +impl Message for StopCounter { + type Result = u64; +} + +impl Actor for Counter {} + +impl Handler for Counter { + fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); + } +} + +impl Handler for Counter { + fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { + self.count += 1; + self.count + }; + reply.reply(__reply_value); + } +} + +impl Handler for Counter { + fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); + } +} + +#[test] +fn basic_send_and_request() { + crate::test_wait::bounded( + concat!(module_path!(), "::basic_send_and_request"), + move || { + let actor = Counter { count: 0 }.start(); + assert_eq!(actor.request(GetCount).unwrap(), 0); + assert_eq!(actor.request(Increment).unwrap(), 1); + // Same mailbox: the request is handled after the send. + actor.send(Increment).unwrap(); + assert_eq!(actor.request(GetCount).unwrap(), 2); + actor.request(StopCounter).unwrap(); + }, + ); +} + +#[test] +fn join_waits_for_completion() { + crate::test_wait::bounded( + concat!(module_path!(), "::join_waits_for_completion"), + move || { + struct SlowStop; + struct StopSlow; + impl Message for StopSlow { + type Result = (); + } + impl Actor for SlowStop { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(300)); + } + } + impl Handler for SlowStop { + fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } + + let actor = SlowStop.start(); + actor.send(StopSlow).unwrap(); + actor.join(); + // If join() returned, stopped() has completed + }, + ); +} + +#[test] +fn join_multiple_callers() { + crate::test_wait::bounded( + concat!(module_path!(), "::join_multiple_callers"), + move || { + struct SlowStop2; + struct StopSlow2; + impl Message for StopSlow2 { + type Result = (); + } + impl Actor for SlowStop2 { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(200)); + } + } + impl Handler for SlowStop2 { + fn handle( + &mut self, + _msg: StopSlow2, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } + + let actor = SlowStop2.start(); + let a1 = actor.clone(); + let a2 = actor.clone(); + let t1 = thread::spawn(move || { + a1.join(); + 1u32 + }); + let t2 = thread::spawn(move || { + a2.join(); + 2u32 + }); + actor.send(StopSlow2).unwrap(); + assert_eq!(t1.join().unwrap(), 1); + assert_eq!(t2.join().unwrap(), 2); + }, + ); +} + +#[test] +fn panic_in_started_stops_actor() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + struct PanicOnStart; + struct PingThread; + impl Message for PingThread { + type Result = (); + } + impl Actor for PanicOnStart { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + fn handle( + &mut self, + _msg: PingThread, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } + + let actor = PanicOnStart.start(); + actor.join(); + let result = actor.send(PingThread); + assert!(result.is_err()); + }, + ); +} + +#[test] +fn panic_in_handler_stops_actor() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + struct PanicOnMsg; + struct ExplodeThread; + impl Message for ExplodeThread { + type Result = (); + } + struct CheckThread; + impl Message for CheckThread { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: ExplodeThread, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: CheckThread, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } + } + + let actor = PanicOnMsg.start(); + actor.send(ExplodeThread).unwrap(); + actor.join(); + let result = actor.request(CheckThread); + assert!(result.is_err()); + }, + ); +} + +#[test] +fn panic_in_stopped_still_completes() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + struct PanicOnStop; + struct StopMeThread; + impl Message for StopMeThread { + type Result = (); + } + impl Actor for PanicOnStop { + fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + fn handle( + &mut self, + _msg: StopMeThread, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } + + let actor = PanicOnStop.start(); + actor.send(StopMeThread).unwrap(); + actor.join(); + }, + ); +} + +#[test] +fn recipient_type_erasure() { + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let actor = Counter { count: 42 }.start(); + let recipient: Recipient = actor.recipient(); + let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); + assert_eq!(result, 42); + }, + ); +} + +#[test] +fn send_message_on_delivers() { + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let actor = Counter { count: 0 }.start(); + let ctx = actor.context(); + send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); + let count = + crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))) + .unwrap(); + assert_eq!(count, 1); + }, + ); +} + +// --- ExitReason tests --- + +#[test] +fn exit_reason_normal_on_clean_stop() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).unwrap(); + let reason = actor.wait_exit(); + assert!(matches!(reason, ExitReason::Normal)); + }, + ); +} + +#[test] +fn exit_reason_panic_in_started() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStartThread; + struct PingThread2; + impl Message for PingThread2 { + type Result = (); + } + impl Actor for PanicStartThread { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStartThread { + fn handle( + &mut self, + _msg: PingThread2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } + + let actor = PanicStartThread.start(); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) + ); + }, + ); +} + +#[test] +fn exit_reason_panic_in_handler() { + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandlerThread; + struct ExplodeThread2; + impl Message for ExplodeThread2 { + type Result = (); + } + impl Actor for PanicHandlerThread {} + impl Handler for PanicHandlerThread { + fn handle( + &mut self, + _msg: ExplodeThread2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + + let actor = PanicHandlerThread.start(); + let _ = actor.send(ExplodeThread2); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) + ); + }, + ); +} + +#[test] +fn exit_reason_poll_returns_none_while_running() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).unwrap(); + actor.join(); + assert!(actor.exit_reason().is_some()); + }, + ); +} + +// --- Monitor tests --- + +struct GetDowns; +impl Message for GetDowns { + type Result = Vec; +} + +struct Watcher { + downs: Arc>>, +} + +struct StartMonitor(crate::ChildHandle); +impl Message for StartMonitor { + type Result = crate::monitor::MonitorRef; +} +struct CallDemonitor(crate::monitor::MonitorRef); +impl Message for CallDemonitor { + type Result = (); +} +/// Monitor, then demonitor in the same turn. +struct MonitorThenDemonitor(crate::ChildHandle); +impl Message for MonitorThenDemonitor { + type Result = (); +} + +impl Actor for Watcher {} + +impl Handler for Watcher { + fn handle( + &mut self, + msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); + } +} + +impl Handler for Watcher { + fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); + } +} + +impl Handler for Watcher { + fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.demonitor(msg.0); + reply.reply(()); + } +} + +impl Handler for Watcher { + fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + self.downs.lock().unwrap().push(msg); + reply.reply(()); + } +} + +impl Handler for Watcher { + fn handle( + &mut self, + _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.downs.lock().unwrap().clone()); + } +} + +fn make_watcher() -> ActorRef { + Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start() +} + +#[test] +fn monitor_running_actor_delivers_down_on_exit() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }, + ); +} + +/// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox +/// (here the `NoProc` a monitor of a dead target queues at once) is +/// never handled once `demonitor` returned. +#[test] +fn demonitor_flushes_a_queued_down() { + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let watcher = make_watcher(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }, + ); +} + +#[test] +fn monitor_already_dead_actor_delivers_down_immediately() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let target_handle = target.child_handle(); + + let watcher = make_watcher(); + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + }, + ); +} + +#[test] +fn demonitor_before_target_dies_suppresses_down() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + watcher.request(CallDemonitor(monitor_ref)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + rt::sleep(Duration::from_millis(150)); + + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty()); + }, + ); +} + +#[test] +fn multiple_monitors_each_get_own_ref_and_down() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }, + ); +} + +#[test] +fn monitor_observes_panic_reason() { + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } + } + + let target = PanicMe.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let _ = target.send(PanicMsg); + + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }, + ); +} + +// --- Link tests --- + +struct TrapActor { + exits: Arc>>, + trap: bool, +} + +struct GetExits; +impl Message for GetExits { + type Result = Vec; +} +struct LinkTo(crate::ChildHandle); +impl Message for LinkTo { + type Result = (); +} + +impl Actor for TrapActor { + fn started(&mut self, ctx: &Context) { + ctx.trap_exit(self.trap); + } + fn exit_received(&mut self, exit: Exit, _ctx: &Context) { + self.exits.lock().unwrap().push(exit); + } +} + +impl Handler for TrapActor { + fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.link(&msg.0); + reply.reply(()); + } +} + +impl Handler for TrapActor { + fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.exits.lock().unwrap().clone()); + } +} + +fn make_trapper(trap: bool) -> ActorRef { + TrapActor { + exits: Arc::new(Mutex::new(Vec::new())), + trap, + } + .start() +} + +#[test] +fn link_propagates_panic_to_non_trapping_peer_threads() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } + + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).unwrap(); + + let _ = a.send(Boom); + let reason_a = a.wait_exit(); + let reason_b = b.wait_exit(); + assert!(matches!(reason_a, ExitReason::Panic(_))); + assert!(matches!(reason_b, ExitReason::Panic(_))); + }, + ); +} + +#[test] +fn link_delivers_exit_to_trapping_peer_threads() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_delivers_exit_to_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } + + let a = make_trapper(false); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).unwrap(); + + let _ = a.send(Boom); + a.wait_exit(); + let exits = crate::test_wait::wait_for( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .unwrap(); + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_blocking(); + }, + ); +} + +#[test] +fn start_linked_threads() { + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_threads"), + move || { + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } + } + + struct Child; + impl Actor for Child {} + + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); + + let _ = parent.send(PanicParent); + parent.wait_exit(); + let reason = child.wait_exit(); + assert!(matches!(reason, ExitReason::Panic(_))); + }, + ); +} diff --git a/docs/TESTING.md b/docs/TESTING.md index 0748ff92..7762641b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -532,7 +532,7 @@ SPAWNED_SIM_TURN_BUDGET=ms sets the budget, 0 disables it) ``` The actor dispatch loop marks each turn (a `#[cfg(spawned_sim)]` guard in -`tasks/actor.rs`); the executor times only the stretches of a poll that run +`tasks/actor/lifecycle.rs`); the executor times only the stretches of a poll that run inside a turn and keeps the turn of a task suspended at an `.await`, so a poll that handles several messages charges each turn separately and the budget applies to each stretch between awaits, not to the whole turn. A @@ -1170,7 +1170,7 @@ is asserted; allocator retention can differ between platforms. At most one is now sent per actor (`concurrent_cancellation_enqueues_exactly_one_shutdown`, `repeated_stop_during_a_handler_discards_queued_requests`). The #168 tests are - in `concurrency/src/threads/actor.rs`. + in `concurrency/src/threads/actor/`. Reviews and evidence: [PR #166](https://github.com/lambdaclass/spawned/blob/efba24340a01df82f872ec422b9dbaf48f897b04/legacy/docs/reviews/pr-166-links.md) and [PR #168](https://github.com/lambdaclass/spawned/blob/efba24340a01df82f872ec422b9dbaf48f897b04/legacy/docs/reviews/pr-168-security.md). @@ -1467,7 +1467,7 @@ updates the model in the same PR, and drift is a bug. The `model-sync` job - **Paths.** `verification/model-map.toml` maps code to the models that cover it (for example `actor_cleanup.rs`, `link.rs`, `monitor.rs` and the - actor loops to `Lifecycle`; `supervision.rs` to `Supervision`, `Cohorts` + actor loops to `Lifecycle`; `supervision.rs` and `supervision/` to `Supervision`, `Cohorts` and `NestedSupervision`; `system.rs` to `Coordinator`). If a PR changes a mapped file but none of its models (the model, its MC configurations, inductive or coverage modules, or trace spec; a comment-only model edit diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 23eab4c4..a2f962a7 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -100,12 +100,12 @@ not wait for the current turn (X4). A handler stopping its own actor prevents the next dispatch. Queued messages are dropped one by one and every queued request is answered (callers get `ActorStopped` with the final reason) before the exit is published. -Checked by: `src/tasks/actor.rs::stop_receive_probe` (untraced stop/receive gap, +Checked by: `src/tasks/actor/stop_receive_probe.rs` (untraced stop/receive gap, caller-before-payload ordering and runtime drop), `stop_token_loom` (queue publication versus post-receive admission), -`src/tasks/actor.rs::stop_while_idle_does_not_dispatch_a_queued_request_on_resume`, -`src/threads/actor.rs::repeated_stop_during_a_handler_discards_queued_requests`, -`src/threads/actor.rs::stop_while_receiving_does_not_dispatch_a_later_request`, +`src/tasks/actor/idle_stop_regression.rs::stop_while_idle_does_not_dispatch_a_queued_request_on_resume`, +`src/threads/actor/stop_wakeup_regression.rs::repeated_stop_during_a_handler_discards_queued_requests`, +`src/threads/actor/idle_stop_regression.rs::stop_while_receiving_does_not_dispatch_a_later_request`, [`shutdown_publication`][shutdown_publication], [`mailbox_drop_panic`][mailbox_drop_panic], [`semantics_gaps`][semantics_gaps]`::{native,tasks}_contract::stop_window_accepts_then_discards_and_releases_before_join` (payloads dropped and reply senders closed when `join()` returns, every placement), @@ -133,7 +133,7 @@ the owner's final exit reason, the one `wait_exit` and `Down` report), `NoProc`; a slow teardown can still produce a timeout first. `ActorStopped` is reported only at the owner's actual death, never while `stopped()` still runs (otp-exit-signals.md, "Exit errors wait for D"). -Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor.rs::request_timeout`, +Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor/tests.rs::request_timeout`, [`timeout_outcomes`][ex_timeout_outcomes] example. **R2. Reply identity.** Each request has its own reply channel; a reply reaches @@ -295,7 +295,7 @@ returns and the reason becomes `Panic`, replacing an earlier requested reason unless a kill was received (then `Killed` stands). `stopped()` does not run after a kill, an untrapped exit signal or link notice, or a runtime drop (see X3, X4); one that already began finishes. -Checked by: `src/{tasks,threads}/actor.rs::panic_in_started_stops_actor`, +Checked by: `src/{tasks,threads}/actor/tests.rs::panic_in_started_stops_actor`, `panic_in_handler_stops_actor`, `panic_in_stopped_still_completes`, `panic_during_early_stop_is_reported`; [`panic_safety`][panic_safety], [`foreign_panics::panicking_subscriber_keeps_handler_reason_and_runs_stopped`][foreign_panics], @@ -339,7 +339,7 @@ Checked by: `Lifecycle!DownAtMostOnce`, `Lifecycle!NoProcIffLate`, `Lifecycle!Do [`link_fence_loom`][link_fence_loom]`::monitor_racing_death_gets_exactly_one_down`; [`exit_signals_sim::link_and_monitor_race_the_death_point`][exit_signals_sim]; [`actor_scenarios::demonitor_before_exit_suppresses_notification_and_races_do_not_duplicate`][actor_scenarios], -[`exit_signal_matrix`][exit_signal_matrix], `src/*/actor.rs::monitor_*`, +[`exit_signal_matrix`][exit_signal_matrix], `src/*/actor/tests.rs::monitor_*`, `multiple_monitors_each_get_own_ref_and_down`, [`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals], [`cancellation_safety::monitor_and_link_setup_requests_are_cancel_safe`][cancellation_safety]. @@ -358,7 +358,7 @@ and `try_link` returns `Ok(())` (a dead process cannot call `link` in OTP; design otp-exit-signals.md, "Sender lifetime"). Checked by: `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`; the `NoDemonitorFlush` control runs a flushed `Down`); -`src/*/actor.rs::{demonitor_before_target_dies_suppresses_down, demonitor_flushes_a_queued_down}`, +`src/*/actor/tests.rs::{demonitor_before_target_dies_suppresses_down, demonitor_flushes_a_queued_down}`, `src/monitor.rs::tests::demonitor_flushes_a_queued_down`; [`link_fence_loom`][link_fence_loom]`::demonitor_flushes_a_down_in_dispatch` (a dispatch racing `demonitor` on another thread); @@ -395,7 +395,7 @@ Checked by: `Lifecycle!TrapRespected`, `Lifecycle!NormalSignalIgnored`, `Lifecyc `threads_mode::kill_exit_and_noproc_in_threads_mode`; `failpoints::an_exit_signal_is_not_overtaken_by_its_senders_death_notice`; `src/link.rs::tests::signal_rules_follow_exit_signal_2`; -`src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, +`src/tasks/actor/tests.rs::link_propagates_panic_to_non_trapping_peer`, `link_delivers_exit_to_trapping_peer`, `link_normal_exit_not_propagated_to_non_trapping_peer`, `chain_propagation_through_non_trapping_middle` (and threads equivalents), [`exit_signal_matrix`][exit_signal_matrix], [`linked_exit_macro`][linked_exit_macro], @@ -433,7 +433,7 @@ fence under Loom: `unlink_fences_a_death_notice`, `link_racing_death_gets_exactl [`exit_signals_sim`][exit_signals_sim]`::{link_and_monitor_race_the_death_point, unlink_and_demonitor_fence_delivery}`; `src/link.rs::tests::{closure_is_terminal_and_delivers_each_edge_once, try_link_to_a_dead_actor_keeps_no_record, unlink_fences_a_notice_taken_at_death}`; [`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals]; -`src/tasks/actor.rs::duplicate_link_is_idempotent`, `unlink_prevents_signal_delivery`; +`src/tasks/actor/tests.rs::duplicate_link_is_idempotent`, `unlink_prevents_signal_delivery`; `src/native_link_regression_tests.rs::start_linked_registers_before_threads_started`; lifecycle traces `late_registrations_get_noproc`, `unlink_fences_a_notice_taken_at_death`, `demonitor_fences_a_down`. @@ -536,7 +536,7 @@ reverse order. Checked by: `Supervision!BudgetBounded`; Kani `restart_window_admits_exactly_within_budget`; [`supervision::restart_budget_is_shared_between_siblings_and_exhaustion_closes_tree`][supervision], `disabled_retries_fail_without_waiting_for_a_long_backoff`, -`src/supervision.rs::retry_reservations_exclude_siblings_and_refund_interrupted_backoff`, +`src/supervision/policy_tests.rs::retry_reservations_exclude_siblings_and_refund_interrupted_backoff`, `sliding_window_expires_only_old_grants_before_the_next_retry`, [`supervision_strategies::cohort_reconstruction_consumes_shared_factory_attempt_budget`][supervision_strategies]. @@ -546,7 +546,7 @@ zero through the bound, so `initial` is a cap, not a minimum. Without jitter the delay equals the bound. The exponent resets after the same generation stays ready for `stable_after` (default 30 s); collateral children add no delay. Checked by: Kani `backoff_bound_is_capped`, `backoff_jitter_stays_in_range`, -`backoff_bound_is_monotone_*`, `jitter_*`; `src/supervision.rs::capped_backoff_and_seeded_jitter_remain_bounded_after_long_failure_streaks`, +`backoff_bound_is_monotone_*`, `jitter_*`; `src/supervision/policy_tests.rs::capped_backoff_and_seeded_jitter_remain_bounded_after_long_failure_streaks`, [`supervision::backoff_exponent_resets_only_after_a_stable_generation`][supervision], [`semantics_gaps`][semantics_gaps]`::supervision_contract::collateral_children_add_no_backoff`. @@ -689,7 +689,7 @@ checked by construction only. `Context` created it and never retargets a supervised replacement. Managed timers run on the actor's group. `send_message_on` does not send if the actor stopped first. -Checked by: `src/tasks/actor.rs::send_message_on_cancelled`, +Checked by: `src/tasks/actor/tests.rs::send_message_on_cancelled`, [`semantics_gaps`][semantics_gaps]`::tasks_contract::managed_timers_run_on_the_group_and_die_with_the_actor` (the timer task runs on the actor's async group), [`semantics_gaps`][semantics_gaps]`::tasks::timers_are_not_carried_to_a_supervised_replacement` (supervision diff --git a/docs/research/supervision-models.md b/docs/research/supervision-models.md index 045c68a4..c0a69aa2 100644 --- a/docs/research/supervision-models.md +++ b/docs/research/supervision-models.md @@ -115,7 +115,7 @@ Recovery of state is the application's job through persistence. ### Spawned v0.7.0 -Read from `concurrency/src/supervision.rs`, `supervision/nested.rs` and the +Read from `concurrency/src/supervision.rs`, `supervision/` (`run.rs`, `nested.rs`) and the [guide](../GUIDE.md#construct-fresh-children-after-failure): - **Strategies:** `OneForOne` (default), `OneForAll`, `RestForOne`. Cohorts stop diff --git a/macros/src/actor.rs b/macros/src/actor.rs new file mode 100644 index 00000000..218191c8 --- /dev/null +++ b/macros/src/actor.rs @@ -0,0 +1,751 @@ +//! `#[actor]`: `Actor` and `Handler` impls from an annotated impl block. +use super::*; + +// Only inline a body whose suspension and wrapping behavior is visible here. +// Unknown macros/attributes can introduce awaits or wrappers after expansion. +pub(super) fn await_free_handler(method: &ImplItemFn) -> bool { + fn inert_attribute(attr: &Attribute) -> bool { + ["cfg", "doc", "allow", "warn", "deny", "forbid", "expect"] + .iter() + .any(|name| attr.path().is_ident(name)) + } + // An explicit/shared receiver can carry different borrow promises than + // Handler's &mut self. Keep its original call adapter unchanged. + let ordinary_mut_receiver = matches!(method.sig.inputs.first(), + Some(FnArg::Receiver(r)) if r.attrs.is_empty() + && r.mutability.is_some() + && r.reference.as_ref().is_some_and(|(_, lifetime)| lifetime.is_none()) + && r.colon_token.is_none()); + let plain_parameters = method.sig.inputs.iter().all(|p| match p { + FnArg::Receiver(r) => r.attrs.is_empty(), + FnArg::Typed(p) => p.attrs.is_empty(), + }); + if !ordinary_mut_receiver + || !plain_parameters + || !method.sig.generics.params.is_empty() + || method.sig.generics.where_clause.is_some() + || method.sig.asyncness.is_none() + || method.sig.unsafety.is_some() + || method.sig.abi.is_some() + || !method.attrs.iter().all(inert_attribute) + { + return false; + } + struct Check { + blocked: bool, + } + impl<'ast> syn::visit::Visit<'ast> for Check { + fn visit_expr_await(&mut self, _: &'ast syn::ExprAwait) { + self.blocked = true; + } + fn visit_expr_yield(&mut self, _: &'ast syn::ExprYield) { + self.blocked = true; + } + fn visit_macro(&mut self, _: &'ast syn::Macro) { + self.blocked = true; + } + fn visit_item(&mut self, _: &'ast syn::Item) { + // A copied static (including one inside a nested function) has + // a different identity from the inherent method's original. + self.blocked = true; + } + fn visit_expr_closure(&mut self, _: &'ast syn::ExprClosure) { + // Anonymous types also have observable identity through TypeId. + self.blocked = true; + } + fn visit_expr_async(&mut self, _: &'ast syn::ExprAsync) { + self.blocked = true; + } + fn visit_type_impl_trait(&mut self, _: &'ast syn::TypeImplTrait) { + self.blocked = true; + } + fn visit_attribute(&mut self, attr: &'ast Attribute) { + if !inert_attribute(attr) { + self.blocked = true; + } + } + } + let mut check = Check { blocked: false }; + syn::visit::Visit::visit_block(&mut check, &method.block); + syn::visit::Visit::visit_signature(&mut check, &method.sig); + !check.blocked +} + +pub(super) fn is_handler_attribute(attr: &Attribute) -> bool { + let path = attr.path(); + path.is_ident("handler") + || path.is_ident("send_handler") + || path.is_ident("request_handler") + || path.is_ident("deferred_handler") +} + +pub(super) fn is_actor_method_attribute(attr: &Attribute) -> bool { + let path = attr.path(); + is_handler_attribute(attr) + || path.is_ident("started") + || path.is_ident("stopped") + || path.is_ident("exit_received") +} + +pub(crate) fn expand(attr: TokenStream, item: TokenStream) -> TokenStream { + let mut impl_block = parse_macro_input!(item as ItemImpl); + + if let Some((_, path, _)) = &impl_block.trait_ { + return syn::Error::new_spanned( + path, + "#[actor] must be applied to an inherent impl block (`impl MyActor { ... }`), \ + not a trait impl", + ) + .to_compile_error() + .into(); + } + + // Tasks actors use `async fn` for every handler and hook, threads actors + // plain `fn`; the runtime is inferred from them. A sync method returning + // `impl Future` is valid in tasks mode, and cfg-gated methods may differ. + let mut runtime_methods = impl_block.items.iter().filter_map(|item| match item { + ImplItem::Fn(method) + if method.attrs.iter().any(is_actor_method_attribute) + && !method.attrs.iter().any(|a| a.path().is_ident("cfg")) + && !matches!(&method.sig.output, ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_))) => + { + Some(&method.sig) + } + _ => None, + }); + if let Some(first) = runtime_methods.next() { + let asynchronous = first.asyncness.is_some(); + if let Some(other) = runtime_methods.find(|sig| sig.asyncness.is_some() != asynchronous) { + return syn::Error::new_spanned( + other, + "cannot mix async and sync handlers or lifecycle hooks in one #[actor] block; \ + tasks actors use `async fn` throughout, threads actors use plain `fn`", + ) + .to_compile_error() + .into(); + } + } + + let self_ty = &impl_block.self_ty; + let (impl_generics, _, where_clause) = impl_block.generics.split_for_impl(); + + // --- Parse named parameters: #[actor(protocol = X)], #[actor(protocol(X, Y))], + // the deferred-reply bounds `max_outstanding = N`, `max_native_jobs = N`, + // and the lint opt-outs `allow_shared_state` and `allow_panic_abort` --- + let mut max_outstanding: Option = None; + let mut max_native_jobs: Option = None; + let mut allow_shared_state = false; + let mut allow_panic_abort = false; + let bridge_traits: Vec = if attr.is_empty() { + Vec::new() + } else { + // A protocol is named by a single identifier; report paths explicitly + // instead of failing on the `::` token. + fn protocol_name(input: syn::parse::ParseStream) -> syn::Result { + let path: syn::Path = input.parse()?; + path.get_ident().cloned().ok_or_else(|| { + syn::Error::new_spanned( + &path, + "protocol must be named by a single identifier; \ + import the trait first (`use path::to::Protocol;`)", + ) + }) + } + let parser = |input: syn::parse::ParseStream| -> syn::Result> { + let mut protocols = Vec::new(); + while !input.is_empty() { + let key: Ident = input.parse()?; + if key == "max_outstanding" || key == "max_native_jobs" { + let _: syn::Token![=] = input.parse()?; + let value: syn::Expr = input.parse()?; + let slot = if key == "max_outstanding" { + &mut max_outstanding + } else { + &mut max_native_jobs + }; + if slot.replace(value).is_some() { + return Err(syn::Error::new(key.span(), format!("`{key}` given twice"))); + } + } else if key == "allow_shared_state" { + allow_shared_state = true; + } else if key == "allow_panic_abort" { + allow_panic_abort = true; + } else if key != "protocol" { + return Err(syn::Error::new( + key.span(), + "unknown parameter, expected `protocol`, `max_outstanding`, \ + `max_native_jobs`, `allow_shared_state` or `allow_panic_abort`", + )); + } else if input.peek(syn::Token![=]) { + // protocol = TraitName + let _: syn::Token![=] = input.parse()?; + protocols.push(protocol_name(input)?); + } else if input.peek(syn::token::Paren) { + // protocol(Trait1, Trait2) + let content; + syn::parenthesized!(content in input); + let punctuated = content.parse_terminated(protocol_name, syn::Token![,])?; + protocols.extend(punctuated); + } else { + return Err(syn::Error::new( + key.span(), + "expected `protocol = Trait` or `protocol(TraitA, TraitB)`", + )); + } + if !input.is_empty() { + let _: syn::Token![,] = input.parse().map_err(|e| { + syn::Error::new(e.span(), "expected `,` between #[actor] parameters") + })?; + } + } + Ok(protocols) + }; + match syn::parse::Parser::parse(parser, attr) { + Ok(traits) => traits, + Err(e) => return e.to_compile_error().into(), + } + }; + + // --- Compile-time lints (ROADMAP rank 6b); no runtime code is generated --- + let lints = actor_lints(&impl_block, allow_shared_state, allow_panic_abort); + + // --- Extract #[started] and #[stopped] lifecycle methods --- + let mut started_method: Option = None; + let mut stopped_method: Option = None; + let mut exit_method: Option = None; + let mut has_async = false; + + let mut items_to_keep = Vec::new(); + for item in impl_block.items.drain(..) { + if let ImplItem::Fn(ref method) = item { + let is_started = method.attrs.iter().any(|a| a.path().is_ident("started")); + let is_stopped = method.attrs.iter().any(|a| a.path().is_ident("stopped")); + + let is_exit = method + .attrs + .iter() + .any(|a| a.path().is_ident("exit_received")); + if is_exit { + if exit_method.is_some() { + return syn::Error::new_spanned( + &method.sig, + "only one #[exit_received] method is allowed per actor", + ) + .to_compile_error() + .into(); + } + if is_started || is_stopped || method.attrs.iter().any(is_handler_attribute) { + return syn::Error::new_spanned(&method.sig, "#[exit_received] cannot be combined with other lifecycle or handler attributes").to_compile_error().into(); + } + if method.sig.inputs.len() != 3 + || !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) + { + return syn::Error::new_spanned( + &method.sig, + "#[exit_received] must take (&mut self, Exit, &Context)", + ) + .to_compile_error() + .into(); + } + let mut m = method.clone(); + m.attrs.retain(|a| !a.path().is_ident("exit_received")); + m.vis = syn::Visibility::Inherited; + m.sig.ident = format_ident!("exit_received"); + if m.sig.asyncness.is_some() { + has_async = true; + } + exit_method = Some(m); + continue; + } + + if is_started && is_stopped { + return syn::Error::new_spanned( + &method.sig, + "#[started] and #[stopped] cannot be combined on one method", + ) + .to_compile_error() + .into(); + } + + if is_started { + if started_method.is_some() { + return syn::Error::new_spanned( + &method.sig, + "only one #[started] method is allowed per actor", + ) + .to_compile_error() + .into(); + } + if method.attrs.iter().any(is_handler_attribute) { + return syn::Error::new_spanned( + &method.sig, + "#[started] cannot be combined with handler attributes", + ) + .to_compile_error() + .into(); + } + // Expect: fn started(&mut self, ctx: &Context) + if method.sig.inputs.len() != 2 { + return syn::Error::new_spanned( + &method.sig, + "#[started] method must take exactly (&mut self, &Context)", + ) + .to_compile_error() + .into(); + } + if !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) + { + return syn::Error::new_spanned( + &method.sig, + "#[started] method's first parameter must be `&mut self`", + ) + .to_compile_error() + .into(); + } + let mut m = method.clone(); + m.attrs.retain(|a| !a.path().is_ident("started")); + m.vis = syn::Visibility::Inherited; + m.sig.ident = format_ident!("started"); + if m.sig.asyncness.is_some() { + has_async = true; + } + started_method = Some(m); + continue; + } + + if is_stopped { + if stopped_method.is_some() { + return syn::Error::new_spanned( + &method.sig, + "only one #[stopped] method is allowed per actor", + ) + .to_compile_error() + .into(); + } + if method.attrs.iter().any(is_handler_attribute) { + return syn::Error::new_spanned( + &method.sig, + "#[stopped] cannot be combined with handler attributes", + ) + .to_compile_error() + .into(); + } + // Expect: fn stopped(&mut self, ctx: &Context) + if method.sig.inputs.len() != 2 { + return syn::Error::new_spanned( + &method.sig, + "#[stopped] method must take exactly (&mut self, &Context)", + ) + .to_compile_error() + .into(); + } + if !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) + { + return syn::Error::new_spanned( + &method.sig, + "#[stopped] method's first parameter must be `&mut self`", + ) + .to_compile_error() + .into(); + } + let mut m = method.clone(); + m.attrs.retain(|a| !a.path().is_ident("stopped")); + m.vis = syn::Visibility::Inherited; + m.sig.ident = format_ident!("stopped"); + if m.sig.asyncness.is_some() { + has_async = true; + } + stopped_method = Some(m); + continue; + } + } + items_to_keep.push(item); + } + impl_block.items = items_to_keep; + + // --- Process handler methods --- + let mut handler_impls = Vec::new(); + let mut lifetime_name = "__spawned_reply_turn".to_owned(); + while impl_block + .generics + .lifetimes() + .any(|p| p.lifetime.ident == lifetime_name) + { + lifetime_name.push('_'); + } + let turn_lifetime = syn::Lifetime::new( + &format!("'{lifetime_name}"), + proc_macro2::Span::mixed_site(), + ); + + for item in &mut impl_block.items { + if let ImplItem::Fn(method) = item { + let handler_idx = method.attrs.iter().position(is_handler_attribute); + + if let Some(idx) = handler_idx { + let deferred = method.attrs[idx].path().is_ident("deferred_handler"); + let send = method.attrs[idx].path().is_ident("send_handler"); + if method.attrs[idx].path().is_ident("handler") { + return syn::Error::new_spanned( + &method.attrs[idx], + "#[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value)", + ) + .to_compile_error() + .into(); + } + method.attrs.remove(idx); + if let Some(extra) = method.attrs.iter().find(|a| is_handler_attribute(a)) { + return syn::Error::new_spanned( + extra, + "a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler]", + ) + .to_compile_error() + .into(); + } + if !method.sig.generics.params.is_empty() { + return syn::Error::new_spanned( + &method.sig.generics, + "handler methods cannot be generic; write one handler per message type", + ) + .to_compile_error() + .into(); + } + + // Collect remaining attributes (e.g. #[cfg(...)]) to propagate + // to the generated Handler impl block. + let extra_attrs: Vec<_> = method + .attrs + .iter() + .filter(|a| { + !is_handler_attribute(a) + && !a.path().is_ident("started") + && !a.path().is_ident("stopped") + }) + .cloned() + .collect(); + + let method_name = &method.sig.ident; + if method.sig.asyncness.is_some() { + has_async = true; + } + + // Validate parameters: (&mut self, msg, ctx), or for a deferred + // handler (&mut self, msg, reply, ctx). + let param_count = method.sig.inputs.len(); + let ctx_index = if deferred { 3 } else { 2 }; + if deferred && param_count != 4 { + return syn::Error::new_spanned( + &method.sig, + format!( + "#[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found {param_count}" + ), + ) + .to_compile_error() + .into(); + } + if !deferred && param_count != 3 { + return syn::Error::new_spanned( + &method.sig, + format!( + "handler method must have 3 parameters (&mut self, msg: M, ctx: &Context), found {param_count}" + ), + ) + .to_compile_error() + .into(); + } + if deferred { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned( + ty, + "#[deferred_handler] answers through its PendingReply; remove the return type", + ) + .to_compile_error() + .into(); + } + } + } + // A send handler that returned a value would silently answer a + // `request` with it. (`impl Future` returns are checked by the + // `Result = ()` bound of the message instead.) + if send { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) && !matches!(**ty, Type::ImplTrait(_)) { + return syn::Error::new_spanned( + ty, + "#[send_handler] methods return `()`; use #[request_handler] to answer a request with a value", + ) + .to_compile_error() + .into(); + } + } + } + + // `&self` is accepted too; a missing or by-value receiver is not. + match method.sig.inputs.first() { + Some(FnArg::Receiver(r)) + if r.reference.is_some() || r.colon_token.is_some() => {} + Some(first) => { + return syn::Error::new_spanned( + first, + "handler method's first parameter must be `&mut self`", + ) + .to_compile_error() + .into(); + } + None => unreachable!("the parameters were checked above"), + } + if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(ctx_index) { + if !matches!(&*ctx.ty, Type::Reference(r) if r.mutability.is_none()) { + return syn::Error::new_spanned( + &ctx.ty, + "handler context parameter must be `&Context`", + ) + .to_compile_error() + .into(); + } + } + + // Extract message type from 2nd parameter (index 1, after &mut self) + let msg_ty = match method.sig.inputs.iter().nth(1) { + Some(FnArg::Typed(pat_type)) => &*pat_type.ty, + _ => { + return syn::Error::new_spanned( + &method.sig, + "handler method must have signature: fn(&mut self, msg: M, ctx: &Context) -> R", + ) + .to_compile_error() + .into(); + } + }; + + // The generated `handle` is the envelope dispatch: it takes the + // request's reply obligation and answers it with the method's + // result (a deferred handler receives the obligation itself). + let reply_ty = quote! { + spawned_concurrency::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> + }; + // Mixed-site locals cannot shadow free names in a user body + // that is specialized into the generated dispatch adapter. + let msg_name = Ident::new("__spawned_msg", proc_macro2::Span::mixed_site()); + let reply_name = Ident::new("__spawned_reply", proc_macro2::Span::mixed_site()); + let ctx_name = Ident::new( + "__spawned_ctx", + proc_macro2::Span::mixed_site().located_at( + method + .sig + .inputs + .iter() + .nth(ctx_index) + .expect("validated context arity") + .span(), + ), + ); + // The parameter list carries the user's parentheses span. + let mut params = proc_macro2::Group::new( + proc_macro2::Delimiter::Parenthesis, + quote! { &mut self, #msg_name: #msg_ty, #reply_name: #reply_ty, #ctx_name: &Context }, + ); + params.set_span(method.sig.paren_token.span.join()); + // Span the generated impl at the method so trait errors + // (conflicting impls, signature mismatches) point at it. + let span = method.sig.span(); + // The reply's type must equal the message's `Result`; a + // mismatch is reported at the user's return type. + let ret_span = match &method.sig.output { + ReturnType::Default => method.sig.paren_token.span.join(), + ReturnType::Type(_, ty) => ty.span(), + }; + // A tasks handler may also be a plain `fn` returning + // `impl Future`; its result is awaited like an `async fn`'s. + let returns_future = matches!( + &method.sig.output, + ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_)) + ); + let asynchronous = method.sig.asyncness.is_some() || returns_future; + let immediate = asynchronous && !deferred && await_free_handler(method); + let call = match (deferred, asynchronous) { + (true, true) => { + quote! { self.#method_name(#msg_name, #reply_name, #ctx_name).await } + } + (true, false) => { + quote! { self.#method_name(#msg_name, #reply_name, #ctx_name) } + } + (false, true) if immediate => { + let body = &method.block; + let msg_pat = match method + .sig + .inputs + .iter() + .nth(1) + .expect("validated message arity") + { + FnArg::Typed(arg) => &arg.pat, + _ => unreachable!(), + }; + let (ctx_pat, ctx_ty) = match method + .sig + .inputs + .iter() + .nth(ctx_index) + .expect("validated context arity") + { + FnArg::Typed(arg) => (&arg.pat, &arg.ty), + _ => unreachable!(), + }; + let output = match &method.sig.output { + ReturnType::Default => quote! { () }, + ReturnType::Type(_, ty) => quote! { #ty }, + }; + // Dispatch has already begun when this adapter is + // constructed. Keep the inherent async method lazy, + // but evaluate this await-free dispatch without a + // heap-stored future. A closure preserves return/?. + let answer = quote::quote_spanned! { ret_span => + let _ = #reply_name.reply(value); + }; + quote! { + // Function/closure parameters retain ignored fields + // until the body returns. A destructuring let here + // would drop them before the handler body runs. + let value = (|#msg_pat: #msg_ty, #ctx_pat: #ctx_ty| -> #output #body)( + #msg_name, #ctx_name + ); + #answer + } + } + (false, true) => { + let value = quote::quote_spanned! { ret_span => self.#method_name(#msg_name, #ctx_name).await }; + quote! { let _ = #reply_name.reply(#value); } + } + (false, false) => { + let value = quote::quote_spanned! { ret_span => self.#method_name(#msg_name, #ctx_name) }; + quote! { let _ = #reply_name.reply(#value); } + } + }; + + let handler_impl = if immediate { + quote::quote_spanned! { span => + #(#extra_attrs)* + impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { + async fn handle #params { + let _ = #reply_name.reply(self.#method_name(#msg_name, #ctx_name).await); + } + + fn __spawned_handle_immediate<#turn_lifetime>( + &mut self, + #msg_name: #msg_ty, + #reply_name: spawned_concurrency::PendingReply<#turn_lifetime, <#msg_ty as spawned_concurrency::message::Message>::Result>, + #ctx_name: &Context, + ) -> ::core::option::Option<(#msg_ty, spawned_concurrency::PendingReply<#turn_lifetime, <#msg_ty as spawned_concurrency::message::Message>::Result>)> { + // Keep payload destructors inside the original + // future, whose drop is fenced by DeathFirst. + // A refused result is dropped by reply() in + // either adapter, after the method returned; + // unlike a parameter, it adds no unwind capture. + // This constant branch disappears for messages + // without drop glue. + if ::core::mem::needs_drop::<#msg_ty>() { + return ::core::option::Option::Some((#msg_name, #reply_name)); + } + #call + ::core::option::Option::None + } + } + } + } else if asynchronous { + quote::quote_spanned! { span => + #(#extra_attrs)* + impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { + async fn handle #params { + #call + } + } + } + } else { + quote::quote_spanned! { span => + #(#extra_attrs)* + impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { + fn handle #params { + #call + } + } + } + }; + + handler_impls.push(handler_impl); + } + } + } + + // --- Generate impl Actor --- + let lifecycle_methods: Vec<&ImplItemFn> = [ + started_method.as_ref(), + stopped_method.as_ref(), + exit_method.as_ref(), + ] + .into_iter() + .flatten() + .collect(); + + let protocol_doc = if bridge_traits.is_empty() { + quote! {} + } else { + let lines: Vec = bridge_traits.iter().map(|t| format!("- [`{t}`]")).collect(); + let doc_body = format!( + "# Protocol\n\n\ + When started, `ActorRef<{ty}>` implements:\n\n\ + {lines}\n\n\ + See the protocol trait docs for the full API.", + ty = quote!(#self_ty), + lines = lines.join("\n"), + ); + quote! { #[doc = #doc_body] } + }; + + let bound_consts = { + let outstanding = max_outstanding.map(|n| quote! { const MAX_OUTSTANDING: usize = #n; }); + let native = max_native_jobs + .map(|n| quote! { const MAX_NATIVE_JOBS: ::core::option::Option = ::core::option::Option::Some(#n); }); + quote! { #outstanding #native } + }; + + let actor_impl = quote! { + #protocol_doc + impl #impl_generics Actor for #self_ty #where_clause { + #bound_consts + #(#lifecycle_methods)* + } + }; + + // --- Generate bridge assertions --- + let runtime_path = if has_async { + quote! { spawned_concurrency::tasks } + } else { + quote! { spawned_concurrency::threads } + }; + + let bridge_asserts: Vec<_> = bridge_traits + .iter() + .map(|trait_name| { + quote! { + const _: () = { + fn _assert_bridge<__T: #trait_name>() {} + fn _check #impl_generics () #where_clause { + _assert_bridge::<#runtime_path::ActorRef<#self_ty>>(); + } + }; + } + }) + .collect(); + + let output = quote! { + #actor_impl + #impl_block + #(#handler_impls)* + #(#bridge_asserts)* + #lints + }; + + output.into() +} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index ff50a5b1..e0229c9f 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -6,649 +6,13 @@ use syn::{ ImplItemFn, ItemImpl, ItemTrait, Pat, PathArguments, ReturnType, TraitItem, Type, TypePath, }; -// --- Helpers for #[protocol] --- - -fn to_snake_case(s: &str) -> String { - let mut result = String::new(); - let chars: Vec = s.chars().collect(); - for (i, &ch) in chars.iter().enumerate() { - if ch.is_uppercase() { - // Insert underscore before uppercase if: - // - not at start, AND - // - previous char is lowercase, OR next char is lowercase (handles acronyms) - if i > 0 { - let prev_lower = chars[i - 1].is_lowercase(); - let next_lower = chars.get(i + 1).is_some_and(|c| c.is_lowercase()); - if prev_lower || next_lower { - result.push('_'); - } - } - result.push(ch.to_ascii_lowercase()); - } else { - result.push(ch); - } - } - result -} - -fn to_pascal_case(s: &str) -> String { - s.split('_') - .filter(|part| !part.is_empty()) - .map(|part| { - let mut chars = part.chars(); - match chars.next() { - None => String::new(), - Some(c) => c.to_uppercase().to_string() + chars.as_str(), - } - }) - .collect() -} - -fn strip_protocol_suffix(name: &str) -> String { - name.strip_suffix("Protocol").unwrap_or(name).to_string() -} - -// A method-level cfg_attr has not been expanded when the enclosing protocol -// macro runs. Reject conditional timeout attributes even for false predicates; -// otherwise Rust sees an unknown attribute, or silently omits the timeout. -fn conditional_timeout(meta: &syn::Meta) -> bool { - if let syn::Meta::List(list) = meta { - if list.path.is_ident("cfg_attr") { - use syn::parse::Parser; - let arguments = - syn::punctuated::Punctuated::::parse_terminated; - if let Ok(arguments) = arguments.parse2(list.tokens.clone()) { - return arguments.iter().skip(1).any(|meta| { - meta.path().is_ident("request_timeout") || conditional_timeout(meta) - }); - } - } - } - false -} - -enum MethodKind { - Send, - Request(Box), -} - -#[derive(Clone, Copy)] -enum RuntimeMode { - Tasks, - Threads, -} - -fn classify_return_type(ret: &ReturnType) -> Result { - match ret { - ReturnType::Default => Ok(MethodKind::Send), - ReturnType::Type(_, ty) => { - if is_unit_type(ty) { - return Ok(MethodKind::Send); - } - if let Some(inner) = extract_response_inner(ty) { - return Ok(MethodKind::Request(inner)); - } - if let Some(inner) = extract_result_inner(ty) { - if is_unit_type(&inner) { - return Ok(MethodKind::Send); - } - // Result where T ≠ () is no longer supported; - // use Response which works in both modes. - return Err(ty); - } - Err(ty) - } - } -} - -fn extract_response_inner(ty: &Type) -> Option> { - if let Type::Path(TypePath { path, .. }) = ty { - let seg = path.segments.last()?; - if seg.ident == "Response" { - if let PathArguments::AngleBracketed(args) = &seg.arguments { - if let Some(GenericArgument::Type(inner)) = args.args.first() { - return Some(Box::new(inner.clone())); - } - } - } - } - None -} - -fn extract_result_inner(ty: &Type) -> Option> { - if let Type::Path(TypePath { path, .. }) = ty { - let seg = path.segments.last()?; - if seg.ident == "Result" { - if let PathArguments::AngleBracketed(args) = &seg.arguments { - if let Some(GenericArgument::Type(inner)) = args.args.first() { - return Some(Box::new(inner.clone())); - } - } - } - } - None -} - -/// Protocol parameters become fields of a `'static` message struct. Report -/// borrows and `impl Trait` here; rustc's own errors point into the expansion. -fn unsupported_param_type(ty: &Type) -> Option { - const OWNED: &str = "protocol parameters become message fields and must be owned ('static); \ - use an owned type such as String or Arc instead of a borrow"; - match ty { - Type::Reference(r) => match &r.lifetime { - Some(lifetime) if lifetime.ident == "static" => unsupported_param_type(&r.elem), - _ => Some(syn::Error::new_spanned(r, OWNED)), - }, - Type::ImplTrait(t) => Some(syn::Error::new_spanned( - t, - "`impl Trait` parameters are not supported in protocol methods; \ - use a concrete type or Box", - )), - Type::Path(TypePath { qself, path }) => { - if let Some(error) = qself.as_ref().and_then(|q| unsupported_param_type(&q.ty)) { - return Some(error); - } - path.segments - .iter() - .find_map(|segment| match &segment.arguments { - PathArguments::AngleBracketed(args) => { - args.args.iter().find_map(|arg| match arg { - GenericArgument::Type(inner) => unsupported_param_type(inner), - GenericArgument::Lifetime(lifetime) if lifetime.ident != "static" => { - Some(syn::Error::new_spanned(lifetime, OWNED)) - } - _ => None, - }) - } - _ => None, - }) - } - Type::Tuple(t) => t.elems.iter().find_map(unsupported_param_type), - Type::Array(a) => unsupported_param_type(&a.elem), - Type::Slice(s) => unsupported_param_type(&s.elem), - Type::Paren(p) => unsupported_param_type(&p.elem), - Type::Group(g) => unsupported_param_type(&g.elem), - // fn pointers and trait objects may use higher-ranked borrows. - _ => None, - } -} - -fn is_unit_type(ty: &Type) -> bool { - if let Type::Tuple(tuple) = ty { - return tuple.elems.is_empty(); - } - false -} - -/// Returns true for types available without explicit import (prelude + primitives). -/// These can't be accessed via `super::` so must be left unqualified. -fn is_prelude_or_primitive(name: &str) -> bool { - matches!( - name, - // Primitives - "bool" | "char" | "str" - | "i8" | "i16" | "i32" | "i64" | "i128" | "isize" - | "u8" | "u16" | "u32" | "u64" | "u128" | "usize" - | "f32" | "f64" - // Prelude types (Rust 2021) - | "Box" | "String" | "Vec" - | "Option" | "Some" | "None" - | "Result" | "Ok" | "Err" - | "ToString" | "ToOwned" - ) -} - -/// Qualify a type with `super::` so it resolves to the parent module's scope. -/// This prevents name collisions when a generated struct name shadows an imported type -/// via `use super::*`. -/// -/// Prelude/primitive types are left unqualified (they can't be accessed via `super::`). -/// User-defined types get `super::` prepended: `Event` → `super::Event`. -/// Generic args are recursively qualified: `Vec` → `Vec`. -fn qualify_type_with_super(ty: &Type) -> Type { - match ty { - Type::Path(TypePath { qself, path }) => { - // Leave qualified paths as-is: ::Y, ::abs::path, crate::, super::, self:: - if qself.is_some() || path.leading_colon.is_some() { - return ty.clone(); - } - if let Some(first) = path.segments.first() { - let s = first.ident.to_string(); - if matches!( - s.as_str(), - "crate" | "super" | "self" | "std" | "core" | "alloc" - ) { - return ty.clone(); - } - } - - // Recursively qualify generic arguments in all segments - let qualified_segments: syn::punctuated::Punctuated<_, _> = path - .segments - .iter() - .map(|seg| { - let mut new_seg = seg.clone(); - if let PathArguments::AngleBracketed(ref mut args) = new_seg.arguments { - for arg in &mut args.args { - if let GenericArgument::Type(ref mut inner) = arg { - *inner = qualify_type_with_super(inner); - } - } - } - new_seg - }) - .collect(); - - // Only prepend super:: for non-prelude types - if let Some(first) = path.segments.first() { - if is_prelude_or_primitive(&first.ident.to_string()) { - return Type::Path(TypePath { - qself: None, - path: syn::Path { - leading_colon: None, - segments: qualified_segments, - }, - }); - } - } - - // Prepend super:: for user-defined types - let mut segments = syn::punctuated::Punctuated::new(); - segments.push(syn::PathSegment { - ident: format_ident!("super"), - arguments: PathArguments::None, - }); - for seg in qualified_segments { - segments.push(seg); - } - - Type::Path(TypePath { - qself: None, - path: syn::Path { - leading_colon: None, - segments, - }, - }) - } - Type::Reference(r) => { - let mut new = r.clone(); - new.elem = Box::new(qualify_type_with_super(&r.elem)); - Type::Reference(new) - } - Type::Tuple(t) => { - let mut new = t.clone(); - for elem in &mut new.elems { - *elem = qualify_type_with_super(elem); - } - Type::Tuple(new) - } - Type::Array(a) => { - let mut new = a.clone(); - *new.elem = qualify_type_with_super(&a.elem); - Type::Array(new) - } - Type::Slice(s) => { - let mut new = s.clone(); - *new.elem = qualify_type_with_super(&s.elem); - Type::Slice(new) - } - Type::Paren(p) => { - let mut new = p.clone(); - *new.elem = qualify_type_with_super(&p.elem); - Type::Paren(new) - } - Type::TraitObject(t) => { - let mut new = t.clone(); - for bound in &mut new.bounds { - if let syn::TypeParamBound::Trait(tb) = bound { - qualify_path_with_super(&mut tb.path); - } - } - Type::TraitObject(new) - } - Type::ImplTrait(t) => { - let mut new = t.clone(); - for bound in &mut new.bounds { - if let syn::TypeParamBound::Trait(tb) = bound { - qualify_path_with_super(&mut tb.path); - } - } - Type::ImplTrait(new) - } - Type::BareFn(f) => { - let mut new = f.clone(); - for arg in &mut new.inputs { - arg.ty = qualify_type_with_super(&arg.ty); - } - if let ReturnType::Type(_, ref mut ty) = new.output { - **ty = qualify_type_with_super(ty); - } - Type::BareFn(new) - } - _ => ty.clone(), - } -} - -/// Qualify a path's generic arguments with `super::`, used for trait bounds -/// in `dyn Trait` and `impl Trait` types. -fn qualify_path_with_super(path: &mut syn::Path) { - for seg in &mut path.segments { - match &mut seg.arguments { - PathArguments::AngleBracketed(ref mut args) => { - for arg in &mut args.args { - if let GenericArgument::Type(ref mut inner) = arg { - *inner = qualify_type_with_super(inner); - } - } - } - PathArguments::Parenthesized(ref mut args) => { - for input in &mut args.inputs { - *input = qualify_type_with_super(input); - } - if let syn::ReturnType::Type(_, ref mut ty) = args.output { - **ty = qualify_type_with_super(ty); - } - } - PathArguments::None => {} - } - } -} - -struct ProtocolInfo<'a> { - trait_name: &'a Ident, - mod_name: &'a Ident, - ref_name: &'a Ident, - converter_trait: &'a Ident, - converter_method: &'a Ident, - target_trait: &'a Ident, -} - -/// One transport signature used in declarations, runtime impls and delegation. -fn transport_signature( - info: &ProtocolInfo, - method: &ProtocolMethodInfo, - index: usize, -) -> (Ident, proc_macro2::TokenStream) { - let name = format_ident!("__dispatch_{}", index); - let module = info.mod_name; - let message = &method.struct_name; - let (timeout, result) = match &method.kind { - MethodKind::Send => ( - quote! {}, - quote! { ::std::result::Result<(), spawned_concurrency::ActorError> }, - ), - MethodKind::Request(result) => ( - quote! { , timeout: ::std::time::Duration }, - quote! { spawned_concurrency::Response<#result> }, - ), - }; - let signature = - quote! { fn #name(target: &Self, message: #module::#message #timeout) -> #result }; - (name, signature) -} - -/// Runtime-specific transport primitives, shared by plain and configured handles. -/// Cfg-gated Handler bounds are represented by one marker set per runtime. -fn generate_target_impl( - info: &ProtocolInfo, - methods: &[ProtocolMethodInfo], - runtime_path: &proc_macro2::TokenStream, - mode: RuntimeMode, -) -> proc_macro2::TokenStream { - let ProtocolInfo { - trait_name, - mod_name, - target_trait, - .. - } = info; - - // Unconditional methods: Handler bounds go directly on the impl block. - let handler_bounds: Vec<_> = methods - .iter() - .filter(|m| m.cfg_attrs.is_empty()) - .map(|m| { - let sn = &m.struct_name; - quote! { #runtime_path::Handler<#mod_name::#sn> } - }) - .collect(); - - // Group cfg-gated methods by their cfg predicate. - // Each unique group gets a marker trait that conditionally requires the Handler bounds. - let mut cfg_groups: Vec<(String, Vec<&ProtocolMethodInfo>)> = Vec::new(); - for m in methods.iter().filter(|m| !m.cfg_attrs.is_empty()) { - let key: String = m - .cfg_attrs - .iter() - .map(|a| quote!(#a).to_string()) - .collect::>() - .join(","); - if let Some(group) = cfg_groups.iter_mut().find(|(k, _)| k == &key) { - group.1.push(m); - } else { - cfg_groups.push((key, vec![m])); - } - } - - let mode_suffix = match mode { - RuntimeMode::Tasks => format_ident!("Tasks"), - RuntimeMode::Threads => format_ident!("Threads"), - }; - - let mut marker_trait_defs = Vec::new(); - let mut marker_trait_bounds = Vec::new(); - - for (i, (_key, group_methods)) in cfg_groups.iter().enumerate() { - let marker_name = format_ident!("__{}Cfg{}{}", trait_name, i, mode_suffix); - let cfg_attrs = &group_methods[0].cfg_attrs; - let group_handler_bounds: Vec<_> = group_methods - .iter() - .map(|m| { - let sn = &m.struct_name; - quote! { #runtime_path::Handler<#mod_name::#sn> } - }) - .collect(); - - // Combine cfg predicates and negate for the fallback impl. - // #[cfg(A)] #[cfg(B)] -> active: all(A, B), inactive: not(all(A, B)) - let cfg_predicates: Vec = cfg_attrs - .iter() - .filter(|a| a.path().is_ident("cfg")) - .filter_map(|a| a.parse_args::().ok()) - .collect(); - - let (positive_cfg, negated_cfg) = if cfg_predicates.len() == 1 { - let pred = &cfg_predicates[0]; - (quote! { #[cfg(#pred)] }, quote! { #[cfg(not(#pred))] }) - } else { - ( - quote! { #[cfg(all(#(#cfg_predicates),*))] }, - quote! { #[cfg(not(all(#(#cfg_predicates),*)))] }, - ) - }; - - marker_trait_defs.push(quote! { - #positive_cfg - #[doc(hidden)] - trait #marker_name: #(#group_handler_bounds)+* {} - #positive_cfg - impl<__T: #(#group_handler_bounds)+*> #marker_name for __T {} - - #negated_cfg - #[doc(hidden)] - trait #marker_name {} - #negated_cfg - impl<__T> #marker_name for __T {} - }); - - marker_trait_bounds.push(quote! { + #marker_name }); - } - - let method_impls = methods.iter().enumerate().map(|(index, m)| { - let (_, signature) = transport_signature(info, m, index); - let cfg = &m.cfg_attrs; - let body = match &m.kind { - MethodKind::Send => quote! { target.send(message) }, - MethodKind::Request(_) => match mode { - // The timeout was chosen before the message was built; the - // deadline is fixed now, at send, and travels in the slot. - RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.send_request(message, spawned_concurrency::Deadline::after(timeout))) }, - RuntimeMode::Threads => quote! { spawned_concurrency::Response::ready(target.request_with_timeout(message, timeout)) }, - }, - }; - quote! { #(#cfg)* #signature { #body } } - }); - quote! { - #(#marker_trait_defs)* - impl<__A: #runtime_path::Actor #(+ #handler_bounds)* #(#marker_trait_bounds)*> - #target_trait for #runtime_path::ActorRef<__A> - { - #(#method_impls)* - } - } -} - -/// Share transport bodies and one method template while keeping public protocol -/// impls concrete. Supertraits constrain each nominal target independently. -fn generate_protocol_adapters( - info: &ProtocolInfo, - methods: &[ProtocolMethodInfo], - definition: &ItemTrait, -) -> proc_macro2::TokenStream { - let ProtocolInfo { - trait_name, - mod_name, - ref_name, - converter_trait, - converter_method, - target_trait, - } = info; - let signatures = methods.iter().enumerate().map(|(index, m)| { - let (_, signature) = transport_signature(info, m, index); - let cfg = &m.cfg_attrs; - quote! { #(#cfg)* #signature; } - }); - let delegates = methods.iter().enumerate().map(|(index, m)| { - let (name, signature) = transport_signature(info, m, index); - let cfg = &m.cfg_attrs; - let timeout = match &m.kind { MethodKind::Send => quote! {}, MethodKind::Request(_) => quote! { , timeout } }; - quote! { - #(#cfg)* #signature { - <__T as #target_trait>::#name( - spawned_concurrency::response::RequestOptions::actor_ref(target), message #timeout - ) - } - } - }); - let method_impls: Vec<_> = methods.iter().enumerate().map(|(index, m)| { - let method = &m.method_name; - let dispatch = format_ident!("__dispatch_{}", index); - let params = &m.params; - let attrs = &m.method_attrs; - let result = &m.ret_type; - let message = &m.struct_name; - let fields = &m.field_names; - let construct = if fields.is_empty() { - quote! { #mod_name::#message } - } else { - quote! { #mod_name::#message { #(#fields),* } } - }; - let body = match &m.kind { - MethodKind::Send => { - let call = quote! { ::#dispatch(self, #construct) }; - let unit = match result { - ReturnType::Default => true, - ReturnType::Type(_, ty) => is_unit_type(ty), - }; - if unit { quote! { let _ = #call; } } else { call } - } - MethodKind::Request(_) => { - let timeout = &m.timeout; - let binding = syn::Ident::new("__spawned_request_timeout", proc_macro2::Span::mixed_site()); - quote! { - // Evaluate before message construction can move arguments; - // an explicit override does not evaluate the default expression. - let #binding: ::std::time::Duration = match ::__timeout_override(self) { - Some(timeout) => timeout, - None => #timeout, - }; - ::#dispatch(self, #construct, #binding) - } - } - }; - quote! { - #(#attrs)* - fn #method(&self, #(#params),*) #result { #body } - } - }).collect(); - // Keep public impls nominal. A blanket impl over the hidden target trait - // conflicts with existing Box/&T protocol forwarding implementations. - let mut public_impls = Vec::new(); - for (runtime, asynchronous) in [ - (quote! { spawned_concurrency::tasks }, true), - (quote! { spawned_concurrency::threads }, false), - ] { - let actor = quote! { #runtime::ActorRef<__A> }; - for target in [ - actor.clone(), - quote! { spawned_concurrency::response::RequestOptions<#actor> }, - ] { - let mut predicates: Vec<_> = definition - .generics - .where_clause - .iter() - .flat_map(|clause| clause.predicates.iter()) - .map(|p| quote! { #p }) - .collect(); - let supertraits = &definition.supertraits; - if !supertraits.is_empty() { - predicates.push(quote! { Self: #supertraits }); - } - let implementation = quote! { - impl<__A: #runtime::Actor> #trait_name for #target - where Self: #target_trait, #(#predicates,)* - { #(#method_impls)* } - impl<__A: #runtime::Actor> #converter_trait for #target - where Self: #target_trait + #trait_name - { - fn #converter_method(&self) -> #ref_name { ::std::sync::Arc::new(self.clone()) } - } - }; - public_impls.push(if asynchronous { - quote! { spawned_concurrency::__spawned_async_items! { #implementation } } - } else { - implementation - }); - } - } - quote! { - #[doc(hidden)] - trait #target_trait: Clone + Send + Sync + 'static { - fn __timeout_override(_target: &Self) -> Option<::std::time::Duration> { None } - #(#signatures)* - } - impl<__T: #target_trait> #target_trait for spawned_concurrency::response::RequestOptions<__T> { - fn __timeout_override(target: &Self) -> Option<::std::time::Duration> { - Some(spawned_concurrency::response::RequestOptions::timeout(target)) - } - #(#delegates)* - } - #(#public_impls)* - } -} - -struct ProtocolMethodInfo { - method_name: Ident, - struct_name: Ident, - field_names: Vec, - field_types: Vec, - kind: MethodKind, - params: Vec, - ret_type: ReturnType, - /// Doc + cfg attributes to propagate to blanket impl methods. - method_attrs: Vec, - cfg_attrs: Vec, - timeout: syn::Expr, -} +mod actor; +mod lints; +mod protocol; +mod send_handler; +mod sim_test; +use lints::*; +use protocol::*; /// Generates message types and blanket implementations from a protocol trait. /// @@ -729,436 +93,7 @@ struct ProtocolMethodInfo { /// immediately; async responses wait until that deadline when polled. #[proc_macro_attribute] pub fn protocol(attr: TokenStream, item: TokenStream) -> TokenStream { - let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); - let mut protocol_timeout = None; - for option in options { - if !option.path().is_ident("timeout") { - return syn::Error::new_spanned( - option.path(), - "unsupported protocol option; expected timeout = Duration expression", - ) - .to_compile_error() - .into(); - } - let syn::Meta::NameValue(option) = option else { - return syn::Error::new_spanned(option, "expected `timeout = `") - .to_compile_error() - .into(); - }; - if protocol_timeout.replace(option.value).is_some() { - return syn::Error::new_spanned(option.path, "duplicate protocol timeout") - .to_compile_error() - .into(); - } - } - let mut trait_def = parse_macro_input!(item as ItemTrait); - let trait_name = &trait_def.ident; - let trait_vis = &trait_def.vis; - - if !trait_def.generics.params.is_empty() { - return syn::Error::new_spanned( - &trait_def.generics, - "generic parameters (types, lifetimes, consts) on protocol traits are not supported", - ) - .to_compile_error() - .into(); - } - - let base_name = strip_protocol_suffix(&trait_name.to_string()); - let mod_name = format_ident!("{}", to_snake_case(&trait_name.to_string())); - let ref_name = format_ident!("{}Ref", base_name); - let converter_trait = format_ident!("To{}Ref", base_name); - let converter_method = format_ident!("to_{}_ref", to_snake_case(&base_name)); - - let mut methods: Vec = Vec::new(); - - for item in &mut trait_def.items { - if !matches!(item, TraitItem::Fn(_)) { - return syn::Error::new_spanned( - item, - "protocol traits may only contain methods; \ - associated types, constants, and other items are not supported", - ) - .to_compile_error() - .into(); - } - if let TraitItem::Fn(method) = item { - if let Some(attribute) = method.attrs.iter().find(|a| conditional_timeout(&a.meta)) { - return syn::Error::new_spanned( - attribute, - "conditional request_timeout is not supported; use #[request_timeout(...)] directly", - ).to_compile_error().into(); - } - if method.sig.asyncness.is_some() { - return syn::Error::new_spanned( - &method.sig, - "protocol methods must not be async; \ - use Response as the return type for requests", - ) - .to_compile_error() - .into(); - } - - // Verify first param is &self - match method.sig.inputs.first() { - Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_none() => {} - _ => { - return syn::Error::new_spanned( - &method.sig, - "protocol methods must take `&self` as the first parameter", - ) - .to_compile_error() - .into(); - } - } - - let sig = &method.sig; - let qualifier = [ - (sig.unsafety.as_ref().map(|t| quote!(#t)), "unsafe"), - (sig.abi.as_ref().map(|t| quote!(#t)), "extern"), - (sig.constness.as_ref().map(|t| quote!(#t)), "const"), - ] - .into_iter() - .find_map(|(token, name)| Some((token?, name))); - if let Some((token, name)) = qualifier { - return syn::Error::new_spanned( - token, - format!("`{name}` protocol methods are not supported; use a plain `fn`"), - ) - .to_compile_error() - .into(); - } - if !method.sig.generics.params.is_empty() { - return syn::Error::new_spanned( - &method.sig.generics, - "protocol methods cannot have generic parameters; \ - each method becomes one concrete message struct", - ) - .to_compile_error() - .into(); - } - for arg in method.sig.inputs.iter().skip(1) { - if let FnArg::Typed(pat_type) = arg { - if let Some(error) = unsupported_param_type(&pat_type.ty) { - return error.to_compile_error().into(); - } - } - } - - let method_name = method.sig.ident.clone(); - let struct_name = format_ident!("{}", to_pascal_case(&method_name.to_string())); - // cfg-gated methods may legitimately share a name across exclusive - // predicates; only unconditional collisions are reported here. - let conditional = |attrs: &[Attribute]| { - attrs - .iter() - .any(|a| a.path().is_ident("cfg") || a.path().is_ident("cfg_attr")) - }; - if !conditional(&method.attrs) { - if let Some(previous) = methods - .iter() - .find(|m| m.cfg_attrs.is_empty() && m.struct_name == struct_name) - { - let message = if previous.method_name == method_name { - format!("duplicate protocol method `{method_name}`") - } else { - format!( - "protocol method `{method_name}` generates message struct \ - `{struct_name}`, which `{}` already uses; rename one of them", - previous.method_name - ) - }; - return syn::Error::new_spanned(&method.sig.ident, message) - .to_compile_error() - .into(); - } - } - - let mut field_names: Vec = Vec::new(); - let mut field_types: Vec = Vec::new(); - let mut params: Vec = Vec::new(); - - for arg in method.sig.inputs.iter().skip(1) { - if let FnArg::Typed(pat_type) = arg { - if let Pat::Ident(pat_ident) = &*pat_type.pat { - field_names.push(pat_ident.ident.clone()); - field_types.push((*pat_type.ty).clone()); - } else { - return syn::Error::new_spanned( - &pat_type.pat, - "protocol methods only support simple identifier patterns \ - (e.g., `name: Type`)", - ) - .to_compile_error() - .into(); - } - } - params.push(arg.clone()); - } - - let kind = match classify_return_type(&method.sig.output) { - Ok(kind) => kind, - Err(ty) => { - return syn::Error::new_spanned( - ty, - "unsupported return type in protocol method; \ - use Response for requests (works in both async and sync modes), \ - Result<(), ActorError> for sends, or no return type for fire-and-forget", - ) - .to_compile_error() - .into(); - } - }; - - let mut method_timeout = None; - for attr in method - .attrs - .iter() - .filter(|a| a.path().is_ident("request_timeout")) - { - if !matches!(kind, MethodKind::Request(_)) { - return syn::Error::new_spanned( - attr, - "request_timeout is only valid on Response methods", - ) - .to_compile_error() - .into(); - } - let expression = match attr.parse_args::() { - Ok(expression) => expression, - Err(error) => return error.to_compile_error().into(), - }; - if method_timeout.replace(expression).is_some() { - return syn::Error::new_spanned(attr, "duplicate request_timeout") - .to_compile_error() - .into(); - } - } - method - .attrs - .retain(|a| !a.path().is_ident("request_timeout")); - let timeout = method_timeout - .or_else(|| protocol_timeout.clone()) - .unwrap_or_else( - || syn::parse_quote! { spawned_concurrency::response::DEFAULT_REQUEST_TIMEOUT }, - ); - - let method_attrs: Vec = method - .attrs - .iter() - .filter(|a| { - a.path().is_ident("doc") - || a.path().is_ident("cfg") - || a.path().is_ident("cfg_attr") - }) - .cloned() - .collect(); - let cfg_attrs: Vec = method - .attrs - .iter() - .filter(|a| a.path().is_ident("cfg") || a.path().is_ident("cfg_attr")) - .cloned() - .collect(); - - methods.push(ProtocolMethodInfo { - method_name, - struct_name, - field_names, - field_types, - kind, - params, - ret_type: method.sig.output.clone(), - method_attrs, - cfg_attrs, - timeout, - }); - } - } - - // Generate message structs - // Field types and result types are qualified with `super::` to prevent - // name collisions when a generated struct name shadows an imported type. - // e.g., `fn event(&self, event: Event)` generates `struct Event { pub event: super::Event }` - let msg_structs: Vec<_> = methods - .iter() - .map(|m| { - let struct_name = &m.struct_name; - let field_names = &m.field_names; - let qualified_field_types: Vec = - m.field_types.iter().map(qualify_type_with_super).collect(); - let method_attrs = &m.method_attrs; - let cfg_attrs = &m.cfg_attrs; - let msg_result_ty: Type = match &m.kind { - MethodKind::Send => syn::parse_quote! { () }, - MethodKind::Request(inner) => qualify_type_with_super(inner), - }; - - if field_names.is_empty() { - quote! { - #(#method_attrs)* - #[derive(Clone)] - pub struct #struct_name; - #(#cfg_attrs)* - impl Message for #struct_name { - type Result = #msg_result_ty; - } - } - } else { - quote! { - #(#method_attrs)* - pub struct #struct_name { - #(pub #field_names: #qualified_field_types,)* - } - #(#cfg_attrs)* - impl Message for #struct_name { - type Result = #msg_result_ty; - } - } - } - }) - .collect(); - - // Gate async impls using the dependency feature namespace, not the consumer. - let target_trait = format_ident!("__{}Target", trait_name); - let tasks = quote! { spawned_concurrency::tasks }; - let threads = quote! { spawned_concurrency::threads }; - let proto_info = ProtocolInfo { - trait_name, - mod_name: &mod_name, - ref_name: &ref_name, - converter_trait: &converter_trait, - converter_method: &converter_method, - target_trait: &target_trait, - }; - let tasks_impl = generate_target_impl(&proto_info, &methods, &tasks, RuntimeMode::Tasks); - let threads_impl = generate_target_impl(&proto_info, &methods, &threads, RuntimeMode::Threads); - let adapters = generate_protocol_adapters(&proto_info, &methods, &trait_def); - let blanket_impls = quote! { - #adapters - spawned_concurrency::__spawned_async_items! { #tasks_impl } - #threads_impl - }; - - let ref_doc = format!( - "Type-erased reference to any actor implementing [`{trait_name}`].\n\n\ - Use this type to store protocol references without depending on the concrete actor type." - ); - - // Protocol traits are consumed via blanket impls and dyn references, - // so rustc may report methods as unused during development. - let output = quote! { - #[allow(dead_code)] - #trait_def - - #[doc = #ref_doc] - #trait_vis type #ref_name = ::std::sync::Arc; - - #trait_vis mod #mod_name { - use super::*; - use spawned_concurrency::message::Message; - #(#msg_structs)* - } - - #trait_vis trait #converter_trait { - fn #converter_method(&self) -> #ref_name; - } - - impl #converter_trait for #ref_name { - fn #converter_method(&self) -> #ref_name { - ::std::sync::Arc::clone(self) - } - } - - #blanket_impls - }; - - output.into() -} - -// Only inline a body whose suspension and wrapping behavior is visible here. -// Unknown macros/attributes can introduce awaits or wrappers after expansion. -fn await_free_handler(method: &ImplItemFn) -> bool { - fn inert_attribute(attr: &Attribute) -> bool { - ["cfg", "doc", "allow", "warn", "deny", "forbid", "expect"] - .iter() - .any(|name| attr.path().is_ident(name)) - } - // An explicit/shared receiver can carry different borrow promises than - // Handler's &mut self. Keep its original call adapter unchanged. - let ordinary_mut_receiver = matches!(method.sig.inputs.first(), - Some(FnArg::Receiver(r)) if r.attrs.is_empty() - && r.mutability.is_some() - && r.reference.as_ref().is_some_and(|(_, lifetime)| lifetime.is_none()) - && r.colon_token.is_none()); - let plain_parameters = method.sig.inputs.iter().all(|p| match p { - FnArg::Receiver(r) => r.attrs.is_empty(), - FnArg::Typed(p) => p.attrs.is_empty(), - }); - if !ordinary_mut_receiver - || !plain_parameters - || !method.sig.generics.params.is_empty() - || method.sig.generics.where_clause.is_some() - || method.sig.asyncness.is_none() - || method.sig.unsafety.is_some() - || method.sig.abi.is_some() - || !method.attrs.iter().all(inert_attribute) - { - return false; - } - struct Check { - blocked: bool, - } - impl<'ast> syn::visit::Visit<'ast> for Check { - fn visit_expr_await(&mut self, _: &'ast syn::ExprAwait) { - self.blocked = true; - } - fn visit_expr_yield(&mut self, _: &'ast syn::ExprYield) { - self.blocked = true; - } - fn visit_macro(&mut self, _: &'ast syn::Macro) { - self.blocked = true; - } - fn visit_item(&mut self, _: &'ast syn::Item) { - // A copied static (including one inside a nested function) has - // a different identity from the inherent method's original. - self.blocked = true; - } - fn visit_expr_closure(&mut self, _: &'ast syn::ExprClosure) { - // Anonymous types also have observable identity through TypeId. - self.blocked = true; - } - fn visit_expr_async(&mut self, _: &'ast syn::ExprAsync) { - self.blocked = true; - } - fn visit_type_impl_trait(&mut self, _: &'ast syn::TypeImplTrait) { - self.blocked = true; - } - fn visit_attribute(&mut self, attr: &'ast Attribute) { - if !inert_attribute(attr) { - self.blocked = true; - } - } - } - let mut check = Check { blocked: false }; - syn::visit::Visit::visit_block(&mut check, &method.block); - syn::visit::Visit::visit_signature(&mut check, &method.sig); - !check.blocked -} - -fn is_handler_attribute(attr: &Attribute) -> bool { - let path = attr.path(); - path.is_ident("handler") - || path.is_ident("send_handler") - || path.is_ident("request_handler") - || path.is_ident("deferred_handler") -} - -fn is_actor_method_attribute(attr: &Attribute) -> bool { - let path = attr.path(); - is_handler_attribute(attr) - || path.is_ident("started") - || path.is_ident("stopped") - || path.is_ident("exit_received") + protocol::expand(attr, item) } /// Generates `impl Actor` and `Handler` implementations from an annotated impl block. @@ -1264,828 +199,9 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// ``` #[proc_macro_attribute] pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { - let mut impl_block = parse_macro_input!(item as ItemImpl); - - if let Some((_, path, _)) = &impl_block.trait_ { - return syn::Error::new_spanned( - path, - "#[actor] must be applied to an inherent impl block (`impl MyActor { ... }`), \ - not a trait impl", - ) - .to_compile_error() - .into(); - } - - // Tasks actors use `async fn` for every handler and hook, threads actors - // plain `fn`; the runtime is inferred from them. A sync method returning - // `impl Future` is valid in tasks mode, and cfg-gated methods may differ. - let mut runtime_methods = impl_block.items.iter().filter_map(|item| match item { - ImplItem::Fn(method) - if method.attrs.iter().any(is_actor_method_attribute) - && !method.attrs.iter().any(|a| a.path().is_ident("cfg")) - && !matches!(&method.sig.output, ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_))) => - { - Some(&method.sig) - } - _ => None, - }); - if let Some(first) = runtime_methods.next() { - let asynchronous = first.asyncness.is_some(); - if let Some(other) = runtime_methods.find(|sig| sig.asyncness.is_some() != asynchronous) { - return syn::Error::new_spanned( - other, - "cannot mix async and sync handlers or lifecycle hooks in one #[actor] block; \ - tasks actors use `async fn` throughout, threads actors use plain `fn`", - ) - .to_compile_error() - .into(); - } - } - - let self_ty = &impl_block.self_ty; - let (impl_generics, _, where_clause) = impl_block.generics.split_for_impl(); - - // --- Parse named parameters: #[actor(protocol = X)], #[actor(protocol(X, Y))], - // the deferred-reply bounds `max_outstanding = N`, `max_native_jobs = N`, - // and the lint opt-outs `allow_shared_state` and `allow_panic_abort` --- - let mut max_outstanding: Option = None; - let mut max_native_jobs: Option = None; - let mut allow_shared_state = false; - let mut allow_panic_abort = false; - let bridge_traits: Vec = if attr.is_empty() { - Vec::new() - } else { - // A protocol is named by a single identifier; report paths explicitly - // instead of failing on the `::` token. - fn protocol_name(input: syn::parse::ParseStream) -> syn::Result { - let path: syn::Path = input.parse()?; - path.get_ident().cloned().ok_or_else(|| { - syn::Error::new_spanned( - &path, - "protocol must be named by a single identifier; \ - import the trait first (`use path::to::Protocol;`)", - ) - }) - } - let parser = |input: syn::parse::ParseStream| -> syn::Result> { - let mut protocols = Vec::new(); - while !input.is_empty() { - let key: Ident = input.parse()?; - if key == "max_outstanding" || key == "max_native_jobs" { - let _: syn::Token![=] = input.parse()?; - let value: syn::Expr = input.parse()?; - let slot = if key == "max_outstanding" { - &mut max_outstanding - } else { - &mut max_native_jobs - }; - if slot.replace(value).is_some() { - return Err(syn::Error::new(key.span(), format!("`{key}` given twice"))); - } - } else if key == "allow_shared_state" { - allow_shared_state = true; - } else if key == "allow_panic_abort" { - allow_panic_abort = true; - } else if key != "protocol" { - return Err(syn::Error::new( - key.span(), - "unknown parameter, expected `protocol`, `max_outstanding`, \ - `max_native_jobs`, `allow_shared_state` or `allow_panic_abort`", - )); - } else if input.peek(syn::Token![=]) { - // protocol = TraitName - let _: syn::Token![=] = input.parse()?; - protocols.push(protocol_name(input)?); - } else if input.peek(syn::token::Paren) { - // protocol(Trait1, Trait2) - let content; - syn::parenthesized!(content in input); - let punctuated = content.parse_terminated(protocol_name, syn::Token![,])?; - protocols.extend(punctuated); - } else { - return Err(syn::Error::new( - key.span(), - "expected `protocol = Trait` or `protocol(TraitA, TraitB)`", - )); - } - if !input.is_empty() { - let _: syn::Token![,] = input.parse().map_err(|e| { - syn::Error::new(e.span(), "expected `,` between #[actor] parameters") - })?; - } - } - Ok(protocols) - }; - match syn::parse::Parser::parse(parser, attr) { - Ok(traits) => traits, - Err(e) => return e.to_compile_error().into(), - } - }; - - // --- Compile-time lints (ROADMAP rank 6b); no runtime code is generated --- - let lints = actor_lints(&impl_block, allow_shared_state, allow_panic_abort); - - // --- Extract #[started] and #[stopped] lifecycle methods --- - let mut started_method: Option = None; - let mut stopped_method: Option = None; - let mut exit_method: Option = None; - let mut has_async = false; - - let mut items_to_keep = Vec::new(); - for item in impl_block.items.drain(..) { - if let ImplItem::Fn(ref method) = item { - let is_started = method.attrs.iter().any(|a| a.path().is_ident("started")); - let is_stopped = method.attrs.iter().any(|a| a.path().is_ident("stopped")); - - let is_exit = method - .attrs - .iter() - .any(|a| a.path().is_ident("exit_received")); - if is_exit { - if exit_method.is_some() { - return syn::Error::new_spanned( - &method.sig, - "only one #[exit_received] method is allowed per actor", - ) - .to_compile_error() - .into(); - } - if is_started || is_stopped || method.attrs.iter().any(is_handler_attribute) { - return syn::Error::new_spanned(&method.sig, "#[exit_received] cannot be combined with other lifecycle or handler attributes").to_compile_error().into(); - } - if method.sig.inputs.len() != 3 - || !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) - { - return syn::Error::new_spanned( - &method.sig, - "#[exit_received] must take (&mut self, Exit, &Context)", - ) - .to_compile_error() - .into(); - } - let mut m = method.clone(); - m.attrs.retain(|a| !a.path().is_ident("exit_received")); - m.vis = syn::Visibility::Inherited; - m.sig.ident = format_ident!("exit_received"); - if m.sig.asyncness.is_some() { - has_async = true; - } - exit_method = Some(m); - continue; - } - - if is_started && is_stopped { - return syn::Error::new_spanned( - &method.sig, - "#[started] and #[stopped] cannot be combined on one method", - ) - .to_compile_error() - .into(); - } - - if is_started { - if started_method.is_some() { - return syn::Error::new_spanned( - &method.sig, - "only one #[started] method is allowed per actor", - ) - .to_compile_error() - .into(); - } - if method.attrs.iter().any(is_handler_attribute) { - return syn::Error::new_spanned( - &method.sig, - "#[started] cannot be combined with handler attributes", - ) - .to_compile_error() - .into(); - } - // Expect: fn started(&mut self, ctx: &Context) - if method.sig.inputs.len() != 2 { - return syn::Error::new_spanned( - &method.sig, - "#[started] method must take exactly (&mut self, &Context)", - ) - .to_compile_error() - .into(); - } - if !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) - { - return syn::Error::new_spanned( - &method.sig, - "#[started] method's first parameter must be `&mut self`", - ) - .to_compile_error() - .into(); - } - let mut m = method.clone(); - m.attrs.retain(|a| !a.path().is_ident("started")); - m.vis = syn::Visibility::Inherited; - m.sig.ident = format_ident!("started"); - if m.sig.asyncness.is_some() { - has_async = true; - } - started_method = Some(m); - continue; - } - - if is_stopped { - if stopped_method.is_some() { - return syn::Error::new_spanned( - &method.sig, - "only one #[stopped] method is allowed per actor", - ) - .to_compile_error() - .into(); - } - if method.attrs.iter().any(is_handler_attribute) { - return syn::Error::new_spanned( - &method.sig, - "#[stopped] cannot be combined with handler attributes", - ) - .to_compile_error() - .into(); - } - // Expect: fn stopped(&mut self, ctx: &Context) - if method.sig.inputs.len() != 2 { - return syn::Error::new_spanned( - &method.sig, - "#[stopped] method must take exactly (&mut self, &Context)", - ) - .to_compile_error() - .into(); - } - if !matches!(method.sig.inputs.first(), Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_some()) - { - return syn::Error::new_spanned( - &method.sig, - "#[stopped] method's first parameter must be `&mut self`", - ) - .to_compile_error() - .into(); - } - let mut m = method.clone(); - m.attrs.retain(|a| !a.path().is_ident("stopped")); - m.vis = syn::Visibility::Inherited; - m.sig.ident = format_ident!("stopped"); - if m.sig.asyncness.is_some() { - has_async = true; - } - stopped_method = Some(m); - continue; - } - } - items_to_keep.push(item); - } - impl_block.items = items_to_keep; - - // --- Process handler methods --- - let mut handler_impls = Vec::new(); - let mut lifetime_name = "__spawned_reply_turn".to_owned(); - while impl_block - .generics - .lifetimes() - .any(|p| p.lifetime.ident == lifetime_name) - { - lifetime_name.push('_'); - } - let turn_lifetime = syn::Lifetime::new( - &format!("'{lifetime_name}"), - proc_macro2::Span::mixed_site(), - ); - - for item in &mut impl_block.items { - if let ImplItem::Fn(method) = item { - let handler_idx = method.attrs.iter().position(is_handler_attribute); - - if let Some(idx) = handler_idx { - let deferred = method.attrs[idx].path().is_ident("deferred_handler"); - let send = method.attrs[idx].path().is_ident("send_handler"); - if method.attrs[idx].path().is_ident("handler") { - return syn::Error::new_spanned( - &method.attrs[idx], - "#[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value)", - ) - .to_compile_error() - .into(); - } - method.attrs.remove(idx); - if let Some(extra) = method.attrs.iter().find(|a| is_handler_attribute(a)) { - return syn::Error::new_spanned( - extra, - "a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler]", - ) - .to_compile_error() - .into(); - } - if !method.sig.generics.params.is_empty() { - return syn::Error::new_spanned( - &method.sig.generics, - "handler methods cannot be generic; write one handler per message type", - ) - .to_compile_error() - .into(); - } - - // Collect remaining attributes (e.g. #[cfg(...)]) to propagate - // to the generated Handler impl block. - let extra_attrs: Vec<_> = method - .attrs - .iter() - .filter(|a| { - !is_handler_attribute(a) - && !a.path().is_ident("started") - && !a.path().is_ident("stopped") - }) - .cloned() - .collect(); - - let method_name = &method.sig.ident; - if method.sig.asyncness.is_some() { - has_async = true; - } - - // Validate parameters: (&mut self, msg, ctx), or for a deferred - // handler (&mut self, msg, reply, ctx). - let param_count = method.sig.inputs.len(); - let ctx_index = if deferred { 3 } else { 2 }; - if deferred && param_count != 4 { - return syn::Error::new_spanned( - &method.sig, - format!( - "#[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found {param_count}" - ), - ) - .to_compile_error() - .into(); - } - if !deferred && param_count != 3 { - return syn::Error::new_spanned( - &method.sig, - format!( - "handler method must have 3 parameters (&mut self, msg: M, ctx: &Context), found {param_count}" - ), - ) - .to_compile_error() - .into(); - } - if deferred { - if let ReturnType::Type(_, ty) = &method.sig.output { - if !is_unit_type(ty) { - return syn::Error::new_spanned( - ty, - "#[deferred_handler] answers through its PendingReply; remove the return type", - ) - .to_compile_error() - .into(); - } - } - } - // A send handler that returned a value would silently answer a - // `request` with it. (`impl Future` returns are checked by the - // `Result = ()` bound of the message instead.) - if send { - if let ReturnType::Type(_, ty) = &method.sig.output { - if !is_unit_type(ty) && !matches!(**ty, Type::ImplTrait(_)) { - return syn::Error::new_spanned( - ty, - "#[send_handler] methods return `()`; use #[request_handler] to answer a request with a value", - ) - .to_compile_error() - .into(); - } - } - } - - // `&self` is accepted too; a missing or by-value receiver is not. - match method.sig.inputs.first() { - Some(FnArg::Receiver(r)) - if r.reference.is_some() || r.colon_token.is_some() => {} - Some(first) => { - return syn::Error::new_spanned( - first, - "handler method's first parameter must be `&mut self`", - ) - .to_compile_error() - .into(); - } - None => unreachable!("the parameters were checked above"), - } - if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(ctx_index) { - if !matches!(&*ctx.ty, Type::Reference(r) if r.mutability.is_none()) { - return syn::Error::new_spanned( - &ctx.ty, - "handler context parameter must be `&Context`", - ) - .to_compile_error() - .into(); - } - } - - // Extract message type from 2nd parameter (index 1, after &mut self) - let msg_ty = match method.sig.inputs.iter().nth(1) { - Some(FnArg::Typed(pat_type)) => &*pat_type.ty, - _ => { - return syn::Error::new_spanned( - &method.sig, - "handler method must have signature: fn(&mut self, msg: M, ctx: &Context) -> R", - ) - .to_compile_error() - .into(); - } - }; - - // The generated `handle` is the envelope dispatch: it takes the - // request's reply obligation and answers it with the method's - // result (a deferred handler receives the obligation itself). - let reply_ty = quote! { - spawned_concurrency::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> - }; - // Mixed-site locals cannot shadow free names in a user body - // that is specialized into the generated dispatch adapter. - let msg_name = Ident::new("__spawned_msg", proc_macro2::Span::mixed_site()); - let reply_name = Ident::new("__spawned_reply", proc_macro2::Span::mixed_site()); - let ctx_name = Ident::new( - "__spawned_ctx", - proc_macro2::Span::mixed_site().located_at( - method - .sig - .inputs - .iter() - .nth(ctx_index) - .expect("validated context arity") - .span(), - ), - ); - // The parameter list carries the user's parentheses span. - let mut params = proc_macro2::Group::new( - proc_macro2::Delimiter::Parenthesis, - quote! { &mut self, #msg_name: #msg_ty, #reply_name: #reply_ty, #ctx_name: &Context }, - ); - params.set_span(method.sig.paren_token.span.join()); - // Span the generated impl at the method so trait errors - // (conflicting impls, signature mismatches) point at it. - let span = method.sig.span(); - // The reply's type must equal the message's `Result`; a - // mismatch is reported at the user's return type. - let ret_span = match &method.sig.output { - ReturnType::Default => method.sig.paren_token.span.join(), - ReturnType::Type(_, ty) => ty.span(), - }; - // A tasks handler may also be a plain `fn` returning - // `impl Future`; its result is awaited like an `async fn`'s. - let returns_future = matches!( - &method.sig.output, - ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_)) - ); - let asynchronous = method.sig.asyncness.is_some() || returns_future; - let immediate = asynchronous && !deferred && await_free_handler(method); - let call = match (deferred, asynchronous) { - (true, true) => { - quote! { self.#method_name(#msg_name, #reply_name, #ctx_name).await } - } - (true, false) => { - quote! { self.#method_name(#msg_name, #reply_name, #ctx_name) } - } - (false, true) if immediate => { - let body = &method.block; - let msg_pat = match method - .sig - .inputs - .iter() - .nth(1) - .expect("validated message arity") - { - FnArg::Typed(arg) => &arg.pat, - _ => unreachable!(), - }; - let (ctx_pat, ctx_ty) = match method - .sig - .inputs - .iter() - .nth(ctx_index) - .expect("validated context arity") - { - FnArg::Typed(arg) => (&arg.pat, &arg.ty), - _ => unreachable!(), - }; - let output = match &method.sig.output { - ReturnType::Default => quote! { () }, - ReturnType::Type(_, ty) => quote! { #ty }, - }; - // Dispatch has already begun when this adapter is - // constructed. Keep the inherent async method lazy, - // but evaluate this await-free dispatch without a - // heap-stored future. A closure preserves return/?. - let answer = quote::quote_spanned! { ret_span => - let _ = #reply_name.reply(value); - }; - quote! { - // Function/closure parameters retain ignored fields - // until the body returns. A destructuring let here - // would drop them before the handler body runs. - let value = (|#msg_pat: #msg_ty, #ctx_pat: #ctx_ty| -> #output #body)( - #msg_name, #ctx_name - ); - #answer - } - } - (false, true) => { - let value = quote::quote_spanned! { ret_span => self.#method_name(#msg_name, #ctx_name).await }; - quote! { let _ = #reply_name.reply(#value); } - } - (false, false) => { - let value = quote::quote_spanned! { ret_span => self.#method_name(#msg_name, #ctx_name) }; - quote! { let _ = #reply_name.reply(#value); } - } - }; - - let handler_impl = if immediate { - quote::quote_spanned! { span => - #(#extra_attrs)* - impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - async fn handle #params { - let _ = #reply_name.reply(self.#method_name(#msg_name, #ctx_name).await); - } - - fn __spawned_handle_immediate<#turn_lifetime>( - &mut self, - #msg_name: #msg_ty, - #reply_name: spawned_concurrency::PendingReply<#turn_lifetime, <#msg_ty as spawned_concurrency::message::Message>::Result>, - #ctx_name: &Context, - ) -> ::core::option::Option<(#msg_ty, spawned_concurrency::PendingReply<#turn_lifetime, <#msg_ty as spawned_concurrency::message::Message>::Result>)> { - // Keep payload destructors inside the original - // future, whose drop is fenced by DeathFirst. - // A refused result is dropped by reply() in - // either adapter, after the method returned; - // unlike a parameter, it adds no unwind capture. - // This constant branch disappears for messages - // without drop glue. - if ::core::mem::needs_drop::<#msg_ty>() { - return ::core::option::Option::Some((#msg_name, #reply_name)); - } - #call - ::core::option::Option::None - } - } - } - } else if asynchronous { - quote::quote_spanned! { span => - #(#extra_attrs)* - impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - async fn handle #params { - #call - } - } - } - } else { - quote::quote_spanned! { span => - #(#extra_attrs)* - impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - fn handle #params { - #call - } - } - } - }; - - handler_impls.push(handler_impl); - } - } - } - - // --- Generate impl Actor --- - let lifecycle_methods: Vec<&ImplItemFn> = [ - started_method.as_ref(), - stopped_method.as_ref(), - exit_method.as_ref(), - ] - .into_iter() - .flatten() - .collect(); - - let protocol_doc = if bridge_traits.is_empty() { - quote! {} - } else { - let lines: Vec = bridge_traits.iter().map(|t| format!("- [`{t}`]")).collect(); - let doc_body = format!( - "# Protocol\n\n\ - When started, `ActorRef<{ty}>` implements:\n\n\ - {lines}\n\n\ - See the protocol trait docs for the full API.", - ty = quote!(#self_ty), - lines = lines.join("\n"), - ); - quote! { #[doc = #doc_body] } - }; - - let bound_consts = { - let outstanding = max_outstanding.map(|n| quote! { const MAX_OUTSTANDING: usize = #n; }); - let native = max_native_jobs - .map(|n| quote! { const MAX_NATIVE_JOBS: ::core::option::Option = ::core::option::Option::Some(#n); }); - quote! { #outstanding #native } - }; - - let actor_impl = quote! { - #protocol_doc - impl #impl_generics Actor for #self_ty #where_clause { - #bound_consts - #(#lifecycle_methods)* - } - }; - - // --- Generate bridge assertions --- - let runtime_path = if has_async { - quote! { spawned_concurrency::tasks } - } else { - quote! { spawned_concurrency::threads } - }; - - let bridge_asserts: Vec<_> = bridge_traits - .iter() - .map(|trait_name| { - quote! { - const _: () = { - fn _assert_bridge<__T: #trait_name>() {} - fn _check #impl_generics () #where_clause { - _assert_bridge::<#runtime_path::ActorRef<#self_ty>>(); - } - }; - } - }) - .collect(); - - let output = quote! { - #actor_impl - #impl_block - #(#handler_impls)* - #(#bridge_asserts)* - #lints - }; - - output.into() -} - -// --- #[actor] lints --- -// -// `#[actor]` sees only the impl block it annotates, not the struct, so the -// shared-state lint is syntactic over what that block contains: constructor -// and handler signatures, `Self { .. }` literals, `let` types and -// `Arc::new(Mutex::new(..))`-style expressions. Proc-macro diagnostics are -// unstable, so each warning is a use of a `#[deprecated]` item spanned at the -// offending tokens: rustc prints the note there, `-D warnings` makes it an -// error, and nothing reaches the compiled program. - -const SHARED_STATE_NOTE: &str = "actors should own their state: shared mutable state \ -(`Arc>`, `Arc>`, `Arc`, `Rc>`) breaks the isolation \ -supervision relies on (a restarted actor inherits whatever the old one left behind the lock), \ -and a panic while holding a std lock poisons it for every holder; send messages to the owning \ -actor instead (or, once available, read a published snapshot, ROADMAP rank 10); if the \ -sharing is deliberate, use `#[actor(allow_shared_state)]`"; - -const PANIC_ABORT_NOTE: &str = "built with `panic = \"abort\"`: a panicking handler aborts \ -the whole process instead of stopping one actor, so supervisors, links, monitors and \ -`stopped()` never see the failure; build actor binaries with `panic = \"unwind\"` (the \ -default), or use `#[actor(allow_panic_abort)]` if aborting is deliberate"; - -/// Smart pointers that share their contents between owners. -fn is_shared_pointer(ident: &Ident) -> bool { - ident == "Arc" || ident == "Rc" -} - -/// Interior-mutable types from std, tokio, parking_lot and similar crates, -/// matched by their last path segment. -fn is_interior_mutable(ident: &Ident) -> bool { - ident == "Mutex" - || ident == "RwLock" - || ident == "RefCell" - || ident.to_string().starts_with("Atomic") + actor::expand(attr, item) } -/// `Arc>`, `std::sync::Arc>`, `Rc>`, ... -fn is_shared_mutable_type(path: &syn::Path) -> bool { - let Some(outer) = path.segments.last() else { - return false; - }; - if !is_shared_pointer(&outer.ident) { - return false; - } - let PathArguments::AngleBracketed(args) = &outer.arguments else { - return false; - }; - matches!( - args.args.first(), - Some(GenericArgument::Type(Type::Path(inner))) - if inner.path.segments.last().is_some_and(|s| is_interior_mutable(&s.ident)) - ) -} - -/// The type segment of an associated call such as `Mutex::new` or -/// `std::sync::Arc::::new`. -fn call_type_segment(func: &syn::Expr) -> Option<&Ident> { - let syn::Expr::Path(path) = func else { - return None; - }; - let segments = &path.path.segments; - (segments.len() >= 2).then(|| &segments[segments.len() - 2].ident) -} - -/// `Arc::new(Mutex::new(..))`, `Rc::new(RefCell::new(..))`, -/// `Arc::new(AtomicU64::default())`, ... -fn is_shared_mutable_constructor(call: &syn::ExprCall) -> bool { - let syn::Expr::Path(func) = &*call.func else { - return false; - }; - let wraps_in_pointer = func.path.segments.last().is_some_and(|s| s.ident == "new") - && call_type_segment(&call.func).is_some_and(is_shared_pointer) - && call.args.len() == 1; - wraps_in_pointer - && matches!( - call.args.first(), - Some(syn::Expr::Call(inner)) - if call_type_segment(&inner.func).is_some_and(is_interior_mutable) - ) -} - -#[derive(Default)] -struct SharedStateFinder { - spans: Vec, -} - -impl<'ast> syn::visit::Visit<'ast> for SharedStateFinder { - fn visit_type_path(&mut self, ty: &'ast TypePath) { - if is_shared_mutable_type(&ty.path) { - self.spans.push(ty.span()); - } else { - syn::visit::visit_type_path(self, ty); - } - } - - fn visit_expr_call(&mut self, call: &'ast syn::ExprCall) { - if is_shared_mutable_constructor(call) { - self.spans.push(call.span()); - } else { - syn::visit::visit_expr_call(self, call); - } - } -} - -/// A warning at `span`: the use of a local `#[deprecated]` unit struct. -fn deprecation_warning( - name: &str, - note: &str, - span: proc_macro2::Span, -) -> proc_macro2::TokenStream { - let name = Ident::new(name, span); - quote::quote_spanned! { span => - const _: () = { - #[deprecated(note = #note)] - struct #name; - let _ = #name; - }; - } -} - -/// Set once the panic-abort warning has been emitted in this compilation. A -/// proc macro is loaded once per rustc invocation, so this makes the warning -/// once per crate rather than once per actor. If a compiler ever reloaded the -/// macro, the warning would only repeat. -static PANIC_ABORT_WARNED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -fn actor_lints( - impl_block: &ItemImpl, - allow_shared_state: bool, - allow_panic_abort: bool, -) -> proc_macro2::TokenStream { - let mut lints = proc_macro2::TokenStream::new(); - if !allow_shared_state { - let mut finder = SharedStateFinder::default(); - syn::visit::Visit::visit_item_impl(&mut finder, impl_block); - for span in finder.spans { - lints.extend(deprecation_warning( - "SharedMutableStateInActor", - SHARED_STATE_NOTE, - span, - )); - } - } - if !allow_panic_abort && !PANIC_ABORT_WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { - let warning = deprecation_warning( - "PanicAbortInActorCrate", - PANIC_ABORT_NOTE, - proc_macro2::Span::call_site(), - ); - lints.extend(quote! { - #[cfg(panic = "abort")] - #warning - }); - } - lints -} - -// --- #[sim_test] --- - /// Send-only form of a manual `Handler` impl: write `handle` without the /// reply parameter, and the impl answers `()` once `handle` returns. /// @@ -2114,103 +230,7 @@ fn actor_lints( /// `#[send_handler(crate = my_spawned)]`. #[proc_macro_attribute] pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { - let krate: Option = if attr.is_empty() { - None - } else { - let parser = |input: syn::parse::ParseStream| { - input.parse::()?; - input.parse::()?; - let path: syn::Path = input.parse()?; - if !input.is_empty() { - return Err(input.error("expected only `crate = path`")); - } - Ok(path) - }; - match syn::parse::Parser::parse(parser, attr) { - Ok(path) => Some(path), - Err(error) => { - return syn::Error::new( - error.span(), - format!("#[send_handler] takes no arguments or `crate = path`: {error}"), - ) - .to_compile_error() - .into() - } - } - }; - let mut impl_block = parse_macro_input!(item as ItemImpl); - let is_handler = impl_block - .trait_ - .as_ref() - .and_then(|(_, path, _)| path.segments.last()) - .is_some_and(|segment| segment.ident == "Handler"); - if !is_handler { - return syn::Error::new_spanned( - &impl_block.self_ty, - "#[send_handler] on an item applies to an `impl Handler for Actor` block; \ - inside an #[actor] block, put it on the handler method instead", - ) - .to_compile_error() - .into(); - } - let Some(method) = impl_block.items.iter_mut().find_map(|item| match item { - ImplItem::Fn(method) if method.sig.ident == "handle" => Some(method), - _ => None, - }) else { - return syn::Error::new_spanned( - &impl_block.self_ty, - "#[send_handler] impl must define `handle`", - ) - .to_compile_error() - .into(); - }; - if method.sig.inputs.len() != 3 { - return syn::Error::new_spanned( - &method.sig, - "#[send_handler] `handle` takes (&mut self, msg: M, ctx: &Context), without the reply parameter", - ) - .to_compile_error() - .into(); - } - if let ReturnType::Type(_, ty) = &method.sig.output { - if !is_unit_type(ty) { - return syn::Error::new_spanned(ty, "#[send_handler] `handle` returns nothing") - .to_compile_error() - .into(); - } - } - let reply = format_ident!("__spawned_reply"); - // The `Handler` path's module (`tasks`/`threads`) re-exports - // `PendingReply`; a bare `Handler` falls back to the crate path. - let handler_module = impl_block.trait_.as_ref().and_then(|(_, path, _)| { - (path.segments.len() > 1).then(|| { - let mut module = path.clone(); - module.segments.pop(); - module.segments.pop_punct(); - module - }) - }); - let reply_ty = match (krate, handler_module) { - (Some(krate), _) => quote! { #krate::PendingReply<'_, ()> }, - (None, Some(module)) => quote! { #module::PendingReply<'_, ()> }, - (None, None) => quote! { spawned_concurrency::PendingReply<'_, ()> }, - }; - let reply_param: FnArg = syn::parse_quote! { - #reply: #reply_ty - }; - method.sig.inputs.insert(2, reply_param); - method.sig.output = ReturnType::Default; - let body = &method.block; - let run = if method.sig.asyncness.is_some() { - quote! { async move #body.await } - } else { - quote! { (move || #body)() } - }; - method.block = syn::parse_quote! {{ - let () = #run; - let _ = #reply.reply(()); - }}; - quote! { #impl_block }.into() + send_handler::expand(attr, item) } /// Seeded async test: on Tokio in normal builds, over a seed range on the @@ -2230,111 +250,5 @@ pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { /// ``` #[proc_macro_attribute] pub fn sim_test(attr: TokenStream, item: TokenStream) -> TokenStream { - let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); - let function = parse_macro_input!(item as syn::ItemFn); - let mut multi_thread = false; - let mut workers: Option = None; - let mut seeds: Option = None; - let mut explore: Option = None; - let mut iterations: Option = None; - for option in options { - let key = option - .path - .get_ident() - .map(ToString::to_string) - .unwrap_or_default(); - match key.as_str() { - "flavor" => { - let syn::Expr::Lit(syn::ExprLit { - lit: syn::Lit::Str(flavor), - .. - }) = &option.value - else { - return syn::Error::new_spanned(&option.value, "expected a string flavor") - .to_compile_error() - .into(); - }; - match flavor.value().as_str() { - "multi_thread" => multi_thread = true, - "current_thread" => multi_thread = false, - other => { - return syn::Error::new_spanned(flavor, format!("unknown flavor {other}")) - .to_compile_error() - .into() - } - } - } - "worker_threads" => workers = Some(option.value), - "seeds" => seeds = Some(option.value), - "explore" => explore = Some(option.value), - "iterations" => iterations = Some(option.value), - _ => { - return syn::Error::new_spanned( - &option.path, - "sim_test accepts flavor, worker_threads, seeds, explore and iterations", - ) - .to_compile_error() - .into() - } - } - } - if function.sig.asyncness.is_none() { - return syn::Error::new_spanned(function.sig.fn_token, "sim_test needs an async fn") - .to_compile_error() - .into(); - } - if !matches!(function.sig.output, ReturnType::Default) { - return syn::Error::new_spanned(&function.sig.output, "sim_test functions return ()") - .to_compile_error() - .into(); - } - let seed = match function.sig.inputs.len() { - 0 => quote! { _ }, - 1 => match &function.sig.inputs[0] { - FnArg::Typed(arg) => { - let pat = &arg.pat; - quote! { #pat } - } - FnArg::Receiver(r) => { - return syn::Error::new_spanned(r, "sim_test cannot take self") - .to_compile_error() - .into() - } - }, - _ => { - return syn::Error::new_spanned( - &function.sig.inputs, - "sim_test takes at most one argument, the u64 seed", - ) - .to_compile_error() - .into() - } - }; - let attrs = &function.attrs; - let vis = &function.vis; - let name = &function.sig.ident; - let body = &function.block; - let flavor = if multi_thread { - let workers = workers.unwrap_or_else(|| syn::parse_quote!(2)); - quote! { .multi_thread(#workers) } - } else { - quote! {} - }; - let seeds = seeds.map(|n| quote! { .seeds(0..(#n)) }); - let explore = explore.map(|s| quote! { .explore(#s) }); - let iterations = iterations.map(|n| quote! { .iterations(#n) }); - quote! { - #(#attrs)* - #[test] - #vis fn #name() { - ::spawned_rt::tasks::SimTest::new() - .named(env!("CARGO_PKG_NAME"), module_path!(), stringify!(#name)) - #flavor - #seeds - #explore - #iterations - .run(|#seed: u64| async move #body) - } - } - .into() + sim_test::expand(attr, item) } diff --git a/macros/src/lints.rs b/macros/src/lints.rs new file mode 100644 index 00000000..d6cf76df --- /dev/null +++ b/macros/src/lints.rs @@ -0,0 +1,161 @@ +//! `#[actor]` lints: shared-state and panic-abort warnings as `#[deprecated]` uses. +use super::*; + +// --- #[actor] lints --- +// +// `#[actor]` sees only the impl block it annotates, not the struct, so the +// shared-state lint is syntactic over what that block contains: constructor +// and handler signatures, `Self { .. }` literals, `let` types and +// `Arc::new(Mutex::new(..))`-style expressions. Proc-macro diagnostics are +// unstable, so each warning is a use of a `#[deprecated]` item spanned at the +// offending tokens: rustc prints the note there, `-D warnings` makes it an +// error, and nothing reaches the compiled program. + +pub(super) const SHARED_STATE_NOTE: &str = "actors should own their state: shared mutable state \ +(`Arc>`, `Arc>`, `Arc`, `Rc>`) breaks the isolation \ +supervision relies on (a restarted actor inherits whatever the old one left behind the lock), \ +and a panic while holding a std lock poisons it for every holder; send messages to the owning \ +actor instead (or, once available, read a published snapshot, ROADMAP rank 10); if the \ +sharing is deliberate, use `#[actor(allow_shared_state)]`"; + +pub(super) const PANIC_ABORT_NOTE: &str = + "built with `panic = \"abort\"`: a panicking handler aborts \ +the whole process instead of stopping one actor, so supervisors, links, monitors and \ +`stopped()` never see the failure; build actor binaries with `panic = \"unwind\"` (the \ +default), or use `#[actor(allow_panic_abort)]` if aborting is deliberate"; + +/// Smart pointers that share their contents between owners. +pub(super) fn is_shared_pointer(ident: &Ident) -> bool { + ident == "Arc" || ident == "Rc" +} + +/// Interior-mutable types from std, tokio, parking_lot and similar crates, +/// matched by their last path segment. +pub(super) fn is_interior_mutable(ident: &Ident) -> bool { + ident == "Mutex" + || ident == "RwLock" + || ident == "RefCell" + || ident.to_string().starts_with("Atomic") +} + +/// `Arc>`, `std::sync::Arc>`, `Rc>`, ... +pub(super) fn is_shared_mutable_type(path: &syn::Path) -> bool { + let Some(outer) = path.segments.last() else { + return false; + }; + if !is_shared_pointer(&outer.ident) { + return false; + } + let PathArguments::AngleBracketed(args) = &outer.arguments else { + return false; + }; + matches!( + args.args.first(), + Some(GenericArgument::Type(Type::Path(inner))) + if inner.path.segments.last().is_some_and(|s| is_interior_mutable(&s.ident)) + ) +} + +/// The type segment of an associated call such as `Mutex::new` or +/// `std::sync::Arc::::new`. +pub(super) fn call_type_segment(func: &syn::Expr) -> Option<&Ident> { + let syn::Expr::Path(path) = func else { + return None; + }; + let segments = &path.path.segments; + (segments.len() >= 2).then(|| &segments[segments.len() - 2].ident) +} + +/// `Arc::new(Mutex::new(..))`, `Rc::new(RefCell::new(..))`, +/// `Arc::new(AtomicU64::default())`, ... +pub(super) fn is_shared_mutable_constructor(call: &syn::ExprCall) -> bool { + let syn::Expr::Path(func) = &*call.func else { + return false; + }; + let wraps_in_pointer = func.path.segments.last().is_some_and(|s| s.ident == "new") + && call_type_segment(&call.func).is_some_and(is_shared_pointer) + && call.args.len() == 1; + wraps_in_pointer + && matches!( + call.args.first(), + Some(syn::Expr::Call(inner)) + if call_type_segment(&inner.func).is_some_and(is_interior_mutable) + ) +} + +#[derive(Default)] +pub(super) struct SharedStateFinder { + pub(super) spans: Vec, +} + +impl<'ast> syn::visit::Visit<'ast> for SharedStateFinder { + fn visit_type_path(&mut self, ty: &'ast TypePath) { + if is_shared_mutable_type(&ty.path) { + self.spans.push(ty.span()); + } else { + syn::visit::visit_type_path(self, ty); + } + } + + fn visit_expr_call(&mut self, call: &'ast syn::ExprCall) { + if is_shared_mutable_constructor(call) { + self.spans.push(call.span()); + } else { + syn::visit::visit_expr_call(self, call); + } + } +} + +/// A warning at `span`: the use of a local `#[deprecated]` unit struct. +pub(super) fn deprecation_warning( + name: &str, + note: &str, + span: proc_macro2::Span, +) -> proc_macro2::TokenStream { + let name = Ident::new(name, span); + quote::quote_spanned! { span => + const _: () = { + #[deprecated(note = #note)] + struct #name; + let _ = #name; + }; + } +} + +/// Set once the panic-abort warning has been emitted in this compilation. A +/// proc macro is loaded once per rustc invocation, so this makes the warning +/// once per crate rather than once per actor. If a compiler ever reloaded the +/// macro, the warning would only repeat. +pub(super) static PANIC_ABORT_WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +pub(super) fn actor_lints( + impl_block: &ItemImpl, + allow_shared_state: bool, + allow_panic_abort: bool, +) -> proc_macro2::TokenStream { + let mut lints = proc_macro2::TokenStream::new(); + if !allow_shared_state { + let mut finder = SharedStateFinder::default(); + syn::visit::Visit::visit_item_impl(&mut finder, impl_block); + for span in finder.spans { + lints.extend(deprecation_warning( + "SharedMutableStateInActor", + SHARED_STATE_NOTE, + span, + )); + } + } + if !allow_panic_abort && !PANIC_ABORT_WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { + let warning = deprecation_warning( + "PanicAbortInActorCrate", + PANIC_ABORT_NOTE, + proc_macro2::Span::call_site(), + ); + lints.extend(quote! { + #[cfg(panic = "abort")] + #warning + }); + } + lints +} diff --git a/macros/src/protocol.rs b/macros/src/protocol.rs new file mode 100644 index 00000000..2ce599f1 --- /dev/null +++ b/macros/src/protocol.rs @@ -0,0 +1,993 @@ +//! `#[protocol]`: message structs, blanket protocol impls and request plumbing from a trait. +use super::*; + +// --- Helpers for #[protocol] --- + +pub(super) fn to_snake_case(s: &str) -> String { + let mut result = String::new(); + let chars: Vec = s.chars().collect(); + for (i, &ch) in chars.iter().enumerate() { + if ch.is_uppercase() { + // Insert underscore before uppercase if: + // - not at start, AND + // - previous char is lowercase, OR next char is lowercase (handles acronyms) + if i > 0 { + let prev_lower = chars[i - 1].is_lowercase(); + let next_lower = chars.get(i + 1).is_some_and(|c| c.is_lowercase()); + if prev_lower || next_lower { + result.push('_'); + } + } + result.push(ch.to_ascii_lowercase()); + } else { + result.push(ch); + } + } + result +} + +pub(super) fn to_pascal_case(s: &str) -> String { + s.split('_') + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().to_string() + chars.as_str(), + } + }) + .collect() +} + +pub(super) fn strip_protocol_suffix(name: &str) -> String { + name.strip_suffix("Protocol").unwrap_or(name).to_string() +} + +// A method-level cfg_attr has not been expanded when the enclosing protocol +// macro runs. Reject conditional timeout attributes even for false predicates; +// otherwise Rust sees an unknown attribute, or silently omits the timeout. +pub(super) fn conditional_timeout(meta: &syn::Meta) -> bool { + if let syn::Meta::List(list) = meta { + if list.path.is_ident("cfg_attr") { + use syn::parse::Parser; + let arguments = + syn::punctuated::Punctuated::::parse_terminated; + if let Ok(arguments) = arguments.parse2(list.tokens.clone()) { + return arguments.iter().skip(1).any(|meta| { + meta.path().is_ident("request_timeout") || conditional_timeout(meta) + }); + } + } + } + false +} + +pub(super) enum MethodKind { + Send, + Request(Box), +} + +#[derive(Clone, Copy)] +pub(super) enum RuntimeMode { + Tasks, + Threads, +} + +pub(super) fn classify_return_type(ret: &ReturnType) -> Result { + match ret { + ReturnType::Default => Ok(MethodKind::Send), + ReturnType::Type(_, ty) => { + if is_unit_type(ty) { + return Ok(MethodKind::Send); + } + if let Some(inner) = extract_response_inner(ty) { + return Ok(MethodKind::Request(inner)); + } + if let Some(inner) = extract_result_inner(ty) { + if is_unit_type(&inner) { + return Ok(MethodKind::Send); + } + // Result where T ≠ () is no longer supported; + // use Response which works in both modes. + return Err(ty); + } + Err(ty) + } + } +} + +pub(super) fn extract_response_inner(ty: &Type) -> Option> { + if let Type::Path(TypePath { path, .. }) = ty { + let seg = path.segments.last()?; + if seg.ident == "Response" { + if let PathArguments::AngleBracketed(args) = &seg.arguments { + if let Some(GenericArgument::Type(inner)) = args.args.first() { + return Some(Box::new(inner.clone())); + } + } + } + } + None +} + +pub(super) fn extract_result_inner(ty: &Type) -> Option> { + if let Type::Path(TypePath { path, .. }) = ty { + let seg = path.segments.last()?; + if seg.ident == "Result" { + if let PathArguments::AngleBracketed(args) = &seg.arguments { + if let Some(GenericArgument::Type(inner)) = args.args.first() { + return Some(Box::new(inner.clone())); + } + } + } + } + None +} + +/// Protocol parameters become fields of a `'static` message struct. Report +/// borrows and `impl Trait` here; rustc's own errors point into the expansion. +pub(super) fn unsupported_param_type(ty: &Type) -> Option { + const OWNED: &str = "protocol parameters become message fields and must be owned ('static); \ + use an owned type such as String or Arc instead of a borrow"; + match ty { + Type::Reference(r) => match &r.lifetime { + Some(lifetime) if lifetime.ident == "static" => unsupported_param_type(&r.elem), + _ => Some(syn::Error::new_spanned(r, OWNED)), + }, + Type::ImplTrait(t) => Some(syn::Error::new_spanned( + t, + "`impl Trait` parameters are not supported in protocol methods; \ + use a concrete type or Box", + )), + Type::Path(TypePath { qself, path }) => { + if let Some(error) = qself.as_ref().and_then(|q| unsupported_param_type(&q.ty)) { + return Some(error); + } + path.segments + .iter() + .find_map(|segment| match &segment.arguments { + PathArguments::AngleBracketed(args) => { + args.args.iter().find_map(|arg| match arg { + GenericArgument::Type(inner) => unsupported_param_type(inner), + GenericArgument::Lifetime(lifetime) if lifetime.ident != "static" => { + Some(syn::Error::new_spanned(lifetime, OWNED)) + } + _ => None, + }) + } + _ => None, + }) + } + Type::Tuple(t) => t.elems.iter().find_map(unsupported_param_type), + Type::Array(a) => unsupported_param_type(&a.elem), + Type::Slice(s) => unsupported_param_type(&s.elem), + Type::Paren(p) => unsupported_param_type(&p.elem), + Type::Group(g) => unsupported_param_type(&g.elem), + // fn pointers and trait objects may use higher-ranked borrows. + _ => None, + } +} + +pub(super) fn is_unit_type(ty: &Type) -> bool { + if let Type::Tuple(tuple) = ty { + return tuple.elems.is_empty(); + } + false +} + +/// Returns true for types available without explicit import (prelude + primitives). +/// These can't be accessed via `super::` so must be left unqualified. +pub(super) fn is_prelude_or_primitive(name: &str) -> bool { + matches!( + name, + // Primitives + "bool" | "char" | "str" + | "i8" | "i16" | "i32" | "i64" | "i128" | "isize" + | "u8" | "u16" | "u32" | "u64" | "u128" | "usize" + | "f32" | "f64" + // Prelude types (Rust 2021) + | "Box" | "String" | "Vec" + | "Option" | "Some" | "None" + | "Result" | "Ok" | "Err" + | "ToString" | "ToOwned" + ) +} + +/// Qualify a type with `super::` so it resolves to the parent module's scope. +/// This prevents name collisions when a generated struct name shadows an imported type +/// via `use super::*`. +/// +/// Prelude/primitive types are left unqualified (they can't be accessed via `super::`). +/// User-defined types get `super::` prepended: `Event` → `super::Event`. +/// Generic args are recursively qualified: `Vec` → `Vec`. +pub(super) fn qualify_type_with_super(ty: &Type) -> Type { + match ty { + Type::Path(TypePath { qself, path }) => { + // Leave qualified paths as-is: ::Y, ::abs::path, crate::, super::, self:: + if qself.is_some() || path.leading_colon.is_some() { + return ty.clone(); + } + if let Some(first) = path.segments.first() { + let s = first.ident.to_string(); + if matches!( + s.as_str(), + "crate" | "super" | "self" | "std" | "core" | "alloc" + ) { + return ty.clone(); + } + } + + // Recursively qualify generic arguments in all segments + let qualified_segments: syn::punctuated::Punctuated<_, _> = path + .segments + .iter() + .map(|seg| { + let mut new_seg = seg.clone(); + if let PathArguments::AngleBracketed(ref mut args) = new_seg.arguments { + for arg in &mut args.args { + if let GenericArgument::Type(ref mut inner) = arg { + *inner = qualify_type_with_super(inner); + } + } + } + new_seg + }) + .collect(); + + // Only prepend super:: for non-prelude types + if let Some(first) = path.segments.first() { + if is_prelude_or_primitive(&first.ident.to_string()) { + return Type::Path(TypePath { + qself: None, + path: syn::Path { + leading_colon: None, + segments: qualified_segments, + }, + }); + } + } + + // Prepend super:: for user-defined types + let mut segments = syn::punctuated::Punctuated::new(); + segments.push(syn::PathSegment { + ident: format_ident!("super"), + arguments: PathArguments::None, + }); + for seg in qualified_segments { + segments.push(seg); + } + + Type::Path(TypePath { + qself: None, + path: syn::Path { + leading_colon: None, + segments, + }, + }) + } + Type::Reference(r) => { + let mut new = r.clone(); + new.elem = Box::new(qualify_type_with_super(&r.elem)); + Type::Reference(new) + } + Type::Tuple(t) => { + let mut new = t.clone(); + for elem in &mut new.elems { + *elem = qualify_type_with_super(elem); + } + Type::Tuple(new) + } + Type::Array(a) => { + let mut new = a.clone(); + *new.elem = qualify_type_with_super(&a.elem); + Type::Array(new) + } + Type::Slice(s) => { + let mut new = s.clone(); + *new.elem = qualify_type_with_super(&s.elem); + Type::Slice(new) + } + Type::Paren(p) => { + let mut new = p.clone(); + *new.elem = qualify_type_with_super(&p.elem); + Type::Paren(new) + } + Type::TraitObject(t) => { + let mut new = t.clone(); + for bound in &mut new.bounds { + if let syn::TypeParamBound::Trait(tb) = bound { + qualify_path_with_super(&mut tb.path); + } + } + Type::TraitObject(new) + } + Type::ImplTrait(t) => { + let mut new = t.clone(); + for bound in &mut new.bounds { + if let syn::TypeParamBound::Trait(tb) = bound { + qualify_path_with_super(&mut tb.path); + } + } + Type::ImplTrait(new) + } + Type::BareFn(f) => { + let mut new = f.clone(); + for arg in &mut new.inputs { + arg.ty = qualify_type_with_super(&arg.ty); + } + if let ReturnType::Type(_, ref mut ty) = new.output { + **ty = qualify_type_with_super(ty); + } + Type::BareFn(new) + } + _ => ty.clone(), + } +} + +/// Qualify a path's generic arguments with `super::`, used for trait bounds +/// in `dyn Trait` and `impl Trait` types. +pub(super) fn qualify_path_with_super(path: &mut syn::Path) { + for seg in &mut path.segments { + match &mut seg.arguments { + PathArguments::AngleBracketed(ref mut args) => { + for arg in &mut args.args { + if let GenericArgument::Type(ref mut inner) = arg { + *inner = qualify_type_with_super(inner); + } + } + } + PathArguments::Parenthesized(ref mut args) => { + for input in &mut args.inputs { + *input = qualify_type_with_super(input); + } + if let syn::ReturnType::Type(_, ref mut ty) = args.output { + **ty = qualify_type_with_super(ty); + } + } + PathArguments::None => {} + } + } +} + +pub(super) struct ProtocolInfo<'a> { + pub(super) trait_name: &'a Ident, + pub(super) mod_name: &'a Ident, + pub(super) ref_name: &'a Ident, + pub(super) converter_trait: &'a Ident, + pub(super) converter_method: &'a Ident, + pub(super) target_trait: &'a Ident, +} + +/// One transport signature used in declarations, runtime impls and delegation. +pub(super) fn transport_signature( + info: &ProtocolInfo, + method: &ProtocolMethodInfo, + index: usize, +) -> (Ident, proc_macro2::TokenStream) { + let name = format_ident!("__dispatch_{}", index); + let module = info.mod_name; + let message = &method.struct_name; + let (timeout, result) = match &method.kind { + MethodKind::Send => ( + quote! {}, + quote! { ::std::result::Result<(), spawned_concurrency::ActorError> }, + ), + MethodKind::Request(result) => ( + quote! { , timeout: ::std::time::Duration }, + quote! { spawned_concurrency::Response<#result> }, + ), + }; + let signature = + quote! { fn #name(target: &Self, message: #module::#message #timeout) -> #result }; + (name, signature) +} + +/// Runtime-specific transport primitives, shared by plain and configured handles. +/// Cfg-gated Handler bounds are represented by one marker set per runtime. +pub(super) fn generate_target_impl( + info: &ProtocolInfo, + methods: &[ProtocolMethodInfo], + runtime_path: &proc_macro2::TokenStream, + mode: RuntimeMode, +) -> proc_macro2::TokenStream { + let ProtocolInfo { + trait_name, + mod_name, + target_trait, + .. + } = info; + + // Unconditional methods: Handler bounds go directly on the impl block. + let handler_bounds: Vec<_> = methods + .iter() + .filter(|m| m.cfg_attrs.is_empty()) + .map(|m| { + let sn = &m.struct_name; + quote! { #runtime_path::Handler<#mod_name::#sn> } + }) + .collect(); + + // Group cfg-gated methods by their cfg predicate. + // Each unique group gets a marker trait that conditionally requires the Handler bounds. + let mut cfg_groups: Vec<(String, Vec<&ProtocolMethodInfo>)> = Vec::new(); + for m in methods.iter().filter(|m| !m.cfg_attrs.is_empty()) { + let key: String = m + .cfg_attrs + .iter() + .map(|a| quote!(#a).to_string()) + .collect::>() + .join(","); + if let Some(group) = cfg_groups.iter_mut().find(|(k, _)| k == &key) { + group.1.push(m); + } else { + cfg_groups.push((key, vec![m])); + } + } + + let mode_suffix = match mode { + RuntimeMode::Tasks => format_ident!("Tasks"), + RuntimeMode::Threads => format_ident!("Threads"), + }; + + let mut marker_trait_defs = Vec::new(); + let mut marker_trait_bounds = Vec::new(); + + for (i, (_key, group_methods)) in cfg_groups.iter().enumerate() { + let marker_name = format_ident!("__{}Cfg{}{}", trait_name, i, mode_suffix); + let cfg_attrs = &group_methods[0].cfg_attrs; + let group_handler_bounds: Vec<_> = group_methods + .iter() + .map(|m| { + let sn = &m.struct_name; + quote! { #runtime_path::Handler<#mod_name::#sn> } + }) + .collect(); + + // Combine cfg predicates and negate for the fallback impl. + // #[cfg(A)] #[cfg(B)] -> active: all(A, B), inactive: not(all(A, B)) + let cfg_predicates: Vec = cfg_attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .filter_map(|a| a.parse_args::().ok()) + .collect(); + + let (positive_cfg, negated_cfg) = if cfg_predicates.len() == 1 { + let pred = &cfg_predicates[0]; + (quote! { #[cfg(#pred)] }, quote! { #[cfg(not(#pred))] }) + } else { + ( + quote! { #[cfg(all(#(#cfg_predicates),*))] }, + quote! { #[cfg(not(all(#(#cfg_predicates),*)))] }, + ) + }; + + marker_trait_defs.push(quote! { + #positive_cfg + #[doc(hidden)] + trait #marker_name: #(#group_handler_bounds)+* {} + #positive_cfg + impl<__T: #(#group_handler_bounds)+*> #marker_name for __T {} + + #negated_cfg + #[doc(hidden)] + trait #marker_name {} + #negated_cfg + impl<__T> #marker_name for __T {} + }); + + marker_trait_bounds.push(quote! { + #marker_name }); + } + + let method_impls = methods.iter().enumerate().map(|(index, m)| { + let (_, signature) = transport_signature(info, m, index); + let cfg = &m.cfg_attrs; + let body = match &m.kind { + MethodKind::Send => quote! { target.send(message) }, + MethodKind::Request(_) => match mode { + // The timeout was chosen before the message was built; the + // deadline is fixed now, at send, and travels in the slot. + RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.send_request(message, spawned_concurrency::Deadline::after(timeout))) }, + RuntimeMode::Threads => quote! { spawned_concurrency::Response::ready(target.request_with_timeout(message, timeout)) }, + }, + }; + quote! { #(#cfg)* #signature { #body } } + }); + quote! { + #(#marker_trait_defs)* + impl<__A: #runtime_path::Actor #(+ #handler_bounds)* #(#marker_trait_bounds)*> + #target_trait for #runtime_path::ActorRef<__A> + { + #(#method_impls)* + } + } +} + +/// Share transport bodies and one method template while keeping public protocol +/// impls concrete. Supertraits constrain each nominal target independently. +pub(super) fn generate_protocol_adapters( + info: &ProtocolInfo, + methods: &[ProtocolMethodInfo], + definition: &ItemTrait, +) -> proc_macro2::TokenStream { + let ProtocolInfo { + trait_name, + mod_name, + ref_name, + converter_trait, + converter_method, + target_trait, + } = info; + let signatures = methods.iter().enumerate().map(|(index, m)| { + let (_, signature) = transport_signature(info, m, index); + let cfg = &m.cfg_attrs; + quote! { #(#cfg)* #signature; } + }); + let delegates = methods.iter().enumerate().map(|(index, m)| { + let (name, signature) = transport_signature(info, m, index); + let cfg = &m.cfg_attrs; + let timeout = match &m.kind { MethodKind::Send => quote! {}, MethodKind::Request(_) => quote! { , timeout } }; + quote! { + #(#cfg)* #signature { + <__T as #target_trait>::#name( + spawned_concurrency::response::RequestOptions::actor_ref(target), message #timeout + ) + } + } + }); + let method_impls: Vec<_> = methods.iter().enumerate().map(|(index, m)| { + let method = &m.method_name; + let dispatch = format_ident!("__dispatch_{}", index); + let params = &m.params; + let attrs = &m.method_attrs; + let result = &m.ret_type; + let message = &m.struct_name; + let fields = &m.field_names; + let construct = if fields.is_empty() { + quote! { #mod_name::#message } + } else { + quote! { #mod_name::#message { #(#fields),* } } + }; + let body = match &m.kind { + MethodKind::Send => { + let call = quote! { ::#dispatch(self, #construct) }; + let unit = match result { + ReturnType::Default => true, + ReturnType::Type(_, ty) => is_unit_type(ty), + }; + if unit { quote! { let _ = #call; } } else { call } + } + MethodKind::Request(_) => { + let timeout = &m.timeout; + let binding = syn::Ident::new("__spawned_request_timeout", proc_macro2::Span::mixed_site()); + quote! { + // Evaluate before message construction can move arguments; + // an explicit override does not evaluate the default expression. + let #binding: ::std::time::Duration = match ::__timeout_override(self) { + Some(timeout) => timeout, + None => #timeout, + }; + ::#dispatch(self, #construct, #binding) + } + } + }; + quote! { + #(#attrs)* + fn #method(&self, #(#params),*) #result { #body } + } + }).collect(); + // Keep public impls nominal. A blanket impl over the hidden target trait + // conflicts with existing Box/&T protocol forwarding implementations. + let mut public_impls = Vec::new(); + for (runtime, asynchronous) in [ + (quote! { spawned_concurrency::tasks }, true), + (quote! { spawned_concurrency::threads }, false), + ] { + let actor = quote! { #runtime::ActorRef<__A> }; + for target in [ + actor.clone(), + quote! { spawned_concurrency::response::RequestOptions<#actor> }, + ] { + let mut predicates: Vec<_> = definition + .generics + .where_clause + .iter() + .flat_map(|clause| clause.predicates.iter()) + .map(|p| quote! { #p }) + .collect(); + let supertraits = &definition.supertraits; + if !supertraits.is_empty() { + predicates.push(quote! { Self: #supertraits }); + } + let implementation = quote! { + impl<__A: #runtime::Actor> #trait_name for #target + where Self: #target_trait, #(#predicates,)* + { #(#method_impls)* } + impl<__A: #runtime::Actor> #converter_trait for #target + where Self: #target_trait + #trait_name + { + fn #converter_method(&self) -> #ref_name { ::std::sync::Arc::new(self.clone()) } + } + }; + public_impls.push(if asynchronous { + quote! { spawned_concurrency::__spawned_async_items! { #implementation } } + } else { + implementation + }); + } + } + quote! { + #[doc(hidden)] + trait #target_trait: Clone + Send + Sync + 'static { + fn __timeout_override(_target: &Self) -> Option<::std::time::Duration> { None } + #(#signatures)* + } + impl<__T: #target_trait> #target_trait for spawned_concurrency::response::RequestOptions<__T> { + fn __timeout_override(target: &Self) -> Option<::std::time::Duration> { + Some(spawned_concurrency::response::RequestOptions::timeout(target)) + } + #(#delegates)* + } + #(#public_impls)* + } +} + +pub(super) struct ProtocolMethodInfo { + pub(super) method_name: Ident, + pub(super) struct_name: Ident, + pub(super) field_names: Vec, + pub(super) field_types: Vec, + pub(super) kind: MethodKind, + pub(super) params: Vec, + pub(super) ret_type: ReturnType, + /// Doc + cfg attributes to propagate to blanket impl methods. + pub(super) method_attrs: Vec, + pub(super) cfg_attrs: Vec, + pub(super) timeout: syn::Expr, +} + +pub(crate) fn expand(attr: TokenStream, item: TokenStream) -> TokenStream { + let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); + let mut protocol_timeout = None; + for option in options { + if !option.path().is_ident("timeout") { + return syn::Error::new_spanned( + option.path(), + "unsupported protocol option; expected timeout = Duration expression", + ) + .to_compile_error() + .into(); + } + let syn::Meta::NameValue(option) = option else { + return syn::Error::new_spanned(option, "expected `timeout = `") + .to_compile_error() + .into(); + }; + if protocol_timeout.replace(option.value).is_some() { + return syn::Error::new_spanned(option.path, "duplicate protocol timeout") + .to_compile_error() + .into(); + } + } + let mut trait_def = parse_macro_input!(item as ItemTrait); + let trait_name = &trait_def.ident; + let trait_vis = &trait_def.vis; + + if !trait_def.generics.params.is_empty() { + return syn::Error::new_spanned( + &trait_def.generics, + "generic parameters (types, lifetimes, consts) on protocol traits are not supported", + ) + .to_compile_error() + .into(); + } + + let base_name = strip_protocol_suffix(&trait_name.to_string()); + let mod_name = format_ident!("{}", to_snake_case(&trait_name.to_string())); + let ref_name = format_ident!("{}Ref", base_name); + let converter_trait = format_ident!("To{}Ref", base_name); + let converter_method = format_ident!("to_{}_ref", to_snake_case(&base_name)); + + let mut methods: Vec = Vec::new(); + + for item in &mut trait_def.items { + if !matches!(item, TraitItem::Fn(_)) { + return syn::Error::new_spanned( + item, + "protocol traits may only contain methods; \ + associated types, constants, and other items are not supported", + ) + .to_compile_error() + .into(); + } + if let TraitItem::Fn(method) = item { + if let Some(attribute) = method.attrs.iter().find(|a| conditional_timeout(&a.meta)) { + return syn::Error::new_spanned( + attribute, + "conditional request_timeout is not supported; use #[request_timeout(...)] directly", + ).to_compile_error().into(); + } + if method.sig.asyncness.is_some() { + return syn::Error::new_spanned( + &method.sig, + "protocol methods must not be async; \ + use Response as the return type for requests", + ) + .to_compile_error() + .into(); + } + + // Verify first param is &self + match method.sig.inputs.first() { + Some(FnArg::Receiver(r)) if r.reference.is_some() && r.mutability.is_none() => {} + _ => { + return syn::Error::new_spanned( + &method.sig, + "protocol methods must take `&self` as the first parameter", + ) + .to_compile_error() + .into(); + } + } + + let sig = &method.sig; + let qualifier = [ + (sig.unsafety.as_ref().map(|t| quote!(#t)), "unsafe"), + (sig.abi.as_ref().map(|t| quote!(#t)), "extern"), + (sig.constness.as_ref().map(|t| quote!(#t)), "const"), + ] + .into_iter() + .find_map(|(token, name)| Some((token?, name))); + if let Some((token, name)) = qualifier { + return syn::Error::new_spanned( + token, + format!("`{name}` protocol methods are not supported; use a plain `fn`"), + ) + .to_compile_error() + .into(); + } + if !method.sig.generics.params.is_empty() { + return syn::Error::new_spanned( + &method.sig.generics, + "protocol methods cannot have generic parameters; \ + each method becomes one concrete message struct", + ) + .to_compile_error() + .into(); + } + for arg in method.sig.inputs.iter().skip(1) { + if let FnArg::Typed(pat_type) = arg { + if let Some(error) = unsupported_param_type(&pat_type.ty) { + return error.to_compile_error().into(); + } + } + } + + let method_name = method.sig.ident.clone(); + let struct_name = format_ident!("{}", to_pascal_case(&method_name.to_string())); + // cfg-gated methods may legitimately share a name across exclusive + // predicates; only unconditional collisions are reported here. + let conditional = |attrs: &[Attribute]| { + attrs + .iter() + .any(|a| a.path().is_ident("cfg") || a.path().is_ident("cfg_attr")) + }; + if !conditional(&method.attrs) { + if let Some(previous) = methods + .iter() + .find(|m| m.cfg_attrs.is_empty() && m.struct_name == struct_name) + { + let message = if previous.method_name == method_name { + format!("duplicate protocol method `{method_name}`") + } else { + format!( + "protocol method `{method_name}` generates message struct \ + `{struct_name}`, which `{}` already uses; rename one of them", + previous.method_name + ) + }; + return syn::Error::new_spanned(&method.sig.ident, message) + .to_compile_error() + .into(); + } + } + + let mut field_names: Vec = Vec::new(); + let mut field_types: Vec = Vec::new(); + let mut params: Vec = Vec::new(); + + for arg in method.sig.inputs.iter().skip(1) { + if let FnArg::Typed(pat_type) = arg { + if let Pat::Ident(pat_ident) = &*pat_type.pat { + field_names.push(pat_ident.ident.clone()); + field_types.push((*pat_type.ty).clone()); + } else { + return syn::Error::new_spanned( + &pat_type.pat, + "protocol methods only support simple identifier patterns \ + (e.g., `name: Type`)", + ) + .to_compile_error() + .into(); + } + } + params.push(arg.clone()); + } + + let kind = match classify_return_type(&method.sig.output) { + Ok(kind) => kind, + Err(ty) => { + return syn::Error::new_spanned( + ty, + "unsupported return type in protocol method; \ + use Response for requests (works in both async and sync modes), \ + Result<(), ActorError> for sends, or no return type for fire-and-forget", + ) + .to_compile_error() + .into(); + } + }; + + let mut method_timeout = None; + for attr in method + .attrs + .iter() + .filter(|a| a.path().is_ident("request_timeout")) + { + if !matches!(kind, MethodKind::Request(_)) { + return syn::Error::new_spanned( + attr, + "request_timeout is only valid on Response methods", + ) + .to_compile_error() + .into(); + } + let expression = match attr.parse_args::() { + Ok(expression) => expression, + Err(error) => return error.to_compile_error().into(), + }; + if method_timeout.replace(expression).is_some() { + return syn::Error::new_spanned(attr, "duplicate request_timeout") + .to_compile_error() + .into(); + } + } + method + .attrs + .retain(|a| !a.path().is_ident("request_timeout")); + let timeout = method_timeout + .or_else(|| protocol_timeout.clone()) + .unwrap_or_else( + || syn::parse_quote! { spawned_concurrency::response::DEFAULT_REQUEST_TIMEOUT }, + ); + + let method_attrs: Vec = method + .attrs + .iter() + .filter(|a| { + a.path().is_ident("doc") + || a.path().is_ident("cfg") + || a.path().is_ident("cfg_attr") + }) + .cloned() + .collect(); + let cfg_attrs: Vec = method + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg") || a.path().is_ident("cfg_attr")) + .cloned() + .collect(); + + methods.push(ProtocolMethodInfo { + method_name, + struct_name, + field_names, + field_types, + kind, + params, + ret_type: method.sig.output.clone(), + method_attrs, + cfg_attrs, + timeout, + }); + } + } + + // Generate message structs + // Field types and result types are qualified with `super::` to prevent + // name collisions when a generated struct name shadows an imported type. + // e.g., `fn event(&self, event: Event)` generates `struct Event { pub event: super::Event }` + let msg_structs: Vec<_> = methods + .iter() + .map(|m| { + let struct_name = &m.struct_name; + let field_names = &m.field_names; + let qualified_field_types: Vec = + m.field_types.iter().map(qualify_type_with_super).collect(); + let method_attrs = &m.method_attrs; + let cfg_attrs = &m.cfg_attrs; + let msg_result_ty: Type = match &m.kind { + MethodKind::Send => syn::parse_quote! { () }, + MethodKind::Request(inner) => qualify_type_with_super(inner), + }; + + if field_names.is_empty() { + quote! { + #(#method_attrs)* + #[derive(Clone)] + pub struct #struct_name; + #(#cfg_attrs)* + impl Message for #struct_name { + type Result = #msg_result_ty; + } + } + } else { + quote! { + #(#method_attrs)* + pub struct #struct_name { + #(pub #field_names: #qualified_field_types,)* + } + #(#cfg_attrs)* + impl Message for #struct_name { + type Result = #msg_result_ty; + } + } + } + }) + .collect(); + + // Gate async impls using the dependency feature namespace, not the consumer. + let target_trait = format_ident!("__{}Target", trait_name); + let tasks = quote! { spawned_concurrency::tasks }; + let threads = quote! { spawned_concurrency::threads }; + let proto_info = ProtocolInfo { + trait_name, + mod_name: &mod_name, + ref_name: &ref_name, + converter_trait: &converter_trait, + converter_method: &converter_method, + target_trait: &target_trait, + }; + let tasks_impl = generate_target_impl(&proto_info, &methods, &tasks, RuntimeMode::Tasks); + let threads_impl = generate_target_impl(&proto_info, &methods, &threads, RuntimeMode::Threads); + let adapters = generate_protocol_adapters(&proto_info, &methods, &trait_def); + let blanket_impls = quote! { + #adapters + spawned_concurrency::__spawned_async_items! { #tasks_impl } + #threads_impl + }; + + let ref_doc = format!( + "Type-erased reference to any actor implementing [`{trait_name}`].\n\n\ + Use this type to store protocol references without depending on the concrete actor type." + ); + + // Protocol traits are consumed via blanket impls and dyn references, + // so rustc may report methods as unused during development. + let output = quote! { + #[allow(dead_code)] + #trait_def + + #[doc = #ref_doc] + #trait_vis type #ref_name = ::std::sync::Arc; + + #trait_vis mod #mod_name { + use super::*; + use spawned_concurrency::message::Message; + #(#msg_structs)* + } + + #trait_vis trait #converter_trait { + fn #converter_method(&self) -> #ref_name; + } + + impl #converter_trait for #ref_name { + fn #converter_method(&self) -> #ref_name { + ::std::sync::Arc::clone(self) + } + } + + #blanket_impls + }; + + output.into() +} diff --git a/macros/src/send_handler.rs b/macros/src/send_handler.rs new file mode 100644 index 00000000..a5d4f642 --- /dev/null +++ b/macros/src/send_handler.rs @@ -0,0 +1,102 @@ +//! `#[send_handler]`: the send-only form of a manual `Handler` impl. +use super::*; + +pub(crate) fn expand(attr: TokenStream, item: TokenStream) -> TokenStream { + let krate: Option = if attr.is_empty() { + None + } else { + let parser = |input: syn::parse::ParseStream| { + input.parse::()?; + input.parse::()?; + let path: syn::Path = input.parse()?; + if !input.is_empty() { + return Err(input.error("expected only `crate = path`")); + } + Ok(path) + }; + match syn::parse::Parser::parse(parser, attr) { + Ok(path) => Some(path), + Err(error) => { + return syn::Error::new( + error.span(), + format!("#[send_handler] takes no arguments or `crate = path`: {error}"), + ) + .to_compile_error() + .into() + } + } + }; + let mut impl_block = parse_macro_input!(item as ItemImpl); + let is_handler = impl_block + .trait_ + .as_ref() + .and_then(|(_, path, _)| path.segments.last()) + .is_some_and(|segment| segment.ident == "Handler"); + if !is_handler { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] on an item applies to an `impl Handler for Actor` block; \ + inside an #[actor] block, put it on the handler method instead", + ) + .to_compile_error() + .into(); + } + let Some(method) = impl_block.items.iter_mut().find_map(|item| match item { + ImplItem::Fn(method) if method.sig.ident == "handle" => Some(method), + _ => None, + }) else { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] impl must define `handle`", + ) + .to_compile_error() + .into(); + }; + if method.sig.inputs.len() != 3 { + return syn::Error::new_spanned( + &method.sig, + "#[send_handler] `handle` takes (&mut self, msg: M, ctx: &Context), without the reply parameter", + ) + .to_compile_error() + .into(); + } + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned(ty, "#[send_handler] `handle` returns nothing") + .to_compile_error() + .into(); + } + } + let reply = format_ident!("__spawned_reply"); + // The `Handler` path's module (`tasks`/`threads`) re-exports + // `PendingReply`; a bare `Handler` falls back to the crate path. + let handler_module = impl_block.trait_.as_ref().and_then(|(_, path, _)| { + (path.segments.len() > 1).then(|| { + let mut module = path.clone(); + module.segments.pop(); + module.segments.pop_punct(); + module + }) + }); + let reply_ty = match (krate, handler_module) { + (Some(krate), _) => quote! { #krate::PendingReply<'_, ()> }, + (None, Some(module)) => quote! { #module::PendingReply<'_, ()> }, + (None, None) => quote! { spawned_concurrency::PendingReply<'_, ()> }, + }; + let reply_param: FnArg = syn::parse_quote! { + #reply: #reply_ty + }; + method.sig.inputs.insert(2, reply_param); + method.sig.output = ReturnType::Default; + let body = &method.block; + let run = if method.sig.asyncness.is_some() { + quote! { async move #body.await } + } else { + quote! { (move || #body)() } + }; + method.block = syn::parse_quote! {{ + let () = #run; + let _ = #reply.reply(()); + }}; + quote! { #impl_block }.into() +} diff --git a/macros/src/sim_test.rs b/macros/src/sim_test.rs new file mode 100644 index 00000000..579f1326 --- /dev/null +++ b/macros/src/sim_test.rs @@ -0,0 +1,112 @@ +//! `#[sim_test]`: a test run under the deterministic simulator. +use super::*; + +pub(crate) fn expand(attr: TokenStream, item: TokenStream) -> TokenStream { + let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); + let function = parse_macro_input!(item as syn::ItemFn); + let mut multi_thread = false; + let mut workers: Option = None; + let mut seeds: Option = None; + let mut explore: Option = None; + let mut iterations: Option = None; + for option in options { + let key = option + .path + .get_ident() + .map(ToString::to_string) + .unwrap_or_default(); + match key.as_str() { + "flavor" => { + let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(flavor), + .. + }) = &option.value + else { + return syn::Error::new_spanned(&option.value, "expected a string flavor") + .to_compile_error() + .into(); + }; + match flavor.value().as_str() { + "multi_thread" => multi_thread = true, + "current_thread" => multi_thread = false, + other => { + return syn::Error::new_spanned(flavor, format!("unknown flavor {other}")) + .to_compile_error() + .into() + } + } + } + "worker_threads" => workers = Some(option.value), + "seeds" => seeds = Some(option.value), + "explore" => explore = Some(option.value), + "iterations" => iterations = Some(option.value), + _ => { + return syn::Error::new_spanned( + &option.path, + "sim_test accepts flavor, worker_threads, seeds, explore and iterations", + ) + .to_compile_error() + .into() + } + } + } + if function.sig.asyncness.is_none() { + return syn::Error::new_spanned(function.sig.fn_token, "sim_test needs an async fn") + .to_compile_error() + .into(); + } + if !matches!(function.sig.output, ReturnType::Default) { + return syn::Error::new_spanned(&function.sig.output, "sim_test functions return ()") + .to_compile_error() + .into(); + } + let seed = match function.sig.inputs.len() { + 0 => quote! { _ }, + 1 => match &function.sig.inputs[0] { + FnArg::Typed(arg) => { + let pat = &arg.pat; + quote! { #pat } + } + FnArg::Receiver(r) => { + return syn::Error::new_spanned(r, "sim_test cannot take self") + .to_compile_error() + .into() + } + }, + _ => { + return syn::Error::new_spanned( + &function.sig.inputs, + "sim_test takes at most one argument, the u64 seed", + ) + .to_compile_error() + .into() + } + }; + let attrs = &function.attrs; + let vis = &function.vis; + let name = &function.sig.ident; + let body = &function.block; + let flavor = if multi_thread { + let workers = workers.unwrap_or_else(|| syn::parse_quote!(2)); + quote! { .multi_thread(#workers) } + } else { + quote! {} + }; + let seeds = seeds.map(|n| quote! { .seeds(0..(#n)) }); + let explore = explore.map(|s| quote! { .explore(#s) }); + let iterations = iterations.map(|n| quote! { .iterations(#n) }); + quote! { + #(#attrs)* + #[test] + #vis fn #name() { + ::spawned_rt::tasks::SimTest::new() + .named(env!("CARGO_PKG_NAME"), module_path!(), stringify!(#name)) + #flavor + #seeds + #explore + #iterations + .run(|#seed: u64| async move #body) + } + } + .into() +} diff --git a/verification/model-map.toml b/verification/model-map.toml index 191bd0dc..01e65567 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -40,7 +40,7 @@ coverage_floor = 36 # of 40 action keys [models.Cohorts] # OneForAll/RestForOne rounds (selection, stop, cleanup, join barrier, -# resume, supersession, retirement), in supervision.rs. +# resume, supersession, retirement), in supervision.rs and supervision/. files = [ "verification/tla/Cohorts.tla", "verification/tla/CohortsInd.tla", @@ -59,7 +59,7 @@ coverage_floor = 40 # of 46 action keys [models.NestedSupervision] # Subtree ownership, attachment, parent join after descendants, shutdown -# propagation: supervision.rs (TBegin..TDone) and supervision/nested.rs. +# propagation: supervision.rs and supervision/ (TBegin..TDone), supervision/nested.rs. files = [ "verification/tla/NestedSupervision.tla", "verification/tla/NestedSupervisionInd.tla", @@ -108,7 +108,9 @@ code = [ "concurrency/src/stop_token/core.rs", "concurrency/src/actor_cleanup.rs", "concurrency/src/tasks/actor.rs", + "concurrency/src/tasks/actor/**", "concurrency/src/threads/actor.rs", + "concurrency/src/threads/actor/**", "concurrency/src/link.rs", "concurrency/src/link/table.rs", "concurrency/src/monitor.rs", @@ -135,7 +137,7 @@ coverage_floor = 14 # of 14 action keys (TraceReplies arms) [models.ReplyTimer] # Receiver-side refinement: the shared runtime deadline and shortening mark. files = ["verification/tla/ReplyTimer.tla", "verification/tla/MCReplyTimer*"] -code = ["concurrency/src/reply/mod.rs", "concurrency/src/reply/slot.rs", "concurrency/src/tasks/actor.rs", "rt/src/tasks/tokio/mod.rs", "rt/src/tasks/sim.rs", "rt/src/time.rs"] +code = ["concurrency/src/reply/mod.rs", "concurrency/src/reply/slot.rs", "concurrency/src/tasks/actor.rs", "concurrency/src/tasks/actor/**", "rt/src/tasks/tokio/mod.rs", "rt/src/tasks/sim.rs", "rt/src/time.rs"] [models.ReplyTimer.tlc_only] reason = "receiver-local timer bookkeeping has no actor lifecycle trace events; production slot races are shared with Loom" tracking = "receiver timer projection of rank 11; see tests/reply_slot_loom.rs and tests/request_timer.rs" diff --git a/verification/planted/patches/01-sup-budget-off-by-one.patch b/verification/planted/patches/01-sup-budget-off-by-one.patch index ebd6492c..18ceb638 100644 --- a/verification/planted/patches/01-sup-budget-off-by-one.patch +++ b/verification/planted/patches/01-sup-budget-off-by-one.patch @@ -1,10 +1,9 @@ Planted bug 01 (sup-budget-off-by-one): Restart window admits max+1 restarts (`>=` became `>`) -diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs -index 328ed91b..f82cc670 100644 ---- a/concurrency/src/supervision.rs -+++ b/concurrency/src/supervision.rs -@@ -593,7 +593,7 @@ pub(crate) fn admit_in_window( +diff --git a/concurrency/src/supervision/state.rs b/concurrency/src/supervision/state.rs +--- a/concurrency/src/supervision/state.rs ++++ b/concurrency/src/supervision/state.rs +@@ -40,7 +40,7 @@ while attempts.front().is_some_and(|t| since(now, *t) >= window) { attempts.pop_front(); } diff --git a/verification/planted/patches/02-sup-window-edge.patch b/verification/planted/patches/02-sup-window-edge.patch index c571d07c..6df60f86 100644 --- a/verification/planted/patches/02-sup-window-edge.patch +++ b/verification/planted/patches/02-sup-window-edge.patch @@ -1,10 +1,9 @@ Planted bug 02 (sup-window-edge): A restart stamped exactly `window` ago still counts against the budget -diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs -index 328ed91b..0491d431 100644 ---- a/concurrency/src/supervision.rs -+++ b/concurrency/src/supervision.rs -@@ -590,7 +590,7 @@ pub(crate) fn admit_in_window( +diff --git a/concurrency/src/supervision/state.rs b/concurrency/src/supervision/state.rs +--- a/concurrency/src/supervision/state.rs ++++ b/concurrency/src/supervision/state.rs +@@ -37,7 +37,7 @@ now: T, since: impl Fn(T, T) -> Duration, ) -> bool { diff --git a/verification/planted/patches/03-sup-restforone-wrong-side.patch b/verification/planted/patches/03-sup-restforone-wrong-side.patch index cfc0122f..aaa85cc8 100644 --- a/verification/planted/patches/03-sup-restforone-wrong-side.patch +++ b/verification/planted/patches/03-sup-restforone-wrong-side.patch @@ -1,19 +1,18 @@ Planted bug 03 (sup-restforone-wrong-side): RestForOne rebuilds the failed child and the children *before* it -diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs -index 328ed91b..8ef444b4 100644 ---- a/concurrency/src/supervision.rs -+++ b/concurrency/src/supervision.rs -@@ -1533,7 +1533,11 @@ async fn run_recoveries(core: Arc) { +diff --git a/concurrency/src/supervision/run.rs b/concurrency/src/supervision/run.rs +--- a/concurrency/src/supervision/run.rs ++++ b/concurrency/src/supervision/run.rs +@@ -263,7 +263,11 @@ continue; }; let mut affected = Vec::new(); - for (affected_index, child) in children.iter().enumerate().skip(start) { -+ for (affected_index, child) in children -+ .iter() -+ .enumerate() -+ .take(if start == 0 { children.len() } else { start + 1 }) -+ { ++ for (affected_index, child) in children.iter().enumerate().take(if start == 0 { ++ children.len() ++ } else { ++ start + 1 ++ }) { let slot = RecoverySlot::new(meta.clone()); let accepted = trace::atomic(|| { let accepted = child.assign_recovery(slot.clone()); diff --git a/verification/planted/patches/05-sup-temporary-collateral-rebuilt.patch b/verification/planted/patches/05-sup-temporary-collateral-rebuilt.patch index f82dc10c..2c21e827 100644 --- a/verification/planted/patches/05-sup-temporary-collateral-rebuilt.patch +++ b/verification/planted/patches/05-sup-temporary-collateral-rebuilt.patch @@ -1,10 +1,9 @@ Planted bug 05 (sup-temporary-collateral-rebuilt): A Temporary sibling stopped as collateral by OneForAll/RestForOne is rebuilt -diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs -index 328ed91b..6ab677b2 100644 ---- a/concurrency/src/supervision.rs -+++ b/concurrency/src/supervision.rs -@@ -1695,7 +1695,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc (), } last_recovery = round.meta.id; diff --git a/verification/planted/patches/06-sup-backoff-never-resets.patch b/verification/planted/patches/06-sup-backoff-never-resets.patch index 4dd4f4ce..fed09ee8 100644 --- a/verification/planted/patches/06-sup-backoff-never-resets.patch +++ b/verification/planted/patches/06-sup-backoff-never-resets.patch @@ -1,10 +1,9 @@ Planted bug 06 (sup-backoff-never-resets): Backoff failure streak never resets after a stable generation -diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs -index 328ed91b..4fea1214 100644 ---- a/concurrency/src/supervision.rs -+++ b/concurrency/src/supervision.rs -@@ -2148,9 +2148,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc=` became `>`) -02 sup-window-edge supervision.rs A restart stamped exactly `window` ago still counts against the budget -03 sup-restforone-wrong-side supervision.rs RestForOne rebuilds the failed child and the children *before* it +01 sup-budget-off-by-one supervision/state.rs Restart window admits max+1 restarts (`>=` became `>`) +02 sup-window-edge supervision/state.rs A restart stamped exactly `window` ago still counts against the budget +03 sup-restforone-wrong-side supervision/run.rs RestForOne rebuilds the failed child and the children *before* it 04 sup-transient-restarts-on-shutdown supervision.rs Transient restarts on Shutdown (only Normal treated as clean) -05 sup-temporary-collateral-rebuilt supervision.rs A Temporary sibling stopped as collateral by OneForAll/RestForOne is rebuilt -06 sup-backoff-never-resets supervision.rs Backoff failure streak never resets after a stable generation -07 sup-shutdown-forward-order supervision.rs Supervisor stops children in declaration order instead of reverse -08 sup-budget-exhaustion-silent supervision.rs Exhausting the restart budget retires the child instead of failing the supervisor +05 sup-temporary-collateral-rebuilt supervision/run.rs A Temporary sibling stopped as collateral by OneForAll/RestForOne is rebuilt +06 sup-backoff-never-resets supervision/run.rs Backoff failure streak never resets after a stable generation +07 sup-shutdown-forward-order supervision/run.rs Supervisor stops children in declaration order instead of reverse +08 sup-budget-exhaustion-silent supervision/run.rs Exhausting the restart budget retires the child instead of failing the supervisor 09 nested-restart-mode-ignored supervision/nested.rs A nested supervisor's restart mode is ignored (always Permanent) 10 link-normal-exit-kills link.rs A Normal exit kills a non-trapping linked peer 11 link-unlink-one-sided link.rs unlink removes only the caller's edge; the peer still signals it diff --git a/verification/tla/README.md b/verification/tla/README.md index 4715ee4f..dfe455d3 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -6,7 +6,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | Model | Covers | |---|---| -| `Supervision.tla` | The implemented OneForOne supervisor: `run_supervisor` and `run_child` in `concurrency/src/supervision.rs`, restart modes, rolling restart budget, ordered startup, reverse shutdown, shutdown deadlines, cleanup-stuck failures. | +| `Supervision.tla` | The implemented OneForOne supervisor: `run_supervisor` and `run_child` in `concurrency/src/supervision/run.rs`, restart modes, rolling restart budget, ordered startup, reverse shutdown, shutdown deadlines, cleanup-stuck failures. | | `MCSupervision.tla/.cfg` | Fast instance (CI, every PR): two children (Permanent, Transient), safety, action properties and liveness. | | `MCSupervisionTemporary.tla/.cfg` | Small instance (CI, every PR, `other` leg, seconds): a Temporary and a Permanent child, safety and action properties, so `TemporaryStartsOnce` is not vacuous; `negative_checks.py` restarts the Temporary child and requires a violation. | | `MCSupervisionDeep.tla/.cfg` | Deep instance (weekly, `--deep`): three children, one per restart mode, safety and action properties. |