Skip to content

Commit fef8402

Browse files
committed
Updated telemetry implementation to be non blocking
1 parent 96c8730 commit fef8402

6 files changed

Lines changed: 131 additions & 38 deletions

File tree

‎crates/openshell-core/src/telemetry.rs‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,26 @@
66
use chrono::{SecondsFormat, Utc};
77
use reqwest::blocking::Client;
88
use serde_json::{Value, json};
9+
use std::sync::{OnceLock, mpsc};
910
use std::thread;
1011
use std::time::Duration;
1112

13+
const TELEMETRY_EVENT_QUEUE_CAPACITY: usize = 1024;
1214
const CLIENT_ID: &str = "415437562476676";
1315
const DEFAULT_ENDPOINT: &str = "https://events.telemetry.data-uat.nvidia.com/v1.1/events/json";
1416
const EVENT_SCHEMA_VERSION: &str = "2.0";
1517
const EVENT_PROTOCOL_VERSION: &str = "1.6";
1618
const EVENT_SYSTEM_VERSION: &str = "openshell-telemetry/1.0";
1719
const HTTP_TIMEOUT: Duration = Duration::from_secs(5);
1820
const SOURCE: &str = "openshell";
21+
static TELEMETRY_SENDER: OnceLock<Option<mpsc::SyncSender<TelemetryEvent>>> = OnceLock::new();
22+
23+
#[derive(Debug)]
24+
struct TelemetryEvent {
25+
endpoint: String,
26+
name: &'static str,
27+
event: Value,
28+
}
1929

2030
fn telemetry_enabled() -> bool {
2131
telemetry_enabled_from(std::env::var("OPENSHELL_TELEMETRY_ENABLED").ok().as_deref())
@@ -93,6 +103,26 @@ fn build_payload(name: &str, event: Value, ts: &str) -> Value {
93103
})
94104
}
95105

106+
fn telemetry_sender() -> Option<&'static mpsc::SyncSender<TelemetryEvent>> {
107+
TELEMETRY_SENDER
108+
.get_or_init(|| {
109+
let (tx, rx) = mpsc::sync_channel(TELEMETRY_EVENT_QUEUE_CAPACITY);
110+
thread::Builder::new()
111+
.name("openshell-telemetry".to_string())
112+
.spawn(move || telemetry_worker(rx))
113+
.ok()
114+
.map(|_| tx)
115+
})
116+
.as_ref()
117+
}
118+
119+
fn telemetry_worker(rx: mpsc::Receiver<TelemetryEvent>) {
120+
for event in rx {
121+
let payload = build_payload(event.name, event.event, &timestamp());
122+
let _ = publish_payload(&event.endpoint, payload);
123+
}
124+
}
125+
96126
fn publish_payload(endpoint: &str, payload: Value) -> Result<(), reqwest::Error> {
97127
Client::builder()
98128
.use_rustls_tls()
@@ -106,18 +136,29 @@ fn publish_payload(endpoint: &str, payload: Value) -> Result<(), reqwest::Error>
106136
Ok(())
107137
}
108138

139+
fn try_enqueue_event(sender: &mpsc::SyncSender<TelemetryEvent>, event: TelemetryEvent) -> bool {
140+
sender.try_send(event).is_ok()
141+
}
142+
109143
fn emit_event(name: &'static str, event: Value) {
110144
if !telemetry_enabled() {
111145
return;
112146
}
113147
let Some(endpoint) = telemetry_endpoint() else {
114148
return;
115149
};
150+
let Some(sender) = telemetry_sender() else {
151+
return;
152+
};
116153

117-
thread::spawn(move || {
118-
let payload = build_payload(name, event, &timestamp());
119-
let _ = publish_payload(&endpoint, payload);
120-
});
154+
let _ = try_enqueue_event(
155+
sender,
156+
TelemetryEvent {
157+
endpoint,
158+
name,
159+
event,
160+
},
161+
);
121162
}
122163

123164
pub fn emit_lifecycle(resource: &str, operation: &str, outcome: &str) {
@@ -263,4 +304,22 @@ mod tests {
263304
assert_eq!(payload["events"][0]["parameters"]["nvidiaSource"], SOURCE);
264305
assert_eq!(payload["events"][0]["ts"], "2026-05-18T00:00:00.000Z");
265306
}
307+
308+
#[test]
309+
fn telemetry_enqueue_drops_when_queue_is_full() {
310+
let (tx, _rx) = mpsc::sync_channel(1);
311+
let event = || TelemetryEvent {
312+
endpoint: "https://example.test/events".to_string(),
313+
name: "openshell_lifecycle_event",
314+
event: json!({
315+
"nvidiaSource": SOURCE,
316+
"resource": "sandbox",
317+
"operation": "create",
318+
"outcome": "success",
319+
}),
320+
};
321+
322+
assert!(try_enqueue_event(&tx, event()));
323+
assert!(!try_enqueue_event(&tx, event()));
324+
}
266325
}

‎crates/openshell-sandbox/src/activity_aggregator.rs‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,17 @@ use std::future::Future;
88
use tokio::sync::mpsc;
99
use tracing::debug;
1010

11+
pub const ACTIVITY_EVENT_QUEUE_CAPACITY: usize = 1024;
12+
const ACTIVITY_FLUSH_QUEUE_CAPACITY: usize = 1;
13+
1114
#[derive(Debug, Clone)]
1215
pub struct ActivityEvent {
1316
pub denied: bool,
1417
pub deny_group: &'static str,
1518
}
1619

20+
pub type ActivitySender = mpsc::Sender<ActivityEvent>;
21+
1722
#[derive(Debug, Clone, PartialEq, Eq)]
1823
pub struct FlushableActivitySummary {
1924
pub network_activity_count: u32,
@@ -22,15 +27,15 @@ pub struct FlushableActivitySummary {
2227
}
2328

2429
pub struct ActivityAggregator {
25-
rx: mpsc::UnboundedReceiver<ActivityEvent>,
30+
rx: mpsc::Receiver<ActivityEvent>,
2631
network_activity_count: u32,
2732
denied_action_count: u32,
2833
denials_by_group: HashMap<String, u32>,
2934
flush_interval_secs: u64,
3035
}
3136

3237
impl ActivityAggregator {
33-
pub fn new(rx: mpsc::UnboundedReceiver<ActivityEvent>, flush_interval_secs: u64) -> Self {
38+
pub fn new(rx: mpsc::Receiver<ActivityEvent>, flush_interval_secs: u64) -> Self {
3439
Self {
3540
rx,
3641
network_activity_count: 0,
@@ -42,9 +47,17 @@ impl ActivityAggregator {
4247

4348
pub async fn run<F, Fut>(mut self, flush_callback: F)
4449
where
45-
F: Fn(FlushableActivitySummary) -> Fut,
46-
Fut: Future<Output = ()>,
50+
F: Fn(FlushableActivitySummary) -> Fut + Send + 'static,
51+
Fut: Future<Output = ()> + Send + 'static,
4752
{
53+
let (flush_tx, mut flush_rx) =
54+
mpsc::channel::<FlushableActivitySummary>(ACTIVITY_FLUSH_QUEUE_CAPACITY);
55+
tokio::spawn(async move {
56+
while let Some(summary) = flush_rx.recv().await {
57+
flush_callback(summary).await;
58+
}
59+
});
60+
4861
let mut flush_interval =
4962
tokio::time::interval(std::time::Duration::from_secs(self.flush_interval_secs));
5063
flush_interval.tick().await;
@@ -56,7 +69,7 @@ impl ActivityAggregator {
5669
self.ingest(event);
5770
} else {
5871
if let Some(summary) = self.drain() {
59-
flush_callback(summary).await;
72+
queue_flush_summary(&flush_tx, summary);
6073
}
6174
debug!("ActivityAggregator: channel closed, exiting");
6275
return;
@@ -69,7 +82,7 @@ impl ActivityAggregator {
6982
denied = summary.denied_action_count,
7083
"ActivityAggregator: flushing anonymous activity summary"
7184
);
72-
flush_callback(summary).await;
85+
queue_flush_summary(&flush_tx, summary);
7386
}
7487
}
7588
}
@@ -103,6 +116,17 @@ impl ActivityAggregator {
103116
}
104117
}
105118

119+
pub fn try_record_activity(tx: &ActivitySender, denied: bool, deny_group: &'static str) -> bool {
120+
tx.try_send(ActivityEvent { denied, deny_group }).is_ok()
121+
}
122+
123+
fn queue_flush_summary(
124+
tx: &mpsc::Sender<FlushableActivitySummary>,
125+
summary: FlushableActivitySummary,
126+
) -> bool {
127+
tx.try_send(summary).is_ok()
128+
}
129+
106130
pub fn sanitize_deny_group(raw: &str) -> &'static str {
107131
match raw {
108132
"connect_policy" | "connect" | "l4_deny" => "connect_policy",
@@ -150,4 +174,25 @@ mod tests {
150174
assert_float_eq(denial_rate_pct(4, 1), 25.0);
151175
assert_float_eq(denial_rate_pct(4, 10), 100.0);
152176
}
177+
178+
#[test]
179+
fn activity_send_drops_when_queue_is_full() {
180+
let (tx, _rx) = mpsc::channel(1);
181+
182+
assert!(try_record_activity(&tx, false, "unknown"));
183+
assert!(!try_record_activity(&tx, true, "connect_policy"));
184+
}
185+
186+
#[test]
187+
fn flush_summary_drops_when_queue_is_full() {
188+
let (tx, _rx) = mpsc::channel(1);
189+
let summary = FlushableActivitySummary {
190+
network_activity_count: 1,
191+
denied_action_count: 0,
192+
denials_by_group: Vec::new(),
193+
};
194+
195+
assert!(queue_flush_summary(&tx, summary.clone()));
196+
assert!(!queue_flush_summary(&tx, summary));
197+
}
153198
}

‎crates/openshell-sandbox/src/bypass_monitor.rs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
//! the monitor logs a one-time warning and returns. The iptables REJECT rules
1717
//! still provide fast-fail UX — the monitor only adds diagnostic visibility.
1818
19-
use crate::activity_aggregator::ActivityEvent;
19+
use crate::activity_aggregator::{ActivitySender, try_record_activity};
2020
use crate::denial_aggregator::DenialEvent;
2121
use openshell_ocsf::{
2222
ActionId, ActivityId, ConfidenceId, DetectionFindingBuilder, DispositionId, Endpoint,
@@ -119,7 +119,7 @@ pub fn spawn(
119119
namespace_name: String,
120120
entrypoint_pid: Arc<AtomicU32>,
121121
denial_tx: Option<mpsc::UnboundedSender<DenialEvent>>,
122-
activity_tx: Option<mpsc::UnboundedSender<ActivityEvent>>,
122+
activity_tx: Option<ActivitySender>,
123123
) -> Option<tokio::task::JoinHandle<()>> {
124124
use std::io::BufRead;
125125
use std::process::{Command, Stdio};
@@ -280,10 +280,7 @@ pub fn spawn(
280280
});
281281
}
282282
if let Some(ref tx) = activity_tx {
283-
let _ = tx.send(ActivityEvent {
284-
denied: true,
285-
deny_group: "bypass",
286-
});
283+
let _ = try_record_activity(tx, true, "bypass");
287284
}
288285
}
289286

‎crates/openshell-sandbox/src/l7/relay.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
//! Parses each request within the tunnel, evaluates it against OPA policy,
88
//! and either forwards or denies the request.
99
10-
use crate::activity_aggregator::ActivityEvent;
10+
use crate::activity_aggregator::{ActivitySender, try_record_activity};
1111
use crate::l7::provider::{L7Provider, RelayOutcome};
1212
use crate::l7::rest::WebSocketExtensionMode;
1313
use crate::l7::{EnforcementMode, L7EndpointConfig, L7Protocol, L7RequestInfo};
@@ -20,7 +20,6 @@ use openshell_ocsf::{
2020
};
2121
use std::sync::Arc;
2222
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
23-
use tokio::sync::mpsc;
2423
use tracing::{debug, warn};
2524

2625
/// Context for L7 request policy evaluation.
@@ -40,7 +39,7 @@ pub struct L7EvalContext {
4039
/// Supervisor-only placeholder resolver for outbound headers.
4140
pub(crate) secret_resolver: Option<Arc<SecretResolver>>,
4241
/// Anonymous activity counter channel.
43-
pub(crate) activity_tx: Option<mpsc::UnboundedSender<ActivityEvent>>,
42+
pub(crate) activity_tx: Option<ActivitySender>,
4443
}
4544

4645
#[derive(Default)]
@@ -458,7 +457,7 @@ fn emit_l7_request_log(
458457

459458
fn emit_activity(ctx: &L7EvalContext, denied: bool, deny_group: &'static str) {
460459
if let Some(tx) = &ctx.activity_tx {
461-
let _ = tx.send(ActivityEvent { denied, deny_group });
460+
let _ = try_record_activity(tx, denied, deny_group);
462461
}
463462
}
464463

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -606,7 +606,8 @@ pub async fn run_sandbox(
606606
(None, None, None)
607607
};
608608
let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() {
609-
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
609+
let (tx, rx) =
610+
tokio::sync::mpsc::channel(activity_aggregator::ACTIVITY_EVENT_QUEUE_CAPACITY);
610611
let bypass_tx = tx.clone();
611612
(Some(tx), Some(rx), Some(bypass_tx))
612613
} else {
@@ -1016,7 +1017,7 @@ pub async fn run_sandbox(
10161017

10171018
tokio::spawn(async move {
10181019
aggregator
1019-
.run(|summary| {
1020+
.run(move |summary| {
10201021
let endpoint = agg_endpoint.clone();
10211022
let sandbox_name = agg_name.clone();
10221023
async move {

‎crates/openshell-sandbox/src/proxy.rs‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
//! HTTP CONNECT proxy with OPA policy evaluation and process-identity binding.
55
6-
use crate::activity_aggregator::ActivityEvent;
6+
use crate::activity_aggregator::{ActivitySender, try_record_activity};
77
use crate::denial_aggregator::DenialEvent;
88
use crate::identity::BinaryIdentityCache;
99
use crate::l7::tls::ProxyTlsState;
@@ -179,7 +179,7 @@ impl ProxyHandle {
179179
provider_credentials: Option<ProviderCredentialState>,
180180
policy_local_ctx: Option<Arc<PolicyLocalContext>>,
181181
denial_tx: Option<mpsc::UnboundedSender<DenialEvent>>,
182-
activity_tx: Option<mpsc::UnboundedSender<ActivityEvent>>,
182+
activity_tx: Option<ActivitySender>,
183183
) -> Result<Self> {
184184
// Use override bind_addr, fall back to policy http_addr, then default
185185
// to loopback:3128. The default allows the proxy to function when no
@@ -294,23 +294,15 @@ impl Drop for ProxyHandle {
294294
}
295295
}
296296

297-
fn emit_activity(
298-
tx: &Option<mpsc::UnboundedSender<ActivityEvent>>,
299-
denied: bool,
300-
deny_group: &'static str,
301-
) {
297+
fn emit_activity(tx: &Option<ActivitySender>, denied: bool, deny_group: &'static str) {
302298
if let Some(tx) = tx {
303-
let _ = tx.send(ActivityEvent { denied, deny_group });
299+
let _ = try_record_activity(tx, denied, deny_group);
304300
}
305301
}
306302

307-
fn emit_activity_simple(
308-
tx: Option<&mpsc::UnboundedSender<ActivityEvent>>,
309-
denied: bool,
310-
deny_group: &'static str,
311-
) {
303+
fn emit_activity_simple(tx: Option<&ActivitySender>, denied: bool, deny_group: &'static str) {
312304
if let Some(tx) = tx {
313-
let _ = tx.send(ActivityEvent { denied, deny_group });
305+
let _ = try_record_activity(tx, denied, deny_group);
314306
}
315307
}
316308

@@ -387,7 +379,7 @@ async fn handle_tcp_connection(
387379
trusted_host_gateway: Arc<Option<IpAddr>>,
388380
secret_resolver: Option<Arc<SecretResolver>>,
389381
denial_tx: Option<mpsc::UnboundedSender<DenialEvent>>,
390-
activity_tx: Option<mpsc::UnboundedSender<ActivityEvent>>,
382+
activity_tx: Option<ActivitySender>,
391383
) -> Result<()> {
392384
let mut buf = vec![0u8; MAX_HEADER_BYTES];
393385
let mut used = 0usize;
@@ -2725,7 +2717,7 @@ async fn handle_forward_proxy(
27252717
trusted_host_gateway: Arc<Option<IpAddr>>,
27262718
secret_resolver: Option<Arc<SecretResolver>>,
27272719
denial_tx: Option<&mpsc::UnboundedSender<DenialEvent>>,
2728-
activity_tx: Option<&mpsc::UnboundedSender<ActivityEvent>>,
2720+
activity_tx: Option<&ActivitySender>,
27292721
) -> Result<()> {
27302722
// 1. Parse the absolute-form URI. `path` is marked `mut` so that, when an
27312723
// L7 config applies, the canonicalized form produced below replaces it

0 commit comments

Comments
 (0)