agentguard_telemetry/
metrics.rs

1//! In-memory metrics counters and histograms.
2//!
3//! Thread-safe via interior mutability. Renderable to Prometheus text format
4//! for the `/metrics` endpoint (Stage 7).
5
6use parking_lot::RwLock;
7use std::collections::HashMap;
8use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
9use std::sync::Arc;
10use std::time::Duration;
11
12/// Default cap on the number of distinct label-tuples retained for
13/// any one label-keyed metric. Prevents a misconfigured caller (or
14/// an attacker-controlled `tenant_id`) from growing the map without
15/// bound. When the cap is reached, new label-tuples are dropped and
16/// a warning is logged ONCE per metric.
17pub const DEFAULT_LABEL_CARDINALITY_CAP: usize = 4096;
18
19fn warn_once(flag: &AtomicBool, msg: String) {
20    if !flag.swap(true, Ordering::Relaxed) {
21        tracing::warn!("{}", msg);
22    }
23}
24
25/// Atomic-backed counter for cheap `inc()` from hot paths.
26#[derive(Debug, Default)]
27pub struct Counter {
28    inner: AtomicU64,
29}
30
31impl Counter {
32    pub fn new() -> Self {
33        Self::default()
34    }
35
36    /// Increment and return the new value.
37    pub fn inc(&self) -> u64 {
38        self.inner.fetch_add(1, Ordering::Relaxed) + 1
39    }
40
41    /// Add `n` and return the new value.
42    pub fn add(&self, n: u64) -> u64 {
43        self.inner.fetch_add(n, Ordering::Relaxed) + n
44    }
45
46    pub fn get(&self) -> u64 {
47        self.inner.load(Ordering::Relaxed)
48    }
49}
50
51/// Snapshot of a gauge.
52#[derive(Debug, Default)]
53pub struct Gauge {
54    inner: AtomicU64,
55}
56
57impl Gauge {
58    pub fn new() -> Self {
59        Self::default()
60    }
61
62    pub fn set(&self, v: f64) {
63        self.inner.store(v.to_bits(), Ordering::Relaxed);
64    }
65
66    pub fn get(&self) -> f64 {
67        f64::from_bits(self.inner.load(Ordering::Relaxed))
68    }
69}
70
71/// Histogram with exponential bucket boundaries.
72///
73/// Bucket boundaries are in seconds. Default: `[0.001, 0.01, 0.1, 1, 10]`.
74#[derive(Debug)]
75pub struct Histogram {
76    name: String,
77    buckets: Vec<f64>,
78    /// Counts of observations ≤ the bucket boundary. Last bucket is +Inf.
79    bucket_counts: Vec<Counter>,
80    sum_micros: AtomicU64,
81    count: Counter,
82}
83
84impl Histogram {
85    /// Build a histogram with finite bucket boundaries, sorted ascending and
86    /// deduplicated. Non-finite values are ignored; the implicit `+Inf` bucket
87    /// is always appended by [`Self::bucket_counts`].
88    pub fn new(name: impl Into<String>, buckets: &[f64]) -> Self {
89        let mut sorted: Vec<_> = buckets
90            .iter()
91            .copied()
92            .filter(|bucket| bucket.is_finite())
93            .collect();
94        sorted.sort_by(f64::total_cmp);
95        sorted.dedup_by(|left, right| *left == *right);
96        let bucket_counts = (0..sorted.len() + 1).map(|_| Counter::new()).collect();
97        Self {
98            name: name.into(),
99            buckets: sorted,
100            bucket_counts,
101            sum_micros: AtomicU64::new(0),
102            count: Counter::new(),
103        }
104    }
105
106    pub fn observe(&self, d: Duration) {
107        let secs = d.as_secs_f64();
108        self.count.inc();
109        self.sum_micros
110            .fetch_add(d.as_micros() as u64, Ordering::Relaxed);
111        for (i, b) in self.buckets.iter().enumerate() {
112            if secs <= *b {
113                self.bucket_counts[i].inc();
114            }
115        }
116        // +Inf bucket always counts.
117        self.bucket_counts[self.buckets.len()].inc();
118    }
119
120    pub fn count(&self) -> u64 {
121        self.count.get()
122    }
123
124    pub fn sum_seconds(&self) -> f64 {
125        self.sum_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0
126    }
127
128    pub fn bucket_counts(&self) -> Vec<(f64, u64)> {
129        self.buckets
130            .iter()
131            .zip(self.bucket_counts.iter())
132            .map(|(b, c)| (*b, c.get()))
133            .chain(std::iter::once((
134                f64::INFINITY,
135                self.bucket_counts.last().unwrap().get(),
136            )))
137            .collect()
138    }
139
140    pub fn name(&self) -> &str {
141        &self.name
142    }
143}
144
145/// Standard metrics for agentguard.
146pub struct Metrics {
147    /// `agentguard.decision.total{effect,policy_id,action,tenant_id}`
148    decision_total: RwLock<HashMap<String, Arc<Counter>>>,
149    /// `agentguard.decision.duration_seconds{action,tenant_id}`
150    decision_duration: RwLock<HashMap<String, Arc<Histogram>>>,
151    /// `agentguard.delegation.mint.total`
152    delegation_mint_total: Counter,
153    /// `agentguard.delegation.verify.total{outcome}`
154    delegation_verify_total: RwLock<HashMap<String, Arc<Counter>>>,
155    /// `agentguard.cache.hit.total` / `agentguard.cache.miss.total`
156    cache_hit_total: Counter,
157    cache_miss_total: Counter,
158    /// `agentguard.policy.reload.total`
159    policy_reload_total: Counter,
160    /// `agentguard.pdp.error.total{fallback}`
161    pdp_error_total: RwLock<HashMap<String, Arc<Counter>>>,
162    /// Cardinality cap per label-keyed metric. New label-tuples beyond
163    /// the cap are dropped to bound memory under untrusted label input.
164    cardinality_cap: usize,
165    /// One-shot warn flags so we don't flood the log when a metric
166    /// saturates its cardinality cap.
167    warn_decision_cap: AtomicBool,
168    warn_duration_cap: AtomicBool,
169    warn_pdp_error_cap: AtomicBool,
170}
171
172impl Default for Metrics {
173    fn default() -> Self {
174        Self::with_cap(DEFAULT_LABEL_CARDINALITY_CAP)
175    }
176}
177
178impl Metrics {
179    /// Build a new metrics registry. Returns `Self`; wrap with
180    /// `Arc::new(Metrics::new())` at the call site if shared
181    /// ownership is needed.
182    pub fn new() -> Self {
183        Self::default()
184    }
185
186    /// Construct a registry with a custom cardinality cap. Used in tests
187    /// that want to verify the cap is enforced.
188    pub fn with_cap(cardinality_cap: usize) -> Self {
189        Self {
190            decision_total: RwLock::new(HashMap::new()),
191            decision_duration: RwLock::new(HashMap::new()),
192            delegation_mint_total: Counter::new(),
193            delegation_verify_total: RwLock::new(HashMap::new()),
194            cache_hit_total: Counter::new(),
195            cache_miss_total: Counter::new(),
196            policy_reload_total: Counter::new(),
197            pdp_error_total: RwLock::new(HashMap::new()),
198            cardinality_cap,
199            warn_decision_cap: AtomicBool::new(false),
200            warn_duration_cap: AtomicBool::new(false),
201            warn_pdp_error_cap: AtomicBool::new(false),
202        }
203    }
204
205    pub fn cardinality_cap(&self) -> usize {
206        self.cardinality_cap
207    }
208
209    fn label_key(parts: &[&str]) -> String {
210        parts.join("\x1f")
211    }
212
213    pub fn record_decision(
214        &self,
215        effect: &str,
216        policy_id: &str,
217        action: &str,
218        tenant_id: &str,
219        duration: Duration,
220    ) {
221        let key = Self::label_key(&[effect, policy_id, action, tenant_id]);
222        {
223            let mut totals = self.decision_total.write();
224            if let Some(c) = totals.get(&key) {
225                c.inc();
226            } else if totals.len() < self.cardinality_cap {
227                let c = Arc::new(Counter::new());
228                c.inc();
229                totals.insert(key.clone(), c);
230            } else {
231                warn_once(
232                    &self.warn_decision_cap,
233                    format!(
234                        "agentguard.decision.total reached cardinality cap {}; \
235                         dropping new label-tuples",
236                        self.cardinality_cap
237                    ),
238                );
239                return;
240            }
241        }
242        let dur_key = Self::label_key(&[action, tenant_id]);
243        let mut durations = self.decision_duration.write();
244        if let Some(h) = durations.get(&dur_key) {
245            h.observe(duration);
246        } else if durations.len() < self.cardinality_cap {
247            let h = Arc::new(Histogram::new(
248                "agentguard.decision.duration_seconds",
249                &[0.001, 0.01, 0.1, 1.0, 10.0],
250            ));
251            h.observe(duration);
252            durations.insert(dur_key, h);
253        } else {
254            warn_once(
255                &self.warn_duration_cap,
256                format!(
257                    "agentguard.decision.duration_seconds reached cardinality cap {}; \
258                     dropping new label-tuples",
259                    self.cardinality_cap
260                ),
261            );
262        }
263    }
264
265    pub fn record_delegation_mint(&self) {
266        self.delegation_mint_total.inc();
267    }
268
269    pub fn record_delegation_verify(&self, success: bool) {
270        let key = if success { "success" } else { "failure" };
271        self.delegation_verify_total
272            .write()
273            .entry(key.to_string())
274            .or_insert_with(|| Arc::new(Counter::new()))
275            .inc();
276    }
277
278    pub fn record_cache_hit(&self) {
279        self.cache_hit_total.inc();
280    }
281
282    pub fn record_cache_miss(&self) {
283        self.cache_miss_total.inc();
284    }
285
286    pub fn record_policy_reload(&self) {
287        self.policy_reload_total.inc();
288    }
289
290    pub fn record_pdp_error(&self, fallback: &str) {
291        let key = fallback.to_string();
292        let mut pdp_errors = self.pdp_error_total.write();
293        if let Some(c) = pdp_errors.get(&key) {
294            c.inc();
295        } else if pdp_errors.len() < self.cardinality_cap {
296            let c = Arc::new(Counter::new());
297            c.inc();
298            pdp_errors.insert(key, c);
299        } else {
300            warn_once(
301                &self.warn_pdp_error_cap,
302                format!(
303                    "agentguard.pdp.error.total reached cardinality cap {}; \
304                     dropping new label-tuples",
305                    self.cardinality_cap
306                ),
307            );
308        }
309    }
310
311    /// Render a snapshot of all counters in Prometheus text format.
312    ///
313    /// Uses `std::fmt::Write` to write directly into the output
314    /// `String`, avoiding the dozens of `format!` temporaries the
315    /// previous implementation allocated.
316    pub fn render_prometheus(&self) -> String {
317        use std::fmt::Write as _;
318
319        // 4 KiB is enough for ~200 metrics with labels; the String
320        // grows as needed.
321        let mut out = String::with_capacity(4096);
322
323        writeln!(
324            out,
325            "# HELP agentguard_decision_total Total authorization decisions"
326        )
327        .unwrap();
328        writeln!(out, "# TYPE agentguard_decision_total counter").unwrap();
329        let totals = self.decision_total.read();
330        for (key, c) in totals.iter() {
331            let parts: Vec<&str> = key.split('\x1f').collect();
332            let effect = parts.first().copied().unwrap_or("");
333            let policy_id = parts.get(1).copied().unwrap_or("");
334            let action = parts.get(2).copied().unwrap_or("");
335            let tenant_id = parts.get(3).copied().unwrap_or("");
336            writeln!(
337                out,
338                "agentguard_decision_total{{effect=\"{}\",policy_id=\"{}\",action=\"{}\",tenant_id=\"{}\"}} {}",
339                escape_label(effect),
340                escape_label(policy_id),
341                escape_label(action),
342                escape_label(tenant_id),
343                c.get(),
344            )
345            .unwrap();
346        }
347        drop(totals);
348
349        writeln!(
350            out,
351            "# HELP agentguard_decision_duration_seconds Decision evaluation time"
352        )
353        .unwrap();
354        writeln!(out, "# TYPE agentguard_decision_duration_seconds histogram").unwrap();
355        let durations = self.decision_duration.read();
356        for (key, h) in durations.iter() {
357            let parts: Vec<&str> = key.split('\x1f').collect();
358            let action = parts.first().copied().unwrap_or("");
359            let tenant_id = parts.get(1).copied().unwrap_or("");
360            for (b, count) in h.bucket_counts() {
361                let le = if b.is_infinite() {
362                    "+Inf".to_string()
363                } else {
364                    b.to_string()
365                };
366                writeln!(
367                    out,
368                    "agentguard_decision_duration_seconds_bucket{{action=\"{}\",tenant_id=\"{}\",le=\"{}\"}} {}",
369                    escape_label(action),
370                    escape_label(tenant_id),
371                    le,
372                    count,
373                )
374                .unwrap();
375            }
376            writeln!(
377                out,
378                "agentguard_decision_duration_seconds_count{{action=\"{}\",tenant_id=\"{}\"}} {}",
379                escape_label(action),
380                escape_label(tenant_id),
381                h.count(),
382            )
383            .unwrap();
384            writeln!(
385                out,
386                "agentguard_decision_duration_seconds_sum{{action=\"{}\",tenant_id=\"{}\"}} {}",
387                escape_label(action),
388                escape_label(tenant_id),
389                h.sum_seconds(),
390            )
391            .unwrap();
392        }
393        drop(durations);
394
395        writeln!(
396            out,
397            "# HELP agentguard_delegation_mint_total Total delegation tokens minted"
398        )
399        .unwrap();
400        writeln!(out, "# TYPE agentguard_delegation_mint_total counter").unwrap();
401        writeln!(
402            out,
403            "agentguard_delegation_mint_total {}",
404            self.delegation_mint_total.get(),
405        )
406        .unwrap();
407
408        writeln!(
409            out,
410            "# HELP agentguard_delegation_verify_total Total delegation verifications"
411        )
412        .unwrap();
413        writeln!(out, "# TYPE agentguard_delegation_verify_total counter").unwrap();
414        let verify = self.delegation_verify_total.read();
415        for (outcome, c) in verify.iter() {
416            writeln!(
417                out,
418                "agentguard_delegation_verify_total{{outcome=\"{}\"}} {}",
419                escape_label(outcome),
420                c.get(),
421            )
422            .unwrap();
423        }
424        drop(verify);
425
426        writeln!(out, "# HELP agentguard_cache_hit_total Decision cache hits").unwrap();
427        writeln!(out, "# TYPE agentguard_cache_hit_total counter").unwrap();
428        writeln!(
429            out,
430            "agentguard_cache_hit_total {}",
431            self.cache_hit_total.get()
432        )
433        .unwrap();
434        writeln!(out, "# TYPE agentguard_cache_miss_total counter").unwrap();
435        writeln!(
436            out,
437            "agentguard_cache_miss_total {}",
438            self.cache_miss_total.get()
439        )
440        .unwrap();
441
442        writeln!(out, "# TYPE agentguard_policy_reload_total counter").unwrap();
443        writeln!(
444            out,
445            "agentguard_policy_reload_total {}",
446            self.policy_reload_total.get()
447        )
448        .unwrap();
449
450        writeln!(out, "# TYPE agentguard_pdp_error_total counter").unwrap();
451        let pdp = self.pdp_error_total.read();
452        for (fallback, c) in pdp.iter() {
453            writeln!(
454                out,
455                "agentguard_pdp_error_total{{fallback=\"{}\"}} {}",
456                escape_label(fallback),
457                c.get(),
458            )
459            .unwrap();
460        }
461
462        out
463    }
464
465    /// Build a snapshot of all counters for programmatic access.
466    pub fn snapshot(&self) -> MetricsSnapshot {
467        let decision_total: HashMap<String, u64> = self
468            .decision_total
469            .read()
470            .iter()
471            .map(|(k, v)| (k.clone(), v.get()))
472            .collect();
473        let decision_duration: HashMap<String, (u64, f64)> = self
474            .decision_duration
475            .read()
476            .iter()
477            .map(|(k, h)| (k.clone(), (h.count(), h.sum_seconds())))
478            .collect();
479        MetricsSnapshot {
480            decision_total,
481            decision_duration,
482            delegation_mint_total: self.delegation_mint_total.get(),
483            delegation_verify_total: self
484                .delegation_verify_total
485                .read()
486                .iter()
487                .map(|(k, v)| (k.clone(), v.get()))
488                .collect(),
489            cache_hit_total: self.cache_hit_total.get(),
490            cache_miss_total: self.cache_miss_total.get(),
491            policy_reload_total: self.policy_reload_total.get(),
492            pdp_error_total: self
493                .pdp_error_total
494                .read()
495                .iter()
496                .map(|(k, v)| (k.clone(), v.get()))
497                .collect(),
498        }
499    }
500}
501
502fn escape_label(s: &str) -> String {
503    s.replace('\\', "\\\\")
504        .replace('"', "\\\"")
505        .replace('\n', "\\n")
506        .replace('\r', "\\r")
507}
508
509/// Snapshot of all metrics, suitable for JSON serialization.
510#[derive(Debug, Clone, Serialize, Deserialize)]
511pub struct MetricsSnapshot {
512    pub decision_total: HashMap<String, u64>,
513    pub decision_duration: HashMap<String, (u64, f64)>,
514    pub delegation_mint_total: u64,
515    pub delegation_verify_total: HashMap<String, u64>,
516    pub cache_hit_total: u64,
517    pub cache_miss_total: u64,
518    pub policy_reload_total: u64,
519    pub pdp_error_total: HashMap<String, u64>,
520}
521
522use serde::{Deserialize, Serialize};
523
524#[cfg(test)]
525mod tests {
526    use super::*;
527
528    #[test]
529    fn counter_increments() {
530        let c = Counter::new();
531        assert_eq!(c.inc(), 1);
532        assert_eq!(c.inc(), 2);
533        assert_eq!(c.add(5), 7);
534    }
535
536    #[test]
537    fn histogram_observes_into_buckets() {
538        let h = Histogram::new("test", &[0.1, 1.0, 10.0]);
539        h.observe(Duration::from_millis(5));
540        h.observe(Duration::from_millis(500));
541        h.observe(Duration::from_secs(20));
542        let counts = h.bucket_counts();
543        // 5ms <= 0.1s and <= 1.0s and <= 10s and <= +Inf: all 4 buckets get +1
544        // 500ms <= 1.0s and <= 10s and <= +Inf: 3 buckets
545        // 20s only <= +Inf: 1 bucket
546        let bucket_01 = counts.iter().find(|(b, _)| *b == 0.1).unwrap().1;
547        let bucket_1 = counts.iter().find(|(b, _)| *b == 1.0).unwrap().1;
548        let bucket_10 = counts.iter().find(|(b, _)| *b == 10.0).unwrap().1;
549        let bucket_inf = counts.iter().find(|(b, _)| b.is_infinite()).unwrap().1;
550        assert_eq!(bucket_01, 1);
551        assert_eq!(bucket_1, 2);
552        assert_eq!(bucket_10, 2);
553        assert_eq!(bucket_inf, 3);
554    }
555
556    #[test]
557    fn histogram_ignores_non_finite_and_duplicate_boundaries() {
558        let h = Histogram::new(
559            "test",
560            &[
561                1.0,
562                f64::NAN,
563                f64::INFINITY,
564                0.1,
565                1.0,
566                f64::NEG_INFINITY,
567                -0.0,
568                0.0,
569            ],
570        );
571
572        h.observe(Duration::ZERO);
573        assert_eq!(
574            h.bucket_counts(),
575            vec![(-0.0, 1), (0.1, 1), (1.0, 1), (f64::INFINITY, 1)]
576        );
577    }
578
579    #[test]
580    fn histogram_with_only_non_finite_boundaries_keeps_implicit_infinity_bucket() {
581        let h = Histogram::new("test", &[f64::NAN, f64::INFINITY, f64::NEG_INFINITY]);
582
583        h.observe(Duration::from_secs(1));
584        assert_eq!(h.bucket_counts(), vec![(f64::INFINITY, 1)]);
585    }
586
587    #[test]
588    fn metrics_record_and_render() {
589        let m = Metrics::new();
590        m.record_decision(
591            "allow",
592            "policy0",
593            "send_email",
594            "tenant1",
595            Duration::from_micros(500),
596        );
597        m.record_decision(
598            "deny",
599            "policy3",
600            "shell_exec",
601            "tenant1",
602            Duration::from_micros(2000),
603        );
604        m.record_cache_hit();
605        m.record_cache_hit();
606        m.record_cache_miss();
607
608        let text = m.render_prometheus();
609        assert!(text.contains("agentguard_decision_total"));
610        assert!(text.contains("effect=\"allow\""));
611        assert!(text.contains("agentguard_cache_hit_total 2"));
612        assert!(text.contains("agentguard_cache_miss_total 1"));
613    }
614
615    /// T4: render_prometheus on an empty registry returns the
616    /// HELP/TYPE headers and no series lines for label-keyed metrics
617    /// (Prometheus requires the headers to be emitted even with no
618    /// data). Label-free counters are always emitted with their
619    /// initial 0 value.
620    #[test]
621    fn render_prometheus_empty_registry() {
622        let m = Metrics::new();
623        let text = m.render_prometheus();
624        // Label-keyed metric: headers but no series.
625        assert!(text.contains("# HELP agentguard_decision_total"));
626        assert!(text.contains("# TYPE agentguard_decision_total counter"));
627        assert!(!text.contains("agentguard_decision_total{"));
628        // Label-free counter: always present with value 0.
629        assert!(text.contains("agentguard_cache_hit_total 0"));
630    }
631
632    /// Cardinality invariant: even if a caller records 10× more
633    /// distinct label tuples than the cap, the underlying map never
634    /// grows past `cardinality_cap`. (Older entries are silently
635    /// dropped once the cap is reached.)
636    #[test]
637    fn cardinality_cap_is_enforced() {
638        let m = Metrics::with_cap(8);
639        for i in 0..100 {
640            m.record_decision(
641                "allow",
642                "policy0",
643                &format!("action-{i}"),
644                "",
645                std::time::Duration::from_micros(i as u64),
646            );
647        }
648        let snap = m.snapshot();
649        assert_eq!(
650            snap.decision_total.len(),
651            8,
652            "cardinality cap must be enforced; got {} entries",
653            snap.decision_total.len()
654        );
655    }
656}
agentguard_telemetry/metrics.rs

agentguard_telemetry/
metrics.rs
