diff --git a/crates/health/example/config.example.toml b/crates/health/example/config.example.toml index 436a464e3d..962b0febb9 100644 --- a/crates/health/example/config.example.toml +++ b/crates/health/example/config.example.toml @@ -113,6 +113,13 @@ flush_interval = "2s" # attach them. OTLP exports parent logs normally and keeps diagnostics as # latest-wins per endpoint while the drain is backed up. include_diagnostics = false + +# Include per-alert detail on health report records for this target. Health +# report records otherwise carry only alert and success counts. When enabled, +# each record with alerts gains a `health_report.alerts` attribute holding a +# JSON array of up to 64 alerts; a truncated record also carries +# `health_report.alerts.dropped` with the number omitted. +include_alert_details = false # # TLS-only target with an explicit CA bundle. # The reload interval is optional and defaults to five minutes. @@ -121,6 +128,7 @@ include_diagnostics = false # batch_size = 1024 # flush_interval = "5s" # include_diagnostics = true +# include_alert_details = true # # [sinks.otlp.targets.tls] # ca_cert_path = "/var/run/secrets/central-otlp/ca.crt" @@ -133,6 +141,7 @@ include_diagnostics = false # batch_size = 1024 # flush_interval = "5s" # include_diagnostics = true +# include_alert_details = true # # [sinks.otlp.targets.tls] # ca_cert_path = "/var/run/secrets/central-otlp/ca.crt" diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 2f024f68de..ce95ee60a7 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -537,6 +537,14 @@ pub struct OtlpTargetConfig { /// up. #[serde(default)] pub include_diagnostics: bool, + + /// Emit per-alert detail on health report log records sent to this target. + /// + /// Disabled by default because alert messages are free-form and a fully + /// degraded endpoint can produce many of them. When enabled, health report + /// records carry a `health_report.alerts` attribute holding a JSON array. + #[serde(default)] + pub include_alert_details: bool, } impl OtlpTargetConfig { @@ -1932,6 +1940,7 @@ mod tests { batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, } } @@ -2636,6 +2645,7 @@ endpoint = "https://central.example:4317" batch_size = 1024 flush_interval = "5s" include_diagnostics = true +include_alert_details = true [targets.tls] ca_cert_path = "/central/ca.crt" @@ -2657,6 +2667,8 @@ reload_interval = "30s" assert_eq!(targets[1].batch_size, 1024); assert_eq!(targets[1].flush_interval, Duration::from_secs(5)); assert!(targets[1].include_diagnostics); + assert!(!targets[0].include_alert_details); + assert!(targets[1].include_alert_details); let tls = targets[1] .tls @@ -2892,6 +2904,7 @@ reload_interval = "30s" batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, tls: None, }], }), @@ -2929,6 +2942,7 @@ reload_interval = "30s" batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: true, + include_alert_details: false, tls: None, }], }), @@ -2946,6 +2960,7 @@ reload_interval = "30s" batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, tls: None, }, OtlpTargetConfig { @@ -2953,6 +2968,7 @@ reload_interval = "30s" batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: true, + include_alert_details: false, tls: None, }, ], @@ -2998,6 +3014,7 @@ reload_interval = "30s" batch_size: 512, flush_interval: Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, tls: None, }], }), diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index 0b18e3c6c2..d0bc97a4a1 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -18,6 +18,8 @@ use std::collections::HashMap; use std::time::SystemTime; +use serde::Serialize; + use super::collector_logs::ExportLogsServiceRequest; use super::collector_metrics::ExportMetricsServiceRequest; use super::common::{AnyValue, KeyValue, any_value}; @@ -28,7 +30,15 @@ use super::metrics::{ }; use super::resource::Resource; use crate::endpoint::SwitchEndpointRole; -use crate::sink::{CollectorEvent, EventContext, MetricSample}; +use crate::sink::{CollectorEvent, EventContext, HealthReportAlert, MetricSample}; + +/// Maximum alerts serialized into the `health_report.alerts` attribute. +/// +/// A fully degraded endpoint reports an alert per failing sensor, and an export +/// that exceeds the collector's receive limit fails as `ResourceExhausted`, +/// which the drain retries before dropping the whole batch. Truncating here +/// keeps one degraded endpoint from taking unrelated records down with it. +const MAX_SERIALIZED_ALERTS: usize = 64; fn severity_text_to_number(severity: &str) -> i32 { match severity.to_uppercase().as_str() { @@ -166,7 +176,72 @@ fn convert_log(log: &crate::sink::LogRecord, observed_nanos: u64) -> OtlpLogReco } } -fn convert_event(event: &CollectorEvent, observed_nanos: u64) -> Option { +/// Alert detail as it appears inside the `health_report.alerts` JSON array. +/// +/// Probe and classification identities use the wire names the health API +/// already accepts, so `Probe::GpuInventory` reports as `SkuValidation`. +#[derive(Serialize)] +struct AlertDetail<'a> { + probe_id: &'static str, + + #[serde(skip_serializing_if = "Option::is_none")] + target: Option<&'a str>, + + message: &'a str, + classifications: Vec<&'static str>, +} + +/// Serializes alert detail, truncating to [`MAX_SERIALIZED_ALERTS`]. +/// +/// Returns no attributes when serialization fails so a malformed report costs +/// only the detail, not the record. +fn alert_detail_attributes(alerts: &[HealthReportAlert]) -> Vec { + let details: Vec> = alerts + .iter() + .take(MAX_SERIALIZED_ALERTS) + .map(|alert| AlertDetail { + probe_id: alert.probe_id.as_str(), + target: alert.target.as_deref(), + message: &alert.message, + classifications: alert + .classifications + .iter() + .map(|classification| classification.as_str()) + .collect(), + }) + .collect(); + + let json = match serde_json::to_string(&details) { + Ok(json) => json, + Err(error) => { + tracing::warn!( + ?error, + alert_count = alerts.len(), + "failed to serialize health report alert details" + ); + + return Vec::new(); + } + }; + + let mut attributes = vec![kv("health_report.alerts", json)]; + let dropped = alerts.len().saturating_sub(MAX_SERIALIZED_ALERTS); + + if dropped > 0 { + attributes.push(int_kv( + "health_report.alerts.dropped", + i64::try_from(dropped).unwrap_or(i64::MAX), + )); + } + + attributes +} + +fn convert_event( + event: &CollectorEvent, + observed_nanos: u64, + include_alert_details: bool, +) -> Option { match event { CollectorEvent::Log(log) => Some(convert_log(log, observed_nanos)), CollectorEvent::HealthReport(report) => { @@ -181,13 +256,20 @@ fn convert_event(event: &CollectorEvent, observed_nanos: u64) -> Option Option ExportLogsServiceRequest { +/// +/// `include_alert_details` is the receiving target's policy, so one target can +/// carry per-alert detail while another receives only the report counts. +pub fn build_export_request( + batch: &[(EventContext, CollectorEvent)], + include_alert_details: bool, +) -> ExportLogsServiceRequest { let observed_nanos = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap_or_default() @@ -220,7 +308,7 @@ pub fn build_export_request(batch: &[(EventContext, CollectorEvent)]) -> ExportL let mut by_endpoint: HashMap, Vec)> = HashMap::new(); for (context, event) in batch { - let Some(record) = convert_event(event, observed_nanos) else { + let Some(record) = convert_event(event, observed_nanos, include_alert_details) else { continue; }; by_endpoint @@ -660,7 +748,7 @@ mod tests { diagnostic_record: None, })); - let request = build_export_request(&[(ctx, log)]); + let request = build_export_request(&[(ctx, log)], false); assert_eq!(request.resource_logs.len(), 1); let records = &request.resource_logs[0].scope_logs[0].log_records; @@ -697,7 +785,7 @@ mod tests { diagnostic_record: None, })); - let request = build_export_request(&[(ctx, log)]); + let request = build_export_request(&[(ctx, log)], false); let records = &request.resource_logs[0].scope_logs[0].log_records; let record = &records[0]; @@ -723,32 +811,174 @@ mod tests { (ctx.clone(), CollectorEvent::MetricCollectionStart), (ctx, CollectorEvent::MetricCollectionEnd), ]; - let request = build_export_request(&batch); + let request = build_export_request(&batch, false); assert!(request.resource_logs.is_empty()); } - #[test] - fn health_report_converts_with_alert_severity() { - let ctx = test_context(); + fn sensor_alert() -> HealthReportAlert { + HealthReportAlert { + probe_id: Probe::Sensor, + target: Some("Temp1".to_string()), + message: "critical".to_string(), + classifications: vec![Classification::SensorCritical], + } + } + + fn health_report_record( + alerts: Vec, + include_alert_details: bool, + ) -> OtlpLogRecord { let report = CollectorEvent::HealthReport( HealthReport { source: ReportSource::BmcSensors, target: None, observed_at: None, successes: vec![], - alerts: vec![HealthReportAlert { - probe_id: Probe::Sensor, - target: Some("Temp1".to_string()), - message: "critical".to_string(), - classifications: vec![Classification::SensorCritical], - }], + alerts, } .into(), ); - let request = build_export_request(&[(ctx, report)]); - let records = &request.resource_logs[0].scope_logs[0].log_records; - assert_eq!(records[0].severity_text, "WARN"); + let request = build_export_request(&[(test_context(), report)], include_alert_details); + + request.resource_logs[0].scope_logs[0].log_records[0].clone() + } + + fn alert_details(record: &OtlpLogRecord) -> Vec { + let json = attr_value(&record.attributes, "health_report.alerts") + .expect("alert details attribute"); + + serde_json::from_str::(json) + .expect("alert details parse as JSON") + .as_array() + .expect("alert details are a JSON array") + .clone() + } + + #[test] + fn health_report_converts_with_alert_severity() { + let record = health_report_record(vec![sensor_alert()], false); + + assert_eq!(record.severity_text, "WARN"); + } + + /// Guards the flag-off record against any change from the alert detail work. + #[test] + fn health_report_omits_alert_details_when_disabled() { + let record = health_report_record(vec![sensor_alert()], false); + + assert_eq!(record.attributes.len(), 1); + assert_eq!( + attr_value(&record.attributes, "event.type"), + Some("health_report") + ); + } + + #[test] + fn health_report_serializes_alert_details_when_enabled() { + let alerts = vec![ + HealthReportAlert { + probe_id: Probe::LeakDetection, + target: Some( + "/redfish/v1/Chassis/BMC_0/ThermalSubsystem/LeakDetection/LeakDetectors/1" + .to_string(), + ), + message: "Leak detected: 2 detector alerts reached threshold 1".to_string(), + classifications: vec![Classification::Leak, Classification::PreventAllocations], + }, + sensor_alert(), + ]; + + let record = health_report_record(alerts, true); + + // The body, severity, and event.type stay as they are without the flag. + assert_eq!(record.severity_text, "WARN"); + assert_eq!(record.severity_number, SeverityNumber::Warn as i32); + assert_eq!( + record.body.as_ref().and_then(|body| body.value.as_ref()), + Some(&any_value::Value::StringValue( + "health report: 2 alerts, 0 ok (source: BmcSensors)".to_string() + )) + ); + assert_eq!( + attr_value(&record.attributes, "event.type"), + Some("health_report") + ); + + let details = alert_details(&record); + + assert_eq!(details.len(), 2); + assert_eq!(details[0]["probe_id"], "BmcLeakDetection"); + assert_eq!( + details[0]["target"], + "/redfish/v1/Chassis/BMC_0/ThermalSubsystem/LeakDetection/LeakDetectors/1" + ); + assert_eq!( + details[0]["message"], + "Leak detected: 2 detector alerts reached threshold 1" + ); + assert_eq!( + details[0]["classifications"], + serde_json::json!(["Leak", "PreventAllocations"]) + ); + + assert_eq!(details[1]["probe_id"], "BmcSensor"); + assert_eq!(details[1]["target"], "Temp1"); + assert_eq!(details[1]["message"], "critical"); + assert_eq!( + details[1]["classifications"], + serde_json::json!(["SensorCritical"]) + ); + + assert_eq!( + attr_int_value(&record.attributes, "health_report.alerts.dropped"), + None + ); + } + + #[test] + fn health_report_without_alerts_omits_alert_details() { + let record = health_report_record(vec![], true); + + assert_eq!(record.attributes.len(), 1); + assert_eq!(attr_value(&record.attributes, "health_report.alerts"), None); + } + + #[test] + fn health_report_alert_details_omit_absent_target() { + let alert = HealthReportAlert { + target: None, + ..sensor_alert() + }; + + let record = health_report_record(vec![alert], true); + let details = alert_details(&record); + + assert_eq!(details.len(), 1); + assert!(details[0].get("target").is_none()); + assert_eq!(details[0]["probe_id"], "BmcSensor"); + } + + /// A fully degraded endpoint reports an alert per sensor; the attribute is + /// capped so one endpoint cannot push the export past the receive limit. + #[test] + fn health_report_alert_details_are_capped() { + let alerts = (0..MAX_SERIALIZED_ALERTS + 3) + .map(|index| HealthReportAlert { + target: Some(format!("Temp{index}")), + ..sensor_alert() + }) + .collect(); + + let record = health_report_record(alerts, true); + let details = alert_details(&record); + + assert_eq!(details.len(), MAX_SERIALIZED_ALERTS); + assert_eq!(details[0]["target"], "Temp0"); + assert_eq!( + attr_int_value(&record.attributes, "health_report.alerts.dropped"), + Some(3) + ); } #[test] @@ -783,7 +1013,7 @@ mod tests { }; let batch = vec![log(ctx1.clone()), log(ctx2), log(ctx1)]; - let request = build_export_request(&batch); + let request = build_export_request(&batch, false); assert_eq!(request.resource_logs.len(), 2); let total_records: usize = request diff --git a/crates/health/src/otlp/drain.rs b/crates/health/src/otlp/drain.rs index d015c73d14..7bd8e04b30 100644 --- a/crates/health/src/otlp/drain.rs +++ b/crates/health/src/otlp/drain.rs @@ -171,7 +171,7 @@ impl OtlpDrainTask { return; } - let request = build_export_request(batch); + let request = build_export_request(batch, self.target.include_alert_details); batch.clear(); let record_count = request diff --git a/crates/health/src/otlp/mod.rs b/crates/health/src/otlp/mod.rs index 2df6688723..1ea0065026 100644 --- a/crates/health/src/otlp/mod.rs +++ b/crates/health/src/otlp/mod.rs @@ -264,6 +264,7 @@ mod tests { batch_size: 1, flush_interval: Duration::from_secs(1), include_diagnostics: false, + include_alert_details: false, }; let endpoint = target_endpoint(&target).await?; @@ -303,6 +304,7 @@ mod tests { batch_size: 1, flush_interval: Duration::from_secs(1), include_diagnostics: false, + include_alert_details: false, }; let peer = tokio::spawn(async move { diff --git a/crates/health/src/sink/otlp.rs b/crates/health/src/sink/otlp.rs index 767130256b..bd3f1ed403 100644 --- a/crates/health/src/sink/otlp.rs +++ b/crates/health/src/sink/otlp.rs @@ -521,6 +521,7 @@ mod tests { batch_size: 512, flush_interval: std::time::Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, tls: None, }, OtlpTargetConfig { @@ -528,6 +529,7 @@ mod tests { batch_size: 512, flush_interval: std::time::Duration::from_secs(2), include_diagnostics: false, + include_alert_details: false, tls: None, }, ]; diff --git a/docs/operations/monitoring-health.md b/docs/operations/monitoring-health.md index 23039323e5..fa2c4071c0 100644 --- a/docs/operations/monitoring-health.md +++ b/docs/operations/monitoring-health.md @@ -271,6 +271,17 @@ For leak-related events, look for: report_source=tray-leak-detection ``` +Health report records exported over OTLP carry only these counts by default. +Setting `include_alert_details = true` on a `[[sinks.otlp.targets]]` entry adds +a `health_report.alerts` attribute holding a JSON array of the individual +alerts, each with `probe_id`, `message`, `classifications`, and `target` when +the alert names one. The setting is per target, so a debugging destination can +receive detail while a long-term store receives only counts. At most 64 alerts +are serialized per record; a truncated record also carries +`health_report.alerts.dropped` with the number omitted. Note that `probe_id` +uses health API probe names, so OOB GPU inventory alerts appear as +`SkuValidation` to dedup with the in-band SKU alerts. + ## DPU Health Checks `dpu-agent` runs on managed DPUs and reports DPU health to NICo. The BlueField