diff --git a/dev-infrastructure/modules/metrics/rp-rules.bicep b/dev-infrastructure/modules/metrics/rp-rules.bicep index bc89be721ed..b7e5725d985 100644 --- a/dev-infrastructure/modules/metrics/rp-rules.bicep +++ b/dev-infrastructure/modules/metrics/rp-rules.bicep @@ -16,3 +16,12 @@ module generatedAlerts 'rules/generatedRPPrometheusAlertingRules.bicep' = { severityCeiling: severityCeiling } } + +module generatedHCPAlerts 'rules/generatedRPHCPPrometheusAlertingRules.bicep' = { + name: 'generatedRPHCPPrometheusAlertingRules' + params: { + azureMonitoring: azureMonitoringWorkspaceId + actionGroups: actionGroups + severityCeiling: severityCeiling + } +} diff --git a/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep index 0cf7772a7f9..c34fed6f581 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep @@ -195,3 +195,41 @@ resource hcpKasLatencyRecordingRules 'Microsoft.AlertsManagement/prometheusRuleG ] } } + +resource arohcpSwiftNetworkingSloRecordingRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'arohcp_swift_networking_slo_recording_rules' + location: location + properties: { + scopes: [ + azureMonitoring + ] + enabled: true + interval: 'PT1M' + rules: [ + { + record: 'router:startup_latency:seconds' + expression: '(time() - kube_pod_created{namespace=~"ocm-.*"}) * on (namespace, pod) kube_pod_owner{owner_kind="ReplicaSet",owner_name=~"router-.*"} * on (namespace, pod) (kube_pod_status_phase{phase="Pending"} == 1)' + } + { + record: 'router:startup_latency:p99' + expression: 'quantile(0.99, router:startup_latency:seconds)' + } + { + record: 'router:startup_latency:p99_avg_5m' + expression: 'avg_over_time(router:startup_latency:p99[5m])' + } + { + record: 'router:startup_latency:p99_avg_30m' + expression: 'avg_over_time(router:startup_latency:p99[30m])' + } + { + record: 'router:startup_latency:p99_avg_1h' + expression: 'avg_over_time(router:startup_latency:p99[1h])' + } + { + record: 'router:startup_latency:p99_avg_6h' + expression: 'avg_over_time(router:startup_latency:p99[6h])' + } + ] + } +} diff --git a/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep b/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep index 74c11f5f18c..5ad3f1ed036 100644 --- a/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep +++ b/dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep @@ -9,3 +9,104 @@ param severityCeiling int = 0 #disable-next-line no-unused-params param location string = resourceGroup().location + +resource arohcpSwiftNetworkingAlerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'arohcp_swift_networking_alerts' + location: location + properties: { + interval: 'PT1M' + rules: [ + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneySwiftLatencyP991h5m' + enabled: true + labels: { + burn_rate_tier: 'fast' + long_window: '1h' + severity: '3' + short_window: '5m' + } + annotations: { + correlationId: 'userJourneySwiftLatencyP991h5m/{{ $labels.cluster }}' + description: 'Router pod startup latency p99 has exceeded 300s over the last hour and is still elevated. SWIFT secondary NIC assignment is stalled.' + info: 'Router pod startup latency p99 has exceeded 300s over the last hour and is still elevated. SWIFT secondary NIC assignment is stalled.' + runbook_url: 'https://aka.ms/arohcp-runbook-swift' + summary: 'SWIFT router pod startup latency p99 critically elevated (fast burn)' + title: 'SWIFT router pod startup latency p99 critically elevated (fast burn)' + } + expression: 'router:startup_latency:p99_avg_5m > 300 and router:startup_latency:p99_avg_1h > 300' + for: 'PT2M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneySwiftLatencyP996h30m' + enabled: true + labels: { + burn_rate_tier: 'medium' + long_window: '6h' + severity: '3' + short_window: '30m' + } + annotations: { + correlationId: 'userJourneySwiftLatencyP996h30m/{{ $labels.cluster }}' + description: 'Router pod startup latency p99 has exceeded 300s over the last 6 hours and is still elevated.' + info: 'Router pod startup latency p99 has exceeded 300s over the last 6 hours and is still elevated.' + runbook_url: 'https://aka.ms/arohcp-runbook-swift' + summary: 'SWIFT router pod startup latency p99 elevated (medium burn)' + title: 'SWIFT router pod startup latency p99 elevated (medium burn)' + } + expression: 'router:startup_latency:p99_avg_30m > 300 and router:startup_latency:p99_avg_6h > 300' + for: 'PT15M' + severity: severityCeiling > 0 ? max(3, severityCeiling) : 3 + } + { + actions: [ + for g in actionGroups: { + actionGroupId: g + actionProperties: { + 'IcM.Title': '#$.labels.cluster#: #$.annotations.title#' + 'IcM.CorrelationId': '#$.annotations.correlationId#' + } + } + ] + alert: 'userJourneySwiftLatencyP993d' + enabled: true + labels: { + burn_rate_tier: 'slow' + severity: '4' + } + annotations: { + correlationId: 'userJourneySwiftLatencyP993d/{{ $labels.cluster }}' + description: 'Router pod startup latency p99 has been elevated for an extended period. At current rate the monthly SLO budget will be exhausted before end of month.' + info: 'Router pod startup latency p99 has been elevated for an extended period. At current rate the monthly SLO budget will be exhausted before end of month.' + runbook_url: 'https://aka.ms/arohcp-runbook-swift' + summary: 'SWIFT router pod startup latency p99 elevated (slow burn — SLO budget on track to exhaust by month end)' + title: 'SWIFT router pod startup latency p99 elevated (slow burn — SLO budget on track to exhaust by month end)' + } + expression: 'router:startup_latency:p99_avg_6h > 300' + for: 'PT6H' + severity: severityCeiling > 0 ? max(4, severityCeiling) : 4 + } + ] + scopes: [ + azureMonitoring + ] + } +} diff --git a/docs/drafts/swift-observability-implementation.md b/docs/drafts/swift-observability-implementation.md new file mode 100644 index 00000000000..8edefc13880 --- /dev/null +++ b/docs/drafts/swift-observability-implementation.md @@ -0,0 +1,380 @@ +# SWIFT Observability Implementation Spec + +Implementation details for ARO-25919 (SLIs/SLOs), ARO-25920 (dashboard), and ARO-25978 (alerting). +The runbook (`docs/drafts/uj-runbook-swift.md`) describes the intended behaviour; this doc specifies what to build. + +## Recording rules + +File: `observability/alerts/swift-networking-recordingRule-KSM.yaml` + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: swift-networking-recording-rules + namespace: prometheus +spec: + groups: + - name: arohcp_swift_networking_slo_recording_rules + interval: 1m + rules: + # Per-pod ContainerCreating duration for router pods (no series when healthy) + - record: router:startup_latency:seconds + expr: | + (time() - kube_pod_created{namespace=~"ocm-.*"}) + * on(namespace, pod) kube_pod_owner{owner_kind="ReplicaSet", owner_name=~"router-.*"} + * on(namespace, pod) (kube_pod_status_phase{phase="Pending"} == 1) + + # p99 across all pending router pods (no series when none are stuck) + - record: router:startup_latency:p99 + expr: quantile(0.99, router:startup_latency:seconds) + + # Window averages for alert expressions (threshold lives only in alert) + - record: router:startup_latency:p99_avg_5m + expr: avg_over_time(router:startup_latency:p99[5m]) + - record: router:startup_latency:p99_avg_30m + expr: avg_over_time(router:startup_latency:p99[30m]) + - record: router:startup_latency:p99_avg_1h + expr: avg_over_time(router:startup_latency:p99[1h]) + - record: router:startup_latency:p99_avg_6h + expr: avg_over_time(router:startup_latency:p99[6h]) + + - name: arohcp_swift_cns_latency_recording_rules + interval: 1m + rules: + - record: cns:ip_assignment_latency:p99 + expr: histogram_quantile(0.99, sum(rate(ip_assignment_latency_seconds_bucket[5m])) by (le)) + + - record: cns:ip_assignment_latency:p99_avg_5m + expr: avg_over_time(cns:ip_assignment_latency:p99[5m]) + - record: cns:ip_assignment_latency:p99_avg_30m + expr: avg_over_time(cns:ip_assignment_latency:p99[30m]) + - record: cns:ip_assignment_latency:p99_avg_1h + expr: avg_over_time(cns:ip_assignment_latency:p99[1h]) + - record: cns:ip_assignment_latency:p99_avg_6h + expr: avg_over_time(cns:ip_assignment_latency:p99[6h]) + + - name: arohcp_swift_konnectivity_recording_rules + interval: 1m + rules: + # Stream error rate: errors / packets + - record: konnectivity:stream_error_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_stream_errors_total[5m])) + / + clamp_min(sum(rate(konnectivity_network_proxy_server_stream_packets_total[5m])), 1) + + - record: konnectivity:stream_error_rate:avg_5m + expr: avg_over_time(konnectivity:stream_error_rate:5m[5m]) + - record: konnectivity:stream_error_rate:avg_30m + expr: avg_over_time(konnectivity:stream_error_rate:5m[30m]) + - record: konnectivity:stream_error_rate:avg_1h + expr: avg_over_time(konnectivity:stream_error_rate:5m[1h]) + - record: konnectivity:stream_error_rate:avg_6h + expr: avg_over_time(konnectivity:stream_error_rate:5m[6h]) + + # Dial failure rate: failures / (failures + successful dials) + # Successful dials = DIAL_RSP packets from agents. + - record: konnectivity:dial_failure_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + / + clamp_min( + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + + + sum(rate(konnectivity_network_proxy_server_stream_packets_total{segment="from_agent", packet_type="DIAL_RSP"}[5m])) + , 1) + + - record: konnectivity:dial_failure_rate:avg_5m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[5m]) + - record: konnectivity:dial_failure_rate:avg_30m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[30m]) + - record: konnectivity:dial_failure_rate:avg_1h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[1h]) + - record: konnectivity:dial_failure_rate:avg_6h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[6h]) +``` + +## Alert rules + +File: `observability/alerts/swift-networking-prometheusRule.yaml` + +### Router pod startup latency (RP lane) + +```yaml +# Fast burn: 1h long window + 5m short window +- alert: userJourneySwiftLatencyP991h5m + expr: | + router:startup_latency:p99_avg_5m > 300 + and + router:startup_latency:p99_avg_1h > 300 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + annotations: + summary: "SWIFT router pod startup latency p99 critically elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +# Medium burn: 6h long window + 30m short window +- alert: userJourneySwiftLatencyP996h30m + expr: | + router:startup_latency:p99_avg_30m > 300 + and + router:startup_latency:p99_avg_6h > 300 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + long_window: 6h + short_window: 30m + annotations: + summary: "SWIFT router pod startup latency p99 elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +# Slow burn: 6h for-clause, Sev 4 (ticket, not a page) +- alert: userJourneySwiftLatencyP993d + expr: router:startup_latency:p99_avg_6h > 300 + for: 6h + labels: + severity: "4" + burn_rate_tier: slow + annotations: + summary: "SWIFT router pod startup latency p99 elevated (slow burn — SLO budget on track to exhaust by month end)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +### CNS IP assignment latency (RP lane) + +```yaml +- alert: userJourneySwiftCNSLatencyP991h5m + expr: | + cns:ip_assignment_latency:p99_avg_5m > 10 + and + cns:ip_assignment_latency:p99_avg_1h > 10 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + annotations: + summary: "CNS IP assignment latency p99 critically elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: userJourneySwiftCNSLatencyP996h30m + expr: | + cns:ip_assignment_latency:p99_avg_30m > 10 + and + cns:ip_assignment_latency:p99_avg_6h > 10 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + annotations: + summary: "CNS IP assignment latency p99 elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +### CNS IP assignment error rate (RP lane) + +SLO: <= 1% error rate over 28 days. Error budget = 0.001. +- Fast-burn threshold: 14.4 x 0.001 = 0.0144 +- Medium-burn threshold: 6 x 0.001 = 0.006 + +```yaml +- alert: userJourneySwiftErrors1h5m + expr: | + ( + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[5m])) + / + clamp_min(sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m])), 1) + ) > 0.0144 + and + ( + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[1h])) + / + clamp_min(sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[1h])), 1) + ) > 0.0144 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + annotations: + summary: "CNS IP assignment error rate elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: userJourneySwiftErrors6h30m + expr: | + ( + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[30m])) + / + clamp_min(sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[30m])), 1) + ) > 0.006 + and + ( + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[6h])) + / + clamp_min(sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[6h])), 1) + ) > 0.006 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + annotations: + summary: "CNS IP assignment error rate elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +### Konnectivity tunnel health (RP lane) + +SLO: <= 1% stream error rate and <= 1% dial failure rate over 28 days. + +```yaml +- alert: userJourneySwiftKonnectivityErrors1h5m + expr: | + konnectivity:stream_error_rate:avg_5m > 0.01 + and + konnectivity:stream_error_rate:avg_1h > 0.01 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + annotations: + summary: "Konnectivity tunnel error rate elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: userJourneySwiftKonnectivityErrors6h30m + expr: | + konnectivity:stream_error_rate:avg_30m > 0.01 + and + konnectivity:stream_error_rate:avg_6h > 0.01 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + annotations: + summary: "Konnectivity tunnel error rate elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: userJourneySwiftKonnectivityDialFailures1h5m + expr: | + konnectivity:dial_failure_rate:avg_5m > 0.01 + and + konnectivity:dial_failure_rate:avg_1h > 0.01 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + annotations: + summary: "Konnectivity dial failure rate elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: userJourneySwiftKonnectivityDialFailures6h30m + expr: | + konnectivity:dial_failure_rate:avg_30m > 0.01 + and + konnectivity:dial_failure_rate:avg_6h > 0.01 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + annotations: + summary: "Konnectivity dial failure rate elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +### Component-level alerts (SRE HCP lane) + +```yaml +- alert: SwiftCNSAvailability3d + expr: | + kube_daemonset_status_number_ready{daemonset="azure-cns", namespace="kube-system"} + / + kube_daemonset_status_desired_number_scheduled{daemonset="azure-cns", namespace="kube-system"} + < 0.999 + for: 3d + labels: + severity: "4" + annotations: + summary: "CNS daemonset availability below SLO" + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +- alert: SwiftPendingProgramming + expr: cx_pending_programming_ips_v2 > 0 + for: 15m + labels: + severity: "4" + annotations: + summary: "IPs stuck in PendingProgramming" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +### SLO thresholds + +The 300s latency threshold is a starting point. Validate against production baseline before committing. +Error rate alerts use burn-rate multipliers for a 99.9% SLO (error_budget = 0.001): +- Fast-burn: 14.4x -> threshold = `14.4 * 0.001 = 0.0144` +- Medium-burn: 6x -> threshold = `6 * 0.001 = 0.006` + +## PodMonitors + +### CNS PodMonitor + +File: `observability/prometheus/deploy/templates/azure-cns.podmonitor.yaml` + +Scrapes CNS metrics (`ip_assignment_latency_seconds`, `http_request_latency_seconds`, `cx_*` pool metrics) from the CNS DaemonSet on management cluster nodes. + +### Konnectivity Server PodMonitor + +File: `observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml` + +Scrapes konnectivity-server sidecar metrics (`konnectivity_network_proxy_server_stream_*`, `konnectivity_network_proxy_server_dial_*`) from KAS pods in HCP namespaces on port 8093. + +## Dashboard + +File: `observability/grafana-dashboards/sre/user-journey/swift-networking.json` + +### Dashboard-level annotation + +Add to the `annotations.list` array so alert firing times appear as vertical lines on all panels: + +```json +{ + "datasource": { "type": "prometheus" }, + "enable": true, + "expr": "ALERTS{alertname=~\"userJourneySwift.*|SwiftCNS.*|SwiftPending.*\", alertstate=\"firing\"}", + "name": "SWIFT Alerts", + "hide": false, + "iconColor": "red", + "step": "60s" +} +``` + +### Panels + +Datasource variable regex: `^.*-mgmt-\d+$` + +| Panel | Query | Display | +|-------|-------|---------| +| Router pod startup p99, all windows | `router:startup_latency:p99_avg_5m`, `p99_avg_30m`, `p99_avg_1h`, `p99_avg_6h` | Time series, all overlaid; threshold line at 300s | +| Router pod startup p99, raw | `router:startup_latency:p99` | Time series; shows pre-aggregation signal | +| CNS daemonset availability | `kube_daemonset_status_number_ready{daemonset="azure-cns"} / kube_daemonset_status_desired_number_scheduled{daemonset="azure-cns"}` | Time series; SLO target line at 99.9% | +| SWIFT NIC utilization per node | `sum by (node) (kube_pod_container_resource_requests{resource="aro.openshift.io/swift-nic"}) / kube_node_status_capacity{resource="aro.openshift.io/swift-nic"}` | Time series per node; no threshold | +| IPs stuck in PendingProgramming | `cx_pending_programming_ips_v2` | Threshold alert line at 0 | +| IP assignment error rate | `http_request_latency_seconds` error rate (`cns_return_code != "0"`) | SLO threshold at 1% | +| IP assignment latency p99 | `cns:ip_assignment_latency:p99` | SLO threshold at 10s | +| requestipconfig traffic rate | `rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m])` | No SLO; anomaly detection | +| IP pool state per node | `cx_assigned_ips_v2`, `cx_available_ips_v2`, `cx_allocated_ips_v2` stacked | From starter dashboard | +| IP pool saturation | `sum by (instance) (cx_assigned_ips_v2) / sum by (instance) (cx_ipam_max_ips)` | Threshold at 85% | +| Subnet exhaustion | `cx_ipam_subnet_exhaustion_state > 0` | Binary state panel | +| IP pool convergence | `cx_ipam_requested_ips - cx_ipam_total_ips` | Alert on positive values | +| Konnectivity stream error rate | `konnectivity:stream_error_rate:5m` | SLO threshold at 1% | +| Konnectivity dial failure rate | `konnectivity:dial_failure_rate:5m` | SLO threshold at 1% | + +## References + +- [ARO HCP Alerting Recommendation ADR](https://github.com/openshift-online/architecture/pull/79) +- [docs/alerts.md](https://github.com/Azure/ARO-HCP/blob/main/docs/alerts.md): severity mapping, correlationId, IcM title rendering +- [docs/prometheus-rules.md](https://github.com/Azure/ARO-HCP/blob/main/docs/prometheus-rules.md): authoring, testing, generating Bicep +- [Observability code](https://github.com/Azure/ARO-HCP/tree/main/observability): existing alert and recording rule examples diff --git a/docs/drafts/todo-swift.md b/docs/drafts/todo-swift.md new file mode 100644 index 00000000000..319414579c7 --- /dev/null +++ b/docs/drafts/todo-swift.md @@ -0,0 +1,21 @@ +# SWIFT User Journey: Implementation TODO + +Personal checklist for ARO-25382. Not committed to the PR. + +## This PR + +- [ ] Rebase onto main (PR has `needs-rebase` label) +- [ ] Create aka.ms short-links: `arohcp-runbook-swift`, `arohcp-tsg-swift`, `arohcp-dashboard-swift` +- [x] Remove "Alternative signals to explore" paragraph (done: UJ rewritten to new 4-section format) +- [x] Remove "After Incident" runbook update bullets (done: UJ rewritten to new 4-section format) +- [x] Slow-burn alert tier (done: `userJourneySwiftLatencyP993d` implemented in YAML and specced in uj-alerts-swift.md) +- [x] Konnectivity SLIs fully integrated into design docs (done: uj-slislo-swift.md, uj-alerts-swift.md, uj-dashboard-swift.md, swift-observability-implementation.md) +- [x] Implementation spec updated: correct file names, alert names, konnectivity content, error alerts uncommented + +## Follow-up work (separate PRs / Jira tickets) + +- [ ] Alternative SLIs: Node Ready ratio, Router Pod NIC traffic drought (Brendan Bergen, Jun 2026) +- [ ] Kusto datasource for Grafana historical analysis (blocked on ARO-HCP#4878) +- [ ] MTPNC reconciler metrics: currently no metrics emitted; using `cx_pending_programming_ips_v2` as proxy +- [ ] Update CS references once Backend operator replaces Cluster Service (UJ glossary: CS and ILB entries; TSG: SAL prerequisite in Step 2) +- [ ] ARO SLO Policy ADR (upstream draft: swiencki/architecture#1): update SLO targets doc once ratified diff --git a/docs/drafts/uj-alerts-swift.md b/docs/drafts/uj-alerts-swift.md new file mode 100644 index 00000000000..cf2cc819581 --- /dev/null +++ b/docs/drafts/uj-alerts-swift.md @@ -0,0 +1,199 @@ +# SWIFT Networking: Alerting Design (ARO-25978) + + +## File locations + +``` +observability/alerts/swift-networking-recordingRule-KSM.yaml # recording rules +observability/alerts/swift-networking-recordingRule-KSM_test.yaml # recording rule tests +observability/alerts/swift-networking-prometheusRule.yaml # alert rules +observability/alerts/swift-networking-prometheusRule_test.yaml # alert tests +``` + +Register in the appropriate lane config (see [Routing lanes](#routing-lanes) below). Run `make alerts` in `tooling/prometheus-rules/` to generate Bicep. Format with `az bicep format`. + +See [docs/prometheus-rules.md](https://github.com/Azure/ARO-HCP/blob/main/docs/prometheus-rules.md) for full authoring guide. + +## ADR-001 Requirements Summary + +- **Naming:** `{Scope}{Subject}{Metric}{BurnRateTier}`, e.g. `userJourneySwiftLatencyP991h5m` +- **Multi-window multi-burn-rate** for Availability, Errors, Latency +- **Threshold-based** for Traffic and Saturation +- **Burn-rate tiers:** Fast (1h/5m, 14.4x), Medium (6h/30m, 6x), Slow (3d/6h, 1x) +- **Severity:** Sev 3 for customer-facing journey degradation; Sev 4 for component-internal +- **`runbook_url`:** every alert must link to the **runbook** via an `aka.ms/arohcp-runbook-{shortname}` short-link (the chain is alert → runbook → TSG, per ADR-001 Section 3) +- **`correlationId`:** required on every alert; granularity must be set explicitly: + - Default = per-cluster + - Per-subscription for stuck-operation alerts (so per-customer suppression doesn't mask other customers) + - Per-namespace for per-HCP alerts +- **Summary annotation:** must NOT include `{{ $labels.cluster }}` because the RP bicep template already prepends cluster name to every IcM title; including it doubles the title + +## Routing lanes + +| Lane | Config file | Use for | +|---|---|---| +| RP (per-HCP) | `observability/alerts-rp-hcps.yaml` | Per-HCP alerts carrying `_id`, `namespace`, `subscription_id` | +| RP (fleet/services) | `observability/alerts-rp-services.yaml` | Fleet-aggregate alerts firing per region (`by (cluster)` only) | +| SL | `observability/alerts-sl-services.yaml` | Component-internal, no current customer impact | + +Both RP lanes route to `icm-action-group-rp`. + +**SWIFT routing:** +- `userJourneySwift*` latency/error/konnectivity alerts: **RP per-HCP** (`alerts-rp-hcps.yaml`), fires per router pod namespace or HCP namespace +- `SwiftCNSAvailability3d`, `SwiftPendingProgramming`: **SRE HCP** (`alerts-sre-hcps.yaml`), AKS-managed infrastructure signals, consistent with `MgmtClusterNodeSwiftNICCapacityZero` + +**ADR-001 rule:** fleet-aggregate alerts must NOT filter by subscription. Internal-sub failures indicate the same bugs affecting customers. + +## Alert Inventory + +### Journey-level alerts + +**`userJourneySwiftLatencyP991h5m`** - Sev 3 - RP lane +```yaml +alert: userJourneySwiftLatencyP991h5m +expr: | + router:startup_latency:p99_avg_5m > 300 + and + router:startup_latency:p99_avg_1h > 300 +for: 2m +labels: + severity: "3" + long_window: 1h + short_window: 5m +annotations: + summary: "SWIFT router pod startup latency p99 elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +**`userJourneySwiftLatencyP996h30m`** - Sev 3 - RP lane +```yaml +alert: userJourneySwiftLatencyP996h30m +expr: | + router:startup_latency:p99_avg_30m > 300 + and + router:startup_latency:p99_avg_6h > 300 +for: 15m +labels: + severity: "3" + long_window: 6h + short_window: 30m +annotations: + summary: "SWIFT router pod startup latency p99 elevated (medium burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +**`SwiftCNSAvailability3d`** - Sev 4 - SRE HCP lane (`alerts-sre-hcps.yaml`) +- CNS daemonset replica ratio slow-burn +- Basis: `kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled` +- SLO: ≥ 99.9% over 28 days +- Threshold alert, not burn-rate (binary availability ratio) + +**`SwiftPendingProgramming`** - Sev 4 - SRE HCP lane (`alerts-sre-hcps.yaml`) +- `cx_pending_programming_ips_v2 > 0` sustained for 15m +- Threshold alert, not burn-rate: pending-programming is a binary state (IPs stuck or not), not a ratio +**`userJourneySwiftLatencyP993d`** - Sev 4 - RP lane +```yaml +alert: userJourneySwiftLatencyP993d +expr: router:startup_latency:p99_avg_6h > 300 +for: 6h +labels: + severity: "4" + burn_rate_tier: slow +annotations: + summary: "SWIFT router pod startup latency p99 elevated (slow burn — SLO budget on track to exhaust by month end)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +Per ADR-001: user journey Sev 3 alerts wire Fast + Medium; the Slow tier is Sev 4 (ticket, not a page). The name "P993d" reflects the burn rate: at this rate the monthly SLO budget exhausts in ~3 days. The `for: 6h` clause serves as the effective 6h measurement window. + +--- + +**`userJourneySwiftErrors1h5m`** - Sev 3 - RP lane +- Basis: `http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}` error rate fast-burn +- SLO: <= 1% error rate over 28d. Burn-rate threshold: 14.4 x 0.001 = 0.0144 + +**`userJourneySwiftErrors6h30m`** - Sev 3 - RP lane +- Same, medium-burn. Burn-rate threshold: 6 x 0.001 = 0.006 + +**`userJourneySwiftCNSLatencyP991h5m`** - Sev 3 - RP lane +- Basis: `cns:ip_assignment_latency:p99` recording rule > 10s, fast-burn (1h/5m windows) + +**`userJourneySwiftCNSLatencyP996h30m`** - Sev 3 - RP lane +- Same, medium-burn (6h/30m windows) + +### Konnectivity tunnel alerts + +**`userJourneySwiftKonnectivityErrors1h5m`** - Sev 3 - RP lane +```yaml +alert: userJourneySwiftKonnectivityErrors1h5m +expr: | + konnectivity:stream_error_rate:avg_5m > 0.01 + and + konnectivity:stream_error_rate:avg_1h > 0.01 +for: 2m +labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m +annotations: + summary: "Konnectivity tunnel error rate elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +**`userJourneySwiftKonnectivityErrors6h30m`** - Sev 3 - RP lane +- Same, medium-burn (6h/30m windows) + +**`userJourneySwiftKonnectivityDialFailures1h5m`** - Sev 3 - RP lane +```yaml +alert: userJourneySwiftKonnectivityDialFailures1h5m +expr: | + konnectivity:dial_failure_rate:avg_5m > 0.01 + and + konnectivity:dial_failure_rate:avg_1h > 0.01 +for: 2m +labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m +annotations: + summary: "Konnectivity dial failure rate elevated (fast burn)" + runbook_url: "https://aka.ms/arohcp-runbook-swift" +``` + +**`userJourneySwiftKonnectivityDialFailures6h30m`** - Sev 3 - RP lane +- Same, medium-burn (6h/30m windows) + +Source: konnectivity-server PodMonitor (`observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml`), port 8093 on KAS pods in HCP namespaces. + +## aka.ms Short-links + +| Short-link | Points to | +|---|---| +| `aka.ms/arohcp-runbook-swift` | This runbook (SWIFT Networking User Journey Runbook) | +| `aka.ms/arohcp-tsg-swift` | SWIFT Networking TSG | +| `aka.ms/arohcp-dashboard-swift` | SWIFT Networking Grafana dashboard | + +## promtool Test Requirements + +Every alert must have a `_test.yaml` with passing promtool tests. Tests must cover: +- Alert fires when threshold is breached +- Alert does not fire when metric is healthy (absence of series = no fire for latency alerts) +- For multi-window alerts: both windows must exceed the threshold for the alert to fire + +## Recording Rules + +Recording rules live in `observability/alerts/swift-networking-recordingRule-KSM.yaml`. Three recording rule groups: +- `arohcp_swift_networking_slo_recording_rules`: router pod startup latency chain (`router:startup_latency:*`) +- `arohcp_swift_cns_latency_recording_rules`: CNS IP assignment latency chain (`cns:ip_assignment_latency:*`) +- `arohcp_swift_konnectivity_recording_rules`: konnectivity stream error rate and dial failure rate chains (`konnectivity:stream_error_rate:*`, `konnectivity:dial_failure_rate:*`) + +The full chain definitions are in [uj-slislo-swift.md](uj-slislo-swift.md). + +## References + +- [ARO HCP Alerting Recommendation ADR](https://github.com/openshift-online/architecture/pull/79) +- [docs/alerts.md](https://github.com/Azure/ARO-HCP/blob/main/docs/alerts.md): severity mapping, correlationId, IcM title rendering +- [docs/prometheus-rules.md](https://github.com/Azure/ARO-HCP/blob/main/docs/prometheus-rules.md): authoring, testing, generating Bicep +- [Observability code](https://github.com/Azure/ARO-HCP/tree/main/observability): existing alert and recording rule examples diff --git a/docs/drafts/uj-dashboard-swift.md b/docs/drafts/uj-dashboard-swift.md new file mode 100644 index 00000000000..3ae88a8c25f --- /dev/null +++ b/docs/drafts/uj-dashboard-swift.md @@ -0,0 +1,128 @@ +# SWIFT Networking: Dashboard Design (ARO-25920) + +## File location + +``` +observability/grafana-dashboards/sre/user-journey/swift-networking.json +``` + +Register in `observability/observability.yaml`. The pipeline creates the "SRE User Journey" folder in Grafana and imports the dashboard. + +## Data source + +Datasource variable regex: `^.*-mgmt-\d+$` (management cluster Prometheus) + +SWIFT is management-cluster-only. All panels use this datasource. The dashboard should include a datasource variable so SREs can switch between management clusters. + +> **Future:** Once [ARO-HCP#4878](https://github.com/Azure/ARO-HCP/pull/4878) lands, Kusto (Azure Data Explorer) will be available as a Grafana datasource for historical analysis and capacity planning panels. + +## Starter reference + +The `azure-container-networking` repo ships a starter Grafana dashboard at [`cns/doc/examples/metrics/grafana.json`](https://github.com/Azure/azure-container-networking/blob/master/cns/doc/examples/metrics/grafana.json) covering per-node IP utilization. Use as a base for IP pool state panels. + +## ADR-001 Baseline Coverage Requirements + +Every dashboard must include panels for all five baseline metrics: + +| Baseline metric | Panel | +|---|---| +| Availability | Router pod startup p99 vs SLO threshold | +| Errors | CNS IP assignment error rate | +| Latency | CNS IP assignment p99 | +| Traffic | CNS `requestipconfig` call rate | +| Saturation | SWIFT NIC utilization per node | + +## Panel Design Standards (ADR-001) + +- **SLO target line** on every SLI panel: current value always shown against the objective +- **Color thresholds** (green/amber/red) as the metric approaches or breaches the SLO +- **Error budget panel:** remaining budget for the current measurement window +- **Alerts annotation:** `ALERTS{alertname=~"userJourneySwift.*|SwiftCNS.*|SwiftPending.*", alertstate="firing"}` overlaid on all panels so pre/post-alert context is visible without switching tools + +## Dashboard Structure + +### Section 1: Journey Health + +**Panel: Router pod startup p99** +- Query: `router:startup_latency:p99_avg_1h` (or overlay all four window averages) +- All four window averages overlaid: `p99_avg_5m`, `p99_avg_30m`, `p99_avg_1h`, `p99_avg_6h` +- Threshold reference line at 300s +- Rationale: shows burn rate development. Fast window spiking before slow window = developing incident + +**Panel: Raw per-pod ContainerCreating duration** +- Query: `router:startup_latency:seconds` (per-pod detail) +- Useful for seeing which specific pods are stuck and for how long + +**Panel: CNS daemonset availability** +- Query: + ```promql + kube_daemonset_status_number_ready{daemonset="azure-cns", namespace="kube-system"} + / + kube_daemonset_status_desired_number_scheduled{daemonset="azure-cns", namespace="kube-system"} + ``` +- SLO target line at 99.9% + +**Panel: SWIFT NIC utilization per node** +- Query: + ```promql + sum by (node) (kube_pod_container_resource_requests{resource="aro.openshift.io/swift-nic"}) + / + kube_node_status_capacity{resource="aro.openshift.io/swift-nic"} + ``` +- Capacity planning; no SLO threshold (scale-out is automatic) + +**Panel: IPs stuck in PendingProgramming** +- Query: `cx_pending_programming_ips_v2` +- Threshold alert line at 0: any sustained non-zero is the primary proxy for stuck MTPNCs + +**Panel: Konnectivity stream error rate** +- Query: `konnectivity:stream_error_rate:5m` +- SLO threshold at 1% +- Shows tunnel-layer health complementing NIC assignment metrics + +**Panel: Konnectivity dial failure rate** +- Query: `konnectivity:dial_failure_rate:5m` +- SLO threshold at 1% +- Shows KAS-to-node connection establishment health + +### Section 2: Component Health + +**Panel: IP assignment error rate** +- Query: + ```promql + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[5m])) + / + sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m])) + ``` +- SLO threshold at 1% + +**Panel: IP assignment latency p99** +- Query: `histogram_quantile(0.99, sum(rate(ip_assignment_latency_seconds_bucket[5m])) by (le))` +- SLO threshold at 10s + +**Panel: `requestipconfig` traffic rate** +- Query: `sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m]))` +- No SLO; use for anomaly detection (sustained zero with nodes Ready = something stopped scheduling CX pods) + +**Panel: IP pool state per node** +- Queries: `cx_assigned_ips_v2`, `cx_available_ips_v2`, `cx_allocated_ips_v2` stacked +- From starter dashboard: `cns/doc/examples/metrics/grafana.json` + +**Panel: IP pool saturation** +- Query: `sum by (instance) (cx_assigned_ips_v2) / sum by (instance) (cx_ipam_max_ips)` +- Threshold at 85% for saturation signal + +**Panel: Subnet exhaustion** +- Query: `cx_ipam_subnet_exhaustion_state > 0` +- Binary state panel + +**Panel: IP pool convergence** +- Query: `cx_ipam_requested_ips - cx_ipam_total_ips` +- Healthy: slightly negative (~-1, DNC pre-allocates a buffer). Alert on **positive** values: means CNS requested IPs that DNC hasn't delivered. + +## Authoring Reference + +- [How to create an ARO HCP Dashboard](https://docs.google.com/document/d/1Va9ksYqO2nvhk2d4vjrLu5pNuO-A2I2gnMDohlc9dk4/edit?tab=t.3usr5sfulcj3) +- [docs/grafana-dashboards.md](https://github.com/Azure/ARO-HCP/blob/main/docs/grafana-dashboards.md) +- [ARO HCP Alerting Recommendation ADR](https://github.com/openshift-online/architecture/pull/79): dashboard design requirements +- [Monitoring / Metrics / Alerting Recommendation for ARO HCP](https://docs.google.com/document/d/11WCoJa7E8X9dalgzwtnCiPeRHD72AkoqMLG71gK61OA/edit) diff --git a/docs/drafts/uj-runbook-swift.md b/docs/drafts/uj-runbook-swift.md new file mode 100644 index 00000000000..697863f494c --- /dev/null +++ b/docs/drafts/uj-runbook-swift.md @@ -0,0 +1,131 @@ +# User Journey: SWIFT Networking + +# 1. Journey + +> As an ARO HCP customer, I do not want my network traffic to leave Azure (go over the public internet). This includes: +> 1. Traffic from HCP KAS to NodePools (or workloads running on them) +> 2. Traffic from NodePools (or workloads running on them) to HCP KAS +> 3. Traffic leveraging other private Azure services (e.g. private Key Vaults) +> 4. All traffic for private HCP clusters + +> As an ARO HCP SRE, I want SWIFT secondary NIC assignment to succeed within defined latency targets so that private connectivity failures do not block customer access to their HCP clusters. + +# 2. Architecture + +SWIFT is Azure's multi-tenant networking feature for AKS management clusters. It provides router pods with secondary NICs on the customer's subnet, enabling private routing between the customer VNet and the HCP. Without SWIFT, traffic between customer worker nodes and the HCP would traverse the public internet instead of staying internal to Azure. + +SWIFT is transparent to customers when healthy; they never observe it directly. When it fails, the private network path between the customer VNet and the HCP degrades or breaks entirely, affecting connectivity in both directions (Nodes to HCP and HCP to Nodes). + +When this flow completes, private connectivity between the customer VNet and the HCP is fully operational in both directions. + +> **Note:** Secondary SWIFT NICs appear with empty IP configuration in ARM and the Azure portal by design. `DelegateIPAllocation=true` means DNC-RC programs them via NMAgent. NRP does not touch them. Empty IP config is not a provisioning failure. + +```mermaid +sequenceDiagram + participant Customer + participant CS/Backend + participant CPO + participant Scheduler + participant DNC-RC + participant CNS + participant CNI + Note over Customer,CS/Backend: HCP cluster creation + Customer->>CS/Backend: Create HCP cluster + CS/Backend->>CS/Backend: Create SAL on customer subnet + CS/Backend->>CS/Backend: Create PodNetwork + PodNetworkInstance CRDs + Note over CPO,Scheduler: Router pod scheduling (per HCP, 3 replicas) + CPO->>Scheduler: Create router Deployment with SWIFT label + Scheduler->>Scheduler: Place pod on node with available aro.openshift.io/swift-nic + Note over DNC-RC,CNI: Secondary NIC assignment (per router pod) + DNC-RC->>DNC-RC: Create MTPNC for the pod, reserve IP from 3P subnet + CNI->>CNS: requestIPAddress for the pod + CNS->>CNS: Look up MTPNC, confirm status: provisioned + CNS-->>CNI: Return secondary NIC IP + MAC address + CNI->>CNI: Inject secondary NIC into pod network namespace + Note over Customer: Pod Running. Private path established via ILB. +``` + +## Components + +| Component | Where it runs | Owner | +|-----------|---------------|-------| +| **CNS** | DaemonSet on each management cluster worker node | Azure Cloudnet/Container Networking AKS Swift Control Plane | +| **DNC-RC** | AKS customer control plane (AKS-managed, not a pod in the cluster) | Azure Cloudnet/Container Networking AKS Swift Control Plane | +| **DNC** | Azure Container Networking service (AKS infrastructure layer) | Azure Cloudnet/Container Networking AKS Swift Control Plane | +| **NMA** | Azure physical host beneath the AKS worker VM | Cloudnet/VNET | +| **NRP** | Azure platform (Azure-wide control plane service) | Cloudnet/NRP | +| **CRP** | Azure platform (Azure-wide control plane service) | Azure Compute | +| **SAL** | ARM resource on the customer subnet | ARO-HCP | +| **mgmt-agent (SwiftNICController)** | Deployment on the management cluster | ARO-HCP | +| **NNC** | Management cluster API server (one per worker node) | Azure Cloudnet/Container Networking AKS Swift Control Plane | +| **MTPNC** | Management cluster API server (one per router pod) | Azure Cloudnet/Container Networking AKS Swift Control Plane | + +## Glossary + +| Term | Meaning | +|------|---------| +| **AFEC** | Azure Feature Exposure Control: Microsoft's system for registering preview feature flags on Azure subscriptions. SWIFT requires the `NetworkingMultiTenancyPreview` AFEC flag on the management subscription. | +| **CCP (AKS)** | AKS Customer Control Plane: the AKS-managed control plane for a given AKS cluster (in this context, the management cluster). DNC-RC and DNC run inside the AKS CCP. The CCP ID is a hex identifier used to scope Kusto log queries (e.g. `6834682f64206f000129bd12`), discoverable via [ASI](https://aka.ms/asi) by cluster FQDN. Not to be confused with the ARO-HCP Hosted Control Plane. | +| **HCP** | Hosted Control Plane: a customer's OpenShift cluster control plane (kube-apiserver, etcd, etc.) running as pods inside an OCM namespace on the management cluster. Distinct from the AKS CCP, which is the infrastructure layer that hosts the management cluster itself. | +| **2P subnet** | Second-party subnet: the host/infrastructure subnet used by the management cluster's own workloads. | +| **3P subnet** | Third-party subnet: the customer-exclusive subnet used by CX pods, assigned via secondary NICs. | +| **CX pods** | Customer eXclusive pods: pods belonging to a Hosted Control Plane tenant, networked via secondary NICs on the 3P subnet. In ARO-HCP, the CX pod is specifically the **router pod**, the only hosted cluster pod that receives a SWIFT secondary NIC. | +| **CPO** | Control Plane Operator: the HyperShift component that runs inside each HCP namespace on the management cluster and manages the hosted control plane lifecycle. CPO creates the router Deployment and injects the SWIFT pod network instance label onto router pods. | +| **CS** | Cluster Service: the ARO-HCP service responsible for HCP lifecycle operations. For SWIFT, CS creates the SAL, PodNetwork, PodNetworkInstance, and Internal Load Balancer during HCP cluster provisioning. **Note: CS is actively being decommissioned and will be replaced by a Backend (RP) operator.** | +| **ILB** | Internal Load Balancer: a Standard SKU Azure internal load balancer created in the customer's VNet integration subnet during HCP cluster provisioning. Its backend pool contains the three PNI IP addresses (one per router pod replica). Worker nodes reach the kube-apiserver, ignition, OAuth, and Konnectivity endpoints via the `hypershift.local` private DNS zone, which resolves to the ILB frontend IP. | +| **SWIFT / SWIFT V2** | Azure's multi-tenant networking feature for AKS management clusters, enabled via the `NetworkingMultiTenancyPreview` AFEC flag. These terms are used interchangeably; SWIFT V2 is the current implementation version and the term used in component documentation (CNS, DNC-RC, Azure Container Networking TSGs). | + +# 3. SLIs, SLOs, and TSG Routing + +| SLI | SLO | Alert | TSG | +|-----|-----|-------|-----| +| Router pod startup latency p99 | <= 300s | `userJourneySwiftLatencyP991h5m` / `userJourneySwiftLatencyP996h30m` | [SWIFT Networking TSG](uj-tsg-swift.md#step-1-establish-blast-radius) | +| CNS IP assignment error rate | <= 1% over 28d | `userJourneySwiftErrors1h5m` / `userJourneySwiftErrors6h30m` | [SWIFT Networking TSG](uj-tsg-swift.md#step-3-read-cns-logs-and-classify-the-error) | +| CNS daemonset availability | >= 99.9% over 28d | `SwiftCNSAvailability3d` | [SWIFT Networking TSG](uj-tsg-swift.md#step-2-check-hcp-side-prerequisites) | +| IPs stuck in PendingProgramming | 0 sustained | `SwiftPendingProgramming` | [SWIFT Networking TSG](uj-tsg-swift.md#step-3-read-cns-logs-and-classify-the-error) | +| CNS IP assignment latency p99 | <= 10s over 28d | `userJourneySwiftCNSLatencyP991h5m` / `userJourneySwiftCNSLatencyP996h30m` | [SWIFT Networking TSG](uj-tsg-swift.md#step-3-read-cns-logs-and-classify-the-error) | +| Konnectivity tunnel stream error rate | <= 1% over 28d | `userJourneySwiftKonnectivityErrors1h5m` / `userJourneySwiftKonnectivityErrors6h30m` | [SWIFT Networking TSG](uj-tsg-swift.md#step-1-establish-blast-radius) | +| Konnectivity dial failure rate | <= 1% over 28d | `userJourneySwiftKonnectivityDialFailures1h5m` / `userJourneySwiftKonnectivityDialFailures6h30m` | [SWIFT Networking TSG](uj-tsg-swift.md#step-1-establish-blast-radius) | + +User Journey Dashboard: [SWIFT Networking SLO Overview](https://aka.ms/arohcp-dashboard-swift) + +> **Measurement note:** the managed components (CNS, DNC-RC, NRP) are largely opaque. The SLIs above measure SWIFT's observable effects at two layers: the NIC assignment layer (router pod startup, CNS metrics) and the tunnel layer (konnectivity stream/dial health). + +> **NodePool caveat:** HCP clusters can be provisioned with zero NodePools. A metric reading zero may mean "nothing to schedule" rather than "broken." Interpret in context. + +# 4. Escalation + +SWIFT spans ARO-HCP, Azure AKS, and Azure Cloudnet/NRP ownership boundaries. Severity follows the [Azure CEN](https://aka.ms/AzureCEN). + +- **Who to engage:** Cloudnet/NRP + - **Trigger:** DNC NIC/IP exhaustion (ErrorCode 4/135 in CNS logs) with NRP validation failures upstream, or SAL API failures. + - **Evidence to attach:** CNS log snippet, `cx_pending_programming_ips_v2` graph, blast radius (subscriptions and clusters affected). + - **How to escalate:** File IcM directly to Cloudnet/NRP. Do not route through DNC, RNM, or SdnPubSub first — NRP validation failures are the upstream cause; DNC exhaustion is a downstream symptom. + +- **Who to engage:** Cloudnet/Container Networking AKS Swift Control Plane + - **Trigger:** CNS pod failures, MTPNC not created, DNC-RC errors (connection refused, auth failures), or DNC PubSub context selector loss. + - **Evidence to attach:** `kubectl describe mtpnc` output, CNS logs, `kubectl get nodeinfo` output, DNC-RC Kusto log snippet. + - **How to escalate:** File IcM to Cloudnet/Container Networking AKS Swift Control Plane. Informal contact via #external-wg-aro-hcp. + +- **Who to engage:** Cloudnet/VNET + - **Trigger:** CNS logs show `sync host error` or `failed to get nc version list from nmagent`. + - **Evidence to attach:** CNS log snippet from affected node. + - **How to escalate:** File IcM to Cloudnet/VNET. + +- **Who to engage:** Cloudnet/NRP + - **Trigger:** HCP creation success rate significantly lower in one region than others. NRP Kusto shows VMSS validation failures concentrated on SWIFT-enabled pools. + - **Evidence to attach:** Regional creation success rate comparison, NRP Kusto query results, affected subscription IDs and VMSS names. + - **How to escalate:** File IcM to Cloudnet/NRP. Request force-reconcile rollout for affected VMSS instances. + +For ARO-HCP component ownership questions: [ARO-HCP Component Inventory + Ownership](https://docs.google.com/spreadsheets/d/1Z2uhI3ctbZCF0rOj8B4VsoYx1rR9x_1ZLsbOr6Eug-8/edit?gid=0#gid=0) + +# References + +- [SWIFT V2 CNS implementation](https://github.com/Azure/azure-container-networking/blob/master/docs/feature/swift-v2/cns.md): CRD definitions, IPAM flow, `requestIPAddress` handler +- [ARO HCP Region Buildout](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/runbooks/buildout/region-buildout): how SWIFT is enabled per region via Geneva Actions +- [Azure Container Networking TSGs: SWIFT V2](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-networking/azure-container-networking-caggar/azure-container-networking/azure-container-networking-tsgs/tsgs/aks/scenarios/swiftv2/swiftv2): DNC-RC, MTPNC, and NC deletion debugging +- [DNC-RC Overview](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-networking/azure-container-networking-caggar/azure-container-networking/azure-container-networking-tsgs/tsgs/aks/dnc-rc/overview): DNC-RC architecture, Kusto log access, NNC data flow +- [ADR-001: ARO HCP Alerting Recommendation for SRE](https://github.com/openshift-online/architecture/pull/79): naming convention, burn-rate tiers, routing lanes +- [ARO HCP Access Guide](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/runbooks/aro-hcp-access-guide): accessing management clusters and monitoring systems +- [HCM Incident Management Process](https://source.redhat.com/groups/public/hybridcloudmanagement/service_delivery_wiki/incident_management_process) +- [ARO Incident Management Procedure](https://docs.google.com/document/d/18AMixaTUUd5Rk12z_GfaujTzYUyFxUBpraoKPK8ku6o/edit) diff --git a/docs/drafts/uj-slislo-swift.md b/docs/drafts/uj-slislo-swift.md new file mode 100644 index 00000000000..dbbfc7ad456 --- /dev/null +++ b/docs/drafts/uj-slislo-swift.md @@ -0,0 +1,203 @@ +# SWIFT Networking: SLI/SLO Design (ARO-25919) + +## ADR-001 Baseline Requirements + +Every user journey must have SLIs and SLOs for all five baseline metrics: + +| Baseline metric | SWIFT SLI | +|---|---| +| **Availability** | Router pod startup p99 ≤ 300s (via kube-state-metrics recording rule) | +| **Errors** | CNS `requestipconfig` error rate (`http_request_latency_seconds`, `cns_return_code != "0"`) | +| **Latency** | CNS IP assignment latency p99 (`ip_assignment_latency_seconds`) | +| **Traffic** | CNS `requestipconfig` call rate (`http_request_latency_seconds_count`) | +| **Saturation** | SWIFT NIC utilization per node (`aro.openshift.io/swift-nic` extended resource ratio) | + +SLIs must be expressed as good events / total events (ratio format) per ADR-001. + +## Measurement Challenge (Brendan Bergen, Jun 2026) + +Measuring SWIFT health directly is hard because the managed components (CNS, DNC-RC, NRP) are largely opaque. The SLIs below measure SWIFT's observable effects (router pod startup latency, NIC scheduler capacity, CNS IP assignment) rather than the SWIFT internals. + +**NodePool caveat:** HCPs can be provisioned with zero NodePools. Every metric must be interpreted in context: a zero reading may mean "nothing to schedule" rather than "broken". Gate metrics on NodePools existing and scaled > 0. + +## Signals Under Consideration (future work) + +1. **Node `Ready` condition ratio gated on NodePool count:** can worker nodes talk to KAS? (Node->HCP direction) +2. **Router Pod SWIFT NIC traffic drought or saturation:** is traffic flowing through the SWIFT path? +3. **Router Pods `Pending`/`Unschedulable`:** mgmt-agent didn't report NICs, NICs not provisioned, or scheduler can't place pods + +## Journey-level SLI: Router pod startup latency + +SLI: p99 duration a router pod spends in `Pending`/`ContainerCreating` state across all management clusters. + +**Why this is the journey SLI:** A simple readiness ratio is unsuitable because new HCPs legitimately have pods in `ContainerCreating` during normal provisioning. Startup latency is the authoritative signal that SWIFT NIC assignment is stalled. + +**Why kube-state-metrics is authoritative:** kube-state-metrics reads directly from the Kubernetes API server, which is the system of record for pod state. + +**Recording rule chain:** +```yaml +# Base: per-pod duration in ContainerCreating (no series when healthy) +- record: router:startup_latency:seconds + expr: | + (time() - kube_pod_created{namespace=~"ocm-.*"}) + * on(namespace, pod) kube_pod_owner{owner_kind="ReplicaSet", owner_name=~"router-.*"} + * on(namespace, pod) (kube_pod_status_phase{phase="Pending"} == 1) + +# p99 across all stuck router pods (no series when none stuck) +- record: router:startup_latency:p99 + expr: quantile(0.99, router:startup_latency:seconds) + +# Window averages for multi-burn-rate alert expressions +- record: router:startup_latency:p99_avg_5m + expr: avg_over_time(router:startup_latency:p99[5m]) +- record: router:startup_latency:p99_avg_30m + expr: avg_over_time(router:startup_latency:p99[30m]) +- record: router:startup_latency:p99_avg_1h + expr: avg_over_time(router:startup_latency:p99[1h]) +- record: router:startup_latency:p99_avg_6h + expr: avg_over_time(router:startup_latency:p99[6h]) +``` + +Design decisions: +- No threshold baked into recording rules (follows repo convention: `kas:`, `hostedClusterAPI_kubeapiserver_available`) +- Absence of series = healthy (alert correctly does not fire) +- `kube_pod_status_phase{phase="Pending"}` covers both `Pending` and `ContainerCreating` (both show as Pending in kube-state-metrics) + +**SLO:** router pod startup latency p99 ≤ 300s +- Rationale: a router pod stuck > 5 minutes indicates a SWIFT NIC assignment failure, not normal startup latency + +### Component-level: SWIFT NIC saturation per node + +SLI: ratio of SWIFT NIC slots in use to total NIC capacity per node. + +```promql +sum by (node) (kube_pod_container_resource_requests{resource="aro.openshift.io/swift-nic"}) +/ +kube_node_status_capacity{resource="aro.openshift.io/swift-nic"} +``` + +Use for capacity planning dashboards. Scale-out is triggered automatically by the cluster autoscaler, so this metric does not require an SRE alert. + +### Component-level: CNS daemonset availability + +SLI: fraction of SWIFT-enabled management cluster worker nodes with CNS running. + +```promql +kube_daemonset_status_number_ready{daemonset="azure-cns", namespace="kube-system"} +/ +kube_daemonset_status_desired_number_scheduled{daemonset="azure-cns", namespace="kube-system"} +``` + +**SLO:** ≥ 99.9% of nodes have CNS running (28-day window) + +## CNS PodMonitor SLIs + +**Traffic:** CNS `requestipconfig` call rate +```promql +sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m])) +``` + +**Errors:** CNS IP assignment error rate +```promql +sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*", cns_return_code!="0"}[5m])) +/ +sum(rate(http_request_latency_seconds_count{url=~".*/requestipconfig.*"}[5m])) +``` +Note: CNS endpoint is `POST /network/requestipconfig` (not `requestIPAddress`; the Go function name differs from the URL path). + +**Latency:** CNS IP assignment p99 +```promql +histogram_quantile(0.99, sum(rate(ip_assignment_latency_seconds_bucket[5m])) by (le)) +``` + +**Saturation (proxy):** IPs stuck in PendingProgramming. Sustained non-zero is the best available proxy for stuck MTPNCs while MTPNC reconciler emits no metrics. +```promql +cx_pending_programming_ips_v2 +``` + +**Pool state (diagnostic, no SLO):** +```promql +sum by (instance) (cx_assigned_ips_v2) / sum by (instance) (cx_ipam_max_ips) # IP pool utilization per node +cx_ipam_requested_ips - cx_ipam_total_ips # divergence = DNC not fulfilling requests +cx_ipam_subnet_exhaustion_state > 0 # subnet full +``` + +## Konnectivity Tunnel Health SLIs + +The konnectivity-server sidecar runs inside the KAS pod in each HCP namespace. It proxies all KAS-to-node traffic through the SWIFT router pods. When SWIFT NIC assignment fails, konnectivity tunnels break — making these metrics a second layer of SWIFT health measurement (the tunnel layer, vs. the NIC assignment layer above). + +Source: konnectivity-server PodMonitor on port 8093 (`observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml`). + +### Konnectivity stream error rate + +SLI: ratio of errored stream packets to total stream packets. + +**Recording rule chain:** +```yaml +# Base: 5m stream error rate +- record: konnectivity:stream_error_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_stream_errors_total[5m])) + / + clamp_min(sum(rate(konnectivity_network_proxy_server_stream_packets_total[5m])), 1) + +# Window averages for multi-burn-rate alert expressions +- record: konnectivity:stream_error_rate:avg_5m + expr: avg_over_time(konnectivity:stream_error_rate:5m[5m]) +- record: konnectivity:stream_error_rate:avg_30m + expr: avg_over_time(konnectivity:stream_error_rate:5m[30m]) +- record: konnectivity:stream_error_rate:avg_1h + expr: avg_over_time(konnectivity:stream_error_rate:5m[1h]) +- record: konnectivity:stream_error_rate:avg_6h + expr: avg_over_time(konnectivity:stream_error_rate:5m[6h]) +``` + +**SLO:** <= 1% stream error rate over 28 days + +### Konnectivity dial failure rate + +SLI: ratio of dial failures to total dial attempts (failures + successful DIAL_RSP from agents). + +**Recording rule chain:** +```yaml +# Base: 5m dial failure rate +- record: konnectivity:dial_failure_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + / + clamp_min( + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + + + sum(rate(konnectivity_network_proxy_server_stream_packets_total{segment="from_agent", packet_type="DIAL_RSP"}[5m])) + , 1) + +# Window averages for multi-burn-rate alert expressions +- record: konnectivity:dial_failure_rate:avg_5m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[5m]) +- record: konnectivity:dial_failure_rate:avg_30m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[30m]) +- record: konnectivity:dial_failure_rate:avg_1h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[1h]) +- record: konnectivity:dial_failure_rate:avg_6h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[6h]) +``` + +**SLO:** <= 1% dial failure rate over 28 days + +## SLO Targets + +| SLI | Proposed target | Window | Notes | +|---|---|---|---| +| Router pod startup latency p99 | ≤ 300s | per-event | | +| CNS daemonset availability | ≥ 99.9% | 28 days | | +| IP assignment error rate | ≤ 1% | 28 days | | +| IP assignment latency p99 | ≤ 10s | 28 days | | +| Konnectivity stream error rate | ≤ 1% | 28 days | Tunnel layer; complements NIC assignment SLIs | +| Konnectivity dial failure rate | ≤ 1% | 28 days | KAS-to-node connection establishment | + +## References + +- [ARO HCP Alerting Recommendation ADR](https://github.com/openshift-online/architecture/pull/79): SLI design, baseline metrics, coverage model +- [ARO SLO Policy ADR](https://github.com/swiencki/architecture/pull/1) [Draft]: SLO target-setting, error budgets +- [Monitoring / Metrics / Alerting Recommendation for ARO HCP](https://docs.google.com/document/d/11WCoJa7E8X9dalgzwtnCiPeRHD72AkoqMLG71gK61OA/edit) +- ARO Classic & HCP: SLI/SLO meeting (weekly, 17:00 UTC), #external-wg-aro-hcp diff --git a/docs/drafts/uj-tsg-swift.md b/docs/drafts/uj-tsg-swift.md new file mode 100644 index 00000000000..71ce8745b7f --- /dev/null +++ b/docs/drafts/uj-tsg-swift.md @@ -0,0 +1,644 @@ +# TSG: SWIFT Networking: Router Pod NIC Assignment Failure + +# When to Use This TSG + +- Use this TSG when router pods are stuck in `ContainerCreating` or `Pending` on management cluster worker nodes and the root cause is suspected to be SWIFT secondary NIC assignment failure. +- Use this TSG when `userJourneySwift*` alerts fire (latency, errors, CNS latency, konnectivity). +- Use this TSG when `SwiftCNSAvailability3d` or `SwiftPendingProgramming` alerts fire. +- Use this TSG when HCP cluster creation success rate drops significantly in one region compared to others. + +See the [SWIFT Networking User Journey Runbook](uj-runbook-swift.md) for background on the SWIFT stack, component ownership, and SLI/SLO definitions. + +# Alert Triage + +| Alert | What it means | Start here | +|-------|--------------|------------| +| `userJourneySwiftLatencyP991h5m` / `userJourneySwiftLatencyP996h30m` | Router pods are taking too long to start | [Step 1: Establish blast radius](#step-1-establish-blast-radius) | +| `userJourneySwiftErrors1h5m` / `userJourneySwiftErrors6h30m` | NIC assignment requests are failing | [Step 3: Read CNS logs](#step-3-read-cns-logs-and-classify-the-error) | +| `userJourneySwiftCNSLatencyP991h5m` / `userJourneySwiftCNSLatencyP996h30m` | NIC assignment is taking too long | [Step 3: Read CNS logs](#step-3-read-cns-logs-and-classify-the-error) | +| `userJourneySwiftKonnectivityErrors1h5m` / `userJourneySwiftKonnectivityErrors6h30m` | Konnectivity tunnel is dropping packets | [Step 1: Establish blast radius](#step-1-establish-blast-radius) | +| `userJourneySwiftKonnectivityDialFailures1h5m` / `userJourneySwiftKonnectivityDialFailures6h30m` | KAS cannot connect to worker nodes | [Step 1: Establish blast radius](#step-1-establish-blast-radius) | +| `SwiftCNSAvailability3d` | Some nodes are missing the CNS networking agent | [Step 2: Check HCP-side prerequisites](#step-2-check-hcp-side-prerequisites) | +| `SwiftPendingProgramming` | NICs are reserved but not being attached to pods | [Step 3: Read CNS logs](#step-3-read-cns-logs-and-classify-the-error) | + +# Access Requirements + +Most diagnostic steps do not require breakglass cluster access. The access tiers used in this TSG: + +| Access tier | What it gets you | How to get it | +|---|---|---| +| **Grafana** | Prometheus dashboards and PromQL queries (Steps 1b, 2a) | SRE Grafana access (all SREs have this) | +| **az CLI** | AFEC flag, SAL status, nodepool state (Steps 2b, 4) | AME account with Reader on the management subscription | +| **CNS Kusto** | CNS logs (Step 3a) | [AZNWSDN Kusto access](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-networking/sdn-dbansal/sdn-buildout-and-deployments/sdn-fundamentals/teamdocs/data-science/kusto/kusto-access-for-aznwsdn). Cluster: `aznwsdn.kusto.windows.net`, database: `ACN` | +| **AKS Kusto** | DNC-RC and DNC logs (Step 4) | [AKS Kusto Partners](https://coreidentity.microsoft.com/manage/Entitlement/entitlement/akskustopart-mqif). Cluster: `akshuba.centralus.kusto.windows.net`, database: `AKSccplogs` | +| **Breakglass** | kubectl access to management cluster (Steps 1c, 2c, 3b, mitigation) | JIT via [aka.ms/jitaccess](https://aka.ms/jitaccess) (role: Azure Kubernetes Service RBAC Admin) + `hcpctl` for kubeconfig. See [Breakglass Guide](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/runbooks/breakglass/index.html) | + +See the [ARO HCP Access Guide](https://eng.ms/docs/cloud-ai-platform/azure-core/azure-cloud-native-and-management-platform/control-plane-bburns/azure-red-hat-openshift/azure-redhat-openshift-team-doc/hcp/runbooks/aro-hcp-access-guide) for general access and IcM account setup. + +# Purpose + +SWIFT secondary NIC assignment can fail at several layers: ARO-HCP-owned configuration (SAL, PodNetwork, PNI, mgmt-agent), managed Azure components (CNS, DNC-RC), or the Azure platform itself (NRP, DNC). The primary goal at 2am is to quickly determine which tier owns the failure so the right team can be engaged. + +The single most important decision: **is this on the HCP side (we can fix) or the Azure platform side (requires IcM to MSFT)?** + +# Severity and Impact + +Severity follows the [Azure CEN](https://aka.ms/AzureCEN). SWIFT incidents can arrive via two paths: + +**Platform Incident path** (SRE detects via monitoring/alerts): + +| Scenario | Severity | +|----------|----------| +| Isolated CNS issue on a single management cluster worker node, no confirmed customer impact | Sev 4 | +| SWIFT failure on a single management cluster causing router pod networking to fail (a single management cluster hosts multiple customers) | Sev 2 | +| Multi-region SWIFT failure (multiple management clusters across regions affected simultaneously) | Sev 1 | + +**CRI path** (customer opens a support ticket): + +A single-customer support ticket starts at Sev 3. It becomes Sev 2 if SRE determines that the impact is management-cluster wide (same criteria as the platform incident path above). + +- **Customer impact:** Konnectivity tunnel between HCP KAS and customer worker nodes is broken. Worker nodes go `NotReady`, `oc logs`/`oc exec` fail, webhooks time out, new HCP clusters fail to provision. Existing workloads on worker nodes continue running but cannot be managed (no new scheduling, no rolling updates, no webhook-dependent operations). +- **Business impact:** Management cluster functionally impaired for HCP tenants despite appearing healthy at the ARM level. + +# Customer-Visible Symptoms + +SWIFT failure breaks the Konnectivity tunnel between the HCP kube-apiserver and customer worker nodes. All KAS-to-node communication flows through the SWIFT router pods, so when they fail, the downstream effects are: + +- Worker nodes `NotReady`: kubelet cannot heartbeat to the kube-apiserver +- `oc logs` / `oc exec` / `oc port-forward` returning TCP errors or timing out (KAS cannot reach the node) +- Admission webhooks backed by data plane services failing or timing out +- Aggregated API servers unreachable (OpenShift API, OAuth API, packageserver) +- HCP certificate rotations failing (private Key Vault unreachable) +- Customer operators on worker nodes unable to authenticate to the HCP KAS +- OLM catalog resolution failing (catalogs served from data plane) +- ImageStream imports failing (registry access routed through konnectivity proxy) +- New HCP cluster provisioning failing or stalled + +# Service Symptoms + +- `userJourneySwiftLatencyP99*` alert firing: router pod startup p99 exceeds 300s +- `userJourneySwiftErrors*` alert firing: CNS `requestipconfig` error rate elevated +- `userJourneySwiftCNSLatencyP99*` alert firing: CNS IP assignment latency p99 exceeds 10s +- `userJourneySwiftKonnectivityErrors*` alert firing: konnectivity tunnel stream error rate elevated +- `userJourneySwiftKonnectivityDialFailures*` alert firing: KAS-to-node dial failure rate elevated +- `SwiftPendingProgramming` alert firing: `cx_pending_programming_ips_v2 > 0` sustained +- `kubectl get multitenantpodnetworkconfigs -A | grep -v provisioned` returns output +- CNS logs contain `ErrorCode 4`, `ErrorCode 135`, or DNC-RC error strings +- `cx_pending_programming_ips_v2` sustained > 0 in Grafana +- HCP cluster creation success rate asymmetric across regions + +**Limitations:** The MTPNC reconciler in CNS emits no metrics. MTPNC health must be inferred from `cx_pending_programming_ips_v2` (sustained > 0 = pods waiting for NIC assignment) or from `kubectl get multitenantpodnetworkconfigs` directly. DNC-RC and DNC logs are only available via AKS Kusto, not in-cluster. HCP clusters with zero NodePools produce no SWIFT traffic, so zero-valued metrics may mean "nothing to schedule" rather than "broken." + +# Identify the Problem + +## Phase 1: Triage without breakglass + +Use Grafana, az CLI, and Kusto to determine the blast radius and likely root cause before requesting cluster access. + +### Step 1: Establish blast radius + +- **Goal:** Determine scope of impact to route triage correctly. + +**Step 1a: Read the alert** + +The firing alert carries labels that identify the affected management cluster and HCP: +- `cluster`: management cluster name +- `namespace`: HCP namespace (e.g. `ocm-prod-abc123`) +- `_id`: hosted cluster ID + +Check whether other SWIFT alerts are firing on the same cluster, or on multiple clusters across regions. + +**Step 1b: Check Grafana dashboards** + +Open the [SWIFT Networking dashboard](https://aka.ms/arohcp-dashboard-swift) with the affected management cluster datasource: + +- `router:startup_latency:p99`: is this one HCP or many? +- CNS daemonset availability: is it one node or cluster-wide? +- `cx_pending_programming_ips_v2`: any nodes with sustained non-zero? +- Compare the **Cluster Provisioning SLO (Fleet Wide)** dashboard across regional datasources: is one region significantly worse than others? + +**Routing based on blast radius:** + +| Pattern | Likely root cause | Confidence | Next step | +|---------|-------------------|------------|-----------| +| Single node, all other nodes fine | CNS pod issue or node-level config (HCP side) | High | Step 2 | +| All nodes in one management cluster, one region | HCP config or DNC/NRP regional issue | Medium | Step 2 | +| Multiple clusters across multiple subscriptions | Azure platform issue (no HCP config change spans multiple subs) | High | Step 5 directly | +| All `userswft` pools affected, system pools fine | NRP or DNC (SWIFT-specific code path) | High | Step 5 directly | +| HCP creation success rate asymmetric across regions | Possible AKS RP regression | Medium | Step 5, AKS RP regression signal | + +--- + +### Step 2: Check HCP-side prerequisites + +- **Goal:** Rule out ARO-HCP-owned configuration as the root cause. + +**Grafana checks:** + +```promql +# Is CNS running on all nodes? +kube_daemonset_status_number_ready{daemonset="azure-cns", namespace="kube-system"} +/ +kube_daemonset_status_desired_number_scheduled{daemonset="azure-cns", namespace="kube-system"} +# Expected: 1.0. Below 1.0 = CNS pod missing or not ready on some nodes. + +# Are any nodes reporting zero SWIFT NIC capacity? +kube_node_status_capacity{resource="aro.openshift.io/swift-nic"} == 0 +# Any results = mgmt-agent not reporting NIC capacity on that node. + +# Are router pods stuck? +router:startup_latency:seconds +# Any results = router pods in ContainerCreating. Higher values = stuck longer. + +# Are IPs stuck in PendingProgramming? +cx_pending_programming_ips_v2 +# Sustained > 0 = DNC not attaching NICs. +``` + +**Azure CLI checks:** + +```bash +# Is the AFEC flag registered? +az feature show --namespace Microsoft.ContainerService \ + --name NetworkingMultiTenancyPreview \ + --subscription +# Must show "state": "Registered". + +# Does the SAL exist? +az network vnet subnet show \ + -g --vnet-name -n \ + --subscription \ + --query "serviceAssociationLinks[?name=='RedHatOpenShift'].provisioningState" -o tsv +# Expected: Succeeded. Empty = SAL missing = HCP side. +``` + +- **If any check fails:** likely HCP-side root cause (high confidence). Fix the specific failed check and re-validate. +- **If all checks pass:** proceed to Step 3. + +--- + +### Step 3: Read CNS logs and classify the error + +- **Goal:** Identify the error category to determine whether to fix locally or escalate. + +**CNS logs via Kusto:** + +```kusto +cluster('aznwsdn.kusto.windows.net').database('ACN').DNCCNSLogs +| where PreciseTimeStamp > ago(2h) +| where VMName == "" +| where Message has_any ("ErrorCode", "error code", "reservation", "capacity", + "SubnetFull", "JsonError", "sync host", "nmagent", "mtpnc") +| project PreciseTimeStamp, Message +| sort by PreciseTimeStamp desc +| limit 500 +``` + +**DNC error codes (slot exhaustion):** + +| ErrorCode | Message | Category | Next step | +|-----------|---------|----------|-----------| +| `4` | `no free reservation in set` | IP slot exhaustion | Step 4 then Step 5 | +| `135` | `NodeCapacityExceeded` | NIC slot exhaustion | Step 4 then Step 5 | + +Both codes together = DNC slots not being released. Common upstream cause: NRP validation failure blocking node drains. + +**DNC-RC error strings (in CNS logs and MTPNC events):** + +| Error string | Category | Action | +|---|---|---| +| `JsonError:[Code:SubnetFull...]` | Customer subnet full | Customer must expand subnet or free IPs | +| `network is not ready - mtpnc is not ready` | DNC provisioning in progress | Usually transient; check Step 4 if sustained | +| `network is not ready - failed to get MTPNC` | MTPNC not yet created | Check PNI in Phase 2 | +| `network is not ready - mtpnc for previous pod is being deleted` | MTPNC stuck deletion | See Mitigation: Stuck MTPNC deletion | +| `invalid NIC type for SWIFT v2 scenario` | NIC type config error | HCP configuration issue | +| `connection refused` to DNC (port 9000) | DNC down | Escalate: Cloudnet/Container Networking AKS Swift Control Plane | +| `object has been modified; please apply your changes...` | **Benign** | DNC-RC will auto-retry; ignore | +| `sync host error` / `failed to get nc version list from nmagent` | CNS can't reach NMA | Escalate: Cloudnet/VNET | + +Also check aggregate CNS metrics in Grafana: +```promql +# IPs stuck in pending programming? (sustained > 0 = DNC not releasing NICs) +cx_pending_programming_ips_v2 + +# IP pool divergence: POSITIVE = DNC not fulfilling requests (problem) +# Healthy: total slightly > requested (~-1 buffer); alert on > 0 +cx_ipam_requested_ips - cx_ipam_total_ips +``` + +- **If ErrorCode 4 or 135:** DNC slot exhaustion, likely Azure platform issue (high confidence). Proceed to Step 4, then Step 5. +- **If DNC-RC error string matched:** see table above for routing and confidence level. +- **If no recognizable error pattern:** root cause uncertain. Proceed to Phase 2 for deeper inspection, or escalate to Cloudnet/Container Networking AKS Swift Control Plane with the raw logs. + +--- + +### Step 4: AKS Kusto (DNC-RC logs) + +Find the CCP ID via [ASI](https://aka.ms/asi) by searching the management cluster FQDN under AKS Managed Cluster. + +```kusto +// DNC-RC reconciler errors for a specific management cluster +union + cluster('akshuba.centralus.kusto.windows.net').database('AKSccplogs').ControlPlaneEvents, + cluster('akshuba.centralus.kusto.windows.net').database('AKSccplogs').ControlPlaneEventsNonShoebox +| where category == "requestcontroller" +| where namespace == "" +| extend props=parse_json(properties) +| extend logline=parse_json(tostring(props.log)) +| where logline.error != "" +| project PreciseTimeStamp, logline.component, logline.msg, logline.error +| sort by PreciseTimeStamp desc +| limit 1000 +``` + +```kusto +// Combined DNC + DNC-RC logs around a specific event time +union + cluster('akshuba.centralus.kusto.windows.net').database('AKSccplogs').ControlPlaneEvents, + cluster('akshuba.centralus.kusto.windows.net').database('AKSccplogs').ControlPlaneEventsNonShoebox +| where PreciseTimeStamp between (datetime('') - 10m .. datetime('') + 10m) +| where category in ('requestcontroller', 'dnc') +| where namespace == "" +| extend propsJson=parse_json(properties) +| project PreciseTimeStamp, propsJson.log +| sort by PreciseTimeStamp asc +``` + +**Error routing:** + +| DNC-RC log pattern | Meaning | Escalate to | +|---|---|---| +| `NodeReconciler` errors | DNC-RC can't register/deregister nodes with DNC | Cloudnet/Container Networking AKS Swift Control Plane | +| `NodeNetworkConfigReconciler` + `SubnetFull` | Subnet exhausted | Customer action | +| `NodeNetworkConfigReconciler` + `connection refused` / `status 500` | DNC is down | Cloudnet/Container Networking AKS Swift Control Plane | + +- **If DNC server errors (5xx, connection refused):** escalate to Cloudnet/Container Networking AKS Swift Control Plane. +- **If NRP validation errors upstream:** escalate to Cloudnet/NRP, proceed to Step 5. + +--- + +### Step 5: Azure platform signals + +> At this step, the root cause is in Azure platform infrastructure that Red Hat SREs cannot directly access or modify. File an IcM against the team listed below, then provide the Kusto queries to the Microsoft team on the bridge. + +**Signal: NRP VMSS validation failures** + +```bash +az aks show -g -n --query kubernetesVersion +az aks nodepool show -g -n --nodepool-name userswft --query provisioningState +``` + +File IcM to **Cloudnet/NRP** (not DNC, not RNM, not SdnPubSub) when: +- ErrorCode 4/135 in CNS logs +- `cx_pending_programming_ips_v2` sustained > 0 +- Only `userswft` pools affected; system pools fine +- Management cluster upgrade stuck + +Provide this query to the NRP team: +```kusto +cluster('nrp.kusto.windows.net').database('mdsnrp').QosEtwEvent +| where PreciseTimeStamp > ago(24h) +| where Region =~ "" +| where SubscriptionId in ("", "") +| where ResourceType == "virtualMachineScaleSets" +| where Success == false +| where OperationName in ("PutVMScaleSetOperation", "ValidateVMScaleSetOperation") +| summarize ErrorCount = count(), lastSeen = max(PreciseTimeStamp) + by bin(PreciseTimeStamp, 1h), ErrorCode, SubscriptionId, ResourceName +``` + +--- + +**Signal: DNC PubSub partition errors** + +If NRP query shows nothing but DNC is still failing to release NICs: + +Escalate to **Cloudnet/Container Networking AKS Swift Control Plane**. Provide affected VNet GUIDs and subscription IDs: + +```kusto +cluster('aznwsdn.kusto.windows.net').database('sdnpubsub').PubSubAPICall +| where PreciseTimeStamp > ago(24h) +| where Tenant contains "" +| where isFault == true +| where additional contains "" +``` + +--- + +**Signal: AKS RP regression / NIC delegation failure** + +Suspect this when HCP creation success rate is significantly lower in one region than others. + +Observable detection signals: +- Creation failure rate asymmetric across regions (check Cluster Provisioning SLO dashboard, compare regional `Managed_Prometheus_services-*` datasources) +- Only `userswft` pools affected; system pools fine +- Cordoning and recreating affected nodes does not fix the problem (new nodes inherit the broken VMSS template) +- NRP Kusto shows VMSS validation failures concentrated on SWIFT-enabled pools + +Escalate to **Cloudnet/NRP** via IcM. Request a force-reconcile rollout for the affected management cluster VMSS instances. Provide: subscription IDs, VMSS names, affected region, regional creation success rate data. + +--- + +## Phase 2: Breakglass deep inspection + +If Phase 1 is inconclusive, or you need per-pod/CRD detail to confirm a root cause, request breakglass access (see [Access Requirements](#access-requirements)). + +### Step 6: CRD and pod inspection + +```bash +# Per-pod and node state +kubectl get nodes -o wide +kubectl get pods -A -l 'kubernetes.azure.com/pod-network-instance' | grep -E 'ContainerCreating|Pending' + +# Are MTPNCs being created and reaching provisioned? +kubectl get multitenantpodnetworkconfigs -A | grep -v provisioned +# Any output = stuck MTPNC. No output but pods stuck = MTPNC never created; check PNI. + +# Describe a stuck MTPNC: events are more reliable than CRD status fields. +kubectl describe mtpnc -n +# Look for Events: "CreateOrUpdateFailed" shows the exact DNC-RC error string. +# DeletionTimestamp present + finalizer still set = stuck deletion (see Mitigation). + +# Query CNS IP state directly: +kubectl exec -n kube-system -it -- /usr/local/bin/azure-cns -cmd get -darg All +kubectl exec -n kube-system -it -- /usr/local/bin/azure-cns -cmd get -darg Assigned +kubectl exec -n kube-system -it -- /usr/local/bin/azure-cns -cmd get -darg Available + +# Check NodeNetworkConfig for IP allocation state: +kubectl -n kube-system get nnc +# NC VERSION ahead of what CNS has programmed = IPs stuck in PendingProgramming. + +# Check secondary NICs on the node: +kubectl get nodeinfo -oyaml +# status.deviceInfos lists MAC addresses. Cross-reference with MTPNC .status.macAddress. + +# Do PodNetwork and PodNetworkInstance CRDs exist? +kubectl get podnetwork pn- +kubectl describe podnetwork pn- # check Events for DNC-RC errors + +kubectl get podnetworkinstance -n +kubectl get podnetworkinstance pni- -n \ + -o jsonpath='{.status.podIPAddresses}' +# Expected: 3 IPs. Empty = DNC-RC has not allocated IPs yet (check Step 4). +# Note: the ILB provisioning step is the gate: it retries until IPs appear. + +kubectl describe podnetworkinstance pni- -n # check Events + +# Does the router pod have the SWIFT label? +kubectl get pod -o jsonpath='{.metadata.labels}' +# Must contain: kubernetes.azure.com/pod-network-instance: +``` + +### Step 7: CNS logs via kubectl + +Use if Kusto access is unavailable: + +```bash +kubectl logs -n kube-system | \ + grep -E "ErrorCode|error code|reservation|capacity|SubnetFull|JsonError|sync host|nmagent" +``` + +Classify using the error tables in [Step 3](#step-3-read-cns-logs-and-classify-the-error). + +### Step 8: Azure platform signals via kubectl + +```bash +kubectl get events -n kube-system --field-selector reason=FailedScaleUp +``` + +Use alongside the az CLI checks in [Step 5](#step-5-azure-platform-signals). + +# Mitigation + +## Scenario: Stuck MTPNC deletion (MTPNC has DeletionTimestamp, finalizer not clearing) + +- **Goal:** Unblock a MultiTenantPodNetworkConfig stuck in deletion so the router pod can be recreated and DNC-RC can release its IPs. +- **Preconditions:** Confirmed the MTPNC has a non-null DeletionTimestamp and the finalizer `finalizers.acn.azure.com/dnc-operations` is still present. DNC-RC logs show no active reconciliation attempt in the last 10+ minutes. + +The chain is: router pod stuck terminating -> MTPNC can't be GC'd (ownerRef `blockOwnerDeletion`) -> DNC-RC can't release IPs -> PNI finalizer never clears. + +```bash +# Option A: force-delete the stuck router pod +kubectl delete pod -n --grace-period=0 --force + +# Option B: if the node is NotReady, drain and replace it +kubectl drain --ignore-daemonsets --delete-emptydir-data + +az vmss delete-instances \ + --resource-group \ + --name \ + --instance-ids + +kubectl delete node +``` + +> **Risk:** Draining a node evicts all pods on it. Confirm no critical workloads are pinned and check PodDisruptionBudget constraints before proceeding. + +## Scenario: mgmt-agent reporting zero SWIFT NICs on a node + +- **Goal:** Force mgmt-agent's SwiftNICController to re-query the Azure Compute API and republish the node's SWIFT NIC capacity. Uses Server-Side Apply, so it resyncs existing state without disrupting running pods. +- **Preconditions:** Confirmed via node status that `node.status.capacity["aro.openshift.io/swift-nic"]` is 0 or missing, while the VMSS instance shows secondary NICs attached in Azure. + +Router pods will be `Pending` with "Insufficient aro.openshift.io/swift-nic" on an otherwise healthy SWIFT-enabled node. Alert: `MgmtClusterNodeSwiftNICCapacityZero`. + +```bash +kubectl rollout restart deployment mgmt-agent -n mgmt-agent +``` + +## Scenario: CNS not running on a node (HCP side) + +- **Goal:** Restore the CNS pod to Running so it resumes IP assignment on the affected node. +- **Preconditions:** CNS pod confirmed not Running (CrashLoopBackOff, Pending, or missing entirely). + +```bash +kubectl describe pod -n kube-system # check events and logs +kubectl logs -n kube-system --previous +``` + +Escalate to Cloudnet/Container Networking AKS Swift Control Plane if CNS cannot be restarted or the crash is not caused by HCP configuration. + +## Scenario: Azure platform issue (NRP / DNC / AKS RP regression) + +- **Goal:** Escalate to the correct Microsoft team once ARO-HCP-side causes have been ruled out. +- **Preconditions:** Prior diagnostic steps (Steps 1-3) found no ARO-HCP-owned root cause, and Azure platform signals are showing an anomaly. + +File an IcM per the routing in [Step 5](#step-5-azure-platform-signals). Provide the Kusto query results and the blast radius data collected in Steps 1-3. + +> **Routing principle:** When NRP is rejecting VMSS operations for SWIFT-enabled pools, route directly to Cloudnet/NRP. Do not route through SdnPubSub, RNM, or DNC first — NRP validation failures are the upstream cause; DNC exhaustion and RNM signals are downstream symptoms. + +# Escalation + +| Symptom / signal | Escalate to | Channel | Evidence to attach | +|---|---|---|---| +| CNS pod failure, MTPNC never created, DNC-RC errors | Cloudnet/Container Networking AKS Swift Control Plane | IcM | `kubectl describe mtpnc`, CNS logs, `kubectl get nodeinfo` output | +| NRP rejecting VMSS operations on SWIFT-enabled pools | **Cloudnet/NRP** (not DNC, not RNM, not SdnPubSub) | IcM | NRP Kusto query results, `cx_pending_programming_ips_v2` graph, blast radius (subscriptions/clusters affected) | +| DNC PubSub context selector loss | Cloudnet/Container Networking AKS Swift Control Plane | IcM | Affected VNet GUIDs, subscription IDs | +| AKS RP regression / NIC delegation failure | Cloudnet/NRP | IcM | Regional creation success rate comparison, NRP Kusto query results | +| DNC server errors (5xx, connection refused) | Cloudnet/Container Networking AKS Swift Control Plane | IcM | DNC-RC Kusto log snippet | +| NMA unreachable (`sync host error`) | Cloudnet/VNET | IcM | CNS log snippet | +| ARO-HCP component ownership unclear | [ARO-HCP Component Inventory + Ownership](https://docs.google.com/spreadsheets/d/1Z2uhI3ctbZCF0rOj8B4VsoYx1rR9x_1ZLsbOr6Eug-8/edit?gid=0#gid=0) | - | - | + +# Validation and Confirmation + +System state: +- Re-run `kubectl get multitenantpodnetworkconfigs -A | grep -v provisioned`: should return no output +- Confirm router pods reach `Running`: `kubectl get pods -n -l kubernetes.azure.com/pod-network-instance` +- Confirm `cx_pending_programming_ips_v2` returns to 0 in Grafana +- Confirm `userJourneySwift*` alerts have cleared +- If a management cluster upgrade was stuck: confirm `az aks nodepool show ... --query provisioningState` returns `Succeeded` + +Customer impact resolved: +- Confirm affected HCP cluster's worker nodes return to `Ready` +- Confirm admission webhooks are succeeding (no TCP timeout errors in HCP KAS audit logs) +- Confirm HCP certificate rotations complete if private Key Vault access was affected + +# After Incident + +## PMR required? + +A formal Post-Mortem Review (PMR) is required when: +- The incident was Sev 1 or Sev 2 +- Customer impact lasted more than 30 minutes +- The root cause was a regression in ARO-HCP-owned code or configuration + +See the [HCM Incident Management Process](https://source.redhat.com/groups/public/hybridcloudmanagement/service_delivery_wiki/incident_management_process) for the full PMR process and template. + +## Postmortem notes + +- If the NRP or AKS RP signal was present, document the Kusto query results and the IcM filed +- If the MTPNC stuck deletion was the cause, note whether a node drain was required +- If the root cause required a workaround rather than a permanent fix, request an RCA from the owning Microsoft team via the IcM and file a Jira to track resolution +- Update this TSG if new error strings, failure modes, or triage steps were discovered + +# Appendix: CNS Prometheus Metrics Reference + +These metrics are queryable in Grafana via the **management cluster Prometheus datasources** (`Managed_Prometheus_hcps--mgmt-`). Source: [`azure-container-networking`](https://github.com/Azure/azure-container-networking) repo. + +> Scraped from CNS pods in `kube-system` on each management cluster node. + +## SLI / Alerting + +Metrics that map directly to SLOs and alert rules. + +| Metric | Type | Labels | What it measures | +|--------|------|--------|-----------------| +| `ip_assignment_latency_seconds` | Histogram | - | End-to-end pod IP assignment latency. Primary latency SLI. | +| `http_request_latency_seconds` | Histogram | `url`, `verb`, `cns_return_code` | Per-endpoint request latency. Filter `url=~".*/requestipconfig.*"` and `cns_return_code!="0"` for error rate SLI. | +| `cx_pending_programming_ips_v2` | Gauge | - | IPs in `PendingProgramming`: reserved from DNC but secondary NIC not yet attached. Sustained non-zero is the primary proxy for stuck MTPNCs. | +| `cns_nnc_initialized` | Gauge | - | `1` once the initial NNC reconcile has completed. `0` = node not ready to serve IP requests. | +| `cns_nnc_reconciler_start_failures_total` | Counter | - | NNC reconciler failed to start. A positive rate = CNS actively failing to initialise. | +| `cns_nnc_init_failures_total` | Counter | - | Initial NNC reconcile failed. | + +## IP Pool State + +Node-level pool snapshot (`cx_*_v2`) and per-subnet breakdown (`cx_ipam_*`). Use for pool health dashboards and diagnosing IP provisioning issues. + +| Metric | Type | Labels | What it measures | +|--------|------|--------|-----------------| +| `cx_allocated_ips_v2` | Gauge | - | Total IPs allocated to CNS by DNC | +| `cx_assigned_ips_v2` | Gauge | - | IPs currently assigned to router pods | +| `cx_available_ips_v2` | Gauge | - | IPs ready to assign; drops to 0 under exhaustion | +| `cx_pending_release_ips_v2` | Gauge | - | IPs no longer used but not yet returned to DNC | +| `cx_ipam_total_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | Total IP pool size allocated to CNS by DNC | +| `cx_ipam_requested_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | IPs requested by this node from DNC | +| `cx_ipam_pod_allocated_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | IPs currently in use by pods | +| `cx_ipam_available_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | IPs available for assignment | +| `cx_ipam_max_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | Hardware/SKU cap on secondary IPs for this node | +| `cx_ipam_pending_programming_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | IPs reserved but secondary NIC not yet attached | +| `cx_ipam_pending_release_ips` | Gauge | subnet, subnet_cidr, podnet_arm_id | IPs pending release back to DNC | +| `cx_ipam_subnet_exhaustion_state` | Gauge | subnet, subnet_cidr, podnet_arm_id | `1` = subnet exhausted, `0` = healthy | +| `cx_ipam_subnet_exhaustion_state_count_total` | Counter | subnet, subnet_cidr, podnet_arm_id | Cumulative exhaustion observations | + +> Healthy state: `cx_ipam_total_ips` slightly exceeds `cx_ipam_requested_ips` (DNC pre-allocates a small buffer; `requested - total` of about -1 is normal). Alert on **positive** values: `cx_ipam_requested_ips - cx_ipam_total_ips > 0` means CNS asked for IPs that DNC has not delivered. + +## Diagnostic + +Metrics for investigating specific failure modes. Not primary SLI targets. + +| Metric | Type | Labels | What it measures | +|--------|------|--------|-----------------| +| `ipconfigstatus_state_transition_seconds` | Histogram | `previous_state`, `next_state` | Time an IP config spends in each state transition. Useful for finding where IPs are getting stuck. | +| `ip_pool_inc_latency_seconds` | Histogram | `batch` | Round-trip time to increase the IP pool via NNC. Detects DNC slowness. | +| `ip_pool_dec_latency_seconds` | Histogram | `batch` | Round-trip time to decrease the IP pool via NNC. | +| `nnc_has_nodenetworkconfig` | Gauge | - | `1` if CNS has received its NNC from DNC; `0` if not. | +| `nnc_ncs` | Gauge | - | Number of NetworkContainers in the NNC. Should be >= 1 on a SWIFT-enabled node. | +| `sync_host_nc_version_total` | Counter | `ok` | Host NC sync attempts by success/failure. | +| `sync_host_nc_version_latency_seconds` | Histogram | `ok` | Host NC sync latency. Detects NMA communication issues. | +| `has_networkcontainer` | Gauge | - | NetworkContainers retrieved from NMA. | + +# Appendix: Router Pod and SWIFT NIC Metrics (kube-state-metrics) + +Raw kube-state-metrics signals underlying the `router:startup_latency:*` recording rules and the SWIFT NIC saturation metric. Useful when the aggregated recording rules are hiding per-pod or per-node detail. + +> Datasource: management cluster Prometheus (`^.*-mgmt-\d+$`). + +## Router pod scheduling + +| Metric | Useful filter | What it shows | +|--------|--------------|---------------| +| `kube_pod_status_phase` | `namespace=~"ocm-.*", phase="Pending"` | Which router pods are currently in ContainerCreating/Pending | +| `kube_pod_created` | `namespace=~"ocm-.*"` | Pod creation timestamp; combine with `time()` to get age of any pending pod | +| `kube_pod_owner` | `owner_kind="ReplicaSet", owner_name=~"router-.*"` | Identifies which pods are router pods by their ReplicaSet owner | + +To see the raw per-pod ContainerCreating durations: +```promql +(time() - kube_pod_created{namespace=~"ocm-.*"}) +* on(namespace, pod) kube_pod_owner{owner_kind="ReplicaSet", owner_name=~"router-.*"} +* on(namespace, pod) (kube_pod_status_phase{phase="Pending"} == 1) +``` + +## SWIFT NIC capacity + +| Metric | Useful filter | What it shows | +|--------|--------------|---------------| +| `kube_node_status_capacity` | `resource="aro.openshift.io/swift-nic"` | Total SWIFT NIC slots per node (7 in prod) | +| `kube_pod_container_resource_requests` | `resource="aro.openshift.io/swift-nic"` | NIC slots currently requested by scheduled router pods, per pod | + +## CNS daemonset health + +| Metric | Useful filter | What it shows | +|--------|--------------|---------------| +| `kube_daemonset_status_number_ready` | `daemonset="azure-cns", namespace="kube-system"` | How many CNS pods are currently ready | +| `kube_daemonset_status_desired_number_scheduled` | `daemonset="azure-cns", namespace="kube-system"` | How many CNS pods should be running | + +## Recording rules reference + +> **Tip:** Run alert queries with the threshold removed to see pre- and post-alert trends. E.g. graph `router:startup_latency:p99_avg_1h` without `> 300` to see the full history around an alert window. + +| Recording rule | What it pre-computes | +|----------------|---------------------| +| `router:startup_latency:seconds` | Per-pod ContainerCreating duration for all pending router pods | +| `router:startup_latency:p99` | p99 of the above across all pending router pods | +| `router:startup_latency:p99_avg_5m` | 5-minute average of the p99 (short window for fast-burn alert) | +| `router:startup_latency:p99_avg_30m` | 30-minute average of the p99 (short window for medium-burn alert) | +| `router:startup_latency:p99_avg_1h` | 1-hour average of the p99 (long window for fast-burn alert) | +| `router:startup_latency:p99_avg_6h` | 6-hour average of the p99 (long window for medium-burn alert) | + +# Appendix: Konnectivity Metrics Reference + +Konnectivity-server metrics measure the health of the KAS-to-node tunnel that SWIFT router pods carry. When SWIFT NIC assignment fails, konnectivity tunnels degrade because the router pods that carry them can't start or lose their network path. + +> Scraped from the konnectivity-server sidecar in each KAS pod (one per HCP, in `ocm-*` namespaces on management clusters). + +## Raw metrics + +| Metric | Type | Labels | What it measures | +|--------|------|--------|-----------------| +| `konnectivity_network_proxy_server_stream_errors_total` | Counter | - | Total stream-level errors on the konnectivity tunnel (server side). | +| `konnectivity_network_proxy_server_stream_packets_total` | Counter | `segment`, `packet_type` | Total packets through the tunnel. Filter `segment="from_agent", packet_type="DIAL_RSP"` for successful dial responses. | +| `konnectivity_network_proxy_server_dial_failure_count` | Counter | - | Total failed dial attempts (KAS tried to reach a node and failed). | + +## Recording rules + +| Recording rule | What it pre-computes | +|----------------|---------------------| +| `konnectivity:stream_error_rate:5m` | Stream errors / total packets (5m rate). The primary tunnel health signal. | +| `konnectivity:stream_error_rate:avg_5m` | 5-minute average of the error rate (short window for fast-burn alert) | +| `konnectivity:stream_error_rate:avg_30m` | 30-minute average (short window for medium-burn alert) | +| `konnectivity:stream_error_rate:avg_1h` | 1-hour average (long window for fast-burn alert) | +| `konnectivity:stream_error_rate:avg_6h` | 6-hour average (long window for medium-burn alert) | +| `konnectivity:dial_failure_rate:5m` | Dial failures / (failures + successful dials) (5m rate). Measures whether KAS can reach worker nodes. | +| `konnectivity:dial_failure_rate:avg_5m` | 5-minute average of the dial failure rate | +| `konnectivity:dial_failure_rate:avg_30m` | 30-minute average | +| `konnectivity:dial_failure_rate:avg_1h` | 1-hour average | +| `konnectivity:dial_failure_rate:avg_6h` | 6-hour average | diff --git a/observability/alerts-rp-hcps.yaml b/observability/alerts-rp-hcps.yaml index 5c852ac0bc0..0b348363041 100644 --- a/observability/alerts-rp-hcps.yaml +++ b/observability/alerts-rp-hcps.yaml @@ -1,4 +1,7 @@ prometheusRules: - rulesFolders: [] + rulesFolders: + - alerts/swift-networking-prometheusRule.yaml untestedRules: [] + testDependencies: + - recording-rules-hcps.yaml outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedRPHCPPrometheusAlertingRules.bicep diff --git a/observability/alerts/swift-networking-prometheusRule.yaml b/observability/alerts/swift-networking-prometheusRule.yaml new file mode 100644 index 00000000000..e99191fd86d --- /dev/null +++ b/observability/alerts/swift-networking-prometheusRule.yaml @@ -0,0 +1,190 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/name: kube-prometheus + app.kubernetes.io/part-of: kube-prometheus + prometheus: k8s + role: alert-rules + name: swift-networking-alert-rules + namespace: prometheus +spec: + groups: + # ======================================== + # SWIFT Networking User Journey Alerts + # ======================================== + # These alerts implement multi-window multi-burn-rate alerting for the SWIFT + # Networking user journey per ADR-001 (ARO HCP Alerting Recommendation for SRE). + # + # SLO: router pod startup latency p99 ≤ 300s + # Lane: RP per-HCP (alerts-rp-hcps.yaml) + # Runbook: https://aka.ms/arohcp-runbook-swift + # TSG: https://aka.ms/arohcp-tsg-swift + # + # Naming convention: {Scope}{Subject}{Metric}{BurnRateTier} + # - Scope: userJourney (customer-facing, Sev 3) + # - Subject: Swift + # - Metric: LatencyP99 + # - BurnRateTier: 1h5m (fast), 6h30m (medium), 3d (slow/Sev 4) + - name: arohcp_swift_networking_alerts + rules: + # Fast burn: 1h long window + 5m short window + # Fires when both windows exceed threshold — confirms the issue is real and current. + - alert: userJourneySwiftLatencyP991h5m + expr: | + router:startup_latency:p99_avg_5m > 300 + and + router:startup_latency:p99_avg_1h > 300 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + annotations: + summary: "SWIFT router pod startup latency p99 critically elevated (fast burn)" + description: "Router pod startup latency p99 has exceeded 300s over the last hour and is still elevated. SWIFT secondary NIC assignment is stalled." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + # Medium burn: 6h long window + 30m short window + - alert: userJourneySwiftLatencyP996h30m + expr: | + router:startup_latency:p99_avg_30m > 300 + and + router:startup_latency:p99_avg_6h > 300 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + long_window: 6h + short_window: 30m + annotations: + summary: "SWIFT router pod startup latency p99 elevated (medium burn)" + description: "Router pod startup latency p99 has exceeded 300s over the last 6 hours and is still elevated." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + # Slow tier: 6h window with 6h for-clause — Sev 4 ticket, not a page. + # The name "P993d" reflects the burn rate: at this rate the monthly SLO budget exhausts in ~3 days. + # Per ADR-001: user journey Sev 3 alerts wire Fast + Medium; Slow tier is Sev 4. + - alert: userJourneySwiftLatencyP993d + expr: router:startup_latency:p99_avg_6h > 300 + for: 6h + labels: + severity: "4" + burn_rate_tier: slow + annotations: + summary: "SWIFT router pod startup latency p99 elevated (slow burn — SLO budget on track to exhaust by month end)" + description: "Router pod startup latency p99 has been elevated for an extended period. At current rate the monthly SLO budget will be exhausted before end of month." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + # ======================================== + # CNS IP Assignment Latency Alerts + # ======================================== + # SLO: CNS IP assignment latency p99 <= 10s over 28 days. + # Source: ip_assignment_latency_seconds histogram via CNS PodMonitor. + # Lane: RP per-HCP (alerts-rp-hcps.yaml) + - name: arohcp_swift_cns_latency_alerts + rules: + - alert: userJourneySwiftCNSLatencyP991h5m + expr: | + cns:ip_assignment_latency:p99_avg_5m > 10 + and + cns:ip_assignment_latency:p99_avg_1h > 10 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + annotations: + summary: "CNS IP assignment latency p99 critically elevated (fast burn)" + description: "CNS IP assignment latency p99 has exceeded 10s over the last hour and is still elevated. SWIFT secondary NIC assignment is slow." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + - alert: userJourneySwiftCNSLatencyP996h30m + expr: | + cns:ip_assignment_latency:p99_avg_30m > 10 + and + cns:ip_assignment_latency:p99_avg_6h > 10 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + long_window: 6h + short_window: 30m + annotations: + summary: "CNS IP assignment latency p99 elevated (medium burn)" + description: "CNS IP assignment latency p99 has exceeded 10s over the last 6 hours and is still elevated." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + # ======================================== + # Konnectivity Tunnel Error Alerts + # ======================================== + # SLO: konnectivity tunnel stream error rate <= 1% over 28 days. + # Source: konnectivity-server sidecar metrics via PodMonitor (port 8093). + # Lane: RP per-HCP (alerts-rp-hcps.yaml) + - name: arohcp_swift_konnectivity_alerts + rules: + - alert: userJourneySwiftKonnectivityErrors1h5m + expr: | + konnectivity:stream_error_rate:avg_5m > 0.01 + and + konnectivity:stream_error_rate:avg_1h > 0.01 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + annotations: + summary: "Konnectivity tunnel error rate elevated (fast burn)" + description: "Konnectivity tunnel stream error rate has exceeded 1% over the last hour and is still elevated. KAS-to-node communication may be degraded." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + - alert: userJourneySwiftKonnectivityErrors6h30m + expr: | + konnectivity:stream_error_rate:avg_30m > 0.01 + and + konnectivity:stream_error_rate:avg_6h > 0.01 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + long_window: 6h + short_window: 30m + annotations: + summary: "Konnectivity tunnel error rate elevated (medium burn)" + description: "Konnectivity tunnel stream error rate has exceeded 1% over the last 6 hours and is still elevated." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + - alert: userJourneySwiftKonnectivityDialFailures1h5m + expr: | + konnectivity:dial_failure_rate:avg_5m > 0.01 + and + konnectivity:dial_failure_rate:avg_1h > 0.01 + for: 2m + labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + annotations: + summary: "Konnectivity dial failure rate elevated (fast burn)" + description: "Konnectivity server dial failure rate has exceeded 1%. KAS cannot establish connections to worker nodes." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + + - alert: userJourneySwiftKonnectivityDialFailures6h30m + expr: | + konnectivity:dial_failure_rate:avg_30m > 0.01 + and + konnectivity:dial_failure_rate:avg_6h > 0.01 + for: 15m + labels: + severity: "3" + burn_rate_tier: medium + long_window: 6h + short_window: 30m + annotations: + summary: "Konnectivity dial failure rate elevated (medium burn)" + description: "Konnectivity server dial failure rate has exceeded 1% over the last 6 hours." + runbook_url: "https://aka.ms/arohcp-runbook-swift" diff --git a/observability/alerts/swift-networking-prometheusRule_test.yaml b/observability/alerts/swift-networking-prometheusRule_test.yaml new file mode 100644 index 00000000000..e6a59508c1a --- /dev/null +++ b/observability/alerts/swift-networking-prometheusRule_test.yaml @@ -0,0 +1,138 @@ +rule_files: +- swift-networking-recordingRule-KSM.yaml +- swift-networking-prometheusRule.yaml +evaluation_interval: 1m +tests: +# Test 1: No router pods stuck — no alerts should fire. +- interval: 1m + input_series: [] + alert_rule_test: + - eval_time: 10m + alertname: userJourneySwiftLatencyP991h5m + exp_alerts: [] + - eval_time: 10m + alertname: userJourneySwiftLatencyP996h30m + exp_alerts: [] + - eval_time: 10m + alertname: userJourneySwiftLatencyP993d + exp_alerts: [] + +# Test 2: Router pod stuck for > 300s — fast-burn alert should fire after 2m for window. +# Simulates a router pod that has been in Pending state for a long time. +# At eval_time=70m, pod has been pending 70*60=4200 seconds >> 300s. +# Both 5m and 1h averages of p99 will be >> 300. +- interval: 1m + input_series: + - series: 'kube_pod_created{namespace="ocm-prod-abc123", pod="router-abc-1"}' + values: "0+0x120" + - series: 'kube_pod_owner{namespace="ocm-prod-abc123", pod="router-abc-1", owner_kind="ReplicaSet", owner_name="router-abc"}' + values: "1+0x120" + - series: 'kube_pod_status_phase{namespace="ocm-prod-abc123", pod="router-abc-1", phase="Pending"}' + values: "1+0x120" + alert_rule_test: + # After 70m with pod stuck, both the 5m and 1h windows are far above 300s. + # The alert requires for: 2m, so eval at 72m. + - eval_time: 72m + alertname: userJourneySwiftLatencyP991h5m + exp_alerts: + - exp_labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + exp_annotations: + summary: "SWIFT router pod startup latency p99 critically elevated (fast burn)" + description: "Router pod startup latency p99 has exceeded 300s over the last hour and is still elevated. SWIFT secondary NIC assignment is stalled." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +# Test 3: Pod recovers (leaves Pending) — alert should clear. +- interval: 1m + input_series: + - series: 'kube_pod_created{namespace="ocm-prod-abc123", pod="router-abc-1"}' + values: "0+0x80" + - series: 'kube_pod_owner{namespace="ocm-prod-abc123", pod="router-abc-1", owner_kind="ReplicaSet", owner_name="router-abc"}' + values: "1+0x80" + # Pod is Pending from t=0 to t=70m, then transitions to Running (phase no longer Pending) + - series: 'kube_pod_status_phase{namespace="ocm-prod-abc123", pod="router-abc-1", phase="Pending"}' + values: "1+0x70 0+0x10" + alert_rule_test: + # Alert should not fire after pod recovers (series absence = no alert) + - eval_time: 80m + alertname: userJourneySwiftLatencyP991h5m + exp_alerts: [] + +# ======================================== +# CNS IP Assignment Latency Alert Tests +# ======================================== + +# Test 4: No CNS latency data — no alerts should fire. +- interval: 1m + input_series: [] + alert_rule_test: + - eval_time: 10m + alertname: userJourneySwiftCNSLatencyP991h5m + exp_alerts: [] + - eval_time: 10m + alertname: userJourneySwiftCNSLatencyP996h30m + exp_alerts: [] + +# Test 5: CNS IP assignment latency p99 > 10s — fast-burn alert should fire. +# Simulates ip_assignment_latency_seconds histogram with all observations in the 16s+ bucket. +- interval: 1m + input_series: + - series: 'ip_assignment_latency_seconds_bucket{le="0.001"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="0.01"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="0.1"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="1"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="4"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="8"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="16"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="+Inf"}' + values: "0+10x120" + alert_rule_test: + - eval_time: 72m + alertname: userJourneySwiftCNSLatencyP991h5m + exp_alerts: + - exp_labels: + severity: "3" + burn_rate_tier: fast + long_window: 1h + short_window: 5m + exp_annotations: + summary: "CNS IP assignment latency p99 critically elevated (fast burn)" + description: "CNS IP assignment latency p99 has exceeded 10s over the last hour and is still elevated. SWIFT secondary NIC assignment is slow." + runbook_url: "https://aka.ms/arohcp-runbook-swift" + +# Test 6: CNS IP assignment latency healthy (all under 1s) — no alerts. +- interval: 1m + input_series: + - series: 'ip_assignment_latency_seconds_bucket{le="0.001"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="0.01"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="0.1"}' + values: "0+0x120" + - series: 'ip_assignment_latency_seconds_bucket{le="1"}' + values: "0+10x120" + - series: 'ip_assignment_latency_seconds_bucket{le="4"}' + values: "0+10x120" + - series: 'ip_assignment_latency_seconds_bucket{le="8"}' + values: "0+10x120" + - series: 'ip_assignment_latency_seconds_bucket{le="16"}' + values: "0+10x120" + - series: 'ip_assignment_latency_seconds_bucket{le="+Inf"}' + values: "0+10x120" + alert_rule_test: + - eval_time: 72m + alertname: userJourneySwiftCNSLatencyP991h5m + exp_alerts: [] + - eval_time: 72m + alertname: userJourneySwiftCNSLatencyP996h30m + exp_alerts: [] diff --git a/observability/alerts/swift-networking-recordingRule-KSM.yaml b/observability/alerts/swift-networking-recordingRule-KSM.yaml new file mode 100644 index 00000000000..437b3624c91 --- /dev/null +++ b/observability/alerts/swift-networking-recordingRule-KSM.yaml @@ -0,0 +1,125 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: swift-networking-recording-rules + namespace: prometheus +spec: + groups: + # ======================================== + # SWIFT Networking SLO Recording Rules + # ======================================== + # These rules implement the journey-level SLI for the SWIFT Networking + # user journey: router pod startup latency p99. + # + # SLI: p99 duration a router pod spends in ContainerCreating/Pending state. + # A router pod stuck > 300s indicates a SWIFT secondary NIC assignment failure. + # + # Source: kube-state-metrics on management cluster. + # Router pods are identified by their ReplicaSet owner (always "router-*"), + # created by the HyperShift CPO router Deployment. + # + # Design: absence of series = healthy (no router pods stuck = no latency to measure). + # The alert correctly does not fire when the recording rule produces no series. + - name: arohcp_swift_networking_slo_recording_rules + interval: 1m + rules: + # Per-pod ContainerCreating duration for all pending router pods. + # No series when healthy (no pods stuck). + - record: router:startup_latency:seconds + expr: | + (time() - kube_pod_created{namespace=~"ocm-.*"}) + * on(namespace, pod) kube_pod_owner{owner_kind="ReplicaSet", owner_name=~"router-.*"} + * on(namespace, pod) (kube_pod_status_phase{phase="Pending"} == 1) + + # p99 across all stuck router pods. + # No series when no router pods are stuck — alert does not fire. + - record: router:startup_latency:p99 + expr: quantile(0.99, router:startup_latency:seconds) + + # Window averages for multi-window multi-burn-rate alert expressions. + # Thresholds live only in the alert expressions, not here. + - record: router:startup_latency:p99_avg_5m + expr: avg_over_time(router:startup_latency:p99[5m]) + + - record: router:startup_latency:p99_avg_30m + expr: avg_over_time(router:startup_latency:p99[30m]) + + - record: router:startup_latency:p99_avg_1h + expr: avg_over_time(router:startup_latency:p99[1h]) + + - record: router:startup_latency:p99_avg_6h + expr: avg_over_time(router:startup_latency:p99[6h]) + + # ======================================== + # CNS IP Assignment Latency Recording Rules + # ======================================== + # SLI: p99 latency of CNS IP assignment (ip_assignment_latency_seconds histogram). + # SLO: <= 10s p99 over 28 days. + # Source: CNS PodMonitor on management cluster. + - name: arohcp_swift_cns_latency_recording_rules + interval: 1m + rules: + - record: cns:ip_assignment_latency:p99 + expr: histogram_quantile(0.99, sum(rate(ip_assignment_latency_seconds_bucket[5m])) by (le)) + + - record: cns:ip_assignment_latency:p99_avg_5m + expr: avg_over_time(cns:ip_assignment_latency:p99[5m]) + + - record: cns:ip_assignment_latency:p99_avg_30m + expr: avg_over_time(cns:ip_assignment_latency:p99[30m]) + + - record: cns:ip_assignment_latency:p99_avg_1h + expr: avg_over_time(cns:ip_assignment_latency:p99[1h]) + + - record: cns:ip_assignment_latency:p99_avg_6h + expr: avg_over_time(cns:ip_assignment_latency:p99[6h]) + + # ======================================== + # Konnectivity Tunnel Health Recording Rules + # ======================================== + # SLI: konnectivity tunnel stream error rate (server-side). + # Source: konnectivity-server sidecar in KAS pod (PodMonitor on port 8093). + - name: arohcp_swift_konnectivity_recording_rules + interval: 1m + rules: + - record: konnectivity:stream_error_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_stream_errors_total[5m])) + / + clamp_min(sum(rate(konnectivity_network_proxy_server_stream_packets_total[5m])), 1) + + - record: konnectivity:stream_error_rate:avg_5m + expr: avg_over_time(konnectivity:stream_error_rate:5m[5m]) + + - record: konnectivity:stream_error_rate:avg_30m + expr: avg_over_time(konnectivity:stream_error_rate:5m[30m]) + + - record: konnectivity:stream_error_rate:avg_1h + expr: avg_over_time(konnectivity:stream_error_rate:5m[1h]) + + - record: konnectivity:stream_error_rate:avg_6h + expr: avg_over_time(konnectivity:stream_error_rate:5m[6h]) + + # Dial failure rate: failures / (failures + successful dials). + # Successful dials = DIAL_RSP packets from agents. + - record: konnectivity:dial_failure_rate:5m + expr: | + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + / + clamp_min( + sum(rate(konnectivity_network_proxy_server_dial_failure_count[5m])) + + + sum(rate(konnectivity_network_proxy_server_stream_packets_total{segment="from_agent", packet_type="DIAL_RSP"}[5m])) + , 1) + + - record: konnectivity:dial_failure_rate:avg_5m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[5m]) + + - record: konnectivity:dial_failure_rate:avg_30m + expr: avg_over_time(konnectivity:dial_failure_rate:5m[30m]) + + - record: konnectivity:dial_failure_rate:avg_1h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[1h]) + + - record: konnectivity:dial_failure_rate:avg_6h + expr: avg_over_time(konnectivity:dial_failure_rate:5m[6h]) diff --git a/observability/alerts/swift-networking-recordingRule-KSM_test.yaml b/observability/alerts/swift-networking-recordingRule-KSM_test.yaml new file mode 100644 index 00000000000..51b64b77c69 --- /dev/null +++ b/observability/alerts/swift-networking-recordingRule-KSM_test.yaml @@ -0,0 +1,78 @@ +rule_files: +- swift-networking-recordingRule-KSM.yaml +evaluation_interval: 1m +tests: +# Test 1: When no router pods are stuck, recording rules produce no series. +# This is the healthy state — absence of data means no pods are waiting. +- interval: 1m + input_series: [] + promql_expr_test: + - expr: "router:startup_latency:seconds" + eval_time: 5m + exp_samples: [] + - expr: "router:startup_latency:p99" + eval_time: 5m + exp_samples: [] + - expr: "router:startup_latency:p99_avg_5m" + eval_time: 10m + exp_samples: [] + +# Test 2: One router pod stuck in Pending — p99 should exceed 300s threshold. +# kube_pod_created value of 0 = epoch. At eval_time=400m, time()-0=24000s. +# This tests that a single stuck pod triggers the recording rule. +- interval: 1m + input_series: + - series: 'kube_pod_created{namespace="ocm-prod-abc123", pod="router-abc-1"}' + values: "0+0x420" + - series: 'kube_pod_owner{namespace="ocm-prod-abc123", pod="router-abc-1", owner_kind="ReplicaSet", owner_name="router-abc"}' + values: "1+0x420" + - series: 'kube_pod_status_phase{namespace="ocm-prod-abc123", pod="router-abc-1", phase="Pending"}' + values: "1+0x420" + promql_expr_test: + # At t=400m: time()=24000s, kube_pod_created=0, so latency=24000s >> 300s threshold + - expr: "router:startup_latency:p99 > 300" + eval_time: 400m + exp_samples: + - labels: 'router:startup_latency:p99{}' + value: 24000 + +# Test 4: Multiple router pods stuck at different durations — validates quantile aggregation. +# In promtool tests, time() at eval_time=10m = 600s. +# Pod A: kube_pod_created=0 → latency = 600-0 = 600s +# Pod B: kube_pod_created=300 → latency = 600-300 = 300s +# p99 of {600, 300} = 600 (the higher value, as p99 of 2 samples = max). +- interval: 1m + input_series: + - series: 'kube_pod_created{namespace="ocm-prod-abc123", pod="router-a"}' + values: "0+0x11" + - series: 'kube_pod_owner{namespace="ocm-prod-abc123", pod="router-a", owner_kind="ReplicaSet", owner_name="router-abc"}' + values: "1+0x11" + - series: 'kube_pod_status_phase{namespace="ocm-prod-abc123", pod="router-a", phase="Pending"}' + values: "1+0x11" + - series: 'kube_pod_created{namespace="ocm-prod-abc123", pod="router-b"}' + values: "300+0x11" + - series: 'kube_pod_owner{namespace="ocm-prod-abc123", pod="router-b", owner_kind="ReplicaSet", owner_name="router-abc"}' + values: "1+0x11" + - series: 'kube_pod_status_phase{namespace="ocm-prod-abc123", pod="router-b", phase="Pending"}' + values: "1+0x11" + promql_expr_test: + - expr: "router:startup_latency:p99" + eval_time: 10m + exp_samples: + - labels: 'router:startup_latency:p99{}' + value: 600 + +# Test 3: No pods stuck — non-router pods in Pending should not appear. +# Verifies namespace filter (ocm-.*) and owner_name filter (router-.*). +- interval: 1m + input_series: + - series: 'kube_pod_created{namespace="kube-system", pod="coredns-1"}' + values: "0+0x30" + - series: 'kube_pod_owner{namespace="kube-system", pod="coredns-1", owner_kind="ReplicaSet", owner_name="coredns"}' + values: "1+0x30" + - series: 'kube_pod_status_phase{namespace="kube-system", pod="coredns-1", phase="Pending"}' + values: "1+0x30" + promql_expr_test: + - expr: "router:startup_latency:seconds" + eval_time: 10m + exp_samples: [] diff --git a/observability/prometheus/deploy/templates/azure-cns.podmonitor.yaml b/observability/prometheus/deploy/templates/azure-cns.podmonitor.yaml new file mode 100644 index 00000000000..4abb7eaa70b --- /dev/null +++ b/observability/prometheus/deploy/templates/azure-cns.podmonitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + app.kubernetes.io/name: azure-cns + name: azure-cns + namespace: kube-system +spec: + podMetricsEndpoints: + - port: metrics + path: /metrics + selector: + matchLabels: + k8s-app: azure-cns diff --git a/observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml b/observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml new file mode 100644 index 00000000000..85d39c155c8 --- /dev/null +++ b/observability/prometheus/deploy/templates/konnectivity-server.podmonitor.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + app.kubernetes.io/name: konnectivity-server + name: konnectivity-server + namespace: prometheus +spec: + podMetricsEndpoints: + - targetPort: 8093 + path: /metrics + namespaceSelector: + matchLabels: + hypershift.openshift.io/hosted-control-plane: "true" + selector: + matchLabels: + app: kube-apiserver diff --git a/observability/recording-rules-hcps.yaml b/observability/recording-rules-hcps.yaml index 9a86a9ff4a4..1d1d909b3f3 100644 --- a/observability/recording-rules-hcps.yaml +++ b/observability/recording-rules-hcps.yaml @@ -3,5 +3,6 @@ prometheusRules: - alerts/HCPkasRecord-prometheusRule-KSM.yaml - alerts/HCPkasRecord-prometheusRule-apiserver-requests.yaml - alerts/HCPkasRecord-prometheusRule-latency.yaml + - alerts/swift-networking-recordingRule-KSM.yaml untestedRules: [] outputBicep: ../dev-infrastructure/modules/metrics/rules/generatedHCPRecordingRules.bicep