feat: alert on node conntrack table saturation

dev-infrastructure/modules/metrics/rules/generatedPrometheusAlertingRules.bicep

@@ -2020,6 +2020,32 @@ resource kubeNodeRules 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-
         for: 'PT5M'
         severity: severityCeiling > 0 ? max(3, severityCeiling) : 3
       }
+      {
+        actions: [
+          for g in actionGroups: {
+            actionGroupId: g
+            actionProperties: {
+              'IcM.Title': '#$.labels.cluster#: #$.annotations.title#'
+              'IcM.CorrelationId': '#$.annotations.correlationId#'
+            }
+          }
+        ]
+        alert: 'NodeConntrackTableSaturation'
+        enabled: true
+        labels: {
+          severity: 'warning'
+        }
+        annotations: {
+          correlationId: 'NodeConntrackTableSaturation/{{ $labels.cluster }}/{{ $labels.instance }}'
+          description: 'Node {{ $labels.instance }} on cluster {{ $labels.cluster }} has its netfilter conntrack table {{ $value | humanizePercentage }} full (nf_conntrack_max). Sustained saturation leads to \'nf_conntrack: table full, dropping packet\' (ConntrackFull), which times out kubelet/DNS/IMDS traffic and degrades node health. Consider a larger SKU (nf_conntrack_max = 32768 x vCPU) or scaling the pool out.'
+          info: 'Node {{ $labels.instance }} on cluster {{ $labels.cluster }} has its netfilter conntrack table {{ $value | humanizePercentage }} full (nf_conntrack_max). Sustained saturation leads to \'nf_conntrack: table full, dropping packet\' (ConntrackFull), which times out kubelet/DNS/IMDS traffic and degrades node health. Consider a larger SKU (nf_conntrack_max = 32768 x vCPU) or scaling the pool out.'
+          summary: 'Node conntrack table approaching capacity'
+          title: 'Node conntrack table approaching capacity instance:{{ $labels.instance }} cluster:{{ $labels.cluster }}'
+        }
+        expression: 'node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8'
+        for: 'PT10M'
+        severity: severityCeiling > 0 ? max(3, severityCeiling) : 3
+      }
     ]
     scopes: [
       azureMonitoring

observability/alerts/kubeNode-prometheusRule.yaml

@@ -16,3 +16,12 @@ spec:
       for: 5m
       labels:
         severity: warning
+    - alert: NodeConntrackTableSaturation
+      annotations:
+        summary: "Node conntrack table approaching capacity"
+        description: "Node {{ $labels.instance }} on cluster {{ $labels.cluster }} has its netfilter conntrack table {{ $value | humanizePercentage }} full (nf_conntrack_max). Sustained saturation leads to 'nf_conntrack: table full, dropping packet' (ConntrackFull), which times out kubelet/DNS/IMDS traffic and degrades node health. Consider a larger SKU (nf_conntrack_max = 32768 x vCPU) or scaling the pool out."
+      expr: |
+        node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.80
+      for: 10m
+      labels:
+        severity: warning

observability/alerts/kubeNode-prometheusRule_test.yaml

@@ -39,3 +39,45 @@ tests:
   - eval_time: 6m
     alertname: KubeMemoryPressure
     exp_alerts: []
+# Scenario 7: conntrack table sustained above 80% for >10m - alert should fire
+- interval: 1m
+  input_series:
+  - series: 'node_nf_conntrack_entries{cluster="mgmt-1",instance="10.0.0.4:9100",job="node"}'
+    values: "90000+0x15"
+  - series: 'node_nf_conntrack_entries_limit{cluster="mgmt-1",instance="10.0.0.4:9100",job="node"}'
+    values: "100000+0x15"
+  alert_rule_test:
+  - eval_time: 11m
+    alertname: NodeConntrackTableSaturation
+    exp_alerts:
+    - exp_labels:
+        alertname: NodeConntrackTableSaturation
+        severity: warning
+        cluster: mgmt-1
+        instance: 10.0.0.4:9100
+        job: node
+      exp_annotations:
+        summary: "Node conntrack table approaching capacity"
+        description: "Node 10.0.0.4:9100 on cluster mgmt-1 has its netfilter conntrack table 90% full (nf_conntrack_max). Sustained saturation leads to 'nf_conntrack: table full, dropping packet' (ConntrackFull), which times out kubelet/DNS/IMDS traffic and degrades node health. Consider a larger SKU (nf_conntrack_max = 32768 x vCPU) or scaling the pool out."
+# Scenario 8: conntrack table well below threshold - no alert
+- interval: 1m
+  input_series:
+  - series: 'node_nf_conntrack_entries{cluster="mgmt-1",instance="10.0.0.5:9100",job="node"}'
+    values: "8000+0x15"
+  - series: 'node_nf_conntrack_entries_limit{cluster="mgmt-1",instance="10.0.0.5:9100",job="node"}'
+    values: "524288+0x15"
+  alert_rule_test:
+  - eval_time: 11m
+    alertname: NodeConntrackTableSaturation
+    exp_alerts: []
+# Scenario 9: conntrack spike above 80% but clears before 10m - no alert
+- interval: 1m
+  input_series:
+  - series: 'node_nf_conntrack_entries{cluster="mgmt-1",instance="10.0.0.6:9100",job="node"}'
+    values: "90000 90000 90000 90000 8000 8000 8000 8000 8000 8000 8000 8000"
+  - series: 'node_nf_conntrack_entries_limit{cluster="mgmt-1",instance="10.0.0.6:9100",job="node"}'
+    values: "100000+0x11"
+  alert_rule_test:
+  - eval_time: 11m
+    alertname: NodeConntrackTableSaturation
+    exp_alerts: []