From f0020d2aad8a0febb1b3c5fdbaa503a3ea490ba5 Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Wed, 15 Jul 2026 10:53:53 -0400 Subject: [PATCH] Fix harness egress NetworkPolicy blocking DNS on Kind (Connection timeout) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness egress NetworkPolicy allowed DNS egress ONLY to the openshift-dns namespace on port 5353 (an OVN-Kubernetes post-DNAT specific rule, issue #102), on the documented assumption that kindnet does not enforce NetworkPolicy egress. That assumption is no longer true: modern kindnet enforces egress policy. On Kind, CoreDNS runs in kube-system on the standard port 53, so the harness pod's DNS was fully blocked — it could not resolve authbridge-ab1 or redis, and every LLM call (SH_AUTHBRIDGE=1 /runs and leaf-smoke H1-allow / Claims 2,3,4,6,7) failed with "Connection timeout". The request never reached AB1 (AB1 logs were empty); a direct TCP dial to the AB1 pod IP, bypassing DNS, succeeded. Add a second DNS egress entry allowing UDP+TCP 53 to the kube-system namespace alongside the existing openshift-dns:5353 entry, in both the base policy (harness-egress-policy.yaml) and the tightened AB1 variant (authbridge/harness-egress-ab1.yaml). NetworkPolicy egress rules are additive, so both platforms are covered and the non-matching entry is inert on each. Correct the stale "kindnet does not enforce" comments, and add manifest-shape tests asserting the Kind DNS rule in both policy variants. Verified on a live Kind cluster: after adding the kube-system:53 rule, the same /runs request that previously timed out returns a valid verdict. Assisted-By: Claude (Anthropic AI) Signed-off-by: Paolo Dettori --- .../authbridge/harness-egress-ab1.yaml | 29 +++++++++++---- deploy/knative/harness-egress-policy.yaml | 21 ++++++++--- .../test/harness-egress-policy.test.ts | 36 +++++++++++++++++++ 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/deploy/knative/authbridge/harness-egress-ab1.yaml b/deploy/knative/authbridge/harness-egress-ab1.yaml index e1ae352..9d7bf98 100644 --- a/deploy/knative/authbridge/harness-egress-ab1.yaml +++ b/deploy/knative/authbridge/harness-egress-ab1.yaml @@ -15,14 +15,18 @@ # realizes Z2's "harness reaches only AB1": the base's `0.0.0.0/0:443` rule is REPLACED by # a rule scoped to the AB1 pod on TCP 8080. # -# kindnet (the Kind default CNI) does NOT enforce NetworkPolicy egress, so this manifest -# is a documented no-op on Kind; the live-enforcement verification gate is OCP (OVN-K), -# per the Z2 spec. +# NOTE: modern kindnet (Kind's default CNI) DOES enforce NetworkPolicy egress — the +# earlier assumption that this manifest was a no-op on Kind is false. The DNS rule must +# therefore cover BOTH platforms' DNS or the harness cannot resolve authbridge-ab1/redis +# and every LLM call fails with "Connection timeout". OCP (OVN-K) remains the live- +# enforcement gate per the Z2 spec. # # Allowlist (exactly these; NO 0.0.0.0/0:443): -# - DNS : UDP+TCP 5353 to the openshift-dns namespace — in-cluster service -# resolution for authbridge-ab1/redis. Port is 5353 (CoreDNS target -# port), not 53, because OVN-K enforces egress post-DNAT (issue #102). +# - DNS : two entries. On OCP, UDP+TCP 5353 to the openshift-dns namespace (5353 +# is the CoreDNS target port; OVN-K enforces egress post-DNAT, issue #102). +# On Kind/vanilla, UDP+TCP 53 to the kube-system namespace (CoreDNS on the +# standard port). Both resolve authbridge-ab1/redis; the non-matching entry +# is inert on each platform. # - Redis : TCP 6379, scoped to podSelector app=redis — the durable session log. # - AB1 : TCP 8080, scoped to podSelector app=authbridge-ab1 — the in-cluster LLM # gateway. This is the replacement for the base's external :443 rule. @@ -44,7 +48,7 @@ spec: policyTypes: - Egress egress: - # DNS — in-cluster service resolution (authbridge-ab1, redis). On OVN-Kubernetes + # DNS (OCP) — in-cluster service resolution (authbridge-ab1, redis). On OVN-Kubernetes # egress policy is enforced POST-DNAT: the dns-default Service (172.30.0.10:53) # DNATs to the CoreDNS pods on their real container port 5353 (NOT 53), so a bare # `port: 53` rule never matches and DNS times out (issue #102). Scope to the @@ -57,6 +61,17 @@ spec: ports: - { protocol: UDP, port: 5353 } - { protocol: TCP, port: 5353 } + # DNS (Kind/vanilla) — CoreDNS runs in kube-system on the standard port 53 (no + # post-DNAT port remap as on OVN-K). Modern kindnet enforces egress, so without this + # entry DNS is blocked on Kind and authbridge-ab1/redis never resolve. Inert on OCP + # (no kube-system CoreDNS on 53 there). + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } # Redis — the durable log. Scoped to the in-cluster redis pod by label. - to: - podSelector: diff --git a/deploy/knative/harness-egress-policy.yaml b/deploy/knative/harness-egress-policy.yaml index 6c770ae..748ea2f 100644 --- a/deploy/knative/harness-egress-policy.yaml +++ b/deploy/knative/harness-egress-policy.yaml @@ -8,9 +8,10 @@ # enforced half of that invariant: default-deny egress + an explicit allowlist. # # CNI dependency (Z2 §6): a NetworkPolicy is only as real as the CNI enforcing it. -# The Kind base uses kindnet, which does NOT enforce egress policy — there this -# manifest is a documented no-op and the live-enforcement check runs on OCP -# (OVN-Kubernetes). See the Z2 spec's verification gate. +# Modern kindnet (Kind's default CNI) DOES enforce egress policy, so this manifest is +# NOT a no-op on Kind — the DNS rule must cover both Kind (kube-system CoreDNS on 53) +# and OCP (openshift-dns CoreDNS on 5353) or DNS is blocked. OCP (OVN-Kubernetes) +# remains the live-enforcement gate per the Z2 spec. # # policyTypes is Egress ONLY: ingress is intentionally left untouched so the # Knative activator can still route requests into the pod. @@ -47,8 +48,7 @@ spec: # whose real container port is 5353 (NOT 53). A bare `port: 53` rule therefore # never matches the translated DNS traffic, so every lookup hits default-deny # and times out (issue #102). Scope to the openshift-dns namespace and allow - # the CoreDNS target port 5353. kindnet does not enforce egress, so this is a - # no-op on Kind; the enforcing target is OCP/OVN-K. + # the CoreDNS target port 5353. # Removed entirely at M8 (harness targets the injector by IP), closing the # DNS-tunnel exfil channel. - to: @@ -58,6 +58,17 @@ spec: ports: - { protocol: UDP, port: 5353 } - { protocol: TCP, port: 5353 } + # DNS (Kind/vanilla) — CoreDNS runs in kube-system on the standard port 53 (no + # post-DNAT port remap as on OVN-K). Modern kindnet enforces egress, so without this + # entry DNS is blocked on Kind and every lookup times out. Inert on OCP. Also removed + # at M8 along with the openshift-dns entry above. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } # Redis — the durable log. Scoped to the in-cluster redis pod by label. - to: - podSelector: diff --git a/packages/knative-server/test/harness-egress-policy.test.ts b/packages/knative-server/test/harness-egress-policy.test.ts index 9c7c125..71308a7 100644 --- a/packages/knative-server/test/harness-egress-policy.test.ts +++ b/packages/knative-server/test/harness-egress-policy.test.ts @@ -77,6 +77,24 @@ describe('harness egress NetworkPolicy manifest', () => { expect(protos.has('TCP')).toBe(true); }); + it('allows DNS resolution (UDP+TCP 53 to kube-system — Kind/vanilla CoreDNS, modern kindnet enforces)', () => { + // Modern kindnet enforces egress. Kind's CoreDNS runs in kube-system on the standard + // port 53 (no post-DNAT remap as on OVN-K), so without this rule DNS is blocked on + // Kind and the harness cannot resolve authbridge-ab1/redis (every LLM call then fails + // with "Connection timeout"). The rule must allow 53 scoped to the kube-system namespace. + const egress = np.spec?.egress ?? []; + const dnsRule = egress.find((r: any) => + (r.to ?? []).some( + (peer: any) => + peer.namespaceSelector?.matchLabels?.['kubernetes.io/metadata.name'] === 'kube-system', + ), + ); + expect(dnsRule, 'a rule scoped to the kube-system namespace').toBeDefined(); + const protos = new Set((dnsRule.ports ?? []).filter((p: any) => p.port === 53).map((p: any) => p.protocol)); + expect(protos.has('UDP')).toBe(true); + expect(protos.has('TCP')).toBe(true); + }); + it('allows Redis egress scoped to the redis pod on 6379', () => { const egress = np.spec?.egress ?? []; const redisRule = egress.find((r: any) => (r.ports ?? []).some((p: any) => p.port === 6379)); @@ -170,6 +188,24 @@ describe('tightened AB1 egress variant', () => { expect(protos.has('TCP')).toBe(true); }); + it('allows DNS resolution (UDP+TCP 53 to kube-system — Kind/vanilla CoreDNS, modern kindnet enforces)', () => { + // Modern kindnet enforces egress. Kind's CoreDNS runs in kube-system on the standard + // port 53 (no post-DNAT remap as on OVN-K), so without this rule DNS is blocked on + // Kind and the harness cannot resolve authbridge-ab1/redis (every LLM call then fails + // with "Connection timeout"). The rule must allow 53 scoped to the kube-system namespace. + const egress = np.spec?.egress ?? []; + const dnsRule = egress.find((r: any) => + (r.to ?? []).some( + (peer: any) => + peer.namespaceSelector?.matchLabels?.['kubernetes.io/metadata.name'] === 'kube-system', + ), + ); + expect(dnsRule, 'a rule scoped to the kube-system namespace').toBeDefined(); + const protos = new Set((dnsRule.ports ?? []).filter((p: any) => p.port === 53).map((p: any) => p.protocol)); + expect(protos.has('UDP')).toBe(true); + expect(protos.has('TCP')).toBe(true); + }); + it('allows the apiserver / sandbox control channel on 6443', () => { const egress = np.spec?.egress ?? []; const apiserverRule = egress.find((r: any) =>