Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions deploy/knative/authbridge/harness-egress-ab1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,18 @@
# realizes Z2's "harness reaches only AB1": the base's `0.0.0.0/0:443` rule is REPLACED by
# a rule scoped to the AB1 pod on TCP 8080.
#
# kindnet (the Kind default CNI) does NOT enforce NetworkPolicy egress, so this manifest
# is a documented no-op on Kind; the live-enforcement verification gate is OCP (OVN-K),
# per the Z2 spec.
# NOTE: modern kindnet (Kind's default CNI) DOES enforce NetworkPolicy egress — the
# earlier assumption that this manifest was a no-op on Kind is false. The DNS rule must
# therefore cover BOTH platforms' DNS or the harness cannot resolve authbridge-ab1/redis
# and every LLM call fails with "Connection timeout". OCP (OVN-K) remains the live-
# enforcement gate per the Z2 spec.
#
# Allowlist (exactly these; NO 0.0.0.0/0:443):
# - DNS : UDP+TCP 5353 to the openshift-dns namespace — in-cluster service
# resolution for authbridge-ab1/redis. Port is 5353 (CoreDNS target
# port), not 53, because OVN-K enforces egress post-DNAT (issue #102).
# - DNS : two entries. On OCP, UDP+TCP 5353 to the openshift-dns namespace (5353
# is the CoreDNS target port; OVN-K enforces egress post-DNAT, issue #102).
# On Kind/vanilla, UDP+TCP 53 to the kube-system namespace (CoreDNS on the
# standard port). Both resolve authbridge-ab1/redis; the non-matching entry
# is inert on each platform.
# - Redis : TCP 6379, scoped to podSelector app=redis — the durable session log.
# - AB1 : TCP 8080, scoped to podSelector app=authbridge-ab1 — the in-cluster LLM
# gateway. This is the replacement for the base's external :443 rule.
Expand All @@ -44,7 +48,7 @@ spec:
policyTypes:
- Egress
egress:
# DNS — in-cluster service resolution (authbridge-ab1, redis). On OVN-Kubernetes
# DNS (OCP) — in-cluster service resolution (authbridge-ab1, redis). On OVN-Kubernetes
# egress policy is enforced POST-DNAT: the dns-default Service (172.30.0.10:53)
# DNATs to the CoreDNS pods on their real container port 5353 (NOT 53), so a bare
# `port: 53` rule never matches and DNS times out (issue #102). Scope to the
Expand All @@ -57,6 +61,17 @@ spec:
ports:
- { protocol: UDP, port: 5353 }
- { protocol: TCP, port: 5353 }
# DNS (Kind/vanilla) — CoreDNS runs in kube-system on the standard port 53 (no
# post-DNAT port remap as on OVN-K). Modern kindnet enforces egress, so without this
# entry DNS is blocked on Kind and authbridge-ab1/redis never resolve. Inert on OCP
# (no kube-system CoreDNS on 53 there).
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- { protocol: UDP, port: 53 }
- { protocol: TCP, port: 53 }
# Redis — the durable log. Scoped to the in-cluster redis pod by label.
- to:
- podSelector:
Expand Down
21 changes: 16 additions & 5 deletions deploy/knative/harness-egress-policy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
# enforced half of that invariant: default-deny egress + an explicit allowlist.
#
# CNI dependency (Z2 §6): a NetworkPolicy is only as real as the CNI enforcing it.
# The Kind base uses kindnet, which does NOT enforce egress policy — there this
# manifest is a documented no-op and the live-enforcement check runs on OCP
# (OVN-Kubernetes). See the Z2 spec's verification gate.
# Modern kindnet (Kind's default CNI) DOES enforce egress policy, so this manifest is
# NOT a no-op on Kind — the DNS rule must cover both Kind (kube-system CoreDNS on 53)
# and OCP (openshift-dns CoreDNS on 5353) or DNS is blocked. OCP (OVN-Kubernetes)
# remains the live-enforcement gate per the Z2 spec.
#
# policyTypes is Egress ONLY: ingress is intentionally left untouched so the
# Knative activator can still route requests into the pod.
Expand Down Expand Up @@ -47,8 +48,7 @@ spec:
# whose real container port is 5353 (NOT 53). A bare `port: 53` rule therefore
# never matches the translated DNS traffic, so every lookup hits default-deny
# and times out (issue #102). Scope to the openshift-dns namespace and allow
# the CoreDNS target port 5353. kindnet does not enforce egress, so this is a
# no-op on Kind; the enforcing target is OCP/OVN-K.
# the CoreDNS target port 5353.
# Removed entirely at M8 (harness targets the injector by IP), closing the
# DNS-tunnel exfil channel.
- to:
Expand All @@ -58,6 +58,17 @@ spec:
ports:
- { protocol: UDP, port: 5353 }
- { protocol: TCP, port: 5353 }
# DNS (Kind/vanilla) — CoreDNS runs in kube-system on the standard port 53 (no
# post-DNAT port remap as on OVN-K). Modern kindnet enforces egress, so without this
# entry DNS is blocked on Kind and every lookup times out. Inert on OCP. Also removed
# at M8 along with the openshift-dns entry above.
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- { protocol: UDP, port: 53 }
- { protocol: TCP, port: 53 }
# Redis — the durable log. Scoped to the in-cluster redis pod by label.
- to:
- podSelector:
Expand Down
36 changes: 36 additions & 0 deletions packages/knative-server/test/harness-egress-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,24 @@ describe('harness egress NetworkPolicy manifest', () => {
expect(protos.has('TCP')).toBe(true);
});

it('allows DNS resolution (UDP+TCP 53 to kube-system — Kind/vanilla CoreDNS, modern kindnet enforces)', () => {
// Modern kindnet enforces egress. Kind's CoreDNS runs in kube-system on the standard
// port 53 (no post-DNAT remap as on OVN-K), so without this rule DNS is blocked on
// Kind and the harness cannot resolve authbridge-ab1/redis (every LLM call then fails
// with "Connection timeout"). The rule must allow 53 scoped to the kube-system namespace.
const egress = np.spec?.egress ?? [];
const dnsRule = egress.find((r: any) =>
(r.to ?? []).some(
(peer: any) =>
peer.namespaceSelector?.matchLabels?.['kubernetes.io/metadata.name'] === 'kube-system',
),
);
expect(dnsRule, 'a rule scoped to the kube-system namespace').toBeDefined();
const protos = new Set((dnsRule.ports ?? []).filter((p: any) => p.port === 53).map((p: any) => p.protocol));
expect(protos.has('UDP')).toBe(true);
expect(protos.has('TCP')).toBe(true);
});

it('allows Redis egress scoped to the redis pod on 6379', () => {
const egress = np.spec?.egress ?? [];
const redisRule = egress.find((r: any) => (r.ports ?? []).some((p: any) => p.port === 6379));
Expand Down Expand Up @@ -170,6 +188,24 @@ describe('tightened AB1 egress variant', () => {
expect(protos.has('TCP')).toBe(true);
});

it('allows DNS resolution (UDP+TCP 53 to kube-system — Kind/vanilla CoreDNS, modern kindnet enforces)', () => {
// Modern kindnet enforces egress. Kind's CoreDNS runs in kube-system on the standard
// port 53 (no post-DNAT remap as on OVN-K), so without this rule DNS is blocked on
// Kind and the harness cannot resolve authbridge-ab1/redis (every LLM call then fails
// with "Connection timeout"). The rule must allow 53 scoped to the kube-system namespace.
const egress = np.spec?.egress ?? [];
const dnsRule = egress.find((r: any) =>
(r.to ?? []).some(
(peer: any) =>
peer.namespaceSelector?.matchLabels?.['kubernetes.io/metadata.name'] === 'kube-system',
),
);
expect(dnsRule, 'a rule scoped to the kube-system namespace').toBeDefined();
const protos = new Set((dnsRule.ports ?? []).filter((p: any) => p.port === 53).map((p: any) => p.protocol));
expect(protos.has('UDP')).toBe(true);
expect(protos.has('TCP')).toBe(true);
});

it('allows the apiserver / sandbox control channel on 6443', () => {
const egress = np.spec?.egress ?? [];
const apiserverRule = egress.find((r: any) =>
Expand Down
Loading