[Kafka] Change Kafka installation - multi namespace support

.gitignore

@@ -1,3 +1,5 @@
 .idea/*
 charts/mlrun-ce/charts/*
 .DS_Store
+**/.DS_Store
+*.DS_Store

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0-rc3
+version: 0.11.0-rc4
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/admin_installation_values.yaml

@@ -51,5 +51,9 @@ kube-prometheus-stack:
 timescaledb:
   enabled: false
 
+strimzi-kafka-operator:
+  enabled: true
+  watchAnyNamespace: true
+
 kafka:
   enabled: false

charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml

@@ -59,11 +59,12 @@ pipelines:
   priority_class:
     enabled: false
 
+strimzi-kafka-operator:
+  enabled: false
+
+kafka:
+  rbac:
+    operatorNamespace: controller
+
 kube-prometheus-stack:
-  prometheus-node-exporter:
-    fullnameOverride: node-exporter
-    hostNetwork: false
-    service:
-      port: 9100
-    hostRootFsMount:
-      enabled: false
+  enabled: false

charts/mlrun-ce/non_admin_installation_values.yaml

@@ -58,5 +58,12 @@ pipelines:
   priority_class:
     enabled: false
 
+strimzi-kafka-operator:
+  enabled: false
+
+kafka:
+  rbac:
+    operatorNamespace: controller
+
 kube-prometheus-stack:
   enabled: false

charts/mlrun-ce/requirements.lock

@@ -17,8 +17,8 @@ dependencies:
 - name: kube-prometheus-stack
   repository: https://prometheus-community.github.io/helm-charts
   version: 72.1.1
-- name: kafka
-  repository: https://charts.bitnami.com/bitnami
-  version: 31.3.1
-digest: sha256:d804805eb592a0f9a3be418429cee7c150170ed13f6a03b4c3152f3f8b6ae5ba
-generated: "2025-12-02T13:37:34.112042+02:00"
+- name: strimzi-kafka-operator
+  repository: https://strimzi.io/charts/
+  version: 0.48.0
+digest: sha256:542955b13f2da791768eebe8e25092a8977fba987259ebe6ad85e4b977a7963b
+generated: "2026-01-14T13:52:54.335415+02:00"

charts/mlrun-ce/requirements.yaml

@@ -21,7 +21,7 @@ dependencies:
     repository: "https://prometheus-community.github.io/helm-charts"
     version: "72.1.1"
     condition: kube-prometheus-stack.enabled
-  - name: kafka
-    repository: "https://charts.bitnami.com/bitnami"
-    version: "31.3.1"
-    condition: kafka.enabled
+  - name: strimzi-kafka-operator
+    repository: "https://strimzi.io/charts/"
+    version: "0.48.0"
+    condition: strimzi-kafka-operator.enabled

charts/mlrun-ce/templates/kafka/kafka-bootstrap-alias.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.kafka.bootstrapAlias.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.kafka.bootstrapAlias.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: bootstrap-alias
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+  - name: client
+    port: 9092
+    targetPort: 9092
+    protocol: TCP
+  selector:
+    strimzi.io/cluster: {{ .Values.kafka.name }}
+    strimzi.io/kind: Kafka
+    strimzi.io/name: {{ .Values.kafka.name }}-kafka
+{{- end }}
+

charts/mlrun-ce/templates/kafka/kafka-cluster.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.kafka.enabled }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ .Values.kafka.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: cluster
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+  annotations:
+    # Delay Kafka CR creation until after CRDs are installed
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "5"
+spec:
+  kafka:
+    listeners:
+    {{- range .Values.kafka.listeners }}
+      - name: {{ .name }}
+        port: {{ .port }}
+        type: {{ .type }}
+        tls: {{ .tls }}
+    {{- end }}
+    config:
+      {{- toYaml .Values.kafka.config | nindent 6 }}
+  {{- if gt (.Values.kafka.zookeeper.replicas | int) 0 }}
+  zookeeper:
+    replicas: {{ .Values.kafka.zookeeper.replicas }}
+    storage:
+      type: persistent-claim
+      size: 8Gi
+  {{- end }}
+{{- end }}

charts/mlrun-ce/templates/kafka/kafka-network-policy.yaml

@@ -0,0 +1,58 @@
+{{- if .Values.kafka.rbac.enabled -}}
+{{- $operatorNamespace := .Values.kafka.rbac.operatorNamespace | default .Release.Namespace -}}
+{{- $kafkaName := .Values.kafka.name | default "kafka-stream" -}}
+{{- $currentNamespace := .Release.Namespace -}}
+---
+# NetworkPolicy: Allow egress from this namespace to Kafka namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-kafka-access
+  namespace: {{ $currentNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+spec:
+  # Apply to all pods in this namespace
+  podSelector: {}
+
+  policyTypes:
+  - Egress
+
+  egress:
+  # Allow egress to Kafka namespace
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: {{ $operatorNamespace }}
+      podSelector:
+        matchLabels:
+          strimzi.io/cluster: {{ $kafkaName }}
+    ports:
+    - protocol: TCP
+      port: 9092  # client listener
+    - protocol: TCP
+      port: 9093  # controller listener
+    - protocol: TCP
+      port: 9094  # internal listener
+
+  # Allow DNS resolution (required for service discovery)
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: kube-system
+      podSelector:
+        matchLabels:
+          k8s-app: kube-dns
+    ports:
+    - protocol: UDP
+      port: 53
+    - protocol: TCP
+      port: 53
+
+  # Allow egress within same namespace
+  - to:
+    - podSelector: {}
+{{- end }}
+

charts/mlrun-ce/templates/kafka/kafka-nodepool.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.kafka.enabled }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: {{ .Values.kafka.name }}-pool
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: kafka
+    app.kubernetes.io/component: nodepool
+    strimzi.io/cluster: {{ .Values.kafka.name }}
+    {{- include "mlrun-ce.common.labels" . | nindent 4 }}
+  annotations:
+    # Delay KafkaNodePool CR creation until after CRDs are installed
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "5"
+spec:
+  replicas: {{ .Values.kafka.replicas }}
+  roles:
+    - controller
+    - broker
+  storage:
+    type: {{ .Values.kafka.storage.type }}
+    size: {{ .Values.kafka.storage.size }}
+    {{- if .Values.kafka.storage.class }}
+    class: {{ .Values.kafka.storage.class }}
+    {{- end }}
+  resources:
+    requests:
+      memory: {{ .Values.kafka.resources.requests.memory }}
+      cpu: {{ .Values.kafka.resources.requests.cpu }}
+    limits:
+      memory: {{ .Values.kafka.resources.limits.memory }}
+      cpu: {{ .Values.kafka.resources.limits.cpu }}
+{{- end }}

charts/mlrun-ce/templates/kafka/kafka-rbac.yaml

@@ -0,0 +1,90 @@
+{{- if .Values.kafka.rbac.enabled -}}
+{{- $operatorNamespace := .Values.kafka.rbac.operatorNamespace | default .Release.Namespace -}}
+{{- $kafkaName := .Values.kafka.name | default "kafka-stream" -}}
+{{- $currentNamespace := .Release.Namespace -}}
+---
+# ServiceAccount for Kafka client applications
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kafka-client
+  namespace: {{ $currentNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+---
+# Role: Allow managing Kafka resources via CRDs in the operator namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  namespace: {{ $operatorNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+    user-namespace: {{ $currentNamespace }}
+rules:
+  # Allow creating and managing KafkaTopic CRDs
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkatopics
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  # Allow checking KafkaTopic status
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkatopics/status
+    verbs:
+      - get
+      - list
+      - watch
+  # Allow reading KafkaUser CRDs (if using SCRAM auth)
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkausers
+    verbs:
+      - get
+      - list
+      - watch
+  # Allow reading the Kafka cluster info
+  - apiGroups:
+      - kafka.strimzi.io
+    resources:
+      - kafkas
+    verbs:
+      - get
+      - list
+      - watch
+---
+# RoleBinding: Grant Kafka resource management permissions to ServiceAccount
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  namespace: {{ $operatorNamespace }}
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: kafka-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+    user-namespace: {{ $currentNamespace }}
+subjects:
+  - kind: ServiceAccount
+    name: kafka-client
+    namespace: {{ $currentNamespace }}
+roleRef:
+  kind: Role
+  name: {{ $currentNamespace }}-kafka-resource-manager
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
+