feat(k8s): expose UDP ports on cloud-compose via mixed-protocol LB

docs/docs/reference/service-available-deployment-schemas.md

@@ -86,6 +86,63 @@ services:
       DATABASE_URL: ${DATABASE_HOST}
 ```
 
+### **Exposing UDP ports (e.g. WebRTC / LiveKit)**
+
+By default a `cloud-compose` service is created as a `ClusterIP` Service and
+exposed through the shared Caddy ingress, which reverse-proxies **HTTP/TCP**
+traffic only. Protocols that Caddy cannot proxy — most notably **UDP** — must be
+exposed directly by turning the service's own Kubernetes Service into a
+`LoadBalancer`.
+
+Two things are required:
+
+1. **Declare the UDP port in `docker-compose.yaml`** using the compose
+   `/udp` suffix. The port protocol is now preserved end-to-end and emitted on
+   both the container port and the Service port. Ports without a suffix stay
+   `TCP`, exactly as before.
+2. **Set `serviceType: LoadBalancer`** under `cloudExtras`. This makes the
+   service's own Service a `LoadBalancer` with its own external IP, instead of
+   the default `ClusterIP` fronted by Caddy.
+
+```yaml
+# docker-compose.yaml — a LiveKit SFU media server
+services:
+  livekit:
+    image: livekit/livekit-server:latest
+    ports:
+      - "7880:7880"      # signaling (wss)  → TCP
+      - "7881:7881"      # ICE/TCP          → TCP
+      - "7882:7882/udp"  # RTC media        → UDP
+```
+
+```yaml
+# client.yaml
+stacks:
+  production:
+    type: cloud-compose
+    parent: myproject/devops
+    config:
+      dockerComposeFile: ./docker-compose.yaml
+      runs:
+        - livekit
+      cloudExtras:
+        serviceType: LoadBalancer
+```
+
+**Mixed TCP + UDP on one Service.** When a service exposes both TCP and UDP
+ports, `sc` emits a **single mixed-protocol `LoadBalancer` Service** carrying all
+ports (rather than splitting into separate TCP and UDP Services). This keeps a
+single external IP for the whole service, which is what UDP media servers need
+in order to advertise one address to clients. Mixed-protocol Services are GA in
+Kubernetes since 1.26 (the `MixedProtocolLBService` feature) and are supported
+by recent GKE — your cluster must be on a version that supports them.
+
+> **Announcing the external IP is the application's job.** `sc` provisions the
+> `LoadBalancer` and the UDP port, but a media server such as LiveKit must be
+> told its public address to put into ICE candidates (e.g. LiveKit's
+> `rtc.use_external_ip` / `rtc.node_ip`). Wiring that address into your app's
+> configuration is out of scope for `sc`.
+
 ### **Deploying to Staging**
 ```sh
 sc deploy -s myservice -e staging

pkg/clouds/gcloud/gke_autopilot.go

@@ -107,11 +107,13 @@ func ToGkeAutopilotConfig(tpl any, composeCfg compose.Config, stackCfg *api.Stac
 		deployCfg.DisruptionBudget = k8sCloudExtras.DisruptionBudget
 		deployCfg.NodeSelector = k8sCloudExtras.NodeSelector
 		deployCfg.Tolerations = k8sCloudExtras.Tolerations
-		deployCfg.VPA = k8sCloudExtras.VPA                             // Extract VPA configuration from CloudExtras
-		deployCfg.ReadinessProbe = k8sCloudExtras.ReadinessProbe       // Extract global readiness probe configuration
-		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe         // Extract global liveness probe configuration
-		deployCfg.StartupProbe = k8sCloudExtras.StartupProbe           // Extract global startup probe configuration
-		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.VPA = k8sCloudExtras.VPA                                     // Extract VPA configuration from CloudExtras
+		deployCfg.ReadinessProbe = k8sCloudExtras.ReadinessProbe               // Extract global readiness probe configuration
+		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe                 // Extract global liveness probe configuration
+		deployCfg.StartupProbe = k8sCloudExtras.StartupProbe                   // Extract global startup probe configuration
+		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName         // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.ServiceType = k8sCloudExtras.ServiceType                     // Extract Service type override (e.g. LoadBalancer for UDP)
+		deployCfg.ExternalTrafficPolicy = k8sCloudExtras.ExternalTrafficPolicy // e.g. Local, required for WebRTC return media
 		deployCfg.TopologySpreadConstraints = k8sCloudExtras.TopologySpreadConstraints
 
 		// Process affinity rules and merge with existing NodeSelector if needed

pkg/clouds/k8s/containers_test.go

@@ -67,7 +67,7 @@ func TestConvertComposeToContainers_FullMapping(t *testing.T) {
 	Expect(c.Command).To(Equal([]string{"/bin/entry"}))
 	Expect(c.Args).To(Equal([]string{"serve", "--port", "8080"}))
 	Expect(c.Env).To(HaveKeyWithValue("ENV", "prod"))
-	Expect(c.Ports).To(Equal([]int{8080}))
+	Expect(c.Ports).To(Equal(ContainerPorts(8080)))
 	Expect(c.ComposeDir).To(Equal("/work"))
 	Expect(c.Image.Context).To(Equal("./api"))
 	// Context set, dockerfile empty => defaults to "Dockerfile".
@@ -176,7 +176,7 @@ func TestFindIngressContainer(t *testing.T) {
 			},
 		}}
 		containers := []CloudRunContainer{
-			{Name: "web", Ports: []int{3000}},
+			{Name: "web", Ports: ContainerPorts(3000)},
 			{Name: "worker"},
 		}
 
@@ -199,7 +199,7 @@ func TestFindIngressContainer(t *testing.T) {
 				}},
 			},
 		}}
-		containers := []CloudRunContainer{{Name: "web", Ports: []int{3000}}}
+		containers := []CloudRunContainer{{Name: "web", Ports: ContainerPorts(3000)}}
 
 		ic, err := FindIngressContainer(composeCfg, containers)
 		Expect(err).ToNot(HaveOccurred())
@@ -218,7 +218,7 @@ func TestFindIngressContainer(t *testing.T) {
 				}},
 			},
 		}}
-		containers := []CloudRunContainer{{Name: "web", Ports: []int{3000}}}
+		containers := []CloudRunContainer{{Name: "web", Ports: ContainerPorts(3000)}}
 
 		ic, err := FindIngressContainer(composeCfg, containers)
 		Expect(err).ToNot(HaveOccurred())
@@ -251,7 +251,7 @@ func TestFindIngressContainer(t *testing.T) {
 		composeCfg := compose.Config{Project: &types.Project{
 			Services: []types.ServiceConfig{{Name: "solo"}},
 		}}
-		containers := []CloudRunContainer{{Name: "solo", Ports: []int{5000}}}
+		containers := []CloudRunContainer{{Name: "solo", Ports: ContainerPorts(5000)}}
 
 		ic, err := FindIngressContainer(composeCfg, containers)
 		Expect(err).ToNot(HaveOccurred())
@@ -267,8 +267,8 @@ func TestFindIngressContainer(t *testing.T) {
 			Services: []types.ServiceConfig{{Name: "a"}, {Name: "b"}},
 		}}
 		containers := []CloudRunContainer{
-			{Name: "a", Ports: []int{1000}},
-			{Name: "b", Ports: []int{2000}},
+			{Name: "a", Ports: ContainerPorts(1000)},
+			{Name: "b", Ports: ContainerPorts(2000)},
 		}
 
 		ic, err := FindIngressContainer(composeCfg, containers)
@@ -281,7 +281,7 @@ func TestFindIngressContainer(t *testing.T) {
 		composeCfg := compose.Config{Project: &types.Project{
 			Services: []types.ServiceConfig{{Name: "solo"}},
 		}}
-		containers := []CloudRunContainer{{Name: "solo", Ports: []int{5000, 6000}}}
+		containers := []CloudRunContainer{{Name: "solo", Ports: ContainerPorts(5000, 6000)}}
 
 		ic, err := FindIngressContainer(composeCfg, containers)
 		Expect(err).ToNot(HaveOccurred())

pkg/clouds/k8s/kube_run.go

@@ -31,6 +31,15 @@ type CloudExtras struct {
 	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`
 	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
 	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
+	// ServiceType overrides the Kubernetes Service type for the service itself (default: ClusterIP,
+	// fronted by the shared Caddy ingress). Set to "LoadBalancer" to expose the service's ports
+	// directly, which is required for non-HTTP protocols such as UDP that Caddy cannot proxy.
+	ServiceType *string `json:"serviceType,omitempty" yaml:"serviceType,omitempty"`
+	// ExternalTrafficPolicy sets the LoadBalancer Service's externalTrafficPolicy. "Local" is
+	// required for WebRTC media behind the LB: it preserves the LB IP (and the client source IP)
+	// on the return path, so clients accept the SFU's downlink. The default ("Cluster") SNATs to
+	// a node IP, which WebRTC rejects (dtls timeout, call drops).
+	ExternalTrafficPolicy *string `json:"externalTrafficPolicy,omitempty" yaml:"externalTrafficPolicy,omitempty"`
 
 	TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints" yaml:"topologySpreadConstraints"`
 }
@@ -199,11 +208,13 @@ func ToKubernetesRunConfig(tpl any, composeCfg compose.Config, stackCfg *api.Sta
 		deployCfg.RollingUpdate = k8sCloudExtras.RollingUpdate
 		deployCfg.DisruptionBudget = k8sCloudExtras.DisruptionBudget
 		deployCfg.NodeSelector = k8sCloudExtras.NodeSelector
-		deployCfg.VPA = k8sCloudExtras.VPA                             // Extract VPA configuration from CloudExtras
-		deployCfg.ReadinessProbe = k8sCloudExtras.ReadinessProbe       // Extract global readiness probe configuration
-		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe         // Extract global liveness probe configuration
-		deployCfg.EphemeralVolumes = k8sCloudExtras.EphemeralVolumes   // Extract generic ephemeral volumes configuration
-		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.VPA = k8sCloudExtras.VPA                                     // Extract VPA configuration from CloudExtras
+		deployCfg.ReadinessProbe = k8sCloudExtras.ReadinessProbe               // Extract global readiness probe configuration
+		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe                 // Extract global liveness probe configuration
+		deployCfg.EphemeralVolumes = k8sCloudExtras.EphemeralVolumes           // Extract generic ephemeral volumes configuration
+		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName         // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.ServiceType = k8sCloudExtras.ServiceType                     // Extract Service type override (e.g. LoadBalancer for UDP)
+		deployCfg.ExternalTrafficPolicy = k8sCloudExtras.ExternalTrafficPolicy // e.g. Local, required for WebRTC return media
 		deployCfg.TopologySpreadConstraints = k8sCloudExtras.TopologySpreadConstraints
 
 		// Process affinity rules and merge with existing NodeSelector if needed

pkg/clouds/k8s/types.go

@@ -30,12 +30,17 @@ type DeploymentConfig struct {
 	NodeSelector      map[string]string        `json:"nodeSelector" yaml:"nodeSelector"`
 	Affinity          *AffinityRules           `json:"affinity" yaml:"affinity"`
 	Tolerations       []Toleration             `json:"tolerations" yaml:"tolerations"`
-	VPA               *VPAConfig               `json:"vpa" yaml:"vpa"`                             // Vertical Pod Autoscaler configuration
-	ReadinessProbe    *CloudRunProbe           `json:"readinessProbe" yaml:"readinessProbe"`       // Global readiness probe configuration
-	LivenessProbe     *CloudRunProbe           `json:"livenessProbe" yaml:"livenessProbe"`         // Global liveness probe configuration
-	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`           // Global startup probe configuration
-	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
-	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
+	VPA               *VPAConfig               `json:"vpa" yaml:"vpa"`                                     // Vertical Pod Autoscaler configuration
+	ReadinessProbe    *CloudRunProbe           `json:"readinessProbe" yaml:"readinessProbe"`               // Global readiness probe configuration
+	LivenessProbe     *CloudRunProbe           `json:"livenessProbe" yaml:"livenessProbe"`                 // Global liveness probe configuration
+	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`                   // Global startup probe configuration
+	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`           // Generic ephemeral volumes for large temp storage
+	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"`         // Kubernetes PriorityClass for pod scheduling and preemption
+	ServiceType       *string                  `json:"serviceType,omitempty" yaml:"serviceType,omitempty"` // Kubernetes Service type for the service (default: ClusterIP; LoadBalancer to expose UDP/non-HTTP ports directly)
+	// ExternalTrafficPolicy for the service's LoadBalancer (default: Cluster). "Local" preserves
+	// the LB IP and client source IP on the return path — required for WebRTC media behind the LB
+	// (Cluster SNATs to a node IP, which clients reject: dtls timeout).
+	ExternalTrafficPolicy *string `json:"externalTrafficPolicy,omitempty" yaml:"externalTrafficPolicy,omitempty"`
 
 	TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints" yaml:"topologySpreadConstraints"`
 }
@@ -170,14 +175,31 @@ type ProbeHttpGet struct {
 	HTTPHeaders []HTTPHeader `json:"httpHeaders,omitempty" yaml:"httpHeaders,omitempty"`
 }
 
+// ContainerPort is a single exposed container port together with its L4
+// protocol. Protocol is the Kubernetes protocol name ("TCP" or "UDP"); an empty
+// value means the Kubernetes default (TCP) and is emitted as no protocol at all,
+// so ports without an explicit protocol stay byte-for-byte identical to before
+// protocol support was added.
+type ContainerPort struct {
+	Port     int    `json:"port" yaml:"port"`
+	Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"`
+}
+
+// ContainerPorts builds ContainerPort entries with the default (TCP) protocol.
+func ContainerPorts(ports ...int) []ContainerPort {
+	return lo.Map(ports, func(p int, _ int) ContainerPort {
+		return ContainerPort{Port: p}
+	})
+}
+
 type CloudRunContainer struct {
 	Name            string             `json:"name" yaml:"name"`
 	Command         []string           `json:"command" yaml:"command"`
 	Args            []string           `json:"args" yaml:"args"`
 	Image           api.ContainerImage `json:"image" yaml:"image"`
 	Env             map[string]string  `json:"env" yaml:"env"`
 	Secrets         map[string]string  `json:"secrets" yaml:"secrets"`
-	Ports           []int              `json:"ports" yaml:"ports"`
+	Ports           []ContainerPort    `json:"ports" yaml:"ports"`
 	MainPort        *int               `json:"mainPort" yaml:"mainPort"`
 	ReadinessProbe  *CloudRunProbe     `json:"readinessProbe" yaml:"readinessProbe"`
 	LivenessProbe   *CloudRunProbe     `json:"livenessProbe" yaml:"livenessProbe"`
@@ -490,7 +512,7 @@ func ConvertComposeToContainers(composeCfg compose.Config, stackCfg *api.StackCo
 		if container.MainPort == nil && len(container.Ports) > 1 {
 			container.Warnings = append(container.Warnings, fmt.Sprintf("container %q has multiple ports and no main port specified", container.Name))
 		} else if len(container.Ports) > 0 {
-			container.MainPort = lo.ToPtr(container.Ports[0])
+			container.MainPort = lo.ToPtr(container.Ports[0].Port)
 		}
 		containers = append(containers, container)
 	}
@@ -514,7 +536,7 @@ func FindIngressContainer(composeCfg compose.Config, contaniers []CloudRunContai
 	})
 	if !found && len(contaniers) == 1 && len(contaniers[0].Ports) == 1 {
 		iContainer = contaniers[0]
-		iContainer.MainPort = lo.ToPtr(iContainer.Ports[0])
+		iContainer.MainPort = lo.ToPtr(iContainer.Ports[0].Port)
 		found = true
 	}
 	if !found {
@@ -530,17 +552,31 @@ func FindIngressContainer(composeCfg compose.Config, contaniers []CloudRunContai
 		}
 	}
 	if iContainer.MainPort == nil && len(iContainer.Ports) == 1 {
-		iContainer.MainPort = lo.ToPtr(iContainer.Ports[0])
+		iContainer.MainPort = lo.ToPtr(iContainer.Ports[0].Port)
 	}
 	return &iContainer, nil
 }
 
-func toRunPorts(ports []types.ServicePortConfig) []int {
-	return lo.Map(ports, func(p types.ServicePortConfig, _ int) int {
-		return int(p.Target)
+func toRunPorts(ports []types.ServicePortConfig) []ContainerPort {
+	return lo.Map(ports, func(p types.ServicePortConfig, _ int) ContainerPort {
+		return ContainerPort{
+			Port:     int(p.Target),
+			Protocol: normalizeComposeProtocol(p.Protocol),
+		}
 	})
 }
 
+// normalizeComposeProtocol maps a docker-compose port protocol ("tcp"/"udp"/"")
+// to the Kubernetes protocol stored on ContainerPort. TCP (the default) is
+// stored as an empty string so downstream resources omit the protocol field and
+// remain byte-for-byte identical to specs generated before UDP support.
+func normalizeComposeProtocol(proto string) string {
+	if strings.EqualFold(proto, "udp") {
+		return "UDP"
+	}
+	return ""
+}
+
 func toStartupProbe(check *types.HealthCheckConfig) *CloudRunProbe {
 	if check == nil {
 		return nil

pkg/clouds/k8s/types_conversion_test.go

@@ -474,8 +474,15 @@ func TestToRunPorts(t *testing.T) {
 	RegisterTestingT(t)
 
 	ports := []types.ServicePortConfig{{Target: 8080}, {Target: 9090}}
-	Expect(toRunPorts(ports)).To(Equal([]int{8080, 9090}))
+	Expect(toRunPorts(ports)).To(Equal(ContainerPorts(8080, 9090)))
 	Expect(toRunPorts(nil)).To(BeEmpty())
+
+	// A UDP port keeps its protocol; TCP/unspecified ports normalise to "".
+	udpPorts := []types.ServicePortConfig{{Target: 7880}, {Target: 7882, Protocol: "udp"}}
+	Expect(toRunPorts(udpPorts)).To(Equal([]ContainerPort{
+		{Port: 7880},
+		{Port: 7882, Protocol: "UDP"},
+	}))
 }
 
 func TestToRunEnv(t *testing.T) {

pkg/clouds/pulumi/gcp/gke_autopilot_stack.go

@@ -186,14 +186,16 @@ func GkeAutopilotStack(ctx *sdk.Context, stack api.Stack, input api.ResourceInpu
 		Annotations: map[string]string{
 			"pulumi.com/patchForce": "true",
 		},
-		NodeSelector:   gkeAutopilotInput.Deployment.NodeSelector,
-		Affinity:       gkeAutopilotInput.Deployment.Affinity,
-		Tolerations:    gkeAutopilotInput.Deployment.Tolerations,
-		VPA:            gkeAutopilotInput.Deployment.VPA,            // Pass VPA configuration to Kubernetes deployment
-		ReadinessProbe: gkeAutopilotInput.Deployment.ReadinessProbe, // Pass global readiness probe configuration
-		LivenessProbe:  gkeAutopilotInput.Deployment.LivenessProbe,  // Pass global liveness probe configuration
-		StartupProbe:   gkeAutopilotInput.Deployment.StartupProbe,   // Pass global startup probe configuration
-		EphemeralSize:  ephemeralSize,
+		NodeSelector:          gkeAutopilotInput.Deployment.NodeSelector,
+		Affinity:              gkeAutopilotInput.Deployment.Affinity,
+		Tolerations:           gkeAutopilotInput.Deployment.Tolerations,
+		VPA:                   gkeAutopilotInput.Deployment.VPA,                   // Pass VPA configuration to Kubernetes deployment
+		ReadinessProbe:        gkeAutopilotInput.Deployment.ReadinessProbe,        // Pass global readiness probe configuration
+		LivenessProbe:         gkeAutopilotInput.Deployment.LivenessProbe,         // Pass global liveness probe configuration
+		StartupProbe:          gkeAutopilotInput.Deployment.StartupProbe,          // Pass global startup probe configuration
+		ServiceType:           gkeAutopilotInput.Deployment.ServiceType,           // Pass Service type override (e.g. LoadBalancer for UDP)
+		ExternalTrafficPolicy: gkeAutopilotInput.Deployment.ExternalTrafficPolicy, // e.g. Local, required for WebRTC return media on Autopilot
+		EphemeralSize:         ephemeralSize,
 	}
 
 	params.Log.Info(ctx.Context(), "🔍 DEBUG: kubeArgs.Affinity passed to DeploySimpleContainer: %+v", kubeArgs.Affinity)

pkg/clouds/pulumi/kubernetes/caddy.go

@@ -168,7 +168,7 @@ func DeployCaddyService(ctx *sdk.Context, caddy CaddyDeployment, input api.Resou
 			Name:     caddyImage,
 			Platform: api.ImagePlatformLinuxAmd64,
 		},
-		Ports:     []int{443, 80},
+		Ports:     k8s.ContainerPorts(443, 80),
 		MainPort:  lo.ToPtr(80),
 		Resources: caddy.Resources, // Use custom resources if specified, otherwise defaults will be applied
 	}

pkg/clouds/pulumi/kubernetes/custom_stack_vpa_test.go

@@ -35,7 +35,7 @@ func TestNewSimpleContainer_CustomStackVPATargetsDeployment(t *testing.T) {
 			IngressContainer: &k8s.CloudRunContainer{
 				Name:     "myapp-tenant-a",
 				MainPort: lo.ToPtr(8080),
-				Ports:    []int{8080},
+				Ports:    k8s.ContainerPorts(8080),
 			},
 			Log: logger.New(),
 			Containers: []corev1.ContainerArgs{