From 52f1f28c28292b20be1d2206ee7794eb96765fd0 Mon Sep 17 00:00:00 2001 From: Nishchay Date: Tue, 21 Jul 2026 08:59:22 -0700 Subject: [PATCH] fix: disable scriptless phase2 in case of customcacerts --- e2e/validators.go | 2 +- pkg/agent/baker.go | 16 ++++++-- pkg/agent/baker_test.go | 90 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/e2e/validators.go b/e2e/validators.go index 56f13327d89..b5ff970e6b9 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -2757,7 +2757,7 @@ func ValidateScriptlessCSECmd(ctx context.Context, s *Scenario) { // ValidateScriptlessNBCCSECmd checks if the node has scriptless NBCCSECmd correctly enabled func ValidateScriptlessNBCCSECmd(ctx context.Context, s *Scenario) { nbc := s.Runtime.NBC - if nbc != nil && nbc.EnableScriptlessNBCCSECmd && s.VHD.SupportsScriptless() { + if nbc != nil && nbc.EnableScriptlessNBCCSECmd && s.VHD.SupportsScriptless() && (nbc.CustomCATrustConfig == nil || len(nbc.CustomCATrustConfig.CustomCATrustCerts) == 0) { fileNameToCheck := "/opt/azure/containers/aks-node-controller-nbc-cmd.sh" if enableScriptlessCompilation(s) { fileNameToCheck = "/opt/azure/containers/aks-node-controller-nbc-cmd-hack.sh" diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index e0bb3bbc58c..99cefeb130e 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -93,8 +93,12 @@ func (t *TemplateGenerator) getWindowsNodeBootstrappingPayload(config *datamodel } func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.NodeBootstrappingConfiguration) string { - if config.EnableScriptlessNBCCSECmd && !config.PreProvisionOnly { - return t.getScriptlessNBCCustomData(config) + if config.EnableScriptlessNBCCSECmd { + if supportsScriptlessPhase2(config) { + return t.getScriptlessNBCCustomData(config) + } + // if we cannot enable scriptless phase2, we need to fallback to scriptless phase1 + config.EnableScriptlessCSECmd = true } // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation @@ -163,6 +167,10 @@ func (t *TemplateGenerator) getScriptlessNBCCustomData(config *datamodel.NodeBoo return base64.StdEncoding.EncodeToString([]byte(customData)) } +func supportsScriptlessPhase2(config *datamodel.NodeBootstrappingConfiguration) bool { + return !config.PreProvisionOnly && (config.CustomCATrustConfig == nil || len(config.CustomCATrustConfig.CustomCATrustCerts) == 0) +} + // renderEnabledFeatures serializes the feature toggle map into sorted KEY=VALUE lines for // enabled_features.sh. Keys are sorted so the output is deterministic (Go map iteration is // randomized) and filtered to valid shell identifiers - the same set the aks-node-controller @@ -440,7 +448,7 @@ func (t *TemplateGenerator) getNodeBootstrappingCmd(config *datamodel.NodeBootst if config.AgentPoolProfile.IsWindows() { return t.getWindowsNodeCSECommand(config) } - if config.EnableScriptlessNBCCSECmd && !config.PreProvisionOnly { + if config.EnableScriptlessNBCCSECmd && supportsScriptlessPhase2(config) { return "/opt/azure/containers/aks-node-controller provision-wait" } return t.getLinuxNodeCSECommand(config) @@ -1474,7 +1482,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration }, "GetPreProvisionOnly": func() bool { return config.PreProvisionOnly }, "GetCSETimeout": func() string { return datamodel.GetCSETimeout(config.CSETimeout) }, - "GetSkipWaAgentHold": func() bool { return config.EnableScriptlessNBCCSECmd }, + "GetSkipWaAgentHold": func() bool { return supportsScriptlessPhase2(config) }, "BlockIptables": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.BlockIptables }, diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 4141d642a76..c856268ddd6 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -153,6 +153,34 @@ var _ = Describe("Assert generated customData and cseCmd", func() { }) }) + Describe(".supportsScriptlessPhase2()", func() { + It("given EnableScriptlessNBCCSECmd, PreProvisionOnly is false and an empty list of custom ca certs, it returns true", func() { + config.PreProvisionOnly = false + config.CustomCATrustConfig = &datamodel.CustomCATrustConfig{ + CustomCATrustCerts: []string{}, + } + Expect(supportsScriptlessPhase2(config)).To(BeTrue()) + }) + It("given EnableScriptlessNBCCSECmd, PreProvisionOnly is false and custom ca certs are populated, it returns false", func() { + config.PreProvisionOnly = false + config.CustomCATrustConfig = &datamodel.CustomCATrustConfig{ + CustomCATrustCerts: []string{"mock cert value"}, + } + Expect(supportsScriptlessPhase2(config)).To(BeFalse()) + }) + It("given EnableScriptlessNBCCSECmd, PreProvisionOnly is true and no CustomCATrustConfig, it returns false", func() { + config.PreProvisionOnly = true + Expect(supportsScriptlessPhase2(config)).To(BeFalse()) + }) + It("given EnableScriptlessNBCCSECmd, PreProvisionOnly is true and custom ca certs are populated, it returns false", func() { + config.PreProvisionOnly = true + config.CustomCATrustConfig = &datamodel.CustomCATrustConfig{ + CustomCATrustCerts: []string{"mock cert value"}, + } + Expect(supportsScriptlessPhase2(config)).To(BeFalse()) + }) + }) + Describe(".isMariner()", func() { It("given an empty string, that is not mariner", func() { Expect(isMariner("")).To(BeFalse()) @@ -1838,6 +1866,68 @@ var _ = Describe("getNodeBootstrappingCmd", func() { Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal(templateGenerator.getLinuxNodeCSECommand(config))) Expect(templateGenerator.getNodeBootstrappingCmd(config)).NotTo(Equal("/opt/azure/containers/aks-node-controller provision-wait")) }) + + newScriptlessCmdTestConfig := func() *datamodel.NodeBootstrappingConfiguration { + agentPoolProfile := &datamodel.AgentPoolProfile{ + Name: "nodepool1", + OSType: datamodel.Linux, + Distro: datamodel.AKSUbuntuContainerd2204Gen2, + } + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Location: "eastus", + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: "1.29.0", + OrchestratorType: datamodel.Kubernetes, + KubernetesConfig: &datamodel.KubernetesConfig{ + ContainerRuntimeConfig: map[string]string{}, + }, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + FQDN: "test-cluster.hcp.eastus.azmk8s.io", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{agentPoolProfile}, + }, + }, + AgentPoolProfile: agentPoolProfile, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + } + } + + It("should use the aks-node-controller provision-wait command when scriptless phase2 is supported", func() { + templateGenerator := InitializeTemplateGenerator() + config := newScriptlessCmdTestConfig() + config.EnableScriptlessNBCCSECmd = true + config.PreProvisionOnly = false + + Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal("/opt/azure/containers/aks-node-controller provision-wait")) + }) + + It("should use the regular linux CSE command when EnableScriptlessNBCCSECmd is false", func() { + templateGenerator := InitializeTemplateGenerator() + config := newScriptlessCmdTestConfig() + config.EnableScriptlessNBCCSECmd = false + config.PreProvisionOnly = false + + Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal(templateGenerator.getLinuxNodeCSECommand(config))) + Expect(templateGenerator.getNodeBootstrappingCmd(config)).NotTo(Equal("/opt/azure/containers/aks-node-controller provision-wait")) + }) + + It("should use the regular linux CSE command when custom ca trust certs are populated", func() { + templateGenerator := InitializeTemplateGenerator() + config := newScriptlessCmdTestConfig() + config.EnableScriptlessNBCCSECmd = true + config.PreProvisionOnly = false + config.CustomCATrustConfig = &datamodel.CustomCATrustConfig{ + CustomCATrustCerts: []string{"mock cert value"}, + } + + Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal(templateGenerator.getLinuxNodeCSECommand(config))) + Expect(templateGenerator.getNodeBootstrappingCmd(config)).NotTo(Equal("/opt/azure/containers/aks-node-controller provision-wait")) + }) }) var _ = Describe("cloudInitToButane", func() {