fix: ensure kubelet and containerd in kube.slice via systemd drop-ins

aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto

@@ -416,7 +416,7 @@ message KubeletConfigFileConfig {
 
   /* kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the
    kube-reserved compute resources. When enforce-node-allocatable contains "kube-reserved",
-   this cgroup must exist before kubelet starts. Example: "/kubelet.slice".
+   this cgroup must exist before kubelet starts. Example: "/kubereserved.slice".
    Used by AKS Node Memory Hardening (F2/F5).
    +optional. */
   string kube_reserved_cgroup = 44;

e2e/scenario_test.go

@@ -2677,3 +2677,95 @@ func Test_AzureLinuxV3_MANA(t *testing.T) {
 		},
 	})
 }
+
+// Test_Ubuntu2204_NodeHardening_KubeReservedSlice_ConfigFile validates the config-file
+// kubelet path (kubelet reads /etc/default/kubeletconfig.json), which is selected whenever
+// AgentPoolProfile.CustomKubeletConfig is non-nil (see IsKubeletConfigFileEnabled).
+func Test_Ubuntu2204_NodeHardening_KubeReservedSlice_ConfigFile(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "validates kubelet and containerd run in kubereserved.slice when node hardening cgroup hierarchy is enabled (kubelet config-file mode)",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDUbuntu2204Gen2Containerd,
+			// Force the "default" (non-scriptless) subtest path so that fresh CSE scripts
+			// with the Slice= drop-ins are uploaded via custom data.
+			CustomDataWriteFiles: []CustomDataWriteFile{{Path: "/etc/aks-node-hardening-test", Content: "sentinel"}},
+			BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
+				// AgentBaker (not the RP) now owns --kube-reserved-cgroup/--system-reserved-cgroup;
+				// it derives them from --enforce-node-allocatable (see setNodeHardeningCgroupFlags).
+				nbc.KubeletConfig["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved"
+				// kubelet refuses to enforce limits on a reserved cgroup unless a matching
+				// resource list is also supplied; the base config only sets --kube-reserved,
+				// so --system-reserved must be added here too or kubelet fails to start with
+				// "system.slice cgroup is not configured properly".
+				nbc.KubeletConfig["--system-reserved"] = "cpu=200m,memory=500Mi"
+				// Simulate the RP still sending its legacy (stale) cgroup slice name today;
+				// setNodeHardeningCgroupFlags must overwrite these with the values AgentBaker
+				// owns rather than trusting them, or the node would end up in the wrong slice.
+				nbc.KubeletConfig["--kube-reserved-cgroup"] = "/kubelet.slice"
+				nbc.KubeletConfig["--system-reserved-cgroup"] = "/kubelet.slice"
+				// Non-nil (even empty) CustomKubeletConfig switches AgentBaker to the
+				// config-file (kubeletconfig.json) path instead of CLI flags.
+				nbc.AgentPoolProfile.CustomKubeletConfig = &datamodel.CustomKubeletConfig{}
+				// Disable scriptless CSE so that the current cse_helpers.sh (with kubereserved.slice drop-in)
+				// is uploaded via custom data instead of relying on potentially stale VHD scripts.
+				nbc.EnableScriptlessCSECmd = false
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileExists(ctx, s, "/etc/systemd/system/kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/systemd/system/kubelet.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/systemd/system/containerd.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", `"kubeReservedCgroup": "/kubereserved.slice"`)
+				ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", `"systemReservedCgroup": "/system.slice"`)
+				ValidateServiceInSlice(ctx, s, "kubelet.service", "kubereserved.slice")
+				ValidateServiceInSlice(ctx, s, "containerd.service", "kubereserved.slice")
+			},
+		},
+	})
+}
+
+// Test_Ubuntu2204_NodeHardening_KubeReservedSlice_CLIFlags validates the legacy CLI-flags
+// kubelet path (kubelet reads flags from /etc/default/kubelet's KUBELET_FLAGS), which is
+// used whenever AgentPoolProfile.CustomKubeletConfig/CustomLinuxOSConfig are both nil.
+func Test_Ubuntu2204_NodeHardening_KubeReservedSlice_CLIFlags(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "validates kubelet and containerd run in kubereserved.slice when node hardening cgroup hierarchy is enabled (kubelet CLI-flags mode)",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDUbuntu2204Gen2Containerd,
+			// Force the "default" (non-scriptless) subtest path so that fresh CSE scripts
+			// with the Slice= drop-ins are uploaded via custom data.
+			CustomDataWriteFiles: []CustomDataWriteFile{{Path: "/etc/aks-node-hardening-test", Content: "sentinel"}},
+			BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
+				// AgentBaker (not the RP) now owns --kube-reserved-cgroup/--system-reserved-cgroup;
+				// it derives them from --enforce-node-allocatable (see setNodeHardeningCgroupFlags).
+				nbc.KubeletConfig["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved"
+				// kubelet refuses to enforce limits on a reserved cgroup unless a matching
+				// resource list is also supplied; the base config only sets --kube-reserved,
+				// so --system-reserved must be added here too or kubelet fails to start with
+				// "system.slice cgroup is not configured properly".
+				nbc.KubeletConfig["--system-reserved"] = "cpu=200m,memory=500Mi"
+				// Simulate the RP still sending its legacy (stale) cgroup slice name today;
+				// setNodeHardeningCgroupFlags must overwrite these with the values AgentBaker
+				// owns rather than trusting them, or the node would end up in the wrong slice.
+				nbc.KubeletConfig["--kube-reserved-cgroup"] = "/kubelet.slice"
+				nbc.KubeletConfig["--system-reserved-cgroup"] = "/kubelet.slice"
+				// CustomKubeletConfig/CustomLinuxOSConfig left nil so kubelet reads its
+				// flags from the CLI (KUBELET_FLAGS in /etc/default/kubelet) instead of
+				// the config-file path.
+				// Disable scriptless CSE so that the current cse_helpers.sh (with kubereserved.slice drop-in)
+				// is uploaded via custom data instead of relying on potentially stale VHD scripts.
+				nbc.EnableScriptlessCSECmd = false
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileExists(ctx, s, "/etc/systemd/system/kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/systemd/system/kubelet.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/systemd/system/containerd.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--kube-reserved-cgroup=/kubereserved.slice")
+				ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--system-reserved-cgroup=/system.slice")
+				ValidateServiceInSlice(ctx, s, "kubelet.service", "kubereserved.slice")
+				ValidateServiceInSlice(ctx, s, "containerd.service", "kubereserved.slice")
+			},
+		},
+	})
+}

e2e/validators.go

@@ -3485,3 +3485,18 @@ func ValidateRCV1PNotOptedInWindows(ctx context.Context, s *Scenario) {
 	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
 		"expected no aks-ca-certs-refresh-task scheduled task when not opted in")
 }
+
+// ValidateServiceInSlice asserts that the given systemd service is running in the expected slice.
+func ValidateServiceInSlice(ctx context.Context, s *Scenario, service, expectedSlice string) {
+	s.T.Helper()
+	// Avoid accidental shell injection / option smuggling.
+	if !regexp.MustCompile(`^[A-Za-z0-9_.@:-]+$`).MatchString(service) {
+		s.T.Fatalf("invalid systemd unit name: %q", service)
+	}
+	result := execScriptOnVMForScenarioValidateExitCode(ctx, s,
+		fmt.Sprintf("systemctl show --property=Slice --value -- %s", service), 0,
+		fmt.Sprintf("could not query Slice property of %s", service))
+	actual := strings.TrimSpace(result.stdout)
+	require.Equal(s.T, expectedSlice, actual,
+		"expected %s to be in %s, but got %s", service, expectedSlice, actual)
+}

e2e/vmss.go

@@ -535,9 +535,9 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine
 			customData, err = injectWriteFilesEntriesToCustomData(customData, s.Config.CustomDataWriteFiles)
 			require.NoError(s.T, err, "failed to inject customData write_files entries")
 		}
-		if !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() {
-			// Validate that the custom data doesn't contain any script content,
-			// which indicates that the scriptless CSE is working as intended
+		if s.Runtime.NBC.EnableScriptlessCSECmd && !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() {
+			// Validate that the custom data indicates scriptless CSE is enabled by checking
+			// for the scriptless overrides sentinel file path.
 			decodedCustomData, err := base64.StdEncoding.DecodeString(customData)
 			require.NoError(s.T, err, "failed to decode custom data")
 			reader, err := gzip.NewReader(bytes.NewReader(decodedCustomData))

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -383,6 +383,17 @@ net.ipv6.conf.all.forwarding = 1
 net.bridge.bridge-nf-call-iptables = 1
 EOF
   retrycmd_if_failure 120 5 25 sysctl --system || exit $ERR_SYSCTL_RELOAD
+
+  # Node Memory Hardening: create kubereserved.slice and drop-ins BEFORE starting
+  # containerd/kubelet so both services start in the correct slice from the
+  # beginning — avoids needing a disruptive restart after the fact.
+  resolveKubeletReservedCgroups
+  if [ -n "${KUBE_RESERVED_CGROUP}" ] || [ -n "${SYSTEM_RESERVED_CGROUP}" ]; then
+      if ! logs_to_events "AKS.CSE.ensureKubelet.ensureKubeletCgroupHierarchy" ensureKubeletCgroupHierarchy; then
+          exit $ERR_KUBELET_START_FAIL
+      fi
+  fi
+
   systemctlEnableAndStartNoBlock containerd 30 || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
@@ -855,17 +866,30 @@ EOF
     local tls_bootstrapping_start_time_filepath="/opt/azure/containers/tls-bootstrap-start-time"
     date +"%F %T.%3N" > "${tls_bootstrapping_start_time_filepath}"
 
-    # Node Memory Hardening (F2/F5): if the RP rendered --kube-reserved-cgroup or
-    # --system-reserved-cgroup, ensure the corresponding systemd slices exist before
-    # kubelet starts so its NodeAllocatable enforcement loop can find them. The
-    # helper is a no-op when neither value is present (back-compat with non-hardened pools).
+    # Node Memory Hardening (F2/F5): idempotent refresh for PIS real nodes booting
+    # from older cached VHDs where basePrep (ensureContainerd) may not have created
+    # the Slice= drop-ins. No-op when neither value is set (non-hardened pools).
     resolveKubeletReservedCgroups
     if [ -n "${KUBE_RESERVED_CGROUP}" ] || [ -n "${SYSTEM_RESERVED_CGROUP}" ]; then
         if ! logs_to_events "AKS.CSE.ensureKubelet.ensureKubeletCgroupHierarchy" ensureKubeletCgroupHierarchy; then
             exit $ERR_KUBELET_START_FAIL
         fi
     fi
 
+    # Refresh --runtime-cgroups for PIS nodes where basePrep may have baked an older
+    # 10-containerd-base-flag.conf pointing at /system.slice/containerd.service.
+    local containerd_runtime_cgroups="/system.slice/containerd.service"
+    if [ "${KUBE_RESERVED_CGROUP:-}" = "/kubereserved.slice" ] || [ "${KUBE_RESERVED_CGROUP:-}" = "kubereserved.slice" ]; then
+        containerd_runtime_cgroups="/kubereserved.slice/containerd.service"
+    fi
+    tee "/etc/systemd/system/kubelet.service.d/10-containerd-base-flag.conf" > /dev/null <<EOF
+[Service]
+Environment="KUBELET_CONTAINERD_FLAGS=--runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --runtime-cgroups=${containerd_runtime_cgroups}"
+EOF
+
+    if ! systemctl daemon-reload; then
+        exit $ERR_KUBELET_START_FAIL
+    fi
     # start kubelet.service without waiting for the main process to start, though check whether it has entered a failed state after enablement
     if ! systemctlEnableAndStartNoBlock kubelet 240; then
         # append kubelet status to CSE output to ensure we can see it