diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go index 255c6cb98e7..76b21d32166 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go @@ -7,10 +7,11 @@ package aksnodeconfigv1 import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( @@ -521,7 +522,7 @@ type KubeletConfigFileConfig struct { EvictionMaxPodGracePeriod int32 `protobuf:"varint,43,opt,name=eviction_max_pod_grace_period,json=evictionMaxPodGracePeriod,proto3" json:"eviction_max_pod_grace_period,omitempty"` // kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the // kube-reserved compute resources. When enforce-node-allocatable contains "kube-reserved", - // this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + // this cgroup must exist before kubelet starts. Example: "/kubereserved.slice". // Used by AKS Node Memory Hardening (F2/F5). // +optional. KubeReservedCgroup string `protobuf:"bytes,44,opt,name=kube_reserved_cgroup,json=kubeReservedCgroup,proto3" json:"kube_reserved_cgroup,omitempty"` diff --git a/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto index 6efb39e37ea..31fd5702a4f 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto @@ -416,7 +416,7 @@ message KubeletConfigFileConfig { /* kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the kube-reserved compute resources. When enforce-node-allocatable contains "kube-reserved", - this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + this cgroup must exist before kubelet starts. Example: "/kubereserved.slice". Used by AKS Node Memory Hardening (F2/F5). +optional. */ string kube_reserved_cgroup = 44; diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 906eab5facb..03613c09dbf 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -2677,3 +2677,95 @@ func Test_AzureLinuxV3_MANA(t *testing.T) { }, }) } + +// Test_Ubuntu2204_NodeHardening_KubeReservedSlice_ConfigFile validates the config-file +// kubelet path (kubelet reads /etc/default/kubeletconfig.json), which is selected whenever +// AgentPoolProfile.CustomKubeletConfig is non-nil (see IsKubeletConfigFileEnabled). +func Test_Ubuntu2204_NodeHardening_KubeReservedSlice_ConfigFile(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "validates kubelet and containerd run in kubereserved.slice when node hardening cgroup hierarchy is enabled (kubelet config-file mode)", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2Containerd, + // Force the "default" (non-scriptless) subtest path so that fresh CSE scripts + // with the Slice= drop-ins are uploaded via custom data. + CustomDataWriteFiles: []CustomDataWriteFile{{Path: "/etc/aks-node-hardening-test", Content: "sentinel"}}, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // AgentBaker (not the RP) now owns --kube-reserved-cgroup/--system-reserved-cgroup; + // it derives them from --enforce-node-allocatable (see setNodeHardeningCgroupFlags). + nbc.KubeletConfig["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved" + // kubelet refuses to enforce limits on a reserved cgroup unless a matching + // resource list is also supplied; the base config only sets --kube-reserved, + // so --system-reserved must be added here too or kubelet fails to start with + // "system.slice cgroup is not configured properly". + nbc.KubeletConfig["--system-reserved"] = "cpu=200m,memory=500Mi" + // Simulate the RP still sending its legacy (stale) cgroup slice name today; + // setNodeHardeningCgroupFlags must overwrite these with the values AgentBaker + // owns rather than trusting them, or the node would end up in the wrong slice. + nbc.KubeletConfig["--kube-reserved-cgroup"] = "/kubelet.slice" + nbc.KubeletConfig["--system-reserved-cgroup"] = "/kubelet.slice" + // Non-nil (even empty) CustomKubeletConfig switches AgentBaker to the + // config-file (kubeletconfig.json) path instead of CLI flags. + nbc.AgentPoolProfile.CustomKubeletConfig = &datamodel.CustomKubeletConfig{} + // Disable scriptless CSE so that the current cse_helpers.sh (with kubereserved.slice drop-in) + // is uploaded via custom data instead of relying on potentially stale VHD scripts. + nbc.EnableScriptlessCSECmd = false + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/systemd/system/kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/systemd/system/kubelet.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/systemd/system/containerd.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", `"kubeReservedCgroup": "/kubereserved.slice"`) + ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", `"systemReservedCgroup": "/system.slice"`) + ValidateServiceInSlice(ctx, s, "kubelet.service", "kubereserved.slice") + ValidateServiceInSlice(ctx, s, "containerd.service", "kubereserved.slice") + }, + }, + }) +} + +// Test_Ubuntu2204_NodeHardening_KubeReservedSlice_CLIFlags validates the legacy CLI-flags +// kubelet path (kubelet reads flags from /etc/default/kubelet's KUBELET_FLAGS), which is +// used whenever AgentPoolProfile.CustomKubeletConfig/CustomLinuxOSConfig are both nil. +func Test_Ubuntu2204_NodeHardening_KubeReservedSlice_CLIFlags(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "validates kubelet and containerd run in kubereserved.slice when node hardening cgroup hierarchy is enabled (kubelet CLI-flags mode)", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2Containerd, + // Force the "default" (non-scriptless) subtest path so that fresh CSE scripts + // with the Slice= drop-ins are uploaded via custom data. + CustomDataWriteFiles: []CustomDataWriteFile{{Path: "/etc/aks-node-hardening-test", Content: "sentinel"}}, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // AgentBaker (not the RP) now owns --kube-reserved-cgroup/--system-reserved-cgroup; + // it derives them from --enforce-node-allocatable (see setNodeHardeningCgroupFlags). + nbc.KubeletConfig["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved" + // kubelet refuses to enforce limits on a reserved cgroup unless a matching + // resource list is also supplied; the base config only sets --kube-reserved, + // so --system-reserved must be added here too or kubelet fails to start with + // "system.slice cgroup is not configured properly". + nbc.KubeletConfig["--system-reserved"] = "cpu=200m,memory=500Mi" + // Simulate the RP still sending its legacy (stale) cgroup slice name today; + // setNodeHardeningCgroupFlags must overwrite these with the values AgentBaker + // owns rather than trusting them, or the node would end up in the wrong slice. + nbc.KubeletConfig["--kube-reserved-cgroup"] = "/kubelet.slice" + nbc.KubeletConfig["--system-reserved-cgroup"] = "/kubelet.slice" + // CustomKubeletConfig/CustomLinuxOSConfig left nil so kubelet reads its + // flags from the CLI (KUBELET_FLAGS in /etc/default/kubelet) instead of + // the config-file path. + // Disable scriptless CSE so that the current cse_helpers.sh (with kubereserved.slice drop-in) + // is uploaded via custom data instead of relying on potentially stale VHD scripts. + nbc.EnableScriptlessCSECmd = false + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/systemd/system/kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/systemd/system/kubelet.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/systemd/system/containerd.service.d/10-kubereserved-slice.conf", "Slice=kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--kube-reserved-cgroup=/kubereserved.slice") + ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--system-reserved-cgroup=/system.slice") + ValidateServiceInSlice(ctx, s, "kubelet.service", "kubereserved.slice") + ValidateServiceInSlice(ctx, s, "containerd.service", "kubereserved.slice") + }, + }, + }) +} diff --git a/e2e/validators.go b/e2e/validators.go index abe8f2a3ab3..f3341403d36 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -3485,3 +3485,18 @@ func ValidateRCV1PNotOptedInWindows(ctx context.Context, s *Scenario) { execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "expected no aks-ca-certs-refresh-task scheduled task when not opted in") } + +// ValidateServiceInSlice asserts that the given systemd service is running in the expected slice. +func ValidateServiceInSlice(ctx context.Context, s *Scenario, service, expectedSlice string) { + s.T.Helper() + // Avoid accidental shell injection / option smuggling. + if !regexp.MustCompile(`^[A-Za-z0-9_.@:-]+$`).MatchString(service) { + s.T.Fatalf("invalid systemd unit name: %q", service) + } + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, + fmt.Sprintf("systemctl show --property=Slice --value -- %s", service), 0, + fmt.Sprintf("could not query Slice property of %s", service)) + actual := strings.TrimSpace(result.stdout) + require.Equal(s.T, expectedSlice, actual, + "expected %s to be in %s, but got %s", service, expectedSlice, actual) +} diff --git a/e2e/vmss.go b/e2e/vmss.go index f9ae13bfb69..3ad99e85426 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -535,9 +535,9 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine customData, err = injectWriteFilesEntriesToCustomData(customData, s.Config.CustomDataWriteFiles) require.NoError(s.T, err, "failed to inject customData write_files entries") } - if !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { - // Validate that the custom data doesn't contain any script content, - // which indicates that the scriptless CSE is working as intended + if s.Runtime.NBC.EnableScriptlessCSECmd && !scriptlessNBCCSECmdEnabled && s.VHD.SupportsScriptless() { + // Validate that the custom data indicates scriptless CSE is enabled by checking + // for the scriptless overrides sentinel file path. decodedCustomData, err := base64.StdEncoding.DecodeString(customData) require.NoError(s.T, err, "failed to decode custom data") reader, err := gzip.NewReader(bytes.NewReader(decodedCustomData)) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 330988834ac..01c5b90939b 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -383,6 +383,17 @@ net.ipv6.conf.all.forwarding = 1 net.bridge.bridge-nf-call-iptables = 1 EOF retrycmd_if_failure 120 5 25 sysctl --system || exit $ERR_SYSCTL_RELOAD + + # Node Memory Hardening: create kubereserved.slice and drop-ins BEFORE starting + # containerd/kubelet so both services start in the correct slice from the + # beginning — avoids needing a disruptive restart after the fact. + resolveKubeletReservedCgroups + if [ -n "${KUBE_RESERVED_CGROUP}" ] || [ -n "${SYSTEM_RESERVED_CGROUP}" ]; then + if ! logs_to_events "AKS.CSE.ensureKubelet.ensureKubeletCgroupHierarchy" ensureKubeletCgroupHierarchy; then + exit $ERR_KUBELET_START_FAIL + fi + fi + systemctlEnableAndStartNoBlock containerd 30 || exit $ERR_SYSTEMCTL_START_FAIL } @@ -855,10 +866,9 @@ EOF local tls_bootstrapping_start_time_filepath="/opt/azure/containers/tls-bootstrap-start-time" date +"%F %T.%3N" > "${tls_bootstrapping_start_time_filepath}" - # Node Memory Hardening (F2/F5): if the RP rendered --kube-reserved-cgroup or - # --system-reserved-cgroup, ensure the corresponding systemd slices exist before - # kubelet starts so its NodeAllocatable enforcement loop can find them. The - # helper is a no-op when neither value is present (back-compat with non-hardened pools). + # Node Memory Hardening (F2/F5): idempotent refresh for PIS real nodes booting + # from older cached VHDs where basePrep (ensureContainerd) may not have created + # the Slice= drop-ins. No-op when neither value is set (non-hardened pools). resolveKubeletReservedCgroups if [ -n "${KUBE_RESERVED_CGROUP}" ] || [ -n "${SYSTEM_RESERVED_CGROUP}" ]; then if ! logs_to_events "AKS.CSE.ensureKubelet.ensureKubeletCgroupHierarchy" ensureKubeletCgroupHierarchy; then @@ -866,6 +876,20 @@ EOF fi fi + # Refresh --runtime-cgroups for PIS nodes where basePrep may have baked an older + # 10-containerd-base-flag.conf pointing at /system.slice/containerd.service. + local containerd_runtime_cgroups="/system.slice/containerd.service" + if [ "${KUBE_RESERVED_CGROUP:-}" = "/kubereserved.slice" ] || [ "${KUBE_RESERVED_CGROUP:-}" = "kubereserved.slice" ]; then + containerd_runtime_cgroups="/kubereserved.slice/containerd.service" + fi + tee "/etc/systemd/system/kubelet.service.d/10-containerd-base-flag.conf" > /dev/null < /dev/null <<'EOF' + # /system.slice is a built-in systemd slice; we only need to create kubereserved.slice. + if [ "${KUBE_RESERVED_CGROUP:-}" = "/kubereserved.slice" ] || [ "${KUBE_RESERVED_CGROUP:-}" = "kubereserved.slice" ]; then + # Write all unit/drop-in files unconditionally (idempotent). This ensures + # upgraded nodes that already have an older version of these files get the + # latest content (e.g. the Slice= directive added for kubelet/containerd). + mkdir -p "$(dirname "${kube_reserved_slice_unit}")" + tee "${kube_reserved_slice_unit}" > /dev/null <<'EOF' [Unit] -Description=Slice for kubelet kube-reserved enforcement (AKS Node Memory Hardening) +Description=Slice for kube-reserved enforcement (AKS Node Memory Hardening) Before=slices.target DefaultDependencies=no @@ -1578,31 +1576,44 @@ DefaultDependencies=no [Install] WantedBy=slices.target EOF - chmod 0644 "${kubelet_slice_unit}" + chmod 0644 "${kube_reserved_slice_unit}" - # Drop-in on kubelet.service so systemd starts kubelet.slice first - # on every boot. This survives reboots without depending on the - # one-shot `systemctl start` below. - mkdir -p "${kubelet_dropin_dir}" - tee "${kubelet_dropin_dir}/10-kubelet-slice.conf" > /dev/null <<'EOF' + # Drop-in on kubelet.service so systemd starts kubereserved.slice first + # on every boot and places kubelet inside the slice. + mkdir -p "${kubelet_dropin_dir}" + tee "${kubelet_dropin_dir}/10-kubereserved-slice.conf" > /dev/null <<'EOF' [Unit] -Wants=kubelet.slice -After=kubelet.slice +Wants=kubereserved.slice +After=kubereserved.slice + +[Service] +Slice=kubereserved.slice EOF - chmod 0644 "${kubelet_dropin_dir}/10-kubelet-slice.conf" + chmod 0644 "${kubelet_dropin_dir}/10-kubereserved-slice.conf" - systemctl daemon-reload + # Drop-in on containerd.service to place it in kubereserved.slice. + mkdir -p "${containerd_dropin_dir}" + tee "${containerd_dropin_dir}/10-kubereserved-slice.conf" > /dev/null <<'EOF' +[Unit] +Wants=kubereserved.slice +After=kubereserved.slice - # Enable the slice so it is started on subsequent boots. - if ! systemctl enable kubelet.slice; then - echo "ensureKubeletCgroupHierarchy: failed to enable kubelet.slice" - return 1 - fi +[Service] +Slice=kubereserved.slice +EOF + chmod 0644 "${containerd_dropin_dir}/10-kubereserved-slice.conf" + + if ! systemctl daemon-reload; then + echo "ensureKubeletCgroupHierarchy: failed to daemon-reload systemd" + return 1 fi - # Materialise the cgroup tree at /sys/fs/cgroup/kubelet.slice before kubelet starts on this boot. - if ! systemctl start kubelet.slice; then - echo "ensureKubeletCgroupHierarchy: failed to start kubelet.slice" + # Enable the slice for subsequent boots AND materialise the cgroup tree + # at /sys/fs/cgroup/kubereserved.slice on this boot before kubelet starts. + # systemctlEnableAndStart wraps both operations with retry logic to + # survive transient systemd failures during CSE. + if ! systemctlEnableAndStart kubereserved.slice 30; then + echo "ensureKubeletCgroupHierarchy: failed to enable and start kubereserved.slice" return 1 fi fi diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index a49e1b408af..df7e7df6548 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -670,6 +670,15 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "DynamicKubeletConfig", false) } + // Node Hardening: AgentBaker, not the RP, owns the cgroup slice + // names that --kube-reserved-cgroup/--system-reserved-cgroup resolve to, since + // AgentBaker is what actually creates (or doesn't create) the systemd slice unit + // on the node (see cse_helpers.sh::ensureKubeletCgroupHierarchy). The RP only + // signals intent via --enforce-node-allocatable=pods,kube-reserved,system-reserved; + // any value it may still send for the two cgroup flags themselves is ignored and + // overwritten here so there is a single source of truth for the slice names. + setNodeHardeningCgroupFlags(kubeletFlags) + /* ContainerInsights depends on GPU accelerator Usage metrics from Kubelet cAdvisor endpoint but deprecation of this feature moved to beta which breaks the ContainerInsights customers with K8s version 1.20 or higher */ diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index cd55688ed6f..7beb626631c 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -2326,7 +2326,7 @@ type AKSKubeletConfiguration struct { EnforceNodeAllocatable []string `json:"enforceNodeAllocatable,omitempty"` /* kubeReservedCgroup is the absolute name of the cgroup the kubelet should manage for the kube-reserved compute resources. When enforce-node-allocatable contains - "kube-reserved", this cgroup must exist before kubelet starts. Example: "/kubelet.slice". + this cgroup must exist before kubelet starts. Example: "/kubereserved.slice". +optional. */ KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` /* systemReservedCgroup is the absolute name of the cgroup the kubelet should manage diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 63fa79ab0af..26aa0168f69 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -484,6 +484,62 @@ func IsKubeletServingCertificateRotationEnabled(config *datamodel.NodeBootstrapp return config.KubeletConfig["--rotate-server-certificates"] == "true" } +// Node Hardening cgroup slice names. AgentBaker is the single +// source of truth for these values: cse_helpers.sh::ensureKubeletCgroupHierarchy +// is what actually creates (or validates) the systemd slice unit on the node, so +// the name must be decided here rather than accepted verbatim from the RP. +const ( + nodeHardeningKubeReservedCgroup = "/kubereserved.slice" + nodeHardeningSystemReservedCgroup = "/system.slice" +) + +// isNodeHardeningEnabled reports whether the RP has requested Node Hardening +// cgroup enforcement for this node. The RP signals intent solely via +// --enforce-node-allocatable containing both "kube-reserved" and "system-reserved" +// (see ApplyNodeAllocatableEnforcement in aks-rp); any values it may additionally +// send for --kube-reserved-cgroup/--system-reserved-cgroup are ignored, since +// AgentBaker owns those slice names (see setNodeHardeningCgroupFlags). +// +// TODO: this detection is a proxy inferred from a flag the RP happens to set +// today. If/when Node Hardening's own logic (the enable/disable decision, +// --system-reserved formula, etc.) moves into AgentBaker, this should be +// replaced with a real, explicit signal (e.g. a typed field on +// CustomKubeletConfig) instead of inferring intent from --enforce-node-allocatable. +func isNodeHardeningEnabled(kubeletFlags map[string]string) bool { + raw := strings.TrimSpace(kubeletFlags["--enforce-node-allocatable"]) + raw = strings.TrimPrefix(raw, "[") + raw = strings.TrimSuffix(raw, "]") + enforced := strings.Split(raw, ",") + hasKubeReserved, hasSystemReserved := false, false + for _, v := range enforced { + switch strings.TrimSpace(v) { + case "kube-reserved": + hasKubeReserved = true + case "system-reserved": + hasSystemReserved = true + } + } + return hasKubeReserved && hasSystemReserved +} + +// setNodeHardeningCgroupFlags assigns (or clears) --kube-reserved-cgroup and +// --system-reserved-cgroup based solely on whether Node Hardening is +// enabled (isNodeHardeningEnabled), overwriting/deleting any value the RP +// may have set for these two keys directly. +func setNodeHardeningCgroupFlags(kubeletFlags map[string]string) { + if isNodeHardeningEnabled(kubeletFlags) { + kubeletFlags["--kube-reserved-cgroup"] = nodeHardeningKubeReservedCgroup + kubeletFlags["--system-reserved-cgroup"] = nodeHardeningSystemReservedCgroup + // TODO: this is currently the only piece of Node Hardening logic owned by + // AgentBaker; --enforce-node-allocatable, --system-reserved, --kube-reserved, + // and the eviction-soft* flags are still computed and sent as raw values by + // the RP. If/when that logic moves into AgentBaker too, set those flags here. + return + } + delete(kubeletFlags, "--kube-reserved-cgroup") + delete(kubeletFlags, "--system-reserved-cgroup") +} + func getAKSKubeletConfiguration(kc map[string]string) *datamodel.AKSKubeletConfiguration { kubeletConfig := &datamodel.AKSKubeletConfiguration{ APIVersion: "kubelet.config.k8s.io/v1beta1", diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index baf8613d9ad..4e74e94099c 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -599,7 +599,7 @@ func TestGetKubeletConfigFileNodeMemoryHardeningFields(t *testing.T) { kc["--eviction-soft-grace-period"] = "memory.available=30s,nodefs.available=2m,imagefs.available=2m" kc["--eviction-max-pod-grace-period"] = "60" kc["--enforce-node-allocatable"] = "pods,kube-reserved,system-reserved" - kc["--kube-reserved-cgroup"] = "/kubelet.slice" + kc["--kube-reserved-cgroup"] = "/kubereserved.slice" kc["--system-reserved-cgroup"] = "/system.slice" configFileStr := GetKubeletConfigFileContent(kc, nil) @@ -643,14 +643,87 @@ func TestGetKubeletConfigFileNodeMemoryHardeningFields(t *testing.T) { t.Errorf("enforceNodeAllocatable mismatch (-want +got):\n%s", diff) } - if got.KubeReservedCgroup != "/kubelet.slice" { - t.Errorf("kubeReservedCgroup=%q, want %q", got.KubeReservedCgroup, "/kubelet.slice") + if got.KubeReservedCgroup != "/kubereserved.slice" { + t.Errorf("kubeReservedCgroup=%q, want %q", got.KubeReservedCgroup, "/kubereserved.slice") } if got.SystemReservedCgroup != "/system.slice" { t.Errorf("systemReservedCgroup=%q, want %q", got.SystemReservedCgroup, "/system.slice") } } +func TestSetNodeHardeningCgroupFlags(t *testing.T) { + // AgentBaker, not the RP, must own the cgroup slice names: it overwrites + // --kube-reserved-cgroup/--system-reserved-cgroup based solely on whether + // --enforce-node-allocatable signals hardening is on, regardless of any + // (possibly stale or wrong) value the RP put on those two keys directly. + cases := []struct { + name string + enforceNodeAllocatable string + rpKubeReservedCgroup string + rpSystemReservedCgroup string + wantKubeReservedCgroup string + wantSystemReservedCgroup string + }{ + { + name: "hardening enabled overwrites RP-supplied legacy value", + enforceNodeAllocatable: "pods,kube-reserved,system-reserved", + rpKubeReservedCgroup: "/kubelet.slice", // stale/legacy value the RP might still send + rpSystemReservedCgroup: "/system.slice", + wantKubeReservedCgroup: "/kubereserved.slice", + wantSystemReservedCgroup: "/system.slice", + }, + { + name: "hardening enabled with no RP value set", + enforceNodeAllocatable: "pods,kube-reserved,system-reserved", + wantKubeReservedCgroup: "/kubereserved.slice", + wantSystemReservedCgroup: "/system.slice", + }, + { + name: "hardening disabled clears any stale RP value", + enforceNodeAllocatable: "pods", + rpKubeReservedCgroup: "/kubereserved.slice", + rpSystemReservedCgroup: "/system.slice", + wantKubeReservedCgroup: "", + wantSystemReservedCgroup: "", + }, + { + name: "hardening flags absent entirely", + wantKubeReservedCgroup: "", + wantSystemReservedCgroup: "", + }, + { + name: "hardening enabled with bracketed list format", + enforceNodeAllocatable: "[pods,kube-reserved,system-reserved]", + wantKubeReservedCgroup: "/kubereserved.slice", + wantSystemReservedCgroup: "/system.slice", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + kubeletFlags := map[string]string{} + if c.enforceNodeAllocatable != "" { + kubeletFlags["--enforce-node-allocatable"] = c.enforceNodeAllocatable + } + if c.rpKubeReservedCgroup != "" { + kubeletFlags["--kube-reserved-cgroup"] = c.rpKubeReservedCgroup + } + if c.rpSystemReservedCgroup != "" { + kubeletFlags["--system-reserved-cgroup"] = c.rpSystemReservedCgroup + } + + setNodeHardeningCgroupFlags(kubeletFlags) + + if got := kubeletFlags["--kube-reserved-cgroup"]; got != c.wantKubeReservedCgroup { + t.Errorf("--kube-reserved-cgroup=%q, want %q", got, c.wantKubeReservedCgroup) + } + if got := kubeletFlags["--system-reserved-cgroup"]; got != c.wantSystemReservedCgroup { + t.Errorf("--system-reserved-cgroup=%q, want %q", got, c.wantSystemReservedCgroup) + } + }) + } +} + func TestGetKubeletConfigFileNodeMemoryHardeningFieldsOmittedByDefault(t *testing.T) { // Backward-compat: when the RP does not pass the new flags, the generated // kubelet config must NOT contain the new fields. This guards the 6-month diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index 61f8e57595c..efb439c7db2 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -819,9 +819,9 @@ EOF It 'extracts cgroup names from KUBELET_FLAGS in flag mode' KUBELET_CONFIG_FILE_ENABLED="false" KUBELET_CONFIG_FILE_CONTENT="" - KUBELET_FLAGS="--kube-reserved-cgroup=/kubelet.slice --system-reserved-cgroup=/system.slice --node-ip=10.0.0.1" + KUBELET_FLAGS="--kube-reserved-cgroup=/kubereserved.slice --system-reserved-cgroup=/system.slice --node-ip=10.0.0.1" When call resolveKubeletReservedCgroups - The variable KUBE_RESERVED_CGROUP should equal "/kubelet.slice" + The variable KUBE_RESERVED_CGROUP should equal "/kubereserved.slice" The variable SYSTEM_RESERVED_CGROUP should equal "/system.slice" The status should be success End @@ -832,7 +832,7 @@ EOF # Even though flags carry the values, config-file mode must win. KUBELET_FLAGS="--kube-reserved-cgroup=/wrong.slice --system-reserved-cgroup=/wrong.slice" When call resolveKubeletReservedCgroups - The variable KUBE_RESERVED_CGROUP should equal "/kubelet.slice" + The variable KUBE_RESERVED_CGROUP should equal "/kubereserved.slice" The variable SYSTEM_RESERVED_CGROUP should equal "/system.slice" The status should be success End @@ -856,8 +856,9 @@ EOF setup_paths() { ENSURE_CGROUP_TMPDIR=$(mktemp -d) export CGROUPV2_MARKER_PATH="${ENSURE_CGROUP_TMPDIR}/cgroup.controllers" - export KUBELET_SLICE_UNIT_PATH="${ENSURE_CGROUP_TMPDIR}/kubelet.slice" + export KUBE_RESERVED_SLICE_UNIT_PATH="${ENSURE_CGROUP_TMPDIR}/kubereserved.slice" export KUBELET_SERVICE_DROPIN_DIR="${ENSURE_CGROUP_TMPDIR}/kubelet.service.d" + export CONTAINERD_SERVICE_DROPIN_DIR="${ENSURE_CGROUP_TMPDIR}/containerd.service.d" : > "${CGROUPV2_MARKER_PATH}" # cgroupv2 present by default } cleanup_paths() { @@ -895,7 +896,7 @@ EOF It 'fails when the cgroupv2 unified hierarchy is not detected' setup_paths rm -f "${CGROUPV2_MARKER_PATH}" - KUBE_RESERVED_CGROUP="/kubelet.slice" + KUBE_RESERVED_CGROUP="/kubereserved.slice" SYSTEM_RESERVED_CGROUP="" When call ensureKubeletCgroupHierarchy cleanup_paths @@ -903,52 +904,38 @@ EOF The output should include "cgroupv2 unified hierarchy not detected" End - It 'creates kubelet.slice and the kubelet.service drop-in for /kubelet.slice' + It 'creates kubereserved.slice and the kubelet.service and containerd.service drop-ins' setup_paths systemctl() { return 0; } - KUBE_RESERVED_CGROUP="/kubelet.slice" + systemctlEnableAndStart() { return 0; } + KUBE_RESERVED_CGROUP="/kubereserved.slice" SYSTEM_RESERVED_CGROUP="/system.slice" When call ensureKubeletCgroupHierarchy - slice_contents=$(cat "${KUBELET_SLICE_UNIT_PATH}" 2>/dev/null) - dropin_contents=$(cat "${KUBELET_SERVICE_DROPIN_DIR}/10-kubelet-slice.conf" 2>/dev/null) + slice_contents=$(cat "${KUBE_RESERVED_SLICE_UNIT_PATH}" 2>/dev/null) + dropin_contents=$(cat "${KUBELET_SERVICE_DROPIN_DIR}/10-kubereserved-slice.conf" 2>/dev/null) + containerd_dropin_contents=$(cat "${CONTAINERD_SERVICE_DROPIN_DIR}/10-kubereserved-slice.conf" 2>/dev/null) cleanup_paths The status should be success - The variable slice_contents should include "Description=Slice for kubelet kube-reserved enforcement" + The variable slice_contents should include "Description=Slice for kube-reserved enforcement" The variable slice_contents should include "WantedBy=slices.target" - The variable dropin_contents should include "Wants=kubelet.slice" - The variable dropin_contents should include "After=kubelet.slice" + The variable dropin_contents should include "Wants=kubereserved.slice" + The variable dropin_contents should include "After=kubereserved.slice" + The variable dropin_contents should include "Slice=kubereserved.slice" + The variable containerd_dropin_contents should include "Wants=kubereserved.slice" + The variable containerd_dropin_contents should include "After=kubereserved.slice" + The variable containerd_dropin_contents should include "Slice=kubereserved.slice" End - It 'returns failure when systemctl enable kubelet.slice fails' + It 'returns failure when systemctlEnableAndStart kubereserved.slice fails' setup_paths - systemctl() { - if [ "$1" = "enable" ]; then - return 1 - fi - return 0 - } - KUBE_RESERVED_CGROUP="/kubelet.slice" - SYSTEM_RESERVED_CGROUP="" - When call ensureKubeletCgroupHierarchy - cleanup_paths - The status should be failure - The output should include "failed to enable kubelet.slice" - End - - It 'returns failure when systemctl start kubelet.slice fails' - setup_paths - systemctl() { - if [ "$1" = "start" ]; then - return 1 - fi - return 0 - } - KUBE_RESERVED_CGROUP="/kubelet.slice" + systemctl() { return 0; } + systemctlEnableAndStart() { return 1; } + KUBE_RESERVED_CGROUP="/kubereserved.slice" SYSTEM_RESERVED_CGROUP="" When call ensureKubeletCgroupHierarchy cleanup_paths The status should be failure - The output should include "failed to start kubelet.slice" + The output should include "failed to enable and start kubereserved.slice" End End End diff --git a/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json b/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json index 167d355d10b..a2bb75d4dca 100644 --- a/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json +++ b/spec/parts/linux/cloud-init/artifacts/kubelet_mocks/config_file/node_hardening_enabled.json @@ -1,6 +1,6 @@ { "kind": "KubeletConfiguration", "apiVersion": "kubelet.config.k8s.io/v1beta1", - "kubeReservedCgroup": "/kubelet.slice", + "kubeReservedCgroup": "/kubereserved.slice", "systemReservedCgroup": "/system.slice" }