diff --git a/.github/workflows/no-sudo-check.yml b/.github/workflows/no-sudo-check.yml index d82f6338e53..e22b7c2df53 100644 --- a/.github/workflows/no-sudo-check.yml +++ b/.github/workflows/no-sudo-check.yml @@ -15,6 +15,7 @@ jobs: | grep -v ':[[:space:]]*#' \ | grep -v 'sudo\.log' \ | grep -v 'sudoers' \ + | grep -v 'update-alternatives' \ | grep -v 'logrotate\.d/sudo' || true) if [ -n "$matches" ]; then echo "::error::Found sudo command invocations in CSE provisioning scripts." diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 4012ad10ec4..2858c50af13 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -117,12 +117,12 @@ parameters: displayName: Build 2204 TL Gen2 Containerd type: boolean default: true - - name: build2204minimalgen1containerd - displayName: Build 2204 Minimal Gen1 Containerd + - name: build2604minimalgen2containerd + displayName: Build 2604 Minimal Gen2 Containerd type: boolean default: false - - name: build2204minimalgen2containerd - displayName: Build 2204 Minimal Gen2 Containerd + - name: build2604minimalarm64gen2containerd + displayName: Build 2604 Minimal ARM64 Gen2 Containerd type: boolean default: false - name: buildacltlgen2 @@ -763,21 +763,21 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2204-tl-gen2-containerd - - job: build2204minimalgen1containerd - condition: eq('${{ parameters.build2204minimalgen1containerd }}', true) + - job: build2604minimalgen2containerd + condition: eq('${{ parameters.build2604minimalgen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal' echo '##vso[task.setvariable variable=IMG_VERSION]latest' - echo '##vso[task.setvariable variable=HYPERV_GENERATION]V1' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' @@ -786,23 +786,23 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen1-containerd - - job: build2204minimalgen2containerd - condition: eq('${{ parameters.build2204minimalgen2containerd }}', true) + artifactName: 2604-minimal-gen2-containerd + - job: build2604minimalarm64gen2containerd + condition: eq('${{ parameters.build2604minimalarm64gen2containerd }}', true) dependsOn: [ ] timeoutInMinutes: 360 steps: - bash: | echo '##vso[task.setvariable variable=OS_SKU]Ubuntu' - echo '##vso[task.setvariable variable=OS_VERSION]22.04' + echo '##vso[task.setvariable variable=OS_VERSION]26.04' echo '##vso[task.setvariable variable=IMG_PUBLISHER]Canonical' - echo '##vso[task.setvariable variable=IMG_OFFER]0001-com-ubuntu-minimal-jammy-aks-daily-preview' - echo '##vso[task.setvariable variable=IMG_SKU]minimal-aks-22_04-daily-lts-gen2' + echo '##vso[task.setvariable variable=IMG_OFFER]ubuntu-26_04-lts' + echo '##vso[task.setvariable variable=IMG_SKU]minimal-arm64' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' - echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v5' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal' + echo '##vso[task.setvariable variable=ARCHITECTURE]ARM64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' displayName: Setup Build Variables @@ -810,7 +810,7 @@ stages: parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} - artifactName: 2204-minimal-gen2-containerd + artifactName: 2604-minimal-arm64-gen2-containerd - job: buildacltlgen2 condition: eq('${{ parameters.buildacltlgen2 }}', true) dependsOn: [ ] diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 730505b375f..fd758bb9f59 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -75,12 +75,12 @@ steps: - bash: | SKU_NAME=${OS_VERSION} && \ + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ - if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ if [[ ${OS_SKU} != "CBLMariner" && ${OS_SKU} != "AzureLinux" && ${OS_SKU} != "AzureLinuxOSGuard" && ${OS_SKU} != "Flatcar" && ${OS_SKU} != "AzureContainerLinux" ]]; then SKU_NAME="${SKU_NAME}containerd"; fi && \ SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ diff --git a/e2e/aks_model.go b/e2e/aks_model.go index 28e75f1c1a7..45a80ea1e40 100644 --- a/e2e/aks_model.go +++ b/e2e/aks_model.go @@ -22,57 +22,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" ) -// getLatestGAKubernetesVersion returns the highest GA Kubernetes version for the given location. -func getLatestGAKubernetesVersion(ctx context.Context, location string) (string, error) { - versions, err := config.Azure.AKS.ListKubernetesVersions(context.Background(), location, nil) - if err != nil { - return "", fmt.Errorf("failed to list Kubernetes versions: %w", err) - } - if len(versions.Values) == 0 { - return "", fmt.Errorf("no Kubernetes versions available") - } - - var latestPatchVersion string - msg := fmt.Sprintf("Available Kubernetes versions for location %s:\n", location) - defer func() { toolkit.Logf(ctx, "%s", msg) }() - // Iterate through the available versions to find the latest GA version - for _, k8sVersion := range versions.Values { - if k8sVersion == nil { - continue - } - msg += fmt.Sprintf("- %s\n", *k8sVersion.Version) - - // Skip preview versions - if k8sVersion.IsPreview != nil && *k8sVersion.IsPreview { - msg += " - - is in preview, skipping\n" - continue - } - for patchVersion := range k8sVersion.PatchVersions { - if patchVersion == "" { - continue - } - msg += fmt.Sprintf(" - - %s\n", patchVersion) - // Initialize latestVersion with first GA version found - if latestPatchVersion == "" { - latestPatchVersion = patchVersion - msg += fmt.Sprintf(" - - first latest found, updating to: %s\n", latestPatchVersion) - continue - } - // Compare versions - if agent.IsKubernetesVersionGe(patchVersion, latestPatchVersion) { - latestPatchVersion = patchVersion - msg += fmt.Sprintf(" - - new latest found, updating to: %s\n", latestPatchVersion) - } - } - } - - if latestPatchVersion == "" { - return "", fmt.Errorf("no GA Kubernetes version found") - } - msg += fmt.Sprintf("Latest GA Kubernetes version for location %s: %s\n", location, latestPatchVersion) - return latestPatchVersion, nil -} - // getLatestKubernetesVersionClusterModel returns a cluster model with the latest GA Kubernetes version. func getLatestKubernetesVersionClusterModel(ctx context.Context, name, location, k8sSystemPoolSKU string) (*armcontainerservice.ManagedCluster, error) { version, err := getLatestGAKubernetesVersion(ctx, location) @@ -85,26 +34,52 @@ func getLatestKubernetesVersionClusterModel(ctx context.Context, name, location, } func getKubenetClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { - model := getBaseClusterModel(name, location, k8sSystemPoolSKU) + return kubenetClusterModelMutator(getBaseClusterModel(name, location, k8sSystemPoolSKU)) +} + +func getAzureOverlayNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { + return azureOverlayNetworkClusterModelMutator(getBaseClusterModel(name, location, k8sSystemPoolSKU)) +} + +func getAzureOverlayNetworkDualStackClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { + return azureOverlayNetworkClusterModelMutator(getBaseClusterModel(name, location, k8sSystemPoolSKU)) +} + +func getAzureNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { + return azureNetworkClusterModelMutator(getBaseClusterModel(name, location, k8sSystemPoolSKU)) +} + +func getCiliumNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { + return ciliumNetworkClusterModelMutator(getBaseClusterModel(name, location, k8sSystemPoolSKU)) +} + +func kubenetClusterModelMutator(model *armcontainerservice.ManagedCluster) *armcontainerservice.ManagedCluster { model.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginKubenet) return model } -func getAzureOverlayNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { - model := getBaseClusterModel(name, location, k8sSystemPoolSKU) +func azureNetworkClusterModelMutator(model *armcontainerservice.ManagedCluster) *armcontainerservice.ManagedCluster { model.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure) - model.Properties.NetworkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay) + if model.Properties.AgentPoolProfiles != nil { + for _, app := range model.Properties.AgentPoolProfiles { + app.MaxPods = to.Ptr[int32](30) + } + } return model } -func getAzureOverlayNetworkDualStackClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { - model := getAzureOverlayNetworkClusterModel(name, location, k8sSystemPoolSKU) +func azureOverlayNetworkClusterModelMutator(model *armcontainerservice.ManagedCluster) *armcontainerservice.ManagedCluster { + model.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure) + model.Properties.NetworkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay) + return model +} +func azureOverlayNetworkDualStackClusterModelMutator(model *armcontainerservice.ManagedCluster) *armcontainerservice.ManagedCluster { + model = azureOverlayNetworkClusterModelMutator(model) model.Properties.NetworkProfile.IPFamilies = []*armcontainerservice.IPFamily{ to.Ptr(armcontainerservice.IPFamilyIPv4), to.Ptr(armcontainerservice.IPFamilyIPv6), } - networkProfile := model.Properties.NetworkProfile networkProfile.PodCidr = to.Ptr("10.244.0.0/16") networkProfile.PodCidrs = []*string{ @@ -116,31 +91,19 @@ func getAzureOverlayNetworkDualStackClusterModel(name, location, k8sSystemPoolSK networkProfile.ServiceCidr, to.Ptr("fd12:3456:789a:1::/108"), } - return model } -func getAzureNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { - cluster := getBaseClusterModel(name, location, k8sSystemPoolSKU) - cluster.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure) - if cluster.Properties.AgentPoolProfiles != nil { - for _, app := range cluster.Properties.AgentPoolProfiles { - app.MaxPods = to.Ptr[int32](30) - } - } - return cluster -} -func getCiliumNetworkClusterModel(name, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { - cluster := getBaseClusterModel(name, location, k8sSystemPoolSKU) - cluster.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure) - cluster.Properties.NetworkProfile.NetworkDataplane = to.Ptr(armcontainerservice.NetworkDataplaneCilium) - cluster.Properties.NetworkProfile.NetworkPolicy = to.Ptr(armcontainerservice.NetworkPolicyCilium) - if cluster.Properties.AgentPoolProfiles != nil { - for _, app := range cluster.Properties.AgentPoolProfiles { +func ciliumNetworkClusterModelMutator(model *armcontainerservice.ManagedCluster) *armcontainerservice.ManagedCluster { + model.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginAzure) + model.Properties.NetworkProfile.NetworkDataplane = to.Ptr(armcontainerservice.NetworkDataplaneCilium) + model.Properties.NetworkProfile.NetworkPolicy = to.Ptr(armcontainerservice.NetworkPolicyCilium) + if model.Properties.AgentPoolProfiles != nil { + for _, app := range model.Properties.AgentPoolProfiles { app.MaxPods = to.Ptr[int32](30) } } - return cluster + return model } func getBaseClusterModel(clusterName, location, k8sSystemPoolSKU string) *armcontainerservice.ManagedCluster { @@ -192,6 +155,57 @@ func getBaseClusterModel(clusterName, location, k8sSystemPoolSKU string) *armcon } } +// getLatestGAKubernetesVersion returns the highest GA Kubernetes version for the given location. +func getLatestGAKubernetesVersion(ctx context.Context, location string) (string, error) { + versions, err := config.Azure.AKS.ListKubernetesVersions(context.Background(), location, nil) + if err != nil { + return "", fmt.Errorf("failed to list Kubernetes versions: %w", err) + } + if len(versions.Values) == 0 { + return "", fmt.Errorf("no Kubernetes versions available") + } + + var latestPatchVersion string + msg := fmt.Sprintf("Available Kubernetes versions for location %s:\n", location) + defer func() { toolkit.Logf(ctx, "%s", msg) }() + // Iterate through the available versions to find the latest GA version + for _, k8sVersion := range versions.Values { + if k8sVersion == nil { + continue + } + msg += fmt.Sprintf("- %s\n", *k8sVersion.Version) + + // Skip preview versions + if k8sVersion.IsPreview != nil && *k8sVersion.IsPreview { + msg += " - - is in preview, skipping\n" + continue + } + for patchVersion := range k8sVersion.PatchVersions { + if patchVersion == "" { + continue + } + msg += fmt.Sprintf(" - - %s\n", patchVersion) + // Initialize latestVersion with first GA version found + if latestPatchVersion == "" { + latestPatchVersion = patchVersion + msg += fmt.Sprintf(" - - first latest found, updating to: %s\n", latestPatchVersion) + continue + } + // Compare versions + if agent.IsKubernetesVersionGe(patchVersion, latestPatchVersion) { + latestPatchVersion = patchVersion + msg += fmt.Sprintf(" - - new latest found, updating to: %s\n", latestPatchVersion) + } + } + } + + if latestPatchVersion == "" { + return "", fmt.Errorf("no GA Kubernetes version found") + } + msg += fmt.Sprintf("Latest GA Kubernetes version for location %s: %s\n", location, latestPatchVersion) + return latestPatchVersion, nil +} + func getFirewall(ctx context.Context, location, firewallSubnetID, publicIPID string) *armnetwork.AzureFirewall { var ( natRuleCollections []*armnetwork.AzureFirewallNatRuleCollection diff --git a/e2e/cache.go b/e2e/cache.go index 76514bbb5be..343f3398c1e 100644 --- a/e2e/cache.go +++ b/e2e/cache.go @@ -146,13 +146,46 @@ var ClusterLatestKubernetesVersion = cachedFunc(clusterLatestKubernetesVersion) // clusterLatestKubernetesVersion creates a cluster with the latest available Kubernetes version func clusterLatestKubernetesVersion(ctx context.Context, request ClusterRequest) (*Cluster, error) { - model, err := getLatestKubernetesVersionClusterModel(ctx, "abe2e-latest-kubernetes-version-v2", request.Location, request.K8sSystemPoolSKU) + model, err := getLatestKubernetesVersionClusterModel(ctx, "abe2e-latest-k8s-v2", request.Location, request.K8sSystemPoolSKU) if err != nil { return nil, fmt.Errorf("getting latest kubernetes version cluster model: %w", err) } return prepareCluster(ctx, model, false, false) } +var ClusterLatestKubernetesVersionKubenet = cachedFunc(clusterLatestKubernetesVersionKubenet) + +func clusterLatestKubernetesVersionKubenet(ctx context.Context, request ClusterRequest) (*Cluster, error) { + model, err := getLatestKubernetesVersionClusterModel(ctx, "abe2e-latest-k8s-kubenet-v1", request.Location, request.K8sSystemPoolSKU) + if err != nil { + return nil, fmt.Errorf("getting latest kubernetes version cluster model: %w", err) + } + model = kubenetClusterModelMutator(model) + return prepareCluster(ctx, model, false, false) +} + +var ClusterLatestKubernetesVersionAzureNetwork = cachedFunc(clusterLatestKubernetesVersionAzureNetwork) + +func clusterLatestKubernetesVersionAzureNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) { + model, err := getLatestKubernetesVersionClusterModel(ctx, "abe2e-latest-k8s-azure-network-v1", request.Location, request.K8sSystemPoolSKU) + if err != nil { + return nil, fmt.Errorf("getting latest kubernetes version cluster model: %w", err) + } + model = azureNetworkClusterModelMutator(model) + return prepareCluster(ctx, model, false, false) +} + +var ClusterLatestKubernetesVersionAzureOverlayNetworkDualStack = cachedFunc(clusterLatestKubernetesVersionAzureOverlayNetworkDualStack) + +func clusterLatestKubernetesVersionAzureOverlayNetworkDualStack(ctx context.Context, request ClusterRequest) (*Cluster, error) { + model, err := getLatestKubernetesVersionClusterModel(ctx, "abe2e-latest-k8s-azure-overlay-dualstack-v1", request.Location, request.K8sSystemPoolSKU) + if err != nil { + return nil, fmt.Errorf("getting latest kubernetes version cluster model: %w", err) + } + model = azureOverlayNetworkDualStackClusterModelMutator(model) + return prepareCluster(ctx, model, false, false) +} + var ClusterKubenet = cachedFunc(clusterKubenet) // clusterKubenet creates a basic cluster using kubenet networking with shared VNet @@ -196,7 +229,6 @@ func clusterAzureOverlayNetwork(ctx context.Context, request ClusterRequest) (*C var ClusterAzureOverlayNetworkDualStack = cachedFunc(clusterAzureOverlayNetworkDualStack) -// clusterAzureOverlayNetworkDualStack creates a dual-stack (IPv4+IPv6) Azure CNI Overlay cluster func clusterAzureOverlayNetworkDualStack(ctx context.Context, request ClusterRequest) (*Cluster, error) { model := getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v6", request.Location, request.K8sSystemPoolSKU) return prepareCluster(ctx, model, false, false) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 61f44345e85..a79b7257e97 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -171,6 +171,22 @@ var ( Gallery: imageGalleryLinux, } + VHDUbuntu2604MinimalGen2Containerd = &Image{ + Name: "2604minimalgen2containerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuMinimalContainerd2604Gen2, + Gallery: imageGalleryLinux, + } + + VHDUbuntu2604MinimalArm64Gen2Containerd = &Image{ + Name: "2604minimalgen2arm64containerd", + OS: OSUbuntu, + Arch: "arm64", + Distro: datamodel.AKSUbuntuMinimalArm64Containerd2604Gen2, + Gallery: imageGalleryLinux, + } + VHDAzureLinuxV3Gen2Arm64 = &Image{ Name: "azurelinuxv3gen2arm64", OS: OSAzureLinux, diff --git a/e2e/e2e-local.sh b/e2e/e2e-local.sh index 49f97be5f0e..141c2090254 100755 --- a/e2e/e2e-local.sh +++ b/e2e/e2e-local.sh @@ -5,6 +5,12 @@ set -euxo pipefail : "${TIMEOUT:=90m}" : "${PARALLEL:=100}" +if [ -n "${VHD_BUILD_ID:-}" ]; then + echo "VHD_BUILD_ID is specified (${VHD_BUILD_ID}). Running tests using VHDs from that build" + export SIG_VERSION_TAG_NAME=buildId + export SIG_VERSION_TAG_VALUE=$VHD_BUILD_ID +fi + go version # Note, if you run "go test ./..." you won't see the output of the tests until they finish. # -count 1 disables caching of test results diff --git a/e2e/scenario_localdns_hosts_test.go b/e2e/scenario_localdns_hosts_test.go index 5363d92d503..c83eba792a9 100644 --- a/e2e/scenario_localdns_hosts_test.go +++ b/e2e/scenario_localdns_hosts_test.go @@ -24,6 +24,7 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { }{ {name: "Ubuntu2204", vhd: config.VHDUbuntu2204Gen2Containerd}, {name: "Ubuntu2404", vhd: config.VHDUbuntu2404Gen2Containerd}, + {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) @@ -32,10 +33,14 @@ func Test_LocalDNSHostsPlugin(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + cluster := ClusterKubenet + if tt.name == "Ubuntu2604Minimal" { + cluster = ClusterLatestKubernetesVersionKubenet + } RunScenario(t, &Scenario{ Description: "Tests that localdns hosts plugin works correctly on " + tt.name, Config: Config{ - Cluster: ClusterKubenet, + Cluster: cluster, VHD: tt.vhd, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.LocalDNSProfile.EnableHostsPlugin = true diff --git a/e2e/scenario_rcv1p_test.go b/e2e/scenario_rcv1p_test.go index f6e2ea6c3b9..f6476bd4361 100644 --- a/e2e/scenario_rcv1p_test.go +++ b/e2e/scenario_rcv1p_test.go @@ -310,7 +310,7 @@ func rcv1pOptInVMConfigMutator(vmss *armcompute.VirtualMachineScaleSet) { func Test_RCV1P_Ubuntu2204(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 22.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -325,13 +325,34 @@ func Test_RCV1P_Ubuntu2204(t *testing.T) { }) } +// Test_RCV1P_Ubuntu2604Minimal validates RCV1P cert download and trust store installation on Ubuntu 26.04 minimal. +// Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration +// work correctly across Ubuntu versions. +func Test_RCV1P_Ubuntu2604Minimal(t *testing.T) { + skipIfRCV1PNotConfigured(t) + RunScenario(t, &Scenario{ + Description: "Tests RCV1P cert mode on Ubuntu 26.04 minimal with VM opt-in tag", + Tags: Tags{ + RCV1PCertMode: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: rcv1pVMConfigMutator(), + Validator: func(ctx context.Context, s *Scenario) { + ValidateRCV1PCertMode(ctx, s) + }, + }, + }) +} + // Test_RCV1P_Ubuntu2404 validates RCV1P cert download and trust store installation on Ubuntu 24.04. // Covers the newer Ubuntu LTS release to ensure the cert endpoint and trust store integration // work correctly across Ubuntu versions. func Test_RCV1P_Ubuntu2404(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Ubuntu 24.04 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -352,7 +373,7 @@ func Test_RCV1P_Ubuntu2404(t *testing.T) { func Test_RCV1P_AzureLinuxV3(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", + Description: "Tests RCV1P cert mode on Azure Linux V3 with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -373,7 +394,7 @@ func Test_RCV1P_AzureLinuxV3(t *testing.T) { func Test_RCV1P_ACL(t *testing.T) { skipIfRCV1PNotConfigured(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", + Description: "Tests RCV1P cert mode on ACL with VM opt-in tag", Tags: Tags{ RCV1PCertMode: true, }, @@ -403,7 +424,7 @@ func Test_RCV1P_ACL(t *testing.T) { func Test_RCV1P_NotOptedIn(t *testing.T) { skipIfRCV1PNotExplicit(t) RunScenario(t, &Scenario{ - Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", + Description: "Tests RCV1P cert mode without VM opt-in tag; expects no cert installation", Tags: Tags{ RCV1PCertMode: true, }, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 906eab5facb..3197955e346 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1847,6 +1847,244 @@ func Test_Ubuntu2404Gen2(t *testing.T) { }) } +func Test_Ubuntu2604Minimal(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + // TODO(2604): use latest (1.36) until 1.36 becomes default in test regions since 26.04 requires 1.36+ - applies to all Ubuntu2604Minimal E2E tests + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + // ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} +func Test_Ubuntu2604Minimal_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionAzureNetwork, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + // ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + // configureSecondaryNICs is new and not yet baked into released VHDs. + // The scriptless_nbc path always uses VHD scripts (DisableCustomData=true), + // so it can't pick up the new function until the next VHD release. + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + addSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecondaryNIC_DualStack(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 26.04 minimal", + Config: Config{ + Cluster: ClusterLatestKubernetesVersionAzureOverlayNetworkDualStack, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.StandardSecondaryNICCount = 1 + if nbc.ContainerService.Properties.FeatureFlags == nil { + nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{} + } + nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone) + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay" + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID + nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + DualStackVMConfigMutator(vmss) + addDualStackSecondaryNIC(vmss) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6: true") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6-overrides:") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200") + ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false") + ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s)) + }, + }, + }) +} + +func Test_Ubuntu2604Minimal_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using an Ubuntu 2604 minimal Gen2 VHD can be properly bootstrapped even if secure TLS bootstrapping fails", + Tags: Tags{ + BootstrapTokenFallback: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalGen2Containerd, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{ + Enabled: true, + GetAccessTokenTimeout: (10 * time.Second).String(), + UserAssignedIdentityID: "invalid", // use an unexpected user-assigned identity ID to force a secure TLS bootstrapping failure + } + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") + }, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + // ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_AzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Ubuntu 2604 minimal ARM64 VHD can be properly bootstrapped with containerd v2 on an Azure CNI cluster", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionAzureNetwork, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") + }, + Validator: func(ctx context.Context, s *Scenario) { + containerdVersions := components.GetExpectedPackageVersions("containerd", "ubuntu", "r2604") + runcVersions := components.GetExpectedPackageVersions("runc", "ubuntu", "r2604") + ValidateContainerd2Properties(ctx, s, containerdVersions) + ValidateRuncVersion(ctx, s, runcVersions) + ValidateContainerRuntimePlugins(ctx, s) + // ValidateInstalledPackageVersion(ctx, s, "blobfuse2", components.GetExpectedPackageVersions("blobfuse2", "ubuntu", "r2604")[0]) + ValidateSSHServiceEnabled(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_NPD_Basic(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Test that a node using Ubuntu 26.04 minimal ARM64 with AKS VM Extension enabled can report simulated node problem detector events", + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") + extension, err := createVMExtensionLinuxAKSNode(t.Context(), vmss.Location) + require.NoError(t, err, "creating AKS VM extension") + vmss.Properties = addVMExtensionToVMSS(vmss.Properties, extension) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNodeProblemDetector(ctx, s) + ValidateNPDFilesystemCorruption(ctx, s) + }, + }, + }) +} + +func Test_Ubuntu2604MinimalArm64_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using an Ubuntu 2604 minimal ARM64 Gen2 VHD can be properly bootstrapped even if secure TLS bootstrapping fails", + Tags: Tags{ + BootstrapTokenFallback: true, + }, + Config: Config{ + Cluster: ClusterLatestKubernetesVersionKubenet, + VHD: config.VHDUbuntu2604MinimalArm64Gen2Containerd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") + }, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{ + Enabled: true, + GetAccessTokenTimeout: (10 * time.Second).String(), + UserAssignedIdentityID: "invalid", // use an unexpected user-assigned identity ID to force a secure TLS bootstrapping failure + } + }, + }, + }) +} + func Test_Ubuntu2404Gen2_McrChinaCloud(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ @@ -2085,11 +2323,11 @@ func Test_Ubuntu2404_NPD_Basic(t *testing.T) { } func Test_Ubuntu2404_GPU_H100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96isr_H100_v5", "uaenorth", "")) } func Test_Ubuntu2404_GPU_A100(t *testing.T) { - RunScenario(t, runScenarioGPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) + RunScenario(t, runScenarioUbuntu2404GPUNPD(t, "Standard_ND96asr_v4", "southcentralus", "Standard_D2s_v3")) } func Test_AzureLinux3_PMC_Install(t *testing.T) { diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index b5a4ac2e1ae..d2df3773e63 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -1032,7 +1032,7 @@ func attemptSSHConnection(ctx context.Context, s *Scenario) error { return nil } -func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { +func runScenarioUbuntu2404GPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) *Scenario { t.Helper() return &Scenario{ Description: fmt.Sprintf("Tests that a GPU-enabled node with VM size %s using an Ubuntu 2404 VHD can be properly bootstrapped and NPD tests are valid", vmSize), diff --git a/e2e/validators.go b/e2e/validators.go index abe8f2a3ab3..e9c5c449fa6 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -989,7 +989,7 @@ func ValidateNoFailedSystemdUnits(ctx context.Context, s *Scenario) { Name string `json:"unit,omitempty"` } var failedUnits []systemdUnit - result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, fmt.Sprintf("unable to list failed systemd units")) + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, "unable to list failed systemd units") assert.NoError(s.T, json.Unmarshal([]byte(result.stdout), &failedUnits), `unable to parse and unmarshal "systemctl list-units" command output`) failedUnits = lo.Filter(failedUnits, func(unit systemdUnit, _ int) bool { if unitFailureAllowList[unit.Name] { @@ -2958,8 +2958,11 @@ func ValidateKernelLogs(ctx context.Context, s *Scenario) { patterns := map[string]categoryPattern{ "PANIC/CRASH": { pattern: `(kernel: )?(panic|oops|call trace|backtrace|general protection fault|BUG:|RIP:)`, - // exclude boot parameters like "panic=-1" and dm-verity's "panic-on-corruption" (used by ACL for verified boot) - exclude: `panic[-=]`, + // exclude boot parameters like "panic=-1" and dm-verity's "panic-on-corruption" (used by ACL for verified boot). + // Also exclude the benign DRM panic-handler registration messages ("Registered N planes with drm panic") + // emitted by simple-framebuffer/hyperv_drm on kernels with CONFIG_DRM_PANIC (6.10+, e.g. Ubuntu 26.04 / + // linux-azure 7.0). These log the drm_panic screen handler registering its planes at boot, not a kernel panic. + exclude: `panic[-=]|drm panic`, }, "LOCKUP/STALL": {pattern: `(soft|hard) lockup|rcu.*(stall|detected stalls)|hung task|watchdog.*(detected|stuck)`}, "MEMORY": {pattern: `oom[- ]killer|Out of memory:|page allocation failure|memory corruption`}, diff --git a/parts/common/components.json b/parts/common/components.json index d7a63da8b4a..acd21b59e15 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -953,6 +953,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\aks-secure-tls-bootstrap-client\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.1.4-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1055,6 +1063,14 @@ "latestVersion": "1.4.3-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-runc, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.4.3-ubuntu26.04u2" + } + ] } } } @@ -1065,6 +1081,14 @@ "windowsDownloadLocation": "c:\\akse-cache\\containerd\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=moby-containerd, repository=production, os=ubuntu, release=26.04", + "latestVersion": "2.3.2-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1254,6 +1278,14 @@ "latestVersion": "1.9.0-ubuntu24.04u1" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=containernetworking-plugins, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.0-ubuntu26.04u3" + } + ] } }, "azurelinux": { @@ -1357,6 +1389,14 @@ "latestVersion": "1.34.0-ubuntu24.04u5" } ] + }, + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=kubernetes-cri-tools, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.34.0-ubuntu26.04u5" + } + ] } }, "mariner": { @@ -1415,6 +1455,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubelet\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u2", + "previousLatestVersion": "1.36.1-ubuntu26.04u3" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1513,6 +1563,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\kubectl\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.2-ubuntu26.04u5", + "previousLatestVersion": "1.36.1-ubuntu26.04u9" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1664,6 +1724,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/kubernetes-node:${version}-linux-${CPU_ARCH}", "windowsDownloadURL": "https://packages.aks.azure.com/kubernetes/${version}/windowszip/${version}-1int.zip" } + }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } } } }, @@ -1703,6 +1768,11 @@ "downloadURL": "mcr.microsoft.com/oss/binaries/kubernetes/azure-acr-credential-provider:${version}-linux-${CPU_ARCH}" } }, + "ubuntu": { + "r2604": { + "versionsV2": [] + } + }, "flatcar": { "current": { "versionsV2": [] @@ -1715,6 +1785,14 @@ "downloadLocation": "/opt/azure/ig", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=ig, repository=production, os=ubuntu, release=26.04", + "latestVersion": "0.53.2-ubuntu26.04u4" + } + ] + }, "r2404": { "versionsV2": [ { @@ -1758,6 +1836,16 @@ "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "k8sVersion": "1.36", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.36.3-ubuntu26.04u2", + "previousLatestVersion": "1.36.2-ubuntu26.04u2" + } + ] + }, "r2404": { "versionsV2": [ { @@ -2132,6 +2220,14 @@ "downloadLocation": "/opt/node-exporter", "downloadURIs": { "ubuntu": { + "r2604": { + "versionsV2": [ + { + "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=26.04", + "latestVersion": "1.9.1-ubuntu26.04u21" + } + ] + }, "r2404": { "versionsV2": [ { diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index 07d128156c3..c09c023d9a5 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -88,6 +88,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh index 9193afdfde3..0874dde7d3e 100644 --- a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh +++ b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh @@ -89,6 +89,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/cis.sh b/parts/linux/cloud-init/artifacts/cis.sh index dcdd110c9f6..628494ac193 100644 --- a/parts/linux/cloud-init/artifacts/cis.sh +++ b/parts/linux/cloud-init/artifacts/cis.sh @@ -8,8 +8,18 @@ assignRootPW() { VERSION=$(grep DISTRIB_RELEASE /etc/*-release | cut -f 2 -d "=") SALT=$(openssl rand -base64 5) SECRET=$(openssl rand -base64 37) + CMD="import crypt, getpass, pwd; print(crypt.crypt('$SECRET', '\$6\$$SALT\$'))" - if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ]; then + if [ "${VERSION}" = "26.04" ]; then + # Ubuntu 26.04 ships Python 3.14, which removed the 'crypt' module (removed in 3.13). + # passlib's sha512_crypt has a pure-Python backend that produces the same SHA-512 ($6$) + # hash without needing the crypt module. rounds=5000 matches the crypt(3) default cost. + # SALT is base64 and may contain chars (e.g. '+', '=') outside passlib's salt alphabet, + # so filter it to sha512_crypt.salt_chars before use. + CMD="import getpass, pwd; from passlib.hash import sha512_crypt; salt=''.join(c for c in '$SALT' if c in sha512_crypt.salt_chars); print(sha512_crypt.using(salt=salt, rounds=5000).hash('$SECRET'))" + fi + + if [ "${VERSION}" = "22.04" ] || [ "${VERSION}" = "24.04" ] || [ "${VERSION}" = "26.04" ]; then HASH=$(python3 -c "$CMD") else HASH=$(python -c "$CMD") @@ -269,7 +279,34 @@ EOF systemctl restart ssh } +# Ensures that the classic (Todd Miller) sudo is the selected sudo implementation +# rather than sudo-rs. On Ubuntu 26.04 sudo-rs is installed and enabled +# by default, but sudo-rs does NOT support the `Defaults logfile` sudoers setting that +# configureSudo relies on -- per the sudo-rs docs, "logfile is not supported; logging +# is always done via syslog". Leaving sudo-rs as the default would therefore be a CIS +# regression: we would lose the dedicated /var/log/sudo.log audit trail, and sudo-rs +# would also print "unknown setting: 'logfile'" on every sudo invocation. Both classic +# sudo and sudo-rs ship on the base image and are switchable via update-alternatives, +# so select classic sudo here so the logfile setting below is actually honored. +# This is a no-op on releases where sudo is not an update-alternatives group (e.g. +# 22.04/24.04, which only have classic sudo). +# TODO: revisit if/when sudo-rs gains support for file-based logging. +preferClassicSudo() { + # Classic sudo registers itself as the "/usr/bin/sudo.ws" alternative candidate + # (sudo.ws = Todd Miller's sudo). Only relevant where sudo is update-alternatives + # managed (i.e. sudo-rs is present); a no-op on releases that ship only classic sudo. + local classic_sudo="/usr/bin/sudo.ws" + if [ -x "${classic_sudo}" ] && update-alternatives --list sudo >/dev/null 2>&1; then + update-alternatives --set sudo "${classic_sudo}" >/dev/null 2>&1 || echo "warning: failed to set ${classic_sudo} as the default sudo implementation" + fi +} + configureSudo() { + # sudo-rs (the default on Ubuntu 26.04) can only log to syslog and rejects the + # `Defaults logfile` setting below. Select classic sudo first so CIS file-based + # sudo logging works; see preferClassicSudo for details. + preferClassicSudo + cat </etc/sudoers.d/99-cis Defaults logfile="/var/log/sudo.log" EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 330988834ac..c5c2a350b18 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -321,8 +321,8 @@ disableSystemdResolved() { ls -ltr /etc/resolv.conf cat /etc/resolv.conf UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") - if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ]; then - echo "Ingoring systemd-resolved query service but using its resolv.conf file" + if [ "${UBUNTU_RELEASE}" = "20.04" ] || [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then + echo "Ignoring systemd-resolved query service but using its resolv.conf file" echo "This is the simplest approach to workaround resolved issues without completely uninstall it" [ -f /run/systemd/resolve/resolv.conf ] && ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf ls -ltr /etc/resolv.conf diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 77bffdc48f0..27260f8ac20 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -171,11 +171,11 @@ ERR_SECONDARY_NIC_CONFIG_FAIL=243 # Error configuring secondary NIC network inte # For unit tests, the OS and OS_VERSION will be set in the unit test script. # So whether it's if or else actually doesn't matter to our unit test. if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then - OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') - OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') - OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') + OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') + OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') else -# This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. + # This is only for unit test purpose. For example, a Mac OS dev box doesn't have /etc/*-release, then the unit test will continue. echo "/etc/*-release not found" fi @@ -970,6 +970,13 @@ isUbuntu() { return 1 } +isMinimalImage() { + if grep -q "minimal" <<< "$FEATURE_FLAGS"; then + return 0 + fi + return 1 +} + evalPackageDownloadURL() { local url=${1:-} if [ -n "$url" ]; then diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index e288f501ac5..f3331d66864 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -15,8 +15,8 @@ K8S_PRIVATE_PACKAGES_CACHE_DIR="/opt/kubernetes/downloads/private-packages" K8S_REGISTRY_REPO="oss/binaries/kubernetes" UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "") # For Mariner 2.0, this returns "MARINER" and for AzureLinux 3.0, this returns "AZURELINUX" -OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) -OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi) +OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) +OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]'; fi) SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_DIR="/opt/aks-secure-tls-bootstrap-client/downloads" SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/opt/bin" CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads" diff --git a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh index cf45aba65d3..fbdea58bf58 100755 --- a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh +++ b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh @@ -73,6 +73,10 @@ installDeps() { stub } +installMinimalBuildDeps() { + stub +} + installCriCtlPackage() { stub } diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh index 83d2c6425ad..3ca452e3926 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh @@ -674,6 +674,10 @@ cleanUpGPUDrivers() { done } +installMinimalBuildDeps() { + echo "installMinimalBuildDeps not implemented for mariner" +} + downloadContainerdFromVersion() { echo "downloadContainerdFromVersion not implemented for mariner" } diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index e72ebb3a9fd..0873c8e8fac 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -4,6 +4,33 @@ removeContainerd() { apt_get_purge 10 5 300 moby-containerd } +# Batch install all packages in a single apt_get_install call instead of looping one-by-one. +# On failure, fall back to individual installs for diagnostic clarity. A return code of 2 from +# apt_get_install signals a CSE timeout and is propagated immediately by exiting the script. +aptGetBatchInstallPackagesWithFallback() { + local -a pkg_list=("$@") + + apt_get_install 30 1 600 "${pkg_list[@]}" + local batch_rc=$? + if [ "$batch_rc" -eq 2 ]; then + exit "$batch_rc" + elif [ "$batch_rc" -ne 0 ]; then + echo "Batch install failed, falling back to individual package install" + local apt_package + for apt_package in "${pkg_list[@]}"; do + apt_get_install 30 1 600 "$apt_package" + local pkg_rc=$? + if [ "$pkg_rc" -eq 2 ]; then + exit "$pkg_rc" + elif [ "$pkg_rc" -ne 0 ]; then + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + done + fi +} + blobfuseFallbackPackages() { local OSVERSION="${1}" # blobfuse/blobfuse2 started to be centralized in components.json around April 2026. @@ -40,6 +67,33 @@ blobfuseFallbackPackages() { fi } +# Installs any required dependencies needed to build the particular Ubuntu minimal image (currently only 26.04) +# These dependencies are needed specifically in order to run various commands required to build the VHD. +installMinimalBuildDeps() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + + if [ "${OSVERSION}" = "26.04" ]; then + installUbuntu2604MinimalBuildDeps + return 0 + fi + + echo "Unrecognized Ubuntu minimal version ${OSVERSION} - cannot install minimal build dependencies" + exit 1 +} + +installUbuntu2604MinimalBuildDeps() { + wait_for_apt_locks + retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT + retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL + + holdWALinuxAgent hold + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + + local -a pkg_list=(rsyslog gpg) + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" +} + installDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT @@ -48,44 +102,39 @@ installDeps() { holdWALinuxAgent hold apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r) linux-modules-extra-$(uname -r)) + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + + pkg_list=(apparmor-utils bind9-dnsutils ca-certificates ceph-common cgroup-lite cifs-utils conntrack cracklib-runtime ebtables ethtool glusterfs-client htop init-system-helpers inotify-tools iotop iproute2 ipset iptables nftables jq libpam-pwquality libpwquality-tools mount nfs-common pigz socat sysfsutils sysstat util-linux xz-utils netcat-openbsd zip rng-tools kmod gcc make dkms initramfs-tools linux-headers-$(uname -r)) + + if [ "${OSVERSION}" == "26.04" ]; then + if isMinimalImage; then + # libc6-dev is needed for GPU driver installation at runtime and is not included on the 26.04 minimal base image + pkg_list+=(libc6-dev) + # cron/crontab is needed by init-aks-cloud.sh (RCV1P) since we create a ca-refresh cron job and is not included on the 26.04 minimal base image + # init-aks-cloud.sh should be refactored to use systemd timers instead to align with AzureLinux + pkg_list+=(cron) + fi + else + # linux-modules-extra-* content is bundled into linux-modules-* on the 7.x kernel (Ubuntu 26.04) + pkg_list+=(linux-modules-extra-$(uname -r)) + fi - local OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") while IFS= read -r fallback_pkg; do [ -n "${fallback_pkg}" ] && pkg_list+=("${fallback_pkg}") done < <(blobfuseFallbackPackages "${OSVERSION}") - if [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=(irqbalance) fi - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then pkg_list+=("aznfs=3.0.19") fi - # Batch install all packages in a single apt_get_install call instead of - # looping one-by-one. On failure, fall back to individual installs for - # diagnostic clarity. Exit immediately on return code 2 (CSE timeout). - apt_get_install 30 1 600 "${pkg_list[@]}" - local batch_rc=$? - if [ "$batch_rc" -eq 2 ]; then - exit "$batch_rc" - elif [ "$batch_rc" -ne 0 ]; then - echo "Batch install failed, falling back to individual package install" - for apt_package in "${pkg_list[@]}"; do - apt_get_install 30 1 600 "$apt_package" - local pkg_rc=$? - if [ "$pkg_rc" -eq 2 ]; then - exit "$pkg_rc" - elif [ "$pkg_rc" -ne 0 ]; then - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - done - fi + aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" - if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ]; then + if [ "${OSVERSION}" = "22.04" ] || [ "${OSVERSION}" = "24.04" ] || [ "${OSVERSION}" = "26.04" ]; then # disable aznfswatchdog since aznfs install and enable aznfswatchdog and aznfswatchdogv4 services at the same time while we only need aznfswatchdogv4 systemctl disable aznfswatchdog systemctl stop aznfswatchdog @@ -93,6 +142,9 @@ installDeps() { } updateAptWithMicrosoftPkg() { + local OSVERSION + OSVERSION=$(grep DISTRIB_RELEASE /etc/*-release| cut -f 2 -d "=") + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/prod.list > /tmp/microsoft-prod.list || exit $ERR_MOBY_APT_LIST_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit $ERR_MOBY_APT_LIST_TIMEOUT @@ -100,6 +152,13 @@ updateAptWithMicrosoftPkg() { retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OSVERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_silent 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } @@ -132,7 +191,7 @@ updatePMCRepository() { } updateAptWithNvidiaPkg() { - readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.pub" + readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.gpg" mkdir -p "$(dirname "${nvidia_gpg_keyring_path}")" readonly nvidia_sources_list_path="/etc/apt/sources.list.d/nvidia.list" @@ -153,6 +212,8 @@ updateAptWithNvidiaPkg() { nvidia_ubuntu_release="ubuntu2204" elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then nvidia_ubuntu_release="ubuntu2404" + elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_ubuntu_release="ubuntu2604" else echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" return @@ -162,10 +223,21 @@ updateAptWithNvidiaPkg() { echo "deb [arch=${cpu_arch} signed-by=${nvidia_gpg_keyring_path}] https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch} /" > ${nvidia_sources_list_path} # Add NVIDIA repository - local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/3bf863cc.pub" - - # Download and add the GPG key for the NVIDIA repository - retrycmd_curl_file 120 5 25 ${nvidia_gpg_keyring_path} ${nvidia_gpg_key_url} 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + local nvidia_gpg_key_name="3bf863cc.pub" + if [ "${UBUNTU_RELEASE}" = "26.04" ]; then + nvidia_gpg_key_name="60DF8A40.pub" + fi + local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/${nvidia_gpg_key_name}" + + # Download the armored NVIDIA repo key and dearmor it into a binary keyring. + # apt only accepts a signed-by keyring with a .gpg (binary) or .asc (armored) extension; + # NVIDIA publishes an ASCII-armored *.pub, so a raw .pub file is rejected as an "unsupported + # filetype" and the key is ignored (the repo then fails to verify with NO_PUBKEY). Newer apt + # (e.g. 3.x on Ubuntu 26.04) enforces this strictly, so dearmor to nvidia.gpg. + local nvidia_gpg_key_tmp="/tmp/${nvidia_gpg_key_name}" + retrycmd_curl_file 120 5 25 "${nvidia_gpg_key_tmp}" "${nvidia_gpg_key_url}" 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + gpg --dearmor < "${nvidia_gpg_key_tmp}" > "${nvidia_gpg_keyring_path}" || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT + rm -f "${nvidia_gpg_key_tmp}" apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT } @@ -225,9 +297,13 @@ removeNvidiaRepos() { rm -f /etc/apt/sources.list.d/nvidia.list echo "Removed NVIDIA apt repository" fi + if [ -f /etc/apt/keyrings/nvidia.gpg ]; then + rm -f /etc/apt/keyrings/nvidia.gpg + echo "Removed NVIDIA GPG key (nvidia.gpg)" + fi if [ -f /etc/apt/keyrings/nvidia.pub ]; then rm -f /etc/apt/keyrings/nvidia.pub - echo "Removed NVIDIA GPG key" + echo "Removed NVIDIA GPG key (nvidia.pub)" fi } diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 738a9f7751b..675807d74eb 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -810,7 +810,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration return config.GetOrderedKubeproxyConfigStringForPowershell() }, "IsCgroupV2": func() bool { - return profile.Is2204VHDDistro() || profile.Is2404VHDDistro() || + return profile.Is2204VHDDistro() || profile.Is2404VHDDistro() || profile.Is2604VHDDistro() || config.IsAzureLinux() || config.IsFlatcar() || config.IsACL() }, "GetKubeProxyFeatureGatesPsh": func() string { @@ -867,9 +867,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration sb.WriteString(fmt.Sprintf("LimitMEMLOCK=%s\n", ulimitConfig.MaxLockedMemory)) } if ulimitConfig.NoFile != "" { - // ulimit is removed in containerd 2.0+, which is available only in ubuntu2404 distro + // ulimit is removed in containerd 2.0+, which is available only in ubuntu2404/ubuntu2604 distros // https://github.com/containerd/containerd/blob/main/docs/containerd-2.0.md#limitnofile-configuration-has-been-removed - if !profile.Is2404VHDDistro() { + if !profile.Is2404VHDDistro() && !profile.Is2604VHDDistro() { sb.WriteString(fmt.Sprintf("LimitNOFILE=%s\n", ulimitConfig.NoFile)) } } diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index eda6e5c669a..7134494bec6 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1200,7 +1200,7 @@ func (a *AgentPoolProfile) IsContainerdV2Distro() bool { if a.Distro.IsKataDistro() { return false } - return a.Distro.Is2404VHDDistro() || a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() + return a.Distro.Is2604VHDDistro() || a.Distro.Is2404VHDDistro() || a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() } // IsAzureLinuxCgroupV2VHDDistro returns true if the distro uses Azure Linux CgrpupV2 VHD. diff --git a/schemas/components.cue b/schemas/components.cue index 442f736d96e..88cd1a8902a 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -54,6 +54,7 @@ package components r2004?: #ReleaseDownloadURI r2204?: #ReleaseDownloadURI r2404?: #ReleaseDownloadURI + r2604?: #ReleaseDownloadURI } #DefaultOSDistro: { diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 3762d1141c5..1e184e1f725 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -14,13 +14,13 @@ ACL_OS_NAME="AZURECONTAINERLINUX" # Real world examples from the command outputs # For Azure Linux V3: ID=azurelinux VERSION_ID="3.0" # For Azure Linux V2: ID=mariner VERSION_ID="2.0" -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') IS_KATA="false" if grep -q "kata" <<< "$FEATURE_FLAGS"; then IS_KATA="true" fi -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" @@ -38,207 +38,13 @@ CPU_ARCH=$(getCPUArch) #amd64 or arm64 SYSTEMD_ARCH=$(getSystemdArch) # x86-64 or arm64 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete COMPONENTS_FILEPATH=/opt/azure/components.json +LOCALDNS_BINARY_PATH="/opt/azure/containers/localdns/binary" PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json GRID_COMPATIBILITY_DATA_FILE=/opt/azure/vhd-grid-compatibility-data.json -resolve_packages_source_url - -echo "" -echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" - -echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" -# fix grub issue with cvm by reinstalling before other deps -# other VHDs use grub-pc, not grub-efi -if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - wait_for_apt_locks - apt_get_install 10 2 120 grub-efi || exit 1 -fi -capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" - -if [ "$OS" = "$UBUNTU_OS_NAME" ]; then - # disable and mask all UU timers/services - # save some background io/latency - systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 - systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 - - tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' -Compress=yes -Storage=persistent -SystemMaxUse=1G -RuntimeMaxUse=1G -ForwardToSyslog=yes -EOF -capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" - -if [ "$(isARM64)" -eq 1 ]; then - # shellcheck disable=SC3010 - if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then - echo "No arm64 support on V1 VM, exiting..." - exit 1 - fi -fi - -# Always override network config and disable NTP + Timesyncd and install Chrony -# Mariner does this differently, so only do it for Ubuntu -if ! isMarinerOrAzureLinux "$OS"; then - overrideNetworkConfig || exit 1 - disableNtpAndTimesyncdInstallChrony || exit 1 -fi - -# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, -# so these must be called separately (mirrored in the Mariner/AzureLinux block below). -# Other Mariner functions are safe to skip for ACL: -# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network -# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL -# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm -# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain -# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony -if isACL "$OS" "$OS_VARIANT"; then - # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. - disableSystemdIptables || exit 1 - # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file - # so DNS queries go directly through localdns. - disableSystemdResolvedCache -fi -capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" - -# Configure SSH service during VHD build for Ubuntu 22.10+ -configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" - -CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" -mkdir -p "${CONTAINERD_SERVICE_DIR}" -# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. -# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit -# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. -# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. -# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, -# to avoid compatibility issues between new VHDs and old CSE scripts. -tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null < /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules <, then skip and move on to next package - if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then - echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." - continue +installAndConfigureArtifactStreaming() { + local downloadURL="$1" + local version="$2" + # The arm64 packages have "-arm64" inserted before the file extension, + # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb + if [ "$(isARM64)" -eq 1 ]; then + downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" fi - downloadDir=$(echo "${p}" | jq .downloadLocation -r) - #download the package - case $name in - "kubernetes-cri-tools") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" - done - ;; - "azure-cni") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadAzureCNI "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "cni-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" - unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" - echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containernetworking-plugins") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installCNI "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "runc") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" - echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "containerd") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then - installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" - elif isMarinerOrAzureLinux "$OS"; then - installStandaloneContainerd "${version}" - fi - echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "oras") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installOras "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "aks-secure-tls-bootstrap-client") - for version in ${PACKAGE_VERSIONS[@]}; do - # removed at provisioning time if secure TLS bootstrapping is disabled - if isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isMarinerOrAzureLinux; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - installSecureTLSBootstrapClientSysext "${version}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "azure-acr-credential-provider") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" - echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} - # ORAS will be used to install other packages for network isolated clusters, it must go first. - done - ;; - "inspektor-gadget") - if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then - echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" - else - ig_version="${PACKAGE_VERSIONS[0]}" - if isUbuntu "$OS"; then - # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir - downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" - installIG "${ig_version}" "${downloadDir}" - elif isAzureLinux "$OS"; then - # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads - installIG "${ig_version}" "${downloadDir}" - fi - fi - ;; - "kubernetes-binaries") - # kubelet and kubectl - # need to cover previously supported version for VMAS scale up scenario - # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space - # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version - # Please do not use the .1 suffix, because that's only for the base image patches - # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" - echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - azure-acr-credential-provider-pmc) - name=${name%-pmc} - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - kubelet|kubectl) - for version in ${PACKAGE_VERSIONS[@]}; do - if isMarinerOrAzureLinux || isUbuntu; then - downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" - cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR - elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? - fi - echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} - done + local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" + retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + case "$downloadURL" in + *.deb) + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - "${K8S_DEVICE_PLUGIN_PKG}") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then - downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" - fi - echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} - done + *.rpm) + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD ;; - "dra-driver-nvidia-gpu") - for version in ${PACKAGE_VERSIONS[@]}; do - if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then - downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" - fi - echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-core") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "datacenter-gpu-manager-4-proprietary") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" - echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "dcgm-exporter") - for version in ${PACKAGE_VERSIONS[@]}; do - downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" - echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} - done - ;; - "node-exporter") - # Skipping is handled by empty versionsV2 arrays in components.json - # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. - if [ "${IS_KATA}" = "true" ]; then - echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" - else - # Download and install node-exporter-kubernetes at VHD build time. - # node-exporter is installed on the VHD so CSE only needs to enable+start it. - installNodeExporter "${PACKAGE_VERSIONS[0]}" - fi - ;; - "acr-mirror") - # acr-mirror is handled separately below via installAndConfigureArtifactStreaming. - ;; - "aznfs") - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL "${PACKAGE_DOWNLOAD_URL}") - mkdir -p "${downloadDir}" - aznfsFilename=$(basename "${evaluatedURL}") - echo "Downloading aznfs RPM from ${evaluatedURL} to ${downloadDir}/${aznfsFilename}" - retrycmd_curl_file 120 5 25 "${downloadDir}/${aznfsFilename}" "${evaluatedURL}" || exit $ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT - echo " - aznfs version ${version}" >> ${VHD_LOGS_FILEPATH} - done - installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL - ;; - "blobfuse"|"blobfuse2") - for version in "${PACKAGE_VERSIONS[@]}"; do - if isUbuntu "$OS"; then - if ! apt_get_install 10 2 120 "${name}=${version}"; then - journalctl --no-pager -u "${name}" || true - tail -n 200 /var/log/apt/term.log || true - tail -n 200 /var/log/dpkg.log || true - exit $ERR_APT_INSTALL_TIMEOUT - fi - echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" - else - echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" - fi - done - ;; - *) - echo "Package name: ${name} not supported for download. Please implement the download logic in the script." - # We can add a common function to download a generic package here. - # However, installation could be different for different packages. - ;; - esac - capture_benchmark "${SCRIPT_NAME}_download_${name}" -done <<< "$packages" - -installAndConfigureArtifactStreaming() { - local downloadURL="$1" - local version="$2" - # The arm64 packages have "-arm64" inserted before the file extension, - # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb - if [ "$(isARM64)" -eq 1 ]; then - downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" - fi - local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" - retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - case "$downloadURL" in - *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *) - echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + *) + echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 + exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} ;; esac rm "$MIRROR_DOWNLOAD_PATH" @@ -668,257 +223,6 @@ installAndConfigureArtifactStreaming() { echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} } -# Artifact streaming (acr-mirror) - version and URLs resolved from components.json, -# OS filtering handled declaratively by components.json entries ( for unsupported OSes). -acrMirrorPackage=$(echo "${packages}" | jq -c 'select(.name == "acr-mirror")') -updatePackageVersions "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -updatePackageDownloadURL "${acrMirrorPackage}" "${OS}" "${OS_VERSION}" "${OS_VARIANT}" -if [ "${#PACKAGE_VERSIONS[@]}" -gt 0 ] && [ "${PACKAGE_VERSIONS[0]}" != "" ]; then - for version in ${PACKAGE_VERSIONS[@]}; do - evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) - installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" - done -fi -capture_benchmark "${SCRIPT_NAME}_install_artifact_streaming" - -# k8s will use images in the k8s.io namespaces - create it -ctr namespace create k8s.io -cliTool="ctr" - -INSTALLED_RUNC_VERSION=$(runc --version | head -n1 | sed 's/runc version //') -echo " - runc version ${INSTALLED_RUNC_VERSION}" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_install_crictl" - -GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) - -NVIDIA_DRIVER_IMAGE="" -NVIDIA_DRIVER_IMAGE_TAG="" -NVIDIA_GRID_DRIVER_VERSION="" - -# Extract GRID driver version for release notes (applicable to all Linux distributions) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then - NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - # Continue to extract CUDA driver info as well - fi -done <<< "$GPUContainerImages" - -# For Ubuntu, pre-pull the CUDA driver image -if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now - gpu_action="copy" - - while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') - # shellcheck disable=SC2001 - imageName=$(echo "$downloadURL" | sed 's/:.*$//') - - if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then - latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') - NVIDIA_DRIVER_IMAGE="$imageName" - NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" - break # Exit the loop once we find the image - fi - done <<< "$GPUContainerImages" - - # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found - if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then - echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" - exit 1 - fi - - mkdir -p /opt/{actions,gpu} - - /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" - - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} -EOF - - # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the - # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles - # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, - # performs NO device access (safe on the GPU-less Packer builder), and writes the marker - # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" - # when that marker matches, running only the device-dependent steps. - # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the - # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. - # Dropping the image is a separate, deferred size optimization. - if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then - echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" - # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install - # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships - # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile - # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). - apt_get_install 10 2 300 gcc make libc6-dev || exit 1 - CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" - retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 - if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then - echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" - exit 1 - fi - cat << EOF >> ${VHD_LOGS_FILEPATH} - - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) -EOF - fi -fi - -if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then - # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. - if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then - # Replicate all functionality from github.com/azure/aks-gpu/install.sh. - # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up - # to have all drivers installed at VHD build time. - - # 1. Blacklist nouveau driver - cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf -blacklist nouveau -options nouveau modeset=0 -EOF - update-initramfs -u - - # 2. install drivers - BOM_PATH="gb-mai-bom.json" - - # Install a custom repository if a doca-custom-repo is specified - DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) - if [ -n "$DOCA_CUSTOM_REPO" ]; then - mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup - echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list - apt-get update - fi - - # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. - # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... - # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) - - # 3. Add char device symlinks for NVIDIA devices - mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" - cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules -ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" -EOF - - # Create systemd drop-in to override nvidia-device-plugin dependencies - mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d - cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf -[Unit] -After=kubelet.service - -[Service] -ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins -EOF - - # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. - systemctl enable nvidia-dcgm - systemctl enable nvidia-dcgm-exporter - systemctl enable nvidia-device-plugin - systemctl enable openibd - - # One additional request from MAI: signal that NPD is pre-installed on the VHD. - # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. - mkdir -p /etc/node-problem-detector.d/ - touch /etc/node-problem-detector.d/skip_vhd_npd - fi -fi - -if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then - ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} -fi - -installBpftrace -echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} - -PRESENT_DIR=$(pwd) -# run installBcc in a subshell and continue on with container image pull in order to decrease total build time -( - cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } - installBcc - exit $? -) > /var/log/bcc_installation.log 2>&1 & - -BCC_PID=$! - -# Add a separate section for runtime-installed components -# This clearly distinguishes components installed during CSE from VHD build-time components -# Only add for Ubuntu -if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then - cat << EOF >> ${VHD_LOGS_FILEPATH} -Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): - - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} -EOF -fi - -echo "images pre-pulled:" >> ${VHD_LOGS_FILEPATH} -capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads" - -string_replace() { - echo ${1//\*/$2} -} - -# Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources -# Account for possibility that number of cores is 3 or less -num_proc=$(nproc) -if [ "$num_proc" -gt 3 ]; then - parallel_container_image_pull_limit=$(nproc --ignore=2) -else - parallel_container_image_pull_limit=1 -fi -echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" - -declare -a image_pids=() - -ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) -while IFS= read -r imageToBePulled; do - downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) - amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) - updateMultiArchVersions "${imageToBePulled}" - amd64OnlyVersions="" - if [ "${amd64OnlyVersionsStr}" != "null" ]; then - amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") - fi - - if [ "$(isARM64)" -eq 1 ]; then - versions="${MULTI_ARCH_VERSIONS[*]}" - else - versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" - fi - - for version in ${versions}; do - CONTAINER_IMAGE=$(string_replace $downloadURL $version) - pullContainerImage "${cliTool}" "${CONTAINER_IMAGE}" & - image_pids+=($!) - echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} - while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } - done - done -done <<< "$ContainerImages" -echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" -while [ "$(jobs -p | wc -l)" -gt 0 ]; do - wait -n || { - ret=$? - echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 - for pid in "${image_pids[@]}"; do - kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" - done - exit "${ret}" - } -done -capture_benchmark "${SCRIPT_NAME}_caching_container_images" - retagAKSNodeCAWatcher() { # This function retags the aks-node-ca-watcher image to a static tag # The static tag is used to bootstrap custom CA trust when MCR egress may be intercepted by an untrusted TLS MITM firewall. @@ -935,11 +239,8 @@ retagAKSNodeCAWatcher() { # may be intercepted by an untrusted TLS MITM firewall. watcherStaticImg=${watcherBaseImg//\*/static} - # can't use $cliTool variable because crictl doesn't support retagging. retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg} } -retagAKSNodeCAWatcher -capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" pinPodSandboxImages() { # This function pins the pod sandbox image(s) to avoid Kubelet's Garbage Collector (GC) from removing them. @@ -981,35 +282,6 @@ pinPodSandboxImages() { done <<< "$pause_images" } -pinPodSandboxImages -capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" - -# IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux -if [ $OS = $UBUNTU_OS_NAME ] || isMarinerOrAzureLinux "$OS"; then - systemctlEnableAndStart ipv6_nftables 30 || exit 1 -fi - -mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events - -# Disable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory -if ! isMarinerOrAzureLinux "$OS"; then - systemctlEnableAndStart cgroup-memory-telemetry.timer 30 || exit 1 - systemctl enable cgroup-memory-telemetry.service || exit 1 - systemctl restart cgroup-memory-telemetry.service -fi - -CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) -if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then - systemctlEnableAndStart cgroup-pressure-telemetry.timer 30 || exit 1 - systemctl enable cgroup-pressure-telemetry.service || exit 1 - systemctl restart cgroup-pressure-telemetry.service -fi - -if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then - cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* - rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 -fi -capture_benchmark "${SCRIPT_NAME}_configure_telemetry" # download kubernetes package from the given URL using azcopy # if it is a kube-proxy package, extract image from the downloaded package @@ -1038,7 +310,6 @@ cacheKubePackageFromPrivateUrl() { echo "download private package ${kube_private_binary_url} and store as ${cached_pkg}" if ! ./azcopy copy "${kube_private_binary_url}" "${cached_pkg}"; then - azExitCode=$? # loop through azcopy log files shopt -s nullglob for f in "${AZCOPY_LOG_LOCATION}"/*.log; do @@ -1058,38 +329,53 @@ cacheKubePackageFromPrivateUrl() { fi } -if [ "$OS" = "$UBUNTU_OS_NAME" ]; then - # remove snapd, which is not used by container stack - apt_get_purge 20 30 120 snapd || exit 1 - apt_get_purge 20 30 120 apache2-utils || exit 1 - # CIS: Ensure telnet (/ftp) client is not installed - # CIS: Ufw is not used but interferes with log_martians rule - apt_get_purge 20 30 120 telnet ftp ufw tnftp inetutils-telnet || exit 1 +startEBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + return 0 + fi - apt-get -y autoclean || exit 1 - apt-get -y autoremove --purge || exit 1 - apt-get -y clean || exit 1 - # update message-of-the-day to start after multi-user.target - # multi-user.target usually start at the end of the boot sequence - sed -i 's/After=network-online.target/After=multi-user.target/g' /lib/systemd/system/motd-news.service -fi -capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" + installBpftrace + echo " - $(bpftrace --version)" >> ${VHD_LOGS_FILEPATH} -wait $BCC_PID -BCC_EXIT_CODE=$? -chmod 644 /var/log/bcc_installation.log + PRESENT_DIR=$(pwd) + # run installBcc in a subshell asynchronously so we can do other things in the meantime + ( + cd $PRESENT_DIR || { echo "Subshell in the wrong directory" >&2; exit 1; } + installBcc + exit $? + ) > /var/log/bcc_installation.log 2>&1 & -if [ "$BCC_EXIT_CODE" -eq 0 ]; then - echo "Bcc tools successfully installed." - cat << EOF >> ${VHD_LOGS_FILEPATH} + BCC_PID=$! +} + +finishEBPFToolsInstallation() { + if [ "$OS_VERSION" = "26.04" ]; then + echo "Ubuntu 26.04 - skipping eBPF tools installation" + # write out an empty bcc_installation.log since builds currently expect it to be present + touch /var/log/bcc_installation.log || true + chmod 644 /var/log/bcc_installation.log || true + return 0 + fi + + local BCC_EXIT_CODE=0 + wait "$BCC_PID" || BCC_EXIT_CODE=$? + chmod 644 /var/log/bcc_installation.log || true + + if [ "$BCC_EXIT_CODE" -eq 0 ]; then + echo "Bcc tools successfully installed." + cat << EOF >> ${VHD_LOGS_FILEPATH} - bcc-tools - libbcc-examples EOF -else - echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 - exit $BCC_EXIT_CODE -fi -capture_benchmark "${SCRIPT_NAME}_finish_installing_bcc_tools" + else + echo "Error: installBcc subshell failed with exit code $BCC_EXIT_CODE" >&2 + echo "===== BEGIN /var/log/bcc_installation.log =====" >&2 + cat /var/log/bcc_installation.log >&2 2>/dev/null || echo "(bcc_installation.log missing)" >&2 + echo "===== END /var/log/bcc_installation.log =====" >&2 + exit "$BCC_EXIT_CODE" + fi +} # Configure LSM modules to include BPF configureLsmWithBpf() { @@ -1117,10 +403,10 @@ configureLsmWithBpf() { return 0 fi - local new_lsm="bpf,$current_lsm" + local new_lsm="${current_lsm},bpf" echo "New LSM configuration: $new_lsm" - if [ "$OS" = "$UBUNTU_OS_NAME" ] && [ "$OS_VERSION" = "24.04" ]; then + if [ "$OS" = "$UBUNTU_OS_NAME" ] && { [ "$OS_VERSION" = "24.04" ] || [ "$OS_VERSION" = "26.04" ]; }; then local grub_cfg="/etc/default/grub.d/50-cloudimg-settings.cfg" if [ -f "$grub_cfg" ]; then if grep -q "lsm=" "$grub_cfg"; then @@ -1128,7 +414,7 @@ configureLsmWithBpf() { else sed -i "s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"lsm=$new_lsm /" "$grub_cfg" fi - echo "Updating GRUB configuration for Ubuntu 24.04..." + echo "Updating GRUB configuration for Ubuntu..." update-grub2 /boot/grub/grub.cfg || echo "Warning: Failed to update GRUB configuration" else echo "Warning: $grub_cfg not found, skipping LSM configuration" @@ -1146,7 +432,7 @@ configureLsmWithBpf() { echo "Warning: /etc/default/grub not found, skipping LSM configuration" fi else - echo "LSM BPF configuration is only enabled for Ubuntu 24.04 and Azure Linux 3.0, skipping" + echo "LSM BPF configuration is only enabled for Ubuntu 24.04, Ubuntu 26.04, and Azure Linux 3.0, skipping" fi echo "LSM configuration update completed" @@ -1155,47 +441,571 @@ configureLsmWithBpf() { fi } -configureLsmWithBpf -capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" - -# use the private_packages_url to download and cache packages -if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then - IFS=',' read -ra PRIVATE_URLS <<< "${PRIVATE_PACKAGES_URL}" +setUdevRules() { + # set udev rules + echo "set read ahead size to 15380 KB" + AWK_PATH=$(command -v awk) + cat > /etc/udev/rules.d/99-nfs.rules < /etc/udev/rules.d/80-azure-disk.rules </dev/null - rm -rf "${ctr_temp}" - fi - } - trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM +# Use "resource" type for local SCSI because some VM skus offer NVMe local disks in addition to a SCSI resource disk, e.g. LSv3 family. +# This logic is already in walinuxagent rules but we duplicate it here to avoid an unnecessary dependency for anyone requiring it. +ATTRS{device_id}=="?00000000-0001-*", ENV{AZURE_DISK_TYPE}="resource", ENV{AZURE_DISK_LUN}="", GOTO="azure_disk_symlink" +ATTRS{device_id}=="{f8b3781a-1e82-4818-a1c3-63d806ec15bb}", ENV{AZURE_DISK_LUN}=="1", ENV{AZURE_DISK_TYPE}="resource", ENV{AZURE_DISK_LUN}="", GOTO="azure_disk_symlink" +GOTO="azure_disk_end" - # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. - local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) +LABEL="azure_disk_nvme_direct_v1" +LABEL="azure_disk_nvme_direct_v2" +ATTRS{nsid}=="?*", ENV{AZURE_DISK_TYPE}="local", ENV{AZURE_DISK_SERIAL}="\$env{ID_SERIAL_SHORT}" +GOTO="azure_disk_nvme_id" - # Determine latest version. +LABEL="azure_disk_nvme_remote_v1" +ATTRS{nsid}=="1", ENV{AZURE_DISK_TYPE}="os", GOTO="azure_disk_nvme_id" +ATTRS{nsid}=="?*", ENV{AZURE_DISK_TYPE}="data", PROGRAM="/bin/sh -ec 'echo \$\$((%s{nsid}-2))'", ENV{AZURE_DISK_LUN}="\$result" + +LABEL="azure_disk_nvme_id" +ATTRS{nsid}=="?*", IMPORT{program}="/usr/sbin/azure-nvme-id --udev" + +LABEL="azure_disk_symlink" +# systemd v254 ships an updated 60-persistent-storage.rules that would allow +# these to be deduplicated using \$env{.PART_SUFFIX} +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="os|resource|root", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_INDEX}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-index/\$env{AZURE_DISK_INDEX}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_LUN}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-lun/\$env{AZURE_DISK_LUN}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_NAME}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-name/\$env{AZURE_DISK_NAME}" +ENV{DEVTYPE}=="disk", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_SERIAL}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-serial/\$env{AZURE_DISK_SERIAL}" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="os|resource|root", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_INDEX}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-index/\$env{AZURE_DISK_INDEX}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_LUN}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-lun/\$env{AZURE_DISK_LUN}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_NAME}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-name/\$env{AZURE_DISK_NAME}-part%n" +ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_SERIAL}=="?*", SYMLINK+="disk/azure/\$env{AZURE_DISK_TYPE}/by-serial/\$env{AZURE_DISK_SERIAL}-part%n" +LABEL="azure_disk_end" +EOF + udevadm control --reload +} + +cachePackageAndBinaryComponents() { + # Download/cache all declared packages and binaries within components.json that apply to the respective OS SKU + packages=$(jq ".Packages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + # Iterate over each element in the packages array + while IFS= read -r p; do + #getting metadata for each package + name=$(echo "${p}" | jq .name -r) + os=${OS} + # TODO(mheberling): Remove this once kata uses standard containerd. This OS is referenced + # in file `parts/common/component.json` with the same ${MARINER_KATA_OS_NAME}. + if isMariner "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json marinerkata. + os=${MARINER_KATA_OS_NAME} + fi + if isAzureLinux "${OS}" && [ "${IS_KATA}" = "true" ]; then + # This is temporary for kata-cc because it uses a modified version of containerd and + # name is referenced in parts/common.json azurelinuxkata. + os=${AZURELINUX_KATA_OS_NAME} + fi + updatePackageVersions "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + updatePackageDownloadURL "${p}" "${os}" "${OS_VERSION}" "${OS_VARIANT}" + echo "In components.json, processing components.packages \"${name}\" \"${PACKAGE_VERSIONS[@]}\" \"${PACKAGE_DOWNLOAD_URL}\"" + + # if ${PACKAGE_VERSIONS[@]} count is 0 or if the first element of the array is , then skip and move on to next package + if [ "${#PACKAGE_VERSIONS[@]}" -eq 0 ] || [ "${PACKAGE_VERSIONS[0]}" = "" ]; then + echo "INFO: ${name} package versions array is either empty or the first element is . Skipping ${name} installation." + continue + fi + downloadDir=$(echo "${p}" | jq .downloadLocation -r) + #download the package + case $name in + "kubernetes-cri-tools") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAndInstallCriTools "${downloadDir}" "${evaluatedURL}" "${version}" + done + ;; + "azure-cni") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadAzureCNI "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - Azure CNI version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "cni-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCNIPlugins "${downloadDir}" "${evaluatedURL}" + unpackTgzToCNIDownloadsDIR "${downloadDir}" "${evaluatedURL}" + echo " - CNI plugin version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containernetworking-plugins") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installCNI "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - containernetworking-plugins version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "runc") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + ensureRunc "${version}" "${evaluatedURL}" "${downloadDir}" + echo " - runc version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "containerd") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + if [ "${OS}" = "${UBUNTU_OS_NAME}" ]; then + installContainerd "${downloadDir}" "${evaluatedURL}" "${version}" + elif isMarinerOrAzureLinux "$OS"; then + installStandaloneContainerd "${version}" + fi + echo " - containerd version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "oras") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installOras "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - oras version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "aks-secure-tls-bootstrap-client") + for version in ${PACKAGE_VERSIONS[@]}; do + # removed at provisioning time if secure TLS bootstrapping is disabled + if isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installPackageFromCache "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isMarinerOrAzureLinux; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + installRPMPackageFromFile "${name}" "${version}" "/opt/bin/${name}" || exit $? + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + installSecureTLSBootstrapClientSysext "${version}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "azure-acr-credential-provider") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadCredentialProvider "${downloadDir}" "${evaluatedURL}" "${version}" + echo " - azure-acr-credential-provider version ${version}" >> ${VHD_LOGS_FILEPATH} + # ORAS will be used to install other packages for network isolated clusters, it must go first. + done + ;; + "inspektor-gadget") + if isMariner "$OS" || isFlatcar "$OS" || isACL "$OS" "$OS_VARIANT" || isAzureLinuxOSGuard "$OS" "$OS_VARIANT" || [ "${IS_KATA}" = "true" ]; then + echo "Skipping inspektor-gadget installation for ${OS} ${OS_VARIANT:-default} (IS_KATA=${IS_KATA})" + else + ig_version="${PACKAGE_VERSIONS[0]}" + if isUbuntu "$OS"; then + # Ubuntu: download ig deb via apt; ig_install_deb_stack expects it at downloadDir + downloadPkgFromVersion "ig" "${ig_version}" "${downloadDir}" + installIG "${ig_version}" "${downloadDir}" + elif isAzureLinux "$OS"; then + # Azure Linux 3.0: ig_install_rpm_stack handles its own RPM downloads + installIG "${ig_version}" "${downloadDir}" + fi + fi + ;; + "kubernetes-binaries") + # kubelet and kubectl + # need to cover previously supported version for VMAS scale up scenario + # So keeping as many versions as we can - those unsupported version can be removed when we don't have enough space + # NOTE that we only keep the latest one per k8s patch version as kubelet/kubectl is decided by VHD version + # Please do not use the .1 suffix, because that's only for the base image patches + # regular version >= v1.17.0 or hotfixes >= 20211009 has arm64 binaries. + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + extractKubeBinaries "${version}" "${evaluatedURL}" false "${downloadDir}" + echo " - kubernetes-binaries version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + azure-acr-credential-provider-pmc) + name=${name%-pmc} + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + kubelet|kubectl) + for version in ${PACKAGE_VERSIONS[@]}; do + if isMarinerOrAzureLinux || isUbuntu; then + downloadPkgFromVersion "${name}" "${version}" "${downloadDir}" + cacheVersionedKubernetesPackageBinary "${name}" "${version}" "${downloadDir}" || exit $ERR_K8S_INSTALL_ERR + elif isFlatcar || isACL "$OS" "$OS_VARIANT"; then + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + downloadSysextFromVersion "${name}" "${evaluatedURL}" "${downloadDir}" || exit $? + fi + echo " - ${name} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "${K8S_DEVICE_PLUGIN_PKG}") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isMarinerOrAzureLinux "$OS"; then + downloadPkgFromVersion "${K8S_DEVICE_PLUGIN_PKG}" "${version}" "${downloadDir}" + fi + echo " - ${K8S_DEVICE_PLUGIN_PKG} version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dra-driver-nvidia-gpu") + for version in ${PACKAGE_VERSIONS[@]}; do + if [ "${OS}" = "${UBUNTU_OS_NAME}" ] || isAzureLinux "$OS"; then + downloadPkgFromVersion "dra-driver-nvidia-gpu" "${version}" "${downloadDir}" + fi + echo " - dra-driver-nvidia-gpu version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-core") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-core" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-core version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "datacenter-gpu-manager-4-proprietary") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "datacenter-gpu-manager-4-proprietary" "${version}" "${downloadDir}" + echo " - datacenter-gpu-manager-4-proprietary version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "dcgm-exporter") + for version in ${PACKAGE_VERSIONS[@]}; do + downloadPkgFromVersion "dcgm-exporter" "${version}" "${downloadDir}" + echo " - dcgm-exporter version ${version}" >> ${VHD_LOGS_FILEPATH} + done + ;; + "node-exporter") + # Skipping is handled by empty versionsV2 arrays in components.json + # for mariner, flatcar, acl, and osguard. Kata is skipped explicitly here. + if [ "${IS_KATA}" = "true" ]; then + echo "Skipping node-exporter installation for kata (IS_KATA=${IS_KATA})" + else + # Download and install node-exporter-kubernetes at VHD build time. + # node-exporter is installed on the VHD so CSE only needs to enable+start it. + installNodeExporter "${PACKAGE_VERSIONS[0]}" + fi + ;; + "acr-mirror") + # TODO(2604): install acr-mirror for artifact streaming support once acr-mirror is available in resolute PMC repos + # Artifact streaming (acr-mirror) - version and URLs resolved from components.json, + # OS filtering handled declaratively by components.json entries ( for unsupported OSes). + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL ${PACKAGE_DOWNLOAD_URL}) + installAndConfigureArtifactStreaming "${evaluatedURL}" "${version}" + done + ;; + "aznfs") + for version in ${PACKAGE_VERSIONS[@]}; do + evaluatedURL=$(evalPackageDownloadURL "${PACKAGE_DOWNLOAD_URL}") + mkdir -p "${downloadDir}" + aznfsFilename=$(basename "${evaluatedURL}") + echo "Downloading aznfs RPM from ${evaluatedURL} to ${downloadDir}/${aznfsFilename}" + retrycmd_curl_file 120 5 25 "${downloadDir}/${aznfsFilename}" "${evaluatedURL}" || exit $ERR_AZNFS_RPM_DOWNLOAD_TIMEOUT + echo " - aznfs version ${version}" >> ${VHD_LOGS_FILEPATH} + done + installAznfsPackage || exit $ERR_AZNFS_INSTALL_FAIL + ;; + "blobfuse"|"blobfuse2") + # TODO(2604): install blobfuse2 for azure blob CSI driver support when package is made available in resolute PMC repos + for version in "${PACKAGE_VERSIONS[@]}"; do + if isUbuntu "$OS"; then + if ! apt_get_install 10 2 120 "${name}=${version}"; then + journalctl --no-pager -u "${name}" || true + tail -n 200 /var/log/apt/term.log || true + tail -n 200 /var/log/dpkg.log || true + exit $ERR_APT_INSTALL_TIMEOUT + fi + echo " - ${name} version ${version}" >> "${VHD_LOGS_FILEPATH}" + else + echo " - ${name} installation skipped for ${OS}" >> "${VHD_LOGS_FILEPATH}" + fi + done + ;; + *) + echo "Package name: ${name} not supported for download. Please implement the download logic in the script." + # We can add a common function to download a generic package here. + # However, installation could be different for different packages. + ;; + esac + capture_benchmark "${SCRIPT_NAME}_download_${name}" + done <<< "$packages" +} + +cacheContainerImageComponents() { + # Download/cache all declared container images within components.json that apply to the respective OS SKU + + # Limit number of parallel pulls to 2 less than number of processor cores in order to prevent issues with network, CPU, and disk resources + # Account for possibility that number of cores is 3 or less + num_proc=$(nproc) + if [ "$num_proc" -gt 3 ]; then + parallel_container_image_pull_limit=$(nproc --ignore=2) + else + parallel_container_image_pull_limit=1 + fi + echo "Limit for parallel container image pulls set to $parallel_container_image_pull_limit" + + declare -a image_pids=() + + ContainerImages=$(jq ".ContainerImages" $COMPONENTS_FILEPATH | jq .[] --monochrome-output --compact-output) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq .downloadURL -r) + amd64OnlyVersionsStr=$(echo "${imageToBePulled}" | jq .amd64OnlyVersions -r) + updateMultiArchVersions "${imageToBePulled}" + amd64OnlyVersions="" + if [ "${amd64OnlyVersionsStr}" != "null" ]; then + amd64OnlyVersions=$(echo "${amd64OnlyVersionsStr}" | jq -r ".[]") + fi + + if [ "$(isARM64)" -eq 1 ]; then + versions="${MULTI_ARCH_VERSIONS[*]}" + else + versions="${amd64OnlyVersions} ${MULTI_ARCH_VERSIONS[*]}" + fi + + for version in ${versions}; do + CONTAINER_IMAGE=$(string_replace $downloadURL $version) + pullContainerImage "ctr" "${CONTAINER_IMAGE}" & + image_pids+=($!) + echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} + while [ "$(jobs -p | wc -l)" -ge "$parallel_container_image_pull_limit" ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}, ${CONTAINER_IMAGE}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" + done + exit "${ret}" + } + done + done + done <<< "$ContainerImages" + echo "Waiting for container image pulls to finish. PID: ${image_pids[@]}" + while [ "$(jobs -p | wc -l)" -gt 0 ]; do + wait -n || { + ret=$? + echo "A background job pullContainerImage failed: ${ret}. Exiting..." >&2 + for pid in "${image_pids[@]}"; do + kill -9 "$pid" 2>/dev/null || echo "Failed to kill process $pid" + done + exit "${ret}" + } + done +} + +cacheGPUContainerImageComponents() { + # Download/install GPU container image components + GPUContainerImages=$(jq -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH) + + NVIDIA_DRIVER_IMAGE="" + NVIDIA_DRIVER_IMAGE_TAG="" + NVIDIA_GRID_DRIVER_VERSION="" + + # Extract GRID driver version for release notes (applicable to all Linux distributions) + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-grid" ]; then + NVIDIA_GRID_DRIVER_VERSION=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + # Continue to extract CUDA driver info as well + fi + done <<< "$GPUContainerImages" + + # For Ubuntu, pre-pull the CUDA driver image + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" + + while IFS= read -r imageToBePulled; do + downloadURL=$(echo "${imageToBePulled}" | jq -r '.downloadURL') + # shellcheck disable=SC2001 + imageName=$(echo "$downloadURL" | sed 's/:.*$//') + + if [ "$imageName" = "mcr.microsoft.com/aks/aks-gpu-cuda-lts" ]; then + latestVersion=$(echo "${imageToBePulled}" | jq -r '.gpuVersion.latestVersion') + NVIDIA_DRIVER_IMAGE="$imageName" + NVIDIA_DRIVER_IMAGE_TAG="$latestVersion" + break # Exit the loop once we find the image + fi + done <<< "$GPUContainerImages" + + # Check if the NVIDIA_DRIVER_IMAGE and NVIDIA_DRIVER_IMAGE_TAG were found + if [ -z "$NVIDIA_DRIVER_IMAGE" ] || [ -z "$NVIDIA_DRIVER_IMAGE_TAG" ]; then + echo "Error: Unable to find aks-gpu-cuda-lts image in components.json" + exit 1 + fi + + mkdir -p /opt/{actions,gpu} + + /opt/azure/containers/image-fetcher "$NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG" + + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver=${NVIDIA_DRIVER_IMAGE_TAG} +EOF + fi + + # Add a separate section for runtime-installed components + # This clearly distinguishes components installed during CSE from VHD build-time components + # Only add for Ubuntu + if [ -n "$NVIDIA_GRID_DRIVER_VERSION" ] && [ "$OS" = "$UBUNTU_OS_NAME" ]; then + cat << EOF >> ${VHD_LOGS_FILEPATH} +Components installed at node provisioning time (CSE) for supported GPU VM sizes (example A10 family): + - nvidia-grid-driver=${NVIDIA_GRID_DRIVER_VERSION} +EOF + fi +} + +buildNVIDIAKernelModule() { + if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now + gpu_action="copy" + + # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the + # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles + # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, + # performs NO device access (safe on the GPU-less Packer builder), and writes the marker + # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" + # when that marker matches, running only the device-dependent steps. + # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the + # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. + # Dropping the image is a separate, deferred size optimization. + if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then + echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" + # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install + # them. A boot-time fallback recompile (marker mismatch) still has them: the VHD ships + # build-essential -> libc6-dev (release-notes manifests) -- the toolchain baseline GPU nodes compile + # with at boot (installDeps runs apt --no-install-recommends, so gcc alone doesn't pull libc6-dev). + apt_get_install 10 2 300 gcc make libc6-dev || exit 1 + CTR_GPU_PREBUILD_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind" + retrycmd_if_failure 3 10 600 bash -c "$CTR_GPU_PREBUILD_CMD $NVIDIA_DRIVER_IMAGE:$NVIDIA_DRIVER_IMAGE_TAG gpuprebuild /entrypoint.sh build-only" || exit 1 + if [ ! -f /opt/azure/aks-gpu/dkms-marker ]; then + echo "Error: NVIDIA CUDA prebake did not produce /opt/azure/aks-gpu/dkms-marker" + exit 1 + fi + cat << EOF >> ${VHD_LOGS_FILEPATH} + - nvidia-cuda-driver-prebaked=${NVIDIA_DRIVER_IMAGE_TAG} (kernel $(uname -r)) +EOF + fi + fi +} + +configureGraceBlackwell() { + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then + # NVIDIA GB setup is only supported on arm64 Ubuntu 24.04. + if [ "${CPU_ARCH}" = "arm64" ] && [ "${UBUNTU_RELEASE}" = "24.04" ]; then + # Replicate all functionality from github.com/azure/aks-gpu/install.sh. + # aks-gpu is designed to run at node boot/join time, whereas the NVIDIA GB VHD is set up + # to have all drivers installed at VHD build time. + + # 1. Blacklist nouveau driver + cat << EOF >> /etc/modprobe.d/blacklist-nouveau.conf +blacklist nouveau +options nouveau modeset=0 +EOF + update-initramfs -u + + # 2. install drivers + BOM_PATH="gb-mai-bom.json" + + # Install a custom repository if a doca-custom-repo is specified + DOCA_CUSTOM_REPO=$(jq -r '.["doca-custom-repo"]' $BOM_PATH) + if [ -n "$DOCA_CUSTOM_REPO" ]; then + mv /etc/apt/sources.list.d/doca-net.list /etc/apt/sources.list.d/doca-net.list.backup + echo "deb [arch=arm64 signed-by=/etc/apt/keyrings/doca-net.pub] $DOCA_CUSTOM_REPO ./" > /etc/apt/sources.list.d/doca-net.list + apt-get update + fi + + # Install DOCA/OFED before the GPU driver so nvidia-peermem can build against the RDMA APIs provided by OFED. + # Farcically, nvidia-dkms-580-open cannot be installed together with the CUDA toolkit. Something about that package changes the build environment in an incompatible way. I've seen people mention CUDA including an old version of gcc that somehow makes its way onto the PATH... + # Therefore we install DOCA/OFED first, the GPU driver and its dependencies second, then all downstream reverse-dependencies (CUDA, DCGM, and so forth) third. + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave1"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave2"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + sudo apt-get install -y --allow-downgrades $(jq -r '.["versions-wave3"] | to_entries[] | "\(.key)=\(.value)"' $BOM_PATH) + + # 3. Add char device symlinks for NVIDIA devices + mkdir -p "$(dirname /lib/udev/rules.d/71-nvidia-dev-char.rules)" + cat << EOF >> /lib/udev/rules.d/71-nvidia-dev-char.rules +ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/usr/bin/nvidia-ctk system create-dev-char-symlinks --create-all" +EOF + + # Create systemd drop-in to override nvidia-device-plugin dependencies + mkdir -p /etc/systemd/system/nvidia-device-plugin.service.d + cat << EOF > /etc/systemd/system/nvidia-device-plugin.service.d/override.conf +[Unit] +After=kubelet.service + +[Service] +ExecStartPre=-/usr/bin/mkdir -p /var/lib/kubelet/device-plugins +EOF + + # Now we are off-piste: enable DCGM, DCGM exporter, container device plugin, and the NVIDIA containerd config. + systemctl enable nvidia-dcgm + systemctl enable nvidia-dcgm-exporter + systemctl enable nvidia-device-plugin + systemctl enable openibd + + # One additional request from MAI: signal that NPD is pre-installed on the VHD. + # When this file is present, the Azure AKS VM Extension skips installing NPD at provision time. + mkdir -p /etc/node-problem-detector.d/ + touch /etc/node-problem-detector.d/skip_vhd_npd + fi + fi +} + +# This function extracts CoreDNS binary from cached coredns images (latest version) +# and copies it to - /opt/azure/containers/localdns/binary/coredns. +# The binary is later used by localdns systemd unit. +# The function also handles the cleanup of temporary directories and unmounting of images. +extractAndCacheCoreDnsBinary() { + local coredns_image_list=($(ctr -n k8s.io images list -q | grep coredns)) + if [ "${#coredns_image_list[@]}" -eq 0 ]; then + echo "Error: No coredns images found." + exit 1 + fi + + rm -rf "${LOCALDNS_BINARY_PATH}" || exit 1 + mkdir -p "${LOCALDNS_BINARY_PATH}" || exit 1 + + cleanup_coredns_imports() { + set +e + if [ -n "${ctr_temp}" ]; then + ctr -n k8s.io images unmount "${ctr_temp}" >/dev/null + rm -rf "${ctr_temp}" + fi + } + trap cleanup_coredns_imports EXIT ABRT ERR INT PIPE QUIT TERM + + # Extract available coredns image tags (v1.12.0-1 format) and sort them in descending order. + local sorted_coredns_tags=($(for image in "${coredns_image_list[@]}"; do echo "${image##*:}"; done | sort -V -r)) + + # Determine latest version. local latest_coredns_tag="${sorted_coredns_tags[0]}" # Extract the CoreDNS binary for the latest version. @@ -1240,11 +1050,6 @@ extractAndCacheCoreDnsBinary() { trap - EXIT ABRT ERR INT PIPE QUIT TERM } -extractAndCacheCoreDnsBinary - -rm -f ./azcopy # cleanup immediately after usage will return in two downloads -echo "install-dependencies step completed successfully" - # Collect grid compatibility data (placeholder for now - will be extended later) collect_grid_compatibility_data() { if [ -z "${GRID_COMPATIBILITY_DATA_FILE}" ] ; then @@ -1275,6 +1080,250 @@ collect_grid_compatibility_data() { chmod 755 "${GRID_COMPATIBILITY_DATA_FILE}" } +# --------------------------------------------------------- INSTALL DEPENDENCIES START --------------------------------------------------------- + +# check if COMPONENTS_FILEPATH exists +if [ ! -f "$COMPONENTS_FILEPATH" ]; then + echo "Components file not found at $COMPONENTS_FILEPATH. Exiting..." + exit 1 +fi + +resolve_packages_source_url + +echo "" +echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH} +capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" + +echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)" +# fix grub issue with cvm by reinstalling before other deps +# other VHDs use grub-pc, not grub-efi +if [ "$OS" = "$UBUNTU_OS_NAME" ] && echo "$FEATURE_FLAGS" | grep -q "cvm"; then + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + wait_for_apt_locks + apt_get_install 10 2 120 grub-efi || exit 1 +fi +capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm" + +if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + # disable and mask all UU timers/services + # save some background io/latency + systemctl mask apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.service apt-daily-upgrade.service || exit 1 + systemctl disable apt-daily.timer apt-daily-upgrade.timer || exit 1 + + tee /etc/apt/apt.conf.d/99periodic > /dev/null <> "${CHRONYD_DIR}"/10-chrony-restarts.conf < /dev/null <<'EOF' +Compress=yes +Storage=persistent +SystemMaxUse=1G +RuntimeMaxUse=1G +ForwardToSyslog=yes +EOF +capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs" + +if [ "$(isARM64)" -eq 1 ]; then + # shellcheck disable=SC3010 + if [[ ${HYPERV_GENERATION,,} == "v1" ]]; then + echo "No arm64 support on V1 VM, exiting..." + exit 1 + fi +fi + +# Always override network config and disable NTP + Timesyncd and install Chrony +# Mariner does this differently, so only do it for Ubuntu +if ! isMarinerOrAzureLinux "$OS"; then + overrideNetworkConfig || exit 1 + disableNtpAndTimesyncdInstallChrony || exit 1 +fi + +# ACL inherits Azure Linux behaviors but isMarinerOrAzureLinux returns false, +# so these must be called separately (mirrored in the Mariner/AzureLinux block below). +# Other Mariner functions are safe to skip for ACL: +# setMarinerNetworkdConfig — ACL doesn't ship systemd-bootstrap's 99-dhcp-en.network +# fixCBLMarinerPermissions — product_uuid already 444; no rsyslog on ACL +# addMarinerNvidiaRepo / updateDnfWithNvidiaPkg / disableDNFAutomatic / enableCheckRestart — ACL has no dnf/rpm +# activateNfConntrack — nf_conntrack auto-loads via iptables dependency chain +# disableTimesyncd — ACL handles chrony separately above via disableNtpAndTimesyncdInstallChrony +if isACL "$OS" "$OS_VARIANT"; then + # ACL's iptables.service loads host firewall rules that conflict with Cilium eBPF routing. + disableSystemdIptables || exit 1 + # Repoint /etc/resolv.conf from the stub (127.0.0.53) to the real upstream file + # so DNS queries go directly through localdns. + disableSystemdResolvedCache +fi +capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config" + +# Configure SSH service during VHD build for Ubuntu 22.10+ +configureSSHService "$OS" "$OS_VERSION" || echo "##vso[task.logissue type=warning]SSH Service configuration failed, but continuing VHD build" + +CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d" +mkdir -p "${CONTAINERD_SERVICE_DIR}" +# Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. +# On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit +# (for example 1024:524288), so containerd inherits a very low soft file descriptor limit (1024) unless we override it here. +# On Mariner/AzureLinux this is redundant with the base containerd.service unit but harmless. +# Not removing LimitNOFILE from parts/linux/cloud-init/artifacts/containerd.service, +# to avoid compatibility issues between new VHDs and old CSE scripts. +tee "${CONTAINERD_SERVICE_DIR}/exec_start.conf" > /dev/null < /dev/null <> ${VHD_LOGS_FILEPATH} +cacheContainerImageComponents +capture_benchmark "${SCRIPT_NAME}_caching_container_images" + +configureGraceBlackwell +if [ -d "/opt/gpu" ] && [ "$(ls -A /opt/gpu)" ]; then + ls -ltr /opt/gpu/* >> ${VHD_LOGS_FILEPATH} +fi + +retagAKSNodeCAWatcher +capture_benchmark "${SCRIPT_NAME}_retag_aks_node_ca_watcher" + +pinPodSandboxImages +capture_benchmark "${SCRIPT_NAME}_pin_pod_sandbox_image" + +# IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux +if [ $OS = $UBUNTU_OS_NAME ] || isMarinerOrAzureLinux "$OS"; then + systemctlEnableAndStart ipv6_nftables 30 || exit 1 +fi + +mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events + +# Disable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory +if ! isMarinerOrAzureLinux "$OS"; then + systemctlEnableAndStart cgroup-memory-telemetry.timer 30 || exit 1 + systemctl enable cgroup-memory-telemetry.service || exit 1 + systemctl restart cgroup-memory-telemetry.service +fi + +CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) +if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then + systemctlEnableAndStart cgroup-pressure-telemetry.timer 30 || exit 1 + systemctl enable cgroup-pressure-telemetry.service || exit 1 + systemctl restart cgroup-pressure-telemetry.service +fi + +if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then + cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* + rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 +fi +capture_benchmark "${SCRIPT_NAME}_configure_telemetry" + +if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + # remove snapd, which is not used by container stack + apt_get_purge 20 30 120 snapd || exit 1 + apt_get_purge 20 30 120 apache2-utils || exit 1 + # CIS: Ensure telnet (/ftp) client is not installed + # CIS: Ufw is not used but interferes with log_martians rule + apt_get_purge 20 30 120 telnet ftp ufw tnftp inetutils-telnet || exit 1 + + apt-get -y autoclean || exit 1 + apt-get -y autoremove --purge || exit 1 + apt-get -y clean || exit 1 + # update message-of-the-day to start after multi-user.target + # multi-user.target usually start at the end of the boot sequence + sed -i 's/After=network-online.target/After=multi-user.target/g' /lib/systemd/system/motd-news.service +fi +capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu" + +# Ensure eBPF tools installed successfully +finishEBPFToolsInstallation +capture_benchmark "${SCRIPT_NAME}_finish_installing_ebpf_tools" + +configureLsmWithBpf +capture_benchmark "${SCRIPT_NAME}_configure_lsm_with_bpf" + +# use the private_packages_url to download and cache packages +if [ -n "${PRIVATE_PACKAGES_URL:-}" ]; then + IFS=',' read -ra PRIVATE_URLS <<< "${PRIVATE_PACKAGES_URL}" + + for private_url in "${PRIVATE_URLS[@]}"; do + echo "download kube package from ${private_url}" + cacheKubePackageFromPrivateUrl "$private_url" + done +fi +rm -f ./azcopy # cleanup immediately after usage will return in two downloads + +extractAndCacheCoreDnsBinary + collect_grid_compatibility_data # nvidia repos are non msft public endpoints and should not be present on VHDs. @@ -1283,5 +1332,7 @@ collect_grid_compatibility_data removeNvidiaRepos capture_benchmark "${SCRIPT_NAME}_remove_nvidia_repos" +echo "install-dependencies step completed successfully" + capture_benchmark "${SCRIPT_NAME}_overall" true process_benchmarks diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 1f96833215e..b1b1fe19988 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -409,7 +409,7 @@ copyPackerFiles() { cpAndMode $AKS_CHECK_NETWORK_SCRIPT_SRC $AKS_CHECK_NETWORK_SCRIPT_DEST 755 cpAndMode $AKS_CHECK_NETWORK_SERVICE_SRC $AKS_CHECK_NETWORK_SERVICE_DEST 644 - if [ ${UBUNTU_RELEASE} = "22.04" ] || [ ${UBUNTU_RELEASE} = "24.04" ]; then + if [ "${UBUNTU_RELEASE}" = "22.04" ] || [ "${UBUNTU_RELEASE}" = "24.04" ] || [ "${UBUNTU_RELEASE}" = "26.04" ]; then PAM_D_COMMON_AUTH_SRC=/home/packer/pam-d-common-auth-2204 fi diff --git a/vhdbuilder/packer/post-install-dependencies.sh b/vhdbuilder/packer/post-install-dependencies.sh index 65eee5613c6..72509dae75c 100644 --- a/vhdbuilder/packer/post-install-dependencies.sh +++ b/vhdbuilder/packer/post-install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') UBUNTU_OS_NAME="UBUNTU" FLATCAR_OS_NAME="FLATCAR" ACL_OS_NAME="AZURECONTAINERLINUX" @@ -49,13 +49,15 @@ if [ $OS = $UBUNTU_OS_NAME ]; then # remove apport retrycmd_if_failure 10 2 60 apt-get purge --auto-remove apport open-vm-tools -y || exit 1 - # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) - # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so - # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 - # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, - # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). - # No-op on the minimal image, which does not ship these. - retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + if ! isMinimalImage; then + # Remove PackageKit: an unused desktop D-Bus package manager whose apt hook (/etc/apt/apt.conf.d/20packagekit) + # runs `gdbus call --system` after every apt update. During early-boot CSE the system bus isn't ready yet, so + # that benign `Error connecting: ... Broken pipe` stderr trips the node-bootstrap apt error-check -> CSE exit 99 + # -> node never joins. Purging it also drops packagekit-tools + software-properties-common (add-apt-repository, + # unused at node runtime; the build's only add-apt-repository usage is earlier in pre-install-dependencies.sh). + # No-op on the minimal image, which does not ship these. + retrycmd_if_failure 10 2 60 apt-get purge --auto-remove packagekit packagekit-tools software-properties-common -y || exit 1 + fi # strip old kernels/packages retrycmd_if_failure 10 2 60 apt-get -y autoclean || exit 1 diff --git a/vhdbuilder/packer/pre-install-dependencies.sh b/vhdbuilder/packer/pre-install-dependencies.sh index 89ab9f0b885..12b2eef6a44 100644 --- a/vhdbuilder/packer/pre-install-dependencies.sh +++ b/vhdbuilder/packer/pre-install-dependencies.sh @@ -1,7 +1,7 @@ #!/bin/bash -OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') -OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"') -OS_VARIANT=$(sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"') +OS=$(sort -r /etc/*-release | sed -n 's/^ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VERSION=$(sort -r /etc/*-release | sed -n 's/^VERSION_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') +OS_VARIANT=$(sort -r /etc/*-release | sed -n 's/^VARIANT_ID=//p' | head -n1 | tr -d '"' | tr '[:lower:]' '[:upper:]') THIS_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" #the following sed removes all comments of the format {{/* */}} @@ -36,6 +36,11 @@ capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables" copyPackerFiles +# Install required dependencies needed to build minimal images if needed (currently only Ubuntu 26.04) +if isMinimalImage && isUbuntu "$OS"; then + installMinimalBuildDeps +fi + # Update rsyslog configuration RSYSLOG_CONFIG_FILEPATH="/etc/rsyslog.d/60-CIS.conf" if isMarinerOrAzureLinux "$OS"; then diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 3f2889cfe2a..b891a562676 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -803,6 +803,7 @@ testLtsKernel() { elif [ "$os_version" = "24.04" ]; then expected_kernel="6.8" else + # TODO(2604): update once 26.04 LTS kernel metapackage is available echo "LTS kernel not installed for: $os_version" fi @@ -930,8 +931,8 @@ testLSMBPF() { os_sku=$1 os_version=$2 - # Only test on Ubuntu 24.04 and Azure Linux 3.0 that LSM BPF is configured correctly - if { [ "$os_sku" != "Ubuntu" ] || [ "$os_version" != "24.04" ]; } && { [ "$os_sku" != "AzureLinux" ] || [ "$os_version" != "3.0" ]; }; then + # Only test on Ubuntu 24.04/26.04 and Azure Linux 3.0 that LSM BPF is configured correctly + if { [ "$os_sku" != "Ubuntu" ] || [ "$os_version" != "24.04" ] || [ "$os_version" != "26.04" ] ; } && { [ "$os_sku" != "AzureLinux" ] || [ "$os_version" != "3.0" ]; }; then echo "$test: will not test for BPF to be present within LSM modules for SKU: $os_sku, version: $os_version" echo "$test:Finish" return 0 @@ -1772,11 +1773,16 @@ testContainerImagePrefetchScript() { testBccTools () { local test="BCCInstallTest" os_sku="${1}" + os_version="${2}" echo "$test: checking if BCC tools were successfully installed" if [ "$os_sku" = "AzureLinuxOSGuard" ]; then echo "$test: Skipping check on AzureLinuxOSGuard - BCC tools are not installed" return 0 fi + if [ "$os_version" = "26.04" ]; then + echo "$test: Skipping check on Ubuntu 26.04 - BCC tools are not installed" + return 0 + fi for line in ' - bcc-tools' ' - libbcc-examples'; do if ! grep -F -x -e "$line" $VHD_LOGS_FILEPATH; then err "BCC tools were not successfully installed" @@ -2583,7 +2589,7 @@ testContainerNetworkingPluginsInstalled() { # This will keep the VM alive after the tests are run and we can SSH/Bastion into the VM to run the test manually. # Therefore, for example, you can run "sudo bash /var/lib/waagent/run-command/download/0/script.sh" to run the tests manually. checkPerformanceData -testBccTools $OS_SKU +testBccTools $OS_SKU $OS_VERSION testVHDBuildLogsExist testCriticalTools testPackagesInstalled diff --git a/vhdbuilder/packer/trivy-scan.sh b/vhdbuilder/packer/trivy-scan.sh index 42f301f326b..85e32da0a2a 100644 --- a/vhdbuilder/packer/trivy-scan.sh +++ b/vhdbuilder/packer/trivy-scan.sh @@ -17,6 +17,9 @@ TRIVY_DEB_2204_VERSION="0.70.0-ubuntu22.04u9" # renovate: datasource=custom.deb2404 depName=trivy versioning=deb TRIVY_DEB_2404_VERSION="0.70.0-ubuntu24.04u9" +# renovate: datasource=custom.deb2604 depName=trivy versioning=deb +TRIVY_DEB_2604_VERSION="0.70.0-ubuntu26.04u12" + # renovate: datasource=rpm depName=trivy registryUrl=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata TRIVY_RPM_VERSION="0.70.0-9.azl3" @@ -77,50 +80,77 @@ retrycmd_if_failure() { echo Executed \"$@\" $i times; } +addPMCAptKey() { + local OS_VERSION=${1} + + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + + if [ "${OS_VERSION}" = "26.04" ]; then + # Ubuntu 26.04 (Resolute) PMC repo is signed with Microsoft's newer 2025 gpg key + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft-2025.asc | gpg --dearmor > /tmp/microsoft-2025.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-2025.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + fi +} + +installAzCLIFromUbuntuPMC() { + local ARCHITECTURE=${1} + + apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg + + if [ "${ARCHITECTURE,,}" = "arm64" ]; then + echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + else + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + fi + + apt_get_update + apt_get_install 5 1 60 azure-cli +} + install_azure_cli() { OS_SKU=${1} OS_VERSION=${2} ARCHITECTURE=${3} TEST_VM_ADMIN_USERNAME=${4} - if [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "22.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_update - apt_get_install 5 1 60 python3-pip - pip install azure-cli - export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" - CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then - echo "Azure CLI is not installed properly." - exit 1 - fi - elif [ "$OS_SKU" = "Ubuntu" ] && [ "$OS_VERSION" = "24.04" ] && [ "${ARCHITECTURE,,}" = "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "Ubuntu" ] && { [ "$OS_VERSION" = "20.04" ] || [ "$OS_VERSION" = "22.04" ] || [ "$OS_VERSION" = "24.04" ]; } && [ "${ARCHITECTURE,,}" != "arm64" ]; then - apt_get_install 5 1 60 ca-certificates curl apt-transport-https lsb-release gnupg - curl -sL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/azure-cli.list - apt_get_update - apt_get_install 5 1 60 azure-cli - elif [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then + if [ "$OS_SKU" != "Ubuntu" ] && [ "$OS_SKU" != "CBLMariner" ] && [ "$OS_SKU" != "AzureLinux" ] && [ "$OS_SKU" != "Flatcar" ] && [ "$OS_SKU" != "AzureContainerLinux" ] && [ "$OS_SKU" != "AzureLinuxOSGuard" ]; then + echo "Unrecognized OS SKU for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" + exit 1 + fi + + if [ "$OS_SKU" = "CBLMariner" ] || [ "$OS_SKU" = "AzureLinux" ]; then sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc sudo sh -c 'echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo' sudo dnf install -y azure-cli - elif [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then + return 0 + fi + + if [ "$OS_SKU" = "Flatcar" ] || [ "$OS_SKU" = "AzureContainerLinux" ] || [ "$OS_SKU" = "AzureLinuxOSGuard" ]; then python3 -m venv "/home/$TEST_VM_ADMIN_USERNAME/venv" export PATH="/home/$TEST_VM_ADMIN_USERNAME/venv/bin:$PATH" pip install azure-cli CHECKAZ=$(pip freeze | grep "azure-cli==") - if [ -z $CHECKAZ ]; then + if [ -z "$CHECKAZ" ]; then + echo "Azure CLI is not installed properly." + exit 1 + fi + return 0 + fi + + if [ "$OS_VERSION" = "26.04" ]; then + # TODO(2604): install azcli from PMC once the resolute package is available + apt_get_update + apt_get_install 5 1 60 python3-pip + pip install azure-cli --break-system-packages + export PATH="/home/$TEST_VM_ADMIN_USERNAME/.local/bin:$PATH" + CHECKAZ=$(pip freeze | grep "azure-cli==") + if [ -z "$CHECKAZ" ]; then echo "Azure CLI is not installed properly." exit 1 fi else - echo "Unrecognized SKU, Version, and Architecture combination for downloading az: $OS_SKU $OS_VERSION $ARCHITECTURE" - exit 1 + installAzCLIFromUbuntuPMC "$ARCHITECTURE" fi } @@ -136,15 +166,15 @@ login_with_user_assigned_managed_identity() { echo "logging into azure with flags: $LOGIN_FLAGS" az login $LOGIN_FLAGS } + login_with_umsi_object_id() { login_with_user_assigned_managed_identity "--object-id" "$1" } + login_with_umsi_resource_id() { login_with_user_assigned_managed_identity "--resource-id" "$1" } -install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME - install_trivy_from_github() { # Use the dedicated GitHub fallback version — PMC versions (e.g., 0.68.2) don't have # matching GitHub releases, so we pin to an actual upstream release separately. @@ -181,6 +211,7 @@ install_trivy() { 20.04) deb_version="${TRIVY_DEB_2004_VERSION}" ;; 22.04) deb_version="${TRIVY_DEB_2204_VERSION}" ;; 24.04) deb_version="${TRIVY_DEB_2404_VERSION}" ;; + 26.04) deb_version="${TRIVY_DEB_2604_VERSION}" ;; *) echo "No tracked trivy deb version for Ubuntu $os_version, downloading from GitHub" install_trivy_from_github @@ -201,6 +232,12 @@ install_trivy() { esac } +if [ "$OS_SKU" = "Ubuntu" ]; then + addPMCAptKey $OS_VERSION +fi + +install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME + install_trivy "$OS_SKU" "$OS_VERSION" login_with_umsi_object_id ${UMSI_PRINCIPAL_ID} diff --git a/vhdbuilder/packer/vhd-scanning.sh b/vhdbuilder/packer/vhd-scanning.sh index c1ce9e7e17b..42c24da07ca 100755 --- a/vhdbuilder/packer/vhd-scanning.sh +++ b/vhdbuilder/packer/vhd-scanning.sh @@ -249,10 +249,17 @@ isCISUnsupportedUbuntu() { local os="$1" local version="$2" - # Only 22.04+ are supported - if [ "$os" = "Ubuntu" ] && { [ "$version" = "20.04" ]; }; then + # Only 22.04+ are supported. + if [ "$os" = "Ubuntu" ] && [ "$version" = "20.04" ]; then return 0 fi + + # No CIS benchmarks yet available for Ubuntu 26.04 (resolute) + # TODO(2604): enable 26.04 CIS scanning when support is added by upstream CIS. + if [ "$os" = "Ubuntu" ] && [ "$version" = "26.04" ]; then + return 0 + fi + return 1 } isFlatcar() { diff --git a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh index 807d67bae41..dcadf961c88 100755 --- a/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh +++ b/vhdbuilder/scripts/linux/ubuntu/tool_installs_ubuntu.sh @@ -1,23 +1,21 @@ #!/bin/bash -{{/* FIPS-related error codes */}} -ERR_UA_TOOLS_INSTALL_TIMEOUT=180 {{/* Timeout waiting for ubuntu-advantage-tools install */}} -ERR_ADD_UA_APT_REPO=181 {{/* Error to add UA apt repository */}} -ERR_UA_ATTACH=182 {{/* Error attaching UA */}} -ERR_UA_DISABLE_LIVEPATCH=183 {{/* Error to disable UA livepatch */}} -ERR_UA_ENABLE_FIPS=184 {{/* Error to enable UA FIPS */}} -ERR_UA_DETACH=185 {{/* Error to detach UA */}} -ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 {{/* Timeout to install linux header */}} -ERR_STRONGSWAN_INSTALL_TIMEOUT=187 {{/* Timeout to install strongswan */}} -ERR_UA_ESM_HOOK_CLEANUP=188 {{/* Error removing the apt ESM hook for Ubuntu Pro */}} -ERR_UA_MASK_UNIT=189 {{/* Error stopping/disabling/masking an Ubuntu Pro background unit */}} -ERR_UA_TOKEN_CLEANUP=190 {{/* Error removing the baked-in Ubuntu Pro machine token state */}} - -ERR_NTP_INSTALL_TIMEOUT=10 {{/*Unable to install NTP */}} -ERR_NTP_START_TIMEOUT=11 {{/* Unable to start NTP */}} -ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 {{/* Timeout waiting for systemd-timesyncd stop */}} -ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 {{/* Timeout waiting for ntp stop */}} -ERR_CHRONY_INSTALL_TIMEOUT=14 {{/*Unable to install CHRONY */}} -ERR_CHRONY_START_TIMEOUT=15 {{/* Unable to start CHRONY */}} +ERR_UA_TOOLS_INSTALL_TIMEOUT=180 # Timeout waiting for ubuntu-advantage-tools install +ERR_ADD_UA_APT_REPO=181 # Error to add UA apt repository +ERR_UA_ATTACH=182 # Error attaching UA +ERR_UA_DISABLE_LIVEPATCH=183 # Error to disable UA livepatch +ERR_UA_ENABLE_FIPS=184 # Error to enable UA FIPS +ERR_UA_DETACH=185 # Error to detach UA +ERR_LINUX_HEADER_INSTALL_TIMEOUT=186 # Timeout to install linux header +ERR_STRONGSWAN_INSTALL_TIMEOUT=187 # Timeout to install strongswan +ERR_UA_ESM_HOOK_CLEANUP=188 # Error removing the apt ESM hook for Ubuntu Pro +ERR_UA_MASK_UNIT=189 # Error stopping/disabling/masking an Ubuntu Pro background unit +ERR_UA_TOKEN_CLEANUP=190 # Error removing the baked-in Ubuntu Pro machine token state +ERR_NTP_INSTALL_TIMEOUT=10 # Unable to install NTP +ERR_NTP_START_TIMEOUT=11 # Unable to start NTP +ERR_STOP_OR_DISABLE_SYSTEMD_TIMESYNCD_TIMEOUT=12 # Timeout waiting for systemd-timesyncd stop +ERR_STOP_OR_DISABLE_NTP_TIMEOUT=13 # Timeout waiting for ntp stop +ERR_CHRONY_INSTALL_TIMEOUT=14 # Unable to install CHRONY +ERR_CHRONY_START_TIMEOUT=15 # Unable to start CHRONY echo "Sourcing tool_installs_ubuntu.sh" @@ -44,9 +42,9 @@ installBcc() { fi mkdir -p /tmp/bcc - pushd /tmp/bcc + pushd /tmp/bcc || exit 1 git clone https://github.com/iovisor/bcc.git - mkdir bcc/build; cd bcc/build + mkdir bcc/build; cd bcc/build || exit 1 git checkout v0.29.0 @@ -54,11 +52,12 @@ installBcc() { make sudo make install || exit 1 cmake -DPYTHON_CMD=python3 .. || exit 1 # build python3 binding - pushd src/python/ + pushd src/python/ || exit 1 make sudo make install || exit 1 - popd - popd + popd || exit 1 + popd || exit 1 + # we explicitly do not remove build-essential or python # these are standard packages we want to keep, they should usually be in the final build anyway. # only ensuring they are installed above.