diff --git a/CODEOWNERS b/CODEOWNERS index 6c25970de73..6c921eaee65 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -17,7 +17,10 @@ parts/linux/cloud-init/artifacts/mariner/package-update.timer @yewmsft @YaoC parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer @yewmsft @YaoC +parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @yewmsft @YaoC parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @yewmsft @YaoC +spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @yewmsft @YaoC spec/parts/linux/cloud-init/artifacts/mariner-package-update_spec.sh @yewmsft @YaoC diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index ee0d07c898b..a9664dab157 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -86,7 +86,10 @@ "configure-azure-network.sh": "configureAzureNetworkScript", "init-aks-custom-cloud.sh": "initAKSCustomCloud", # Distro-specific scripts + # The updater and handler share one nodecustomdata block, so a change to + # either file hotfix-delivers both atomically. "ubuntu/ubuntu-snapshot-update.sh": "snapshotUpdateScript", + "ubuntu/security-update.sh": "securityUpdateScript", "mariner/mariner-package-update.sh": "packageUpdateScriptMariner", # Systemd services "kubelet.service": "kubeletSystemdService", @@ -97,6 +100,8 @@ "secure-tls-bootstrap.service": "secureTLSBootstrapService", "ensure-no-dup.service": "ensureNoDupEbtablesService", "measure-tls-bootstrapping-latency.service": "measureTLSBootstrappingLatencyService", + # These existing mappings remain part of the hotfix inventory, but the static + # units have no nodecustomdata write_files block and therefore are not injected. "ubuntu/snapshot-update.service": "snapshotUpdateService", "ubuntu/snapshot-update.timer": "snapshotUpdateTimer", "mariner/package-update.service": "packageUpdateServiceMariner", diff --git a/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh new file mode 100644 index 00000000000..7b451e300f8 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/ubuntu/security-update.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +# Applies the securityPatch component selected for this node's agent pool. This +# preserves the legacy Ubuntu snapshot-update behavior while moving desired state into the +# generic live-patching ConfigMap and status contract owned by the updater loop. + +: "${SECURITY_PATCH_CONFIG_DIR:=/var/lib/security-patch}" +: "${SECURITY_PATCH_DEFAULT_ENDPOINT:=snapshot.ubuntu.com}" + +security_patch_unattended_upgrade() { + local attempt + + for attempt in $(seq 1 10); do + if unattended-upgrade -v; then + echo "executed unattended upgrade ${attempt} times" + return 0 + fi + if [ "${attempt}" -lt 10 ]; then + sleep 5 + fi + done + + return 1 +} + +security_patch_generate_sources_list() { + local endpoint="$1" + local golden_timestamp="$2" + local code_name="$3" + + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" || return 1 + if [ "${endpoint}" = "${SECURITY_PATCH_DEFAULT_ENDPOINT}" ]; then + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe +deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse +EOF + else + cat < "${SECURITY_PATCH_CONFIG_DIR}/sources.list" || return 1 +deb http://${endpoint}/ubuntu ${code_name} main restricted +deb http://${endpoint}/ubuntu ${code_name}-updates main restricted +deb http://${endpoint}/ubuntu ${code_name} universe +deb http://${endpoint}/ubuntu ${code_name}-updates universe +deb http://${endpoint}/ubuntu ${code_name} multiverse +deb http://${endpoint}/ubuntu ${code_name}-updates multiverse +deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse +deb http://${endpoint}/ubuntu ${code_name}-security main restricted +deb http://${endpoint}/ubuntu ${code_name}-security universe +deb http://${endpoint}/ubuntu ${code_name}-security multiverse +EOF + fi + + cat < "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" || return 1 +Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; +Dir::Etc::sourceparts ""; +EOF +} + +security_patch_repo_endpoint() { + local node_json="$1" + local repo_service + local private_ip_regex='^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$' + + if ! repo_service="$(printf '%s' "${node_json}" | jq -r '.metadata.annotations["kubernetes.azure.com/live-patching-repo-service"] // empty')"; then + echo "failed to read live patching repo service annotation" + return 1 + fi + + if [ -z "${repo_service}" ]; then + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + elif printf '%s' "${repo_service}" | grep -Eq "${private_ip_regex}"; then + echo "${repo_service}" + else + echo "ignoring invalid live patching repo service: ${repo_service}" >&2 + echo "${SECURITY_PATCH_DEFAULT_ENDPOINT}" + fi +} + +# Returns success when the desired security patch timestamp for this node's +# agent pool matches the timestamp in the locally checkpointed component state. +securityPatchIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + local node_json="$3" + local agent_pool + local desired_timestamp + local current_timestamp + + if ! agent_pool="$(printf '%s' "${node_json}" | jq -er '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + return 1 + fi + if ! desired_timestamp="$(printf '%s' "${desired_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + if ! current_timestamp="$(printf '%s' "${current_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + return 1 + fi + + [ "${desired_timestamp}" = "${current_timestamp}" ] +} + +# Applies the security patch timestamp selected for this node's agent pool and, +# after patching succeeds, updates the legacy annotation consumed by the RP. +updateSecurityPatch() { + local component_payload="${1:-}" + local node_json="${2:-}" + local agent_pool + local golden_timestamp + local node_name + local repo_endpoint + local code_name + local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" + + if [ -z "${node_json}" ]; then + echo "node JSON is required" + return 1 + fi + if ! agent_pool="$(printf '%s' "${node_json}" | jq -r '.metadata.labels["kubernetes.azure.com/agentpool"] // empty')"; then + echo "failed to read node agent pool label" + return 1 + fi + if [ -z "${agent_pool}" ]; then + echo "node agent pool label is not set" + return 1 + fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name // empty')"; then + echo "node name is not set" + return 1 + fi + + if ! golden_timestamp="$(printf '%s' "${component_payload}" | jq -er --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp // empty')"; then + echo "securityPatch profile is missing for agent pool: ${agent_pool}" + return 1 + fi + if ! printf '%s' "${golden_timestamp}" | grep -Eq '^[0-9]{8}T[0-9]{6}Z$'; then + echo "securityPatch goldenTimestamp is invalid: ${golden_timestamp}" + return 1 + fi + + if ! repo_endpoint="$(security_patch_repo_endpoint "${node_json}")"; then + return 1 + fi + if ! code_name="$(lsb_release -cs)" || [ -z "${code_name}" ]; then + echo "failed to determine Ubuntu codename" + return 1 + fi + if ! security_patch_generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}"; then + echo "failed to generate securityPatch apt configuration" + return 1 + fi + + export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" + if ! apt_get_update_with_opts "${apt_opts}"; then + echo "apt_get_update_with_opts failed" + return 1 + fi + if ! security_patch_unattended_upgrade; then + echo "unattended_upgrade failed" + return 1 + fi + + # Keep the legacy RP status channel current during migration to live-patching-status. + # shellcheck disable=SC2086 + if ! $KUBECTL annotate --overwrite node "${node_name}" "kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp}"; then + echo "failed to update legacy securityPatch status annotation" + return 1 + fi + + echo "securityPatch update completed successfully: ${golden_timestamp}" +} + +${__SOURCED__:+return} + +# shellcheck disable=SC1091 +source /opt/azure/containers/provision_source_distro.sh \ No newline at end of file diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index 6e200a8fb8b..e8545960baf 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -3,151 +3,357 @@ set -o nounset set -e -# Global constants used in this file. -# ------------------------------------------------------------------------------------------------- -SECURITY_PATCH_CONFIG_DIR=/var/lib/security-patch -KUBECONFIG="/var/lib/kubelet/kubeconfig" -KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}" -DEFAULT_ENDPOINT="snapshot.ubuntu.com" - -# Function definitions used in this file. -# functions defined until "${__SOURCED__:+return}" are sourced and tested in - -# spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh. -# ------------------------------------------------------------------------------------------------- -# Execute unattended-upgrade -unattended_upgrade() { - retries=10 - for i in $(seq 1 $retries); do - unattended-upgrade -v && break - if [ $i -eq $retries ]; then - return 1 - else sleep 5 - fi - done - echo Executed unattended upgrade $i times -} +# Runs the generic node-side component reconciliation loop. Component handlers own +# their payload schema and node mutations; this loop owns annotations, ConfigMap +# validation, dispatch, local component state, and final success reporting. -generate_sources_list() { - local endpoint="$1" - local golden_timestamp="$2" - local code_name="$3" - - mkdir -p "${SECURITY_PATCH_CONFIG_DIR}" - if [ "${endpoint}" = "${DEFAULT_ENDPOINT}" ]; then - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name} multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-updates multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-backports main restricted universe multiverse -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security main restricted -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security universe -deb https://${endpoint}/ubuntu/${golden_timestamp} ${code_name}-security multiverse -EOF - else - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/sources.list" -deb http://${endpoint}/ubuntu ${code_name} main restricted -deb http://${endpoint}/ubuntu ${code_name}-updates main restricted -deb http://${endpoint}/ubuntu ${code_name} universe -deb http://${endpoint}/ubuntu ${code_name}-updates universe -deb http://${endpoint}/ubuntu ${code_name} multiverse -deb http://${endpoint}/ubuntu ${code_name}-updates multiverse -deb http://${endpoint}/ubuntu ${code_name}-backports main restricted universe multiverse -deb http://${endpoint}/ubuntu ${code_name}-security main restricted -deb http://${endpoint}/ubuntu ${code_name}-security universe -deb http://${endpoint}/ubuntu ${code_name}-security multiverse -EOF - fi - - cat << EOF > "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" -Dir::Etc::sourcelist "${SECURITY_PATCH_CONFIG_DIR}/sources.list"; -Dir::Etc::sourceparts ""; -EOF - - echo "live patching configuration generated successfully" -} +: "${KUBECONFIG:=/var/lib/kubelet/kubeconfig}" +: "${KUBECTL:=/opt/bin/kubectl --kubeconfig ${KUBECONFIG}}" +: "${KNEAD_COMPONENT_CONFIG_NAMESPACE:=kube-system}" +: "${KNEAD_COMPONENT_CONFIGMAP:=live-patching-config}" +: "${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH:=live-patching-config\.json}" +: "${KNEAD_COMPONENT_GOAL_ANNOTATION:=kubernetes.azure.com/live-patching-config-goal-hash}" +: "${KNEAD_COMPONENT_STATUS_ANNOTATION:=kubernetes.azure.com/live-patching-status}" +: "${KNEAD_COMPONENT_STATE_FILE:=/var/lib/aks/live-patching/current.json}" + +KNEAD_COMPONENT_RESULTS='{}' +KNEAD_COMPONENT_RESULTS_VALID=true -main() { - # At startup, we need to wait for kubelet to finish TLS bootstrapping to create the kubeconfig file. - while [ ! -f ${KUBECONFIG} ]; do - echo 'Waiting for TLS bootstrapping' +# Waits for kubelet credentials so kubectl can read Node and ConfigMap state. +knead_wait_for_kubeconfig() { + while [ ! -f "${KUBECONFIG}" ]; do + echo "waiting for kubelet kubeconfig" sleep 3 done +} + +# Reads this Node using the lowercase hostname expected from cloud provider registration. +knead_read_node() { + local node_name - node_name=$(hostname) + node_name="$(hostname)" if [ -z "${node_name}" ]; then - echo "cannot get node name" - exit 1 + echo "cannot get node name" >&2 + return 1 + fi + + node_name="$(printf '%s' "${node_name}" | tr '[:upper:]' '[:lower:]')" + + # shellcheck disable=SC2086 + $KUBECTL get node "${node_name}" -o json +} + +# Returns the value of the given annotation. +knead_get_node_annotation() { + local node_json="$1" + local annotation="$2" + + printf '%s' "${node_json}" | jq -r --arg annotation "${annotation}" '.metadata.annotations[$annotation] // empty' +} + +# Reads and validates the generic ConfigMap contract before any component handler runs. +# +# Knead validates the component envelope. Each handler owns its decoded +# nodeConfig schema. The goal must be the bare sha256 digest of the exact +# ConfigMap value read by this node. +knead_read_configmap() { + local goal="$1" + local payload + local payload_hash + + # shellcheck disable=SC2086 + if ! payload="$($KUBECTL get cm -n "${KNEAD_COMPONENT_CONFIG_NAMESPACE}" "${KNEAD_COMPONENT_CONFIGMAP}" -o "jsonpath={.data.${KNEAD_COMPONENT_CONFIG_KEY_JSONPATH}}")"; then + echo "failed to read live-patching-config ConfigMap" >&2 + return 1 + fi + + if ! printf '%s' "${payload}" | jq -e ' + (.components | type == "array") and + (.components | all( + (.name | type == "string") and + (.name | length > 0) and + (.nodeConfig | type == "string") + )) and + ([.components[].name] | length) == ([.components[].name] | unique | length) + ' > /dev/null; then + echo "live-patching-config payload has invalid envelope" >&2 + return 1 + fi + + if ! printf '%s' "${goal}" | grep -Eq '^[0-9a-f]{64}$'; then + echo "live-patching goal hash must be a 64-character lowercase sha256 digest" >&2 + return 1 + fi + + payload_hash="$(printf '%s' "${payload}" | sha256sum | awk '{print $1}')" + if [ "${payload_hash}" != "${goal}" ]; then + echo "live-patching goal hash does not match ConfigMap payload: goal=${goal}, payload=${payload_hash}" >&2 + return 1 fi - # Azure cloud provider assigns node name as the lowner case of the hostname - node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') + printf '%s' "${payload}" +} + +# Returns success when this exact component config was applied successfully but +# the overall Node status did not converge, such as when the annotation update or +# a sibling component failed. This local checkpoint prevents repeating successful +# work on the next retry. Missing or malformed state is treated as not current. +knead_component_is_current() { + local component="$1" + local component_payload="$2" + local node_json="$3" + local component_comparator="$4" + local current_payload + + if [ ! -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + return 1 + fi - # retrieve golden timestamp from node annotation - golden_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-golden-timestamp']}") - if [ -z "${golden_timestamp}" ]; then - echo "golden timestamp is not set, skip live patching" - exit 0 + if ! current_payload="$(jq -er --arg component "${component}" \ + '.components | map(select(.name == $component)) | last | .nodeConfig' \ + "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)"; then + return 1 fi - echo "golden timestamp is: ${golden_timestamp}" - current_timestamp=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-current-timestamp']}") - if [ -n "${current_timestamp}" ]; then - echo "current timestamp is: ${current_timestamp}" + "${component_comparator}" "${component_payload}" "${current_payload}" "${node_json}" +} + +# Records one successfully applied component without changing sibling +# state. Missing or malformed state is rebuilt from an empty component list. +knead_write_component_state() { + local component="$1" + local component_payload="$2" + local state='{"components":[]}' + local state_dir + local state_tmp + local updated_at + + state_dir="$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + state_tmp="${KNEAD_COMPONENT_STATE_FILE}.tmp" + updated_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - if [ "${golden_timestamp}" = "${current_timestamp}" ]; then - echo "golden and current timestamp is the same, nothing to patch" - exit 0 + if ! mkdir -p "${state_dir}"; then + echo "failed to create knead-component state directory" + return 1 + fi + if [ -f "${KNEAD_COMPONENT_STATE_FILE}" ]; then + if ! state="$(jq -c 'select(.components | type == "array") | {components: .components}' "${KNEAD_COMPONENT_STATE_FILE}" 2> /dev/null)" || [ -z "${state}" ]; then + state='{"components":[]}' fi fi + if ! printf '%s' "${state}" | jq \ + --arg component "${component}" \ + --arg componentPayload "${component_payload}" \ + --arg updatedAt "${updated_at}" \ + '.updatedAt = $updatedAt | .components = ([.components[] | select(.name != $component)] + [{name: $component, nodeConfig: $componentPayload}])' \ + > "${state_tmp}"; then + echo "failed to render knead-component state" + return 1 + fi + if ! mv "${state_tmp}" "${KNEAD_COMPONENT_STATE_FILE}"; then + echo "failed to write knead-component state" + return 1 + fi +} + +# Adds a component result to the status collection. +knead_set_component_result() { + local component="$1" + local code="$2" + local updated_results + + if ! updated_results="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -ce \ + --arg component "${component}" \ + --arg code "${code}" \ + '.[$component] = {code: $code}')" || [ -z "${updated_results}" ]; then + echo "failed to record component result: ${component}=${code}" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 + fi + + KNEAD_COMPONENT_RESULTS="${updated_results}" +} + +# Dispatches every known component and returns failure only after all have been attempted. +# +# This is the main error boundary for node disruption. A broken component should +# not stop unrelated handlers from making progress, so this function records +# failures and continues dispatching. Unknown components are skipped so older +# VHDs can still converge when a newer component appears in the shared config. +# +# Keep component payload parsing behind each handler. The generic loop should know +# only which handler owns a component name, not the shape of that component's JSON. +knead_apply_components() { + local payload="$1" + local node_json="$2" + local failed_components="" + local component + local component_count + local component_comparator + local component_handler + local component_index=0 + local component_payload + local infrastructure_failed=false - # Network isolated cluster can't access the internet, so we deploy a live patching repo service in the cluster - # The node will use the live patching repo service to download the repo metadata and packages - # If the annotation is not set, we will use the ubuntu snapshot repo - live_patching_repo_service=$($KUBECTL get node ${node_name} -o jsonpath="{.metadata.annotations['kubernetes\.azure\.com/live-patching-repo-service']}") - # Limit the live patching repo service to private IPs in the range of 10.x.x.x, 172.16.x.x - 172.31.x.x, and 192.168.x.x - private_ip_regex="^((10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})|(172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})|(192\.168\.[0-9]{1,3}\.[0-9]{1,3}))$" - # shellcheck disable=SC3010 - if [ -n "${live_patching_repo_service}" ] && [[ ! "${live_patching_repo_service}" =~ $private_ip_regex ]]; then - echo "Ignore invalid live patching repo service: ${live_patching_repo_service}" - live_patching_repo_service="" + KNEAD_COMPONENT_RESULTS='{}' + KNEAD_COMPONENT_RESULTS_VALID=true + + if ! component_count="$(printf '%s' "${payload}" | jq -er '.components | length')"; then + echo "failed to read component count" + KNEAD_COMPONENT_RESULTS_VALID=false + return 1 fi + while [ "${component_index}" -lt "${component_count}" ]; do + if ! component="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].name')"; then + echo "failed to read component name at index: ${component_index}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + component_index=$((component_index + 1)) + continue + fi + if ! component_payload="$(printf '%s' "${payload}" | jq -er --argjson index "${component_index}" '.components[$index].nodeConfig')"; then + echo "failed to read component payload: ${component}" + failed_components="${failed_components} ${component}" + infrastructure_failed=true + KNEAD_COMPONENT_RESULTS_VALID=false + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + continue + fi + echo "applying component: ${component}" - repo_endpoint="${DEFAULT_ENDPOINT}" - if [ -z "${live_patching_repo_service}" ]; then - echo "live patching repo service is not set, use ubuntu snapshot repo" - else - echo "live patching repo service is: ${live_patching_repo_service}" - repo_endpoint="${live_patching_repo_service}" + # Select the handler for this component; unsupported components are ignored. + case "${component}" in + securityPatch) + component_comparator=securityPatchIsCurrent + component_handler=updateSecurityPatch + ;; + *) + echo "unsupported component: ${component}" + component_index=$((component_index + 1)) + continue + ;; + esac + + # Skip previously applied work; otherwise apply the component and checkpoint success. + if knead_component_is_current "${component}" "${component_payload}" "${node_json}" "${component_comparator}"; then + echo "component is already current: ${component}" + if ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + elif ! "${component_handler}" "${component_payload}" "${node_json}"; then + failed_components="${failed_components} ${component}" + echo "component failed: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_write_component_state "${component}" "${component_payload}"; then + failed_components="${failed_components} ${component}" + echo "failed to persist component state: ${component}" + if ! knead_set_component_result "${component}" "Failed"; then + infrastructure_failed=true + fi + elif ! knead_set_component_result "${component}" "Succeeded"; then + infrastructure_failed=true + fi + component_index=$((component_index + 1)) + done + + if [ -n "${failed_components}" ]; then + echo "failed components:${failed_components}" + fi + if [ "${infrastructure_failed}" = true ]; then + echo "component dispatch encountered internal failures" + fi + [ -z "${failed_components}" ] && [ "${infrastructure_failed}" = false ] +} + +# Records the processed hash and per-component results in the node status annotation. +knead_write_status() { + local node_name="$1" + local goal="$2" + local status + + if [ "${KNEAD_COMPONENT_RESULTS_VALID}" != true ]; then + echo "refusing to write incomplete live-patching status" + return 1 + fi + if ! status="$(printf '%s' "${KNEAD_COMPONENT_RESULTS}" | jq -c --arg currentHash "${goal}" '{currentHash: $currentHash, components: .}')" || [ -z "${status}" ]; then + echo "failed to render live-patching status annotation" + return 1 fi - code_name=$(lsb_release -cs) - generate_sources_list "${repo_endpoint}" "${golden_timestamp}" "${code_name}" + # shellcheck disable=SC2086 + $KUBECTL annotate --overwrite node "${node_name}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}=${status}" +} + +knead_main() { + local node_name + local node_json + local goal + local status + local payload + local result=0 - export APT_CONFIG="${SECURITY_PATCH_CONFIG_DIR}/apt.conf" + knead_wait_for_kubeconfig + if ! node_json="$(knead_read_node)"; then + echo "failed to read node" + return 1 + fi + if ! node_name="$(printf '%s' "${node_json}" | jq -er '.metadata.name')"; then + echo "failed to read node name" + return 1 + fi + if ! goal="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_GOAL_ANNOTATION}")"; then + echo "failed to read live-patching goal annotation" + return 1 + fi + if [ -z "${goal}" ]; then + echo "live-patching goal is not set, skip knead-component" + return 0 + fi + echo "live-patching goal is: ${goal}" - local apt_opts="-o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3" - if ! apt_get_update_with_opts "${apt_opts}"; then - echo "apt_get_update_with_opts failed" - exit 1 + if ! status="$(knead_get_node_annotation "${node_json}" "${KNEAD_COMPONENT_STATUS_ANNOTATION}")"; then + echo "failed to read live-patching status annotation" + return 1 fi - if ! unattended_upgrade; then - echo "unattended_upgrade failed" - exit 1 + # Skip reconciliation only when this goal was processed and every component succeeded. + if [ -n "${status}" ] && printf '%s' "${status}" | jq -e --arg goal "${goal}" \ + '.currentHash == $goal and (.components | type == "object") and (.components | all(.code == "Succeeded"))' \ + > /dev/null 2>&1; then + echo "live-patching goal is already converged, nothing to apply" + return 0 fi - # update current timestamp - $KUBECTL annotate --overwrite node ${node_name} kubernetes.azure.com/live-patching-current-timestamp=${golden_timestamp} + # Stop before dispatch when generic inputs are missing or unsafe. Once dispatch + # starts, knead_apply_components owns the continue-after-component-failure + # behavior so one broken handler does not prevent other handlers from running. + if ! payload="$(knead_read_configmap "${goal}")"; then + result=1 + else + if ! knead_apply_components "${payload}" "${node_json}"; then + result=1 + fi + + if ! knead_write_status "${node_name}" "${goal}"; then + echo "failed to update live-patching status annotation" + result=1 + fi + fi - echo snapshot update completed successfully + if [ "${result}" -eq 0 ]; then + echo "knead-component completed successfully" + fi + return "${result}" } ${__SOURCED__:+return} # --------------------------------------- Main Execution starts here -------------------------------------------------- -# source apt_get_update -source /opt/azure/containers/provision_source_distro.sh +# shellcheck disable=SC1091 +source /opt/azure/containers/security-update.sh -main "$@" +knead_main "$@" diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 4175bdfc8c8..b9cc5743255 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -152,6 +152,28 @@ write_files: content: !!binary | {{GetVariableProperty "cloudInitData" "initAKSCustomCloud"}} +# Deliver the generic reconciler and its security handler together so replacing +# the established updater path never leaves it without the dispatched handler. +{{if IsACL }} +{{- else if IsAzlOSGuard}} +{{- else if IsMariner}} +{{- else if IsFlatcar }} +{{- else }} +- path: /opt/azure/containers/ubuntu-snapshot-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "snapshotUpdateScript"}} + +- path: /opt/azure/containers/security-update.sh + permissions: "0544" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "securityUpdateScript"}} +{{end}} + - path: /etc/systemd/system/reconcile-private-hosts.service permissions: "0644" encoding: gzip diff --git a/pkg/agent/const.go b/pkg/agent/const.go index 87688fb64c7..d6fe803d3c0 100644 --- a/pkg/agent/const.go +++ b/pkg/agent/const.go @@ -72,6 +72,7 @@ const ( bindMountScript = "linux/cloud-init/artifacts/bind-mount.sh" bindMountSystemdService = "linux/cloud-init/artifacts/bind-mount.service" snapshotUpdateScript = "linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh" + securityUpdateScript = "linux/cloud-init/artifacts/ubuntu/security-update.sh" snapshotUpdateSystemdService = "linux/cloud-init/artifacts/ubuntu/snapshot-update.service" snapshotUpdateSystemdTimer = "linux/cloud-init/artifacts/ubuntu/snapshot-update.timer" packageUpdateScriptMariner = "linux/cloud-init/artifacts/mariner/mariner-package-update.sh" diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 9f39248e7d1..ed8ccffad53 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -47,6 +47,7 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa "migPartitionScript": getBase64EncodedGzippedCustomScript(migPartitionScript, config), "ensureIMDSRestrictionScript": getBase64EncodedGzippedCustomScript(ensureIMDSRestrictionScript, config), "snapshotUpdateScript": getBase64EncodedGzippedCustomScript(snapshotUpdateScript, config), + "securityUpdateScript": getBase64EncodedGzippedCustomScript(securityUpdateScript, config), "snapshotUpdateService": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdService, config), "snapshotUpdateTimer": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdTimer, config), "packageUpdateScriptMariner": getBase64EncodedGzippedCustomScript(packageUpdateScriptMariner, config), diff --git a/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh new file mode 100644 index 00000000000..014f4f24337 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/security-update_spec.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# shellcheck disable=SC2317 + +Describe 'security-update.sh' + security_patch_test_node_json() { + local repo_service="${1:-}" + + jq -nc --arg repoService "${repo_service}" \ + '{metadata: {name: "aks-node-1", labels: {"kubernetes.azure.com/agentpool": "ap1"}, annotations: {"kubernetes.azure.com/live-patching-repo-service": $repoService}}}' + } + + setup() { + Include ./parts/linux/cloud-init/artifacts/ubuntu/security-update.sh + TEST_DIR="/tmp/security-update-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + + SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}/security-patch" + TEST_NODE_JSON="$(security_patch_test_node_json)" + TEST_APT_UPDATE_STATUS=0 + TEST_ANNOTATE_STATUS=0 + KUBECTL="kubectl" + export SECURITY_PATCH_CONFIG_DIR TEST_NODE_JSON KUBECTL + export TEST_APT_UPDATE_STATUS TEST_ANNOTATE_STATUS + } + + cleanup() { + rm -rf "${TEST_DIR}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + Mock lsb_release + echo "jammy" + End + + Mock unattended-upgrade + echo "unattended-upgrade called" + End + + Mock sleep + echo "sleep called" + End + + Mock kubectl + echo "kubectl called with args: $*" + exit "${TEST_ANNOTATE_STATUS}" + End + + apt_get_update_with_opts() { + echo "apt_get_update_with_opts called with args: $*" + return "${TEST_APT_UPDATE_STATUS}" + } + + It 'applies the timestamp selected for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'apt_get_update_with_opts called with args: -o Acquire::http::Timeout=300 -o Acquire::https::Timeout=300 -o Acquire::Retries=3' + The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb https://snapshot.ubuntu.com/ubuntu/20260710T000000Z jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should include "Dir::Etc::sourcelist \"${SECURITY_PATCH_CONFIG_DIR}/sources.list\";" + End + + It 'compares only the current node agent pool timestamp' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260715T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"},"ap2":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be success + End + + It 'detects a changed timestamp for the current node agent pool' + desired_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' + current_payload='{"agentPools":{"ap1":{"goldenTimestamp":"20260701T000000Z"}}}' + + When call securityPatchIsCurrent "${desired_payload}" "${current_payload}" "${TEST_NODE_JSON}" + The status should be failure + End + + It 'retries unattended upgrade until it succeeds' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + [ "${TEST_UNATTENDED_ATTEMPT}" -ge 3 ] + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be success + The output should include 'unattended-upgrade called: 3' + The output should include 'executed unattended upgrade 3 times' + End + + It 'fails after ten unattended upgrade attempts' + eval 'unattended-upgrade() { + TEST_UNATTENDED_ATTEMPT=$((TEST_UNATTENDED_ATTEMPT + 1)) + echo "unattended-upgrade called: ${TEST_UNATTENDED_ATTEMPT}" + return 1 + }' + TEST_UNATTENDED_ATTEMPT=0 + export TEST_UNATTENDED_ATTEMPT + + When call security_patch_unattended_upgrade + The status should be failure + The output should include 'unattended-upgrade called: 10' + End + + It 'uses the private repository service for a network-isolated cluster' + TEST_NODE_JSON="$(security_patch_test_node_json '10.0.0.1')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'deb http://10.0.0.1/ubuntu jammy main restricted' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should not include 'snapshot.ubuntu.com' + End + + It 'falls back to the snapshot service for an invalid repository annotation' + TEST_NODE_JSON="$(security_patch_test_node_json 'example.com')" + export TEST_NODE_JSON + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be success + The output should include 'securityPatch update completed successfully: 20260710T000000Z' + The stderr should include 'ignoring invalid live patching repo service: example.com' + The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should include 'snapshot.ubuntu.com/ubuntu/20260710T000000Z' + End + + It 'fails when the component has no profile for the node agent pool' + When call updateSecurityPatch '{"agentPools":{"ap2":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch profile is missing for agent pool: ap1' + The output should not include 'apt_get_update_with_opts called' + End + + It 'fails when the selected golden timestamp is malformed' + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"not-a-timestamp"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'securityPatch goldenTimestamp is invalid: not-a-timestamp' + The output should not include 'apt_get_update_with_opts called' + End + + It 'returns failure when apt metadata refresh fails' + TEST_APT_UPDATE_STATUS=1 + export TEST_APT_UPDATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'apt_get_update_with_opts failed' + The output should not include 'unattended-upgrade called' + The output should not include 'kubectl called' + End + + It 'returns failure when the legacy status annotation cannot be updated' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'unattended-upgrade called' + The output should include 'kubectl called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-current-timestamp=20260710T000000Z' + The output should include 'failed to update legacy securityPatch status annotation' + The output should not include 'securityPatch update completed successfully' + End + + It 'returns failure before apt update when apt configuration cannot be written' + SECURITY_PATCH_CONFIG_DIR="/dev/null/security-patch" + export SECURITY_PATCH_CONFIG_DIR + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Not a directory' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'does not mask a sources list write failure when apt config is writable' + mkdir -p "${SECURITY_PATCH_CONFIG_DIR}/sources.list" + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to generate securityPatch apt configuration' + The stderr should include 'Is a directory' + The path "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should not be exist + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End + + It 'returns failure before apt update when the Ubuntu codename is unavailable' + Mock lsb_release + exit 1 + End + + When call updateSecurityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260710T000000Z"}}}' "${TEST_NODE_JSON}" + The status should be failure + The output should include 'failed to determine Ubuntu codename' + The output should not include 'apt_get_update_with_opts called' + The output should not include 'unattended-upgrade called' + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh new file mode 100644 index 00000000000..f67923cb31c --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/snapshot-update-service_spec.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +Describe 'snapshot update service commands' + validate_ubuntu_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-base.json \ + vhdbuilder/packer/vhd-image-builder-cvm.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gen2.json \ + vhdbuilder/packer/vhd-image-builder-arm64-gb.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/security-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.timer")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/ubuntu/knead-component.sh")) == null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_mariner_packer_inputs() { + local packer_file + + for packer_file in \ + vhdbuilder/packer/vhd-image-builder-mariner.json \ + vhdbuilder/packer/vhd-image-builder-mariner-cvm.json \ + vhdbuilder/packer/vhd-image-builder-mariner-arm64.json + do + jq -e ' + [.. | objects | .source? // empty] as $sources | + ($sources | index("parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.service")) != null and + ($sources | index("parts/linux/cloud-init/artifacts/mariner/package-update.timer")) != null + ' "${packer_file}" > /dev/null || return 1 + done + } + + validate_ubuntu_hotfix_transition() { + local template="parts/linux/cloud-init/nodecustomdata.yml" + local generator="hotfix/hotfix_generate.py" + local key + + for key in snapshotUpdateScript securityUpdateScript + do + grep -Fq "GetVariableProperty \"cloudInitData\" \"${key}\"" "${template}" || return 1 + grep -Fq ": \"${key}\"" "${generator}" || return 1 + done + + # Existing VHD-baked units remain unchanged; hotfixes update scripts only. + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateService"' "${template}" || return 1 + ! grep -Fq 'GetVariableProperty "cloudInitData" "snapshotUpdateTimer"' "${template}" + } + + It 'keeps the Ubuntu snapshot service unchanged' + When run grep -Fx 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' parts/linux/cloud-init/artifacts/ubuntu/snapshot-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' + End + + It 'keeps the Azure Linux package updater' + When run grep -Fx 'ExecStart=/opt/azure/containers/mariner-package-update.sh' parts/linux/cloud-init/artifacts/mariner/package-update.service + The status should be success + The output should equal 'ExecStart=/opt/azure/containers/mariner-package-update.sh' + End + + It 'stages the generic updater, handler, and existing units for Ubuntu images' + When call validate_ubuntu_packer_inputs + The status should be success + End + + It 'preserves Azure Linux package updater inputs' + When call validate_mariner_packer_inputs + The status should be success + End + + It 'hotfix-delivers the generic updater and security handler without switching units' + When call validate_ubuntu_hotfix_transition + The status should be success + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh index 8f5cedcc5e2..0916782a840 100644 --- a/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/ubuntu-snapshot-update_spec.sh @@ -1,15 +1,31 @@ #!/bin/bash +# shellcheck disable=SC2089,SC2090 -Describe 'ubuntu-snapshot-update.sh' +Describe 'ubuntu-snapshot-update.sh generic reconciliation' setup() { Include ./parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh - TEST_DIR="/tmp/live-patching-test" - mkdir -p ${TEST_DIR} - SECURITY_PATCH_CONFIG_DIR="${TEST_DIR}" + TEST_DIR="/tmp/knead-component-test" + rm -rf "${TEST_DIR}" + mkdir -p "${TEST_DIR}" + KUBECONFIG="${TEST_DIR}/kubeconfig" touch "${KUBECONFIG}" KUBECTL="kubectl" + KNEAD_COMPONENT_STATE_FILE="${TEST_DIR}/state/current.json" + TEST_COMPONENTS_JSON_FILE="${TEST_DIR}/components.json" + TEST_KUBECTL_ARGS_FILE="${TEST_DIR}/kubectl-args" + + TEST_STATUS="" + TEST_GOAL="" + TEST_AGENT_POOL="ap1" + TEST_REPO_SERVICE="" + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_SECURITY_STATUS=0 + TEST_ANNOTATE_STATUS=0 + export KUBECTL KNEAD_COMPONENT_STATE_FILE TEST_COMPONENTS_JSON_FILE TEST_KUBECTL_ARGS_FILE + export TEST_STATUS TEST_GOAL TEST_AGENT_POOL TEST_REPO_SERVICE TEST_SECURITY_STATUS TEST_ANNOTATE_STATUS } + cleanup() { rm -rf "${TEST_DIR}" } @@ -17,108 +33,317 @@ Describe 'ubuntu-snapshot-update.sh' BeforeEach 'setup' AfterEach 'cleanup' - Mock apt_get_update_with_opts - echo "apt_get_update_with_opts mock called" - End - Mock unattended-upgrade - echo "unattended-upgrade mock called" - End - - It 'should update successfully for regular cluster' - Mock lsb_release - echo "jammy" - End - sources_list=$(cat < "${TEST_KUBECTL_ARGS_FILE}" + cat "${TEST_COMPONENTS_JSON_FILE}" + ;; + esac + End + + updateSecurityPatch() { + echo "updateSecurityPatch called with args: $*" + return "${TEST_SECURITY_STATUS}" + } + + securityPatchIsCurrent() { + local desired_payload="$1" + local current_payload="$2" + local node_json="$3" + local agent_pool + local desired_timestamp + local current_timestamp + + agent_pool="$(printf '%s' "${node_json}" | jq -r '.metadata.labels["kubernetes.azure.com/agentpool"]')" + desired_timestamp="$(printf '%s' "${desired_payload}" | jq -r --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp')" + current_timestamp="$(printf '%s' "${current_payload}" | jq -r --arg agentPool "${agent_pool}" '.agentPools[$agentPool].goldenTimestamp')" + [ "${desired_timestamp}" = "${current_timestamp}" ] + } + + set_payload_goal() { + printf '%s' "$1" > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="$(sha256sum "${TEST_COMPONENTS_JSON_FILE}" | awk '{print $1}')" + export TEST_GOAL + } + + fail_result_then_write_status() { + KNEAD_COMPONENT_RESULTS='not-json' + KNEAD_COMPONENT_RESULTS_VALID=true + knead_set_component_result securityPatch Succeeded || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + fail_component_parse_then_write_status() { + knead_apply_components 'not-json' '{}' || true + echo "results valid: ${KNEAD_COMPONENT_RESULTS_VALID}" + knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + } + + It 'does nothing when goal annotation is not set' + TEST_GOAL="" + export TEST_GOAL + + When call knead_main + The status should be success + The output should include 'live-patching goal is not set, skip knead-component' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'does nothing when status is converged for the goal' + TEST_GOAL="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + TEST_STATUS='{"currentHash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","components":{"securityPatch":{"code":"Succeeded"}}}' + export TEST_GOAL TEST_STATUS + + When call knead_main + The status should be success + The output should include 'live-patching goal is already converged, nothing to apply' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'dispatches securityPatch and writes successful status' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'applying component: securityPatch' + The output should include 'updateSecurityPatch called with args: {"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The output should include '"kubernetes.azure.com/agentpool":"ap1"' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"components":{"securityPatch":{"code":"Succeeded"}}}' + The output should include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'reads the dotted ConfigMap key with an escaped JSONPath' + set_payload_goal '{"components":[]}' + + When call knead_read_configmap "${TEST_GOAL}" + The status should be success + The output should equal '{"components":[]}' + The contents of file "${TEST_KUBECTL_ARGS_FILE}" should equal 'get cm -n kube-system live-patching-config -o jsonpath={.data.live-patching-config\.json}' + End + + It 'fails before dispatch when the goal hash does not match the ConfigMap payload' + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash does not match ConfigMap payload' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'rejects a prefixed goal hash' + printf '%s' '{"components":[]}' > "${TEST_COMPONENTS_JSON_FILE}" + TEST_GOAL="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + export TEST_GOAL + + When call knead_main + The status should be failure + The error should include 'live-patching goal hash must be a 64-character lowercase sha256 digest' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when components is not an array' + set_payload_goal '{"components":{}}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'fails before dispatch when component names are duplicated' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{}}"}]}' + + When call knead_main + The status should be failure + The error should include 'live-patching-config payload has invalid envelope' + The output should not include 'updateSecurityPatch called' + The output should not include 'annotate mock called' + End + + It 'records a failed securityPatch result and advances currentHash' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'component failed: securityPatch' + The output should include 'failed components: securityPatch' + The output should include 'annotate mock called with args: annotate --overwrite node aks-node-1 kubernetes.azure.com/live-patching-status={"currentHash":"' + The output should include '"securityPatch":{"code":"Failed"}' + End + + It 'continues reporting later components after securityPatch fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"},{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' + TEST_SECURITY_STATUS=1 + export TEST_SECURITY_STATUS + + When call knead_main + The status should be failure + The output should include 'component failed: securityPatch' + The output should include 'unsupported component: npd' + The output should include 'failed components: securityPatch' + The output should include '"securityPatch":{"code":"Failed"}' + The output should not include '"npd"' + End + + It 'retries when currentHash matches but a component previously failed' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_STATUS="{\"currentHash\":\"${TEST_GOAL}\",\"components\":{\"securityPatch\":{\"code\":\"Failed\"}}}" + export TEST_STATUS + + When call knead_main The status should be success - The output should include 'live patching repo service is not set, use ubuntu snapshot repo' - The output should include 'apt_get_update_with_opts mock called' - The output should include 'unattended-upgrade mock called' - The output should include 'Executed unattended upgrade 1 times' - The output should include 'snapshot update completed successfully' - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/sources.list" should eq "${sources_list}" - The contents of file "${SECURITY_PATCH_CONFIG_DIR}/apt.conf" should eq "${apt_config}" - End - - It 'should update successfully for ni cluster' - Mock lsb_release - echo "noble" - End - sources_list=$(cat < /dev/null || true + + When call knead_main The status should be success - The output should include 'golden timestamp is not set, skip live patching' + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' End - It 'should do nothing if golden timestamp equals current timestamp' - Mock kubectl - echo "20250820T000000Z" - End - When run main + It 'does not rerun an unchanged component when another component changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260701.0\"}"},{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + + When call knead_main The status should be success - The output should include 'golden timestamp is: 20250820T000000Z' - The output should include 'current timestamp is: 20250820T000000Z' - The output should include 'golden and current timestamp is the same, nothing to patch' + The output should include 'unsupported component: npd' + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should not include '"npd"' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'does not rerun securityPatch when only another agent pool changes' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"},\"ap2\":{\"goldenTimestamp\":\"20260701T000000Z\"}}}"}]}' + + When call knead_main + The status should be success + The output should include 'component is already current: securityPatch' + The output should not include 'updateSecurityPatch called' + The output should include '"securityPatch":{"code":"Succeeded"}' + End + + It 'preserves sibling state when recording a successful component' + mkdir -p "$(dirname "${KNEAD_COMPONENT_STATE_FILE}")" + printf '%s' '{"components":[{"name":"npd","nodeConfig":"{\"version\":\"v20260623.0\"}"}]}' > "${KNEAD_COMPONENT_STATE_FILE}" + + When call knead_write_component_state securityPatch '{"agentPools":{"ap1":{"goldenTimestamp":"20260623T000000Z"}}}' + The status should be success + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"npd"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include 'v20260623.0' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '20260623T000000Z' + End + + It 'fails after handler success when status annotation update fails' + set_payload_goal '{"components":[{"name":"securityPatch","nodeConfig":"{\"agentPools\":{\"ap1\":{\"goldenTimestamp\":\"20260623T000000Z\"}}}"}]}' + TEST_ANNOTATE_STATUS=1 + export TEST_ANNOTATE_STATUS + + When call knead_main + The status should be failure + The output should include 'updateSecurityPatch called' + The output should include 'failed to update live-patching status annotation' + The output should not include 'knead-component completed successfully' + The contents of file "${KNEAD_COMPONENT_STATE_FILE}" should include '"securityPatch"' + End + + It 'fails before annotation when component results cannot be rendered' + KNEAD_COMPONENT_RESULTS='not-json' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'fails before annotation when status rendering produces no output' + KNEAD_COMPONENT_RESULTS='' + export KNEAD_COMPONENT_RESULTS + + When call knead_write_status aks-node-1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + The status should be failure + The output should include 'failed to render live-patching status annotation' + The output should not include 'annotate mock called' + End + + It 'preserves existing results and refuses status after a result update fails' + When call fail_result_then_write_status + The status should be failure + The output should include 'failed to record component result: securityPatch=Succeeded' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' + End + + It 'refuses status after component parsing fails' + When call fail_component_parse_then_write_status + The status should be failure + The output should include 'failed to read component count' + The output should include 'results valid: false' + The output should include 'refusing to write incomplete live-patching status' + The stderr should include 'parse error' + The output should not include 'annotate mock called' End -End +End \ No newline at end of file diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index c2a6e2f4213..89a40370568 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -131,6 +131,8 @@ copyPackerFiles() { KUBELET_SERVICE_DEST=/etc/systemd/system/kubelet.service SECURE_TLS_BOOTSTRAP_SERVICE_SRC=/home/packer/secure-tls-bootstrap.service SECURE_TLS_BOOTSTRAP_SERVICE_DEST=/etc/systemd/system/secure-tls-bootstrap.service + SECURITY_UPDATE_SH_SRC=/home/packer/security-update.sh + SECURITY_UPDATE_SH_DEST=/opt/azure/containers/security-update.sh USU_SH_SRC=/home/packer/ubuntu-snapshot-update.sh USU_SH_DEST=/opt/azure/containers/ubuntu-snapshot-update.sh MPU_SH_SRC=/home/packer/mariner-package-update.sh @@ -492,6 +494,7 @@ copyPackerFiles() { cpAndMode $PAM_D_COMMON_ACCOUNT_SRC $PAM_D_COMMON_ACCOUNT_DEST 644 cpAndMode $PAM_D_COMMON_AUTH_SRC $PAM_D_COMMON_AUTH_DEST 644 cpAndMode $PAM_D_COMMON_PASSWORD_SRC $PAM_D_COMMON_PASSWORD_DEST 644 + cpAndMode $SECURITY_UPDATE_SH_SRC $SECURITY_UPDATE_SH_DEST 544 cpAndMode $USU_SH_SRC $USU_SH_DEST 544 if [ "$UBUNTU_RELEASE" = "24.04" ] && [ "$CPU_ARCH" = "arm64" ]; then diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index da466bdda25..8d1a152999b 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2299,6 +2299,37 @@ checkLocaldnsScriptsAndConfigs() { #------------------------ End of test code related to localdns ------------------------ +testKneadSecurityPatchingAssets() { + local test="testKneadSecurityPatchingAssets" + local os_sku="$1" + local file + local permissions + local -A expected_files=( + ["/etc/systemd/system/snapshot-update.service"]=644 + ["/etc/systemd/system/snapshot-update.timer"]=644 + ["/opt/azure/containers/security-update.sh"]=544 + ["/opt/azure/containers/ubuntu-snapshot-update.sh"]=544 + ) + + if [ "$os_sku" != "Ubuntu" ]; then + return 0 + fi + + for file in "${!expected_files[@]}"; do + if [ ! -f "$file" ]; then + err "$test" "Expected file not found: $file" + fi + permissions=$(stat -c "%a" "$file") + if [ "$permissions" != "${expected_files[$file]}" ]; then + err "$test" "Incorrect permissions for $file. Expected ${expected_files[$file]}, got $permissions" + fi + done + + if ! grep -Fxq 'ExecStart=/opt/azure/containers/ubuntu-snapshot-update.sh' /etc/systemd/system/snapshot-update.service; then + err "$test" "snapshot-update.service does not execute the generic reconciler" + fi +} + # Basic sanity check for Inspektor Gadget artifacts baked into the image. testInspektorGadgetAssets() { local test="testInspektorGadgetAssets" @@ -2547,6 +2578,7 @@ testLtsKernel $OS_VERSION $OS_SKU $ENABLE_FIPS testAutologinDisabled $OS_SKU testCorednsBinaryExtractedAndCached $OS_VERSION checkLocaldnsScriptsAndConfigs $OS_SKU +testKneadSecurityPatchingAssets $OS_SKU testInspektorGadgetAssets testPackageDownloadURLFallbackLogic testFileOwnership $OS_SKU diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index fa25d264ec0..0ccc4d9c749 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -317,6 +317,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index cf571b3431c..a6bc67cde66 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -302,6 +302,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index 1bb630a346e..59e1320d063 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -305,6 +305,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index a4b6e438c9d..498752a02dc 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -309,6 +309,11 @@ "source": "parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh", "destination": "/home/packer/setup-custom-search-domains.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/ubuntu/security-update.sh", + "destination": "/home/packer/security-update.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh",