From 0a0fbea10a29b83af721088be42cb57ba08e656e Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 19 Jun 2026 11:13:31 +0200 Subject: [PATCH 01/19] Initial draft of detection of node rollout prow job --- .../Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 11 +++ .../Azure-ARO-HCP-main-presubmits.yaml | 87 +++++++++++++++++++ .../aro-hcp/git/checkout-head/OWNERS | 8 ++ .../aro-hcp-git-checkout-head-commands.sh | 21 +++++ ...ro-hcp-git-checkout-head-ref.metadata.json | 15 ++++ .../aro-hcp-git-checkout-head-ref.yaml | 16 ++++ .../step-registry/aro-hcp/git/checkout/OWNERS | 8 ++ .../checkout/aro-hcp-git-checkout-commands.sh | 20 +++++ .../aro-hcp-git-checkout-ref.metadata.json | 15 ++++ .../checkout/aro-hcp-git-checkout-ref.yaml | 19 ++++ .../aro-hcp/local-e2e-upgrade/OWNERS | 12 +++ ...p-local-e2e-upgrade-workflow.metadata.json | 19 ++++ .../aro-hcp-local-e2e-upgrade-workflow.yaml | 33 +++++++ .../aro-hcp/test/local-post-upgrade/OWNERS | 12 +++ ...ro-hcp-test-local-post-upgrade-commands.sh | 58 +++++++++++++ ...-test-local-post-upgrade-ref.metadata.json | 19 ++++ .../aro-hcp-test-local-post-upgrade-ref.yaml | 52 +++++++++++ .../aro-hcp/test/local-pre-upgrade/OWNERS | 12 +++ ...aro-hcp-test-local-pre-upgrade-commands.sh | 54 ++++++++++++ ...p-test-local-pre-upgrade-ref.metadata.json | 19 ++++ .../aro-hcp-test-local-pre-upgrade-ref.yaml | 57 ++++++++++++ .../aro-hcp/upgrade/infra/OWNERS | 8 ++ .../infra/aro-hcp-upgrade-infra-commands.sh | 76 ++++++++++++++++ .../aro-hcp-upgrade-infra-ref.metadata.json | 15 ++++ .../infra/aro-hcp-upgrade-infra-ref.yaml | 26 ++++++ core-services/prow/02_config/_boskos.yaml | 4 + .../prow/02_config/generate-boskos.py | 3 + 27 files changed, 699 insertions(+) create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS create mode 100755 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS create mode 100755 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 962175e4d9d0c..4aee109534e89 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -463,6 +463,17 @@ tests: MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-upgrade-e2e timeout: 10h0m0s +- as: e2e-parallel-hypershift-upgrade + optional: true + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + steps: + env: + ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 + ARO_HCP_CLOUD: dev + ARO_HCP_DEPLOY_ENV: ci01 + MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 + workflow: aro-hcp-local-e2e-upgrade + timeout: 12h0m0s - as: global-pipeline-postsubmit max_concurrency: 1 postsubmit: true diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 9752006bdf12b..79167c213c548 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -685,6 +685,93 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-parallel,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/e2e-parallel-hypershift-upgrade + decorate: true + decoration_config: + sparse_checkout_files: + - admin/Dockerfile + - backend/Dockerfile + - fleet/Dockerfile + - frontend/Dockerfile + - hcp-recovery/Dockerfile + - image-sync/oc-mirror/Dockerfile + - kube-applier/Dockerfile + - mgmt-agent/Dockerfile + - sessiongate/Dockerfile + - test/Containerfile.e2e + - tooling/aro-hcp-exporter/Dockerfile + timeout: 12h0m0s + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-hypershift-upgrade + optional: true + rerun_command: /test e2e-parallel-hypershift-upgrade + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=e2e-parallel-hypershift-upgrade + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-parallel-hypershift-upgrade,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh new file mode 100644 index 0000000000000..992ad19d4efc2 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +ref="${GIT_REF:-${PULL_PULL_SHA:-}}" +if [[ -z "${ref}" ]]; then + echo "ERROR: PR head ref unknown; set GIT_REF or run on a presubmit with PULL_PULL_SHA" + exit 1 +fi + +echo "Checking out PR head ${ref}" +git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +git fetch --unshallow origin 2>/dev/null || true +git checkout "${ref}" || { + echo "ERROR: failed to checkout ${ref}" + exit 1 +} +git rev-parse HEAD +echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" +git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json new file mode 100644 index 0000000000000..ed31afc6706d1 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml new file mode 100644 index 0000000000000..0608db20bc365 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml @@ -0,0 +1,16 @@ +ref: + as: aro-hcp-git-checkout-head + from: aro-hcp-e2e-tools + commands: aro-hcp-git-checkout-head-commands.sh + grace_period: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + env: + - name: GIT_REF + default: "" + documentation: |- + Override for the PR head. When empty, uses PULL_PULL_SHA from the Prow job. + documentation: |- + Checkout the PR head revision after baseline tests and before in-place operator upgrade. diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh new file mode 100644 index 0000000000000..93339e4e513a9 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +ref="${GIT_REF:-main}" +if [[ -z "${GIT_REF:-}" ]]; then + echo "GIT_REF unset; using default ref=${ref}" +fi + +echo "Checking out ${ref}" +git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +git fetch --unshallow origin 2>/dev/null || true +git checkout "${ref}" || { + echo "ERROR: failed to checkout ${ref}" + exit 1 +} +git rev-parse HEAD +echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" +git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json new file mode 100644 index 0000000000000..f82f672536c8a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml new file mode 100644 index 0000000000000..2b954beaf8125 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml @@ -0,0 +1,19 @@ +ref: + as: aro-hcp-git-checkout + from: aro-hcp-e2e-tools + commands: aro-hcp-git-checkout-commands.sh + grace_period: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + env: + - name: GIT_REF + default: "main" + documentation: |- + Git ref to checkout (branch, tag, or SHA). Defaults to main for the base + provision phase. Override via job env when a different base ref is needed. + documentation: |- + Fetch and checkout the base git ref before regional provision. Defaults to main + so infrastructure is bootstrapped from the stable branch; later steps checkout + the PR head for upgrade validation. diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json new file mode 100644 index 0000000000000..b02830fefd6bf --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml new file mode 100644 index 0000000000000..dfff9c2fc07db --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -0,0 +1,33 @@ +workflow: + as: aro-hcp-local-e2e-upgrade + steps: + allow_best_effort_post_steps: true + leases: + - env: LEASED_MSI_MOCK_SP + resource_type: aro-hcp-msi-mock-cs-sp-dev + pre: + - ref: aro-hcp-lease-acquire + - ref: aro-hcp-git-checkout + - ref: aro-hcp-write-config + - ref: aro-hcp-provision-environment + test: + - ref: aro-hcp-test-local-pre-upgrade + - ref: aro-hcp-git-checkout-head + - ref: aro-hcp-upgrade-infra + - ref: aro-hcp-test-local-post-upgrade + post: + - ref: aro-hcp-gather-provision-failure + - ref: aro-hcp-gather-visualization + - ref: aro-hcp-gather-test-visualization + - ref: aro-hcp-gather-custom-link-tools + - ref: aro-hcp-gather-observability + - ref: aro-hcp-gather-snapshot + - ref: aro-hcp-deprovision-environment + - ref: aro-hcp-lease-release + documentation: |- + Acquire runtime leases, checkout main (or GIT_REF override), provision regional infrastructure, run upgrade/create tests, checkout the PR head, + rerun pipeline/RP.HypershiftOperator using hypershift.image and hypershift.sharedIngressImage + digests from PR-head config/config.yaml, then run upgrade/post-infra tests. + Uses the dedicated aro-hcp-dev-upgrade-westus3-slot pool (Dev - 03, slot_count: 1) + so the provisioned cluster persists isolated across both upgrade suite phases. + Override GIT_REF on git-checkout steps via job env when needed. diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh new file mode 100755 index 0000000000000..f82b92c80d50c --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" + +# Must match aro-hcp-test-local-pre-upgrade: post-infra loads cluster state via setup.go. +export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" +if [[ ! -f "${SETUP_FILEPATH}" ]]; then + printf 'Missing e2e setup file: %s (upgrade/create must run in a prior step)\n' "${SETUP_FILEPATH}" >&2 + exit 1 +fi + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none + +unset GOFLAGS + +# This block prepares the environment to run the tests in. +# It runs against INFRA_SUBSCRIPTION. +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" +make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=kubeconfig +export AZURE_TOKEN_CREDENTIALS=prod +FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" +make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" + +# This block runs the tests against CUSTOMER_SUBSCRIPTION. +az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" + +./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ + --junit-path="${ARTIFACT_DIR}/junit.xml" \ + --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ + --max-concurrency 100 + +junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" +gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json new file mode 100644 index 0000000000000..3aa745358ec50 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml new file mode 100644 index 0000000000000..91696240bbf15 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml @@ -0,0 +1,52 @@ +ref: + as: aro-hcp-test-local-post-upgrade + from: aro-hcp-e2e-tests + commands: aro-hcp-test-local-post-upgrade-commands.sh + credentials: + - namespace: test-credentials + name: aro-hcp-qe-pull-secret + mount_path: /var/run/aro-hcp-qe-pull-secret + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + grace_period: 30s + timeout: 4h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: Selects which environment's cluster secrets to use. + - name: AROHCP_ENV + default: "development" + - name: LOCATION + default: "westus3" + documentation: Azure region for the test environment. + - name: POOLED_IDENTITIES + default: "true" + documentation: Whether to use pooled identities for the test. + - name: ARO_HCP_DEPLOY_ENV + default: "prow" + documentation: Config environment name. + - name: ARO_HCP_SUITE_NAME + default: "upgrade/post-infra" + documentation: |- + Suite to run via aro-hcp-tests run-suite after the infra upgrade step. + - name: SETUP_FILEPATH + default: "" + documentation: |- + Path to e2e-setup.json written by upgrade/create in the pre-upgrade step. When + empty, the step uses ${SHARED_DIR}/e2e-setup.json (same as pre-upgrade). + - name: E2E_JUNIT_SHARED_NAME + default: "junit-e2e-post-upgrade.xml.gz" + documentation: |- + Gzip junit artifact filename under SHARED_DIR (avoids clobbering baseline junit). + - name: COMPRESS_TIMING_METADATA + default: "true" + documentation: Whether to compress timing metadata files with gzip. + documentation: |- + Same local e2e harness as aro-hcp-test-local, run after infra upgrade. + Defaults to the upgrade/post-infra suite, loading e2e-setup.json from + ${SHARED_DIR} written by the pre-upgrade step. diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh new file mode 100755 index 0000000000000..f579a0c57e2ef --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" + +# upgrade/create writes; upgrade/post-infra reads. SHARED_DIR persists across workflow steps. +export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none + +unset GOFLAGS + +# This block prepares the environment to run the tests in. +# It runs against INFRA_SUBSCRIPTION. +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" +make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=kubeconfig +export AZURE_TOKEN_CREDENTIALS=prod +FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" +make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" + +# This block runs the tests against CUSTOMER_SUBSCRIPTION. +az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" + +./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ + --junit-path="${ARTIFACT_DIR}/junit.xml" \ + --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ + --max-concurrency 100 + +junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" +gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json new file mode 100644 index 0000000000000..bea0e79ce8e6a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml new file mode 100644 index 0000000000000..39997ff29b120 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml @@ -0,0 +1,57 @@ +ref: + as: aro-hcp-test-local-pre-upgrade + from: aro-hcp-e2e-tests + commands: aro-hcp-test-local-pre-upgrade-commands.sh + credentials: + - namespace: test-credentials + name: aro-hcp-qe-pull-secret + mount_path: /var/run/aro-hcp-qe-pull-secret + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + grace_period: 30s + timeout: 4h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: Selects which environment's cluster secrets to use. + - name: AROHCP_ENV + default: "development" + - name: LOCATION + default: "westus3" + documentation: Azure region for the test environment. + - name: POOLED_IDENTITIES + default: "true" + documentation: Whether to use pooled identities for the test. + - name: ARO_HCP_DEPLOY_ENV + default: "ci00" + documentation: Config environment name. + - name: ARO_HCP_SUITE_NAME + default: "upgrade/create" + documentation: |- + Suite to run via aro-hcp-tests run-suite before the infra upgrade step. + - name: SETUP_FILEPATH + default: "" + documentation: |- + Path for upgrade/create to write e2e-setup.json. When empty, the step sets + ${SHARED_DIR}/e2e-setup.json so upgrade/post-infra can load it in a later step. + - name: ARO_E2E_SKIP_CLEANUP + default: "true" + documentation: |- + Skip cluster cleanup after the create suite so the cluster remains for + infra upgrade and post-upgrade validation. + - name: E2E_JUNIT_SHARED_NAME + default: "junit-e2e-pre-upgrade.xml.gz" + documentation: |- + Gzip junit artifact filename under SHARED_DIR (avoids clobbering post-upgrade junit). + - name: COMPRESS_TIMING_METADATA + default: "true" + documentation: Whether to compress timing metadata files with gzip. + documentation: |- + Same local e2e harness as aro-hcp-test-local for the pre-upgrade phase of the + upgrade workflow. Defaults to the upgrade/create suite, which writes + ${SHARED_DIR}/e2e-setup.json for upgrade/post-infra. diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS b/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh new file mode 100644 index 0000000000000..a903544ea4d07 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then + echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-environment first" + exit 1 +fi + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export INFRA_SUBSCRIPTION_ID +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" +export AZURE_TOKEN_CREDENTIALS=prod +export SKIP_CONFIRM=true +export PERSIST=true +export DETECT_DIRTY_GIT_WORKTREE=0 + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" + +if ! yq -e ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift" config/config.yaml >/dev/null; then + echo "ERROR: hypershift defaults missing in config/config.yaml for DEPLOY_ENV=${DEPLOY_ENV}" >&2 + exit 1 +fi + +OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" + +yq eval -n " + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift = ( + load(\"config/config.yaml\") | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift + ) +" > "${OVERRIDE_CONFIG_FILE}" + +echo "Created upgrade override at: ${OVERRIDE_CONFIG_FILE}" +cat "${OVERRIDE_CONFIG_FILE}" + +cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml" + +echo "Hypershift operator image (in override, sourced from PR-head config/config.yaml):" +yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image" "${OVERRIDE_CONFIG_FILE}" +echo "Hypershift shared ingress image (in override, sourced from PR-head config/config.yaml):" +yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage" "${OVERRIDE_CONFIG_FILE}" + +unset GOFLAGS + +run_pipeline() { + local target="$1" + echo "Running pipeline/${target}" + make "pipeline/${target}" \ + DEPLOY_ENV="${DEPLOY_ENV}" \ + OVERRIDE_CONFIG_FILE="${OVERRIDE_CONFIG_FILE}" +} + +cd dev-infrastructure && make mgmt.aks.kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" && cd .. +run_pipeline RP.HypershiftOperator + +echo "upgrade" > "${SHARED_DIR}/provision-phase" +date -u +"%Y-%m-%dT%H:%M:%SZ" > "${SHARED_DIR}/infra-upgrade-timestamp-rfc3339" diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json new file mode 100644 index 0000000000000..6c1ce1eff558b --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml new file mode 100644 index 0000000000000..f10b6271cb93a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml @@ -0,0 +1,26 @@ +ref: + as: aro-hcp-upgrade-infra + from: aro-hcp-e2e-tools + commands: aro-hcp-upgrade-infra-commands.sh + grace_period: 15s + timeout: 2h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + credentials: + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: |- + Selects which environment's cluster secrets to use (dev, int, stg, prod). + - name: ARO_HCP_DEPLOY_ENV + default: "ci00" + documentation: Config environment name. + documentation: |- + In-place Hypershift upgrade on the existing regional environment after git-checkout-head. + Builds OVERRIDE_CONFIG_FILE from the hypershift block (operator + sharedIngressImage + digests) in PR-head config/config.yaml, then reruns pipeline/RP.HypershiftOperator. diff --git a/core-services/prow/02_config/_boskos.yaml b/core-services/prow/02_config/_boskos.yaml index f0e55141285b7..f372af44e2e0e 100644 --- a/core-services/prow/02_config/_boskos.yaml +++ b/core-services/prow/02_config/_boskos.yaml @@ -103,6 +103,10 @@ resources: - aro-hcp-dev-shard3-slot-05 state: free type: aro-hcp-dev-shard3-slot +- names: + - aro-hcp-dev-upgrade-westus3-slot-00 + state: free + type: aro-hcp-dev-upgrade-westus3-slot - max-count: 1 min-count: 1 state: free diff --git a/core-services/prow/02_config/generate-boskos.py b/core-services/prow/02_config/generate-boskos.py index 0e6b8907571ad..10256592badd3 100755 --- a/core-services/prow/02_config/generate-boskos.py +++ b/core-services/prow/02_config/generate-boskos.py @@ -330,6 +330,7 @@ 'aro-hcp-dev-shard2-slot': {}, 'aro-hcp-dev-shard3-slot': {}, 'aro-hcp-dev-hypershift-westus3-slot': {}, + 'aro-hcp-dev-upgrade-westus3-slot': {}, 'aro-hcp-int-shard0-slot': {}, 'aro-hcp-prod-shard0-slot': {}, 'aro-hcp-prod-shard1-slot': {}, @@ -811,6 +812,8 @@ CONFIG['aro-hcp-dev-shard3-slot']['aro-hcp-dev-shard3-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1): CONFIG['aro-hcp-dev-hypershift-westus3-slot']['aro-hcp-dev-hypershift-westus3-slot-{i:0>2}'.format(i=i)] = 1 +for i in range(1): + CONFIG['aro-hcp-dev-upgrade-westus3-slot']['aro-hcp-dev-upgrade-westus3-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1): CONFIG['aro-hcp-int-shard0-slot']['aro-hcp-int-shard0-slot-{i:0>2}'.format(i=i)] = 1 for i in range(3): From 4dfe5416eccff4119d63ea480c2aa41020dd000e Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 26 Jun 2026 11:29:32 +0200 Subject: [PATCH 02/19] ARO-HCP: collapse hypershift upgrade workflow to one-shot in-place suite Replace the three-phase local upgrade flow (pre-upgrade, upgrade-infra, post-upgrade) with a single aro-hcp-test-local-upgrade step that runs upgrade/in-place end-to-end in one process. The workflow still provisions regional infrastructure from main, checks out the PR head, builds a hypershift override from config/config.yaml, then validates node pool hash change and cleanup without e2e-setup.json handoff between steps. Remove the dedicated aro-hcp-dev-upgrade-westus3-slot Boskos pool; leasing for the upgrade subscription will be added in a separate PR. --- .../aro-hcp-local-e2e-upgrade-workflow.yaml | 14 ++--- ...ro-hcp-test-local-post-upgrade-commands.sh | 58 ------------------- ...-test-local-post-upgrade-ref.metadata.json | 19 ------ .../aro-hcp/test/local-pre-upgrade/OWNERS | 12 ---- ...aro-hcp-test-local-pre-upgrade-commands.sh | 54 ----------------- .../aro-hcp-test-local-pre-upgrade-ref.yaml | 57 ------------------ .../OWNERS | 0 .../aro-hcp-test-local-upgrade-commands.sh} | 43 ++++++++------ ...-hcp-test-local-upgrade-ref.metadata.json} | 4 +- .../aro-hcp-test-local-upgrade-ref.yaml} | 28 ++++----- .../aro-hcp/upgrade/infra/OWNERS | 8 --- .../aro-hcp-upgrade-infra-ref.metadata.json | 15 ----- .../infra/aro-hcp-upgrade-infra-ref.yaml | 26 --------- core-services/prow/02_config/_boskos.yaml | 4 -- .../prow/02_config/generate-boskos.py | 3 - 15 files changed, 43 insertions(+), 302 deletions(-) delete mode 100755 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh delete mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json delete mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS delete mode 100755 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh delete mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml rename ci-operator/step-registry/aro-hcp/test/{local-post-upgrade => local-upgrade}/OWNERS (100%) rename ci-operator/step-registry/aro-hcp/{upgrade/infra/aro-hcp-upgrade-infra-commands.sh => test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh} (60%) mode change 100644 => 100755 rename ci-operator/step-registry/aro-hcp/test/{local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json => local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json} (71%) rename ci-operator/step-registry/aro-hcp/test/{local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml => local-upgrade/aro-hcp-test-local-upgrade-ref.yaml} (54%) delete mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS delete mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json delete mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index dfff9c2fc07db..aa0e6b877a6c1 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -11,10 +11,8 @@ workflow: - ref: aro-hcp-write-config - ref: aro-hcp-provision-environment test: - - ref: aro-hcp-test-local-pre-upgrade - ref: aro-hcp-git-checkout-head - - ref: aro-hcp-upgrade-infra - - ref: aro-hcp-test-local-post-upgrade + - ref: aro-hcp-test-local-upgrade post: - ref: aro-hcp-gather-provision-failure - ref: aro-hcp-gather-visualization @@ -25,9 +23,7 @@ workflow: - ref: aro-hcp-deprovision-environment - ref: aro-hcp-lease-release documentation: |- - Acquire runtime leases, checkout main (or GIT_REF override), provision regional infrastructure, run upgrade/create tests, checkout the PR head, - rerun pipeline/RP.HypershiftOperator using hypershift.image and hypershift.sharedIngressImage - digests from PR-head config/config.yaml, then run upgrade/post-infra tests. - Uses the dedicated aro-hcp-dev-upgrade-westus3-slot pool (Dev - 03, slot_count: 1) - so the provisioned cluster persists isolated across both upgrade suite phases. - Override GIT_REF on git-checkout steps via job env when needed. + Acquire runtime leases, checkout main (or GIT_REF override), provision regional + infrastructure, checkout the PR head, build a hypershift override from PR + config/config.yaml, then run upgrade/in-place: create hosted cluster, run + pipeline/RP.HypershiftOperator in the test, validate node pool hash change, cleanup. diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh deleted file mode 100755 index f82b92c80d50c..0000000000000 --- a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" - -# Must match aro-hcp-test-local-pre-upgrade: post-infra loads cluster state via setup.go. -export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" -if [[ ! -f "${SETUP_FILEPATH}" ]]; then - printf 'Missing e2e setup file: %s (upgrade/create must run in a prior step)\n' "${SETUP_FILEPATH}" >&2 - exit 1 -fi - -env_file="${SHARED_DIR}/aro-hcp-slot.env" -if [[ ! -f "${env_file}" ]]; then - printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 - exit 1 -fi - -# shellcheck disable=SC1090 -source "${env_file}" - -export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" -: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" - -export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" - -export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") -export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") -export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") -export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") -export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" - -az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none - -unset GOFLAGS - -# This block prepares the environment to run the tests in. -# It runs against INFRA_SUBSCRIPTION. -az account set --subscription "${INFRA_SUBSCRIPTION_ID}" -make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" -export KUBECONFIG=kubeconfig -export AZURE_TOKEN_CREDENTIALS=prod -FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" -make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" - -# This block runs the tests against CUSTOMER_SUBSCRIPTION. -az account set --subscription "${CUSTOMER_SUBSCRIPTION}" -make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" - -./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ - --junit-path="${ARTIFACT_DIR}/junit.xml" \ - --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ - --max-concurrency 100 - -junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" -gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json deleted file mode 100644 index 3aa745358ec50..0000000000000 --- a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "path": "aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml", - "owners": { - "approvers": [ - "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", - "deads2k" - ], - "reviewers": [ - "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", - "deads2k" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS deleted file mode 100644 index 3ff2089d3c778..0000000000000 --- a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS +++ /dev/null @@ -1,12 +0,0 @@ -approvers: -- geoberle -- mmazur -- roivaz -- venkateshsredhat -- deads2k -reviewers: -- geoberle -- mmazur -- roivaz -- venkateshsredhat -- deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh deleted file mode 100755 index f579a0c57e2ef..0000000000000 --- a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" - -# upgrade/create writes; upgrade/post-infra reads. SHARED_DIR persists across workflow steps. -export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" - -env_file="${SHARED_DIR}/aro-hcp-slot.env" -if [[ ! -f "${env_file}" ]]; then - printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 - exit 1 -fi - -# shellcheck disable=SC1090 -source "${env_file}" - -export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" -: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" - -export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" - -export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") -export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") -export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") -export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") -export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" - -az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none - -unset GOFLAGS - -# This block prepares the environment to run the tests in. -# It runs against INFRA_SUBSCRIPTION. -az account set --subscription "${INFRA_SUBSCRIPTION_ID}" -make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" -export KUBECONFIG=kubeconfig -export AZURE_TOKEN_CREDENTIALS=prod -FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" -make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" - -# This block runs the tests against CUSTOMER_SUBSCRIPTION. -az account set --subscription "${CUSTOMER_SUBSCRIPTION}" -make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" - -./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ - --junit-path="${ARTIFACT_DIR}/junit.xml" \ - --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ - --max-concurrency 100 - -junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" -gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml deleted file mode 100644 index 39997ff29b120..0000000000000 --- a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml +++ /dev/null @@ -1,57 +0,0 @@ -ref: - as: aro-hcp-test-local-pre-upgrade - from: aro-hcp-e2e-tests - commands: aro-hcp-test-local-pre-upgrade-commands.sh - credentials: - - namespace: test-credentials - name: aro-hcp-qe-pull-secret - mount_path: /var/run/aro-hcp-qe-pull-secret - - namespace: test-credentials - name: cluster-secrets-aro-hcp-dev - mount_path: /var/run/aro-hcp-dev - grace_period: 30s - timeout: 4h0m0s - resources: - requests: - cpu: 1000m - memory: 1Gi - env: - - name: VAULT_SECRET_PROFILE - default: "dev" - documentation: Selects which environment's cluster secrets to use. - - name: AROHCP_ENV - default: "development" - - name: LOCATION - default: "westus3" - documentation: Azure region for the test environment. - - name: POOLED_IDENTITIES - default: "true" - documentation: Whether to use pooled identities for the test. - - name: ARO_HCP_DEPLOY_ENV - default: "ci00" - documentation: Config environment name. - - name: ARO_HCP_SUITE_NAME - default: "upgrade/create" - documentation: |- - Suite to run via aro-hcp-tests run-suite before the infra upgrade step. - - name: SETUP_FILEPATH - default: "" - documentation: |- - Path for upgrade/create to write e2e-setup.json. When empty, the step sets - ${SHARED_DIR}/e2e-setup.json so upgrade/post-infra can load it in a later step. - - name: ARO_E2E_SKIP_CLEANUP - default: "true" - documentation: |- - Skip cluster cleanup after the create suite so the cluster remains for - infra upgrade and post-upgrade validation. - - name: E2E_JUNIT_SHARED_NAME - default: "junit-e2e-pre-upgrade.xml.gz" - documentation: |- - Gzip junit artifact filename under SHARED_DIR (avoids clobbering post-upgrade junit). - - name: COMPRESS_TIMING_METADATA - default: "true" - documentation: Whether to compress timing metadata files with gzip. - documentation: |- - Same local e2e harness as aro-hcp-test-local for the pre-upgrade phase of the - upgrade workflow. Defaults to the upgrade/create suite, which writes - ${SHARED_DIR}/e2e-setup.json for upgrade/post-infra. diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS similarity index 100% rename from ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS rename to ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh old mode 100644 new mode 100755 similarity index 60% rename from ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh rename to ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index a903544ea4d07..7f889d40749a2 --- a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -3,6 +3,8 @@ set -o errexit set -o nounset set -o pipefail +: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" + if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-environment first" exit 1 @@ -25,8 +27,7 @@ export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") -INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") -export INFRA_SUBSCRIPTION_ID +export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" export AZURE_TOKEN_CREDENTIALS=prod export SKIP_CONFIRM=true @@ -34,14 +35,13 @@ export PERSIST=true export DETECT_DIRTY_GIT_WORKTREE=0 az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none -az account set --subscription "${INFRA_SUBSCRIPTION_ID}" if ! yq -e ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift" config/config.yaml >/dev/null; then echo "ERROR: hypershift defaults missing in config/config.yaml for DEPLOY_ENV=${DEPLOY_ENV}" >&2 exit 1 fi -OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" +export OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" yq eval -n " .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift = ( @@ -61,16 +61,25 @@ yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImag unset GOFLAGS -run_pipeline() { - local target="$1" - echo "Running pipeline/${target}" - make "pipeline/${target}" \ - DEPLOY_ENV="${DEPLOY_ENV}" \ - OVERRIDE_CONFIG_FILE="${OVERRIDE_CONFIG_FILE}" -} - -cd dev-infrastructure && make mgmt.aks.kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" && cd .. -run_pipeline RP.HypershiftOperator - -echo "upgrade" > "${SHARED_DIR}/provision-phase" -date -u +"%Y-%m-%dT%H:%M:%SZ" > "${SHARED_DIR}/infra-upgrade-timestamp-rfc3339" +# Prepare svc cluster access for the test harness (customer tests run below). +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" +make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=kubeconfig +FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" +make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" + +# HypershiftOperator runs on the management cluster; upgrade/in-place invokes +# make pipeline/RP.HypershiftOperator and requires mgmt kubeconfig in KUBECONFIG. +make -C dev-infrastructure/ mgmt.aks.kubeconfig MGMT_KUBECONFIG_FILE=../mgmt-kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=mgmt-kubeconfig + +az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" + +# Single suite: create cluster, run pipeline/RP.HypershiftOperator, hash watch, cleanup. +./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ + --junit-path="${ARTIFACT_DIR}/junit.xml" \ + --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ + --max-concurrency 100 + +gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/junit-e2e-upgrade.xml.gz" diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json similarity index 71% rename from ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json rename to ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json index bea0e79ce8e6a..0c5f11fcfc259 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json @@ -1,5 +1,5 @@ { - "path": "aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml", + "path": "aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml", "owners": { "approvers": [ "geoberle", @@ -16,4 +16,4 @@ "deads2k" ] } -} \ No newline at end of file +} diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml similarity index 54% rename from ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml rename to ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml index 91696240bbf15..fffdfd3adb6b7 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml @@ -1,7 +1,7 @@ ref: - as: aro-hcp-test-local-post-upgrade + as: aro-hcp-test-local-upgrade from: aro-hcp-e2e-tests - commands: aro-hcp-test-local-post-upgrade-commands.sh + commands: aro-hcp-test-local-upgrade-commands.sh credentials: - namespace: test-credentials name: aro-hcp-qe-pull-secret @@ -10,7 +10,7 @@ ref: name: cluster-secrets-aro-hcp-dev mount_path: /var/run/aro-hcp-dev grace_period: 30s - timeout: 4h0m0s + timeout: 8h0m0s resources: requests: cpu: 1000m @@ -28,25 +28,17 @@ ref: default: "true" documentation: Whether to use pooled identities for the test. - name: ARO_HCP_DEPLOY_ENV - default: "prow" + default: "ci01" documentation: Config environment name. - name: ARO_HCP_SUITE_NAME - default: "upgrade/post-infra" + default: "upgrade/in-place" documentation: |- - Suite to run via aro-hcp-tests run-suite after the infra upgrade step. - - name: SETUP_FILEPATH - default: "" - documentation: |- - Path to e2e-setup.json written by upgrade/create in the pre-upgrade step. When - empty, the step uses ${SHARED_DIR}/e2e-setup.json (same as pre-upgrade). - - name: E2E_JUNIT_SHARED_NAME - default: "junit-e2e-post-upgrade.xml.gz" - documentation: |- - Gzip junit artifact filename under SHARED_DIR (avoids clobbering baseline junit). + One-shot suite: create hosted cluster, run pipeline/RP.HypershiftOperator with + OVERRIDE_CONFIG_FILE, validate node pool hash change, cleanup. - name: COMPRESS_TIMING_METADATA default: "true" documentation: Whether to compress timing metadata files with gzip. documentation: |- - Same local e2e harness as aro-hcp-test-local, run after infra upgrade. - Defaults to the upgrade/post-infra suite, loading e2e-setup.json from - ${SHARED_DIR} written by the pre-upgrade step. + After PR-head checkout, builds a hypershift override from config/config.yaml, fetches + svc kubeconfig for frontend discovery and mgmt kubeconfig for pipeline/RP.HypershiftOperator, + then runs the upgrade/in-place suite. Cluster state stays in-process. diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS b/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS deleted file mode 100644 index 8b5c8784266a3..0000000000000 --- a/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -approvers: -- aro-hcp-sl-approvers -- geoberle -- deads2k -reviewers: -- aro-hcp-sl-reviewers -- geoberle -- deads2k diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json deleted file mode 100644 index 6c1ce1eff558b..0000000000000 --- a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "path": "aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml", - "owners": { - "approvers": [ - "aro-hcp-sl-approvers", - "geoberle", - "deads2k" - ], - "reviewers": [ - "aro-hcp-sl-reviewers", - "geoberle", - "deads2k" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml deleted file mode 100644 index f10b6271cb93a..0000000000000 --- a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml +++ /dev/null @@ -1,26 +0,0 @@ -ref: - as: aro-hcp-upgrade-infra - from: aro-hcp-e2e-tools - commands: aro-hcp-upgrade-infra-commands.sh - grace_period: 15s - timeout: 2h0m0s - resources: - requests: - cpu: 1000m - memory: 1Gi - credentials: - - namespace: test-credentials - name: cluster-secrets-aro-hcp-dev - mount_path: /var/run/aro-hcp-dev - env: - - name: VAULT_SECRET_PROFILE - default: "dev" - documentation: |- - Selects which environment's cluster secrets to use (dev, int, stg, prod). - - name: ARO_HCP_DEPLOY_ENV - default: "ci00" - documentation: Config environment name. - documentation: |- - In-place Hypershift upgrade on the existing regional environment after git-checkout-head. - Builds OVERRIDE_CONFIG_FILE from the hypershift block (operator + sharedIngressImage - digests) in PR-head config/config.yaml, then reruns pipeline/RP.HypershiftOperator. diff --git a/core-services/prow/02_config/_boskos.yaml b/core-services/prow/02_config/_boskos.yaml index f372af44e2e0e..f0e55141285b7 100644 --- a/core-services/prow/02_config/_boskos.yaml +++ b/core-services/prow/02_config/_boskos.yaml @@ -103,10 +103,6 @@ resources: - aro-hcp-dev-shard3-slot-05 state: free type: aro-hcp-dev-shard3-slot -- names: - - aro-hcp-dev-upgrade-westus3-slot-00 - state: free - type: aro-hcp-dev-upgrade-westus3-slot - max-count: 1 min-count: 1 state: free diff --git a/core-services/prow/02_config/generate-boskos.py b/core-services/prow/02_config/generate-boskos.py index 10256592badd3..0e6b8907571ad 100755 --- a/core-services/prow/02_config/generate-boskos.py +++ b/core-services/prow/02_config/generate-boskos.py @@ -330,7 +330,6 @@ 'aro-hcp-dev-shard2-slot': {}, 'aro-hcp-dev-shard3-slot': {}, 'aro-hcp-dev-hypershift-westus3-slot': {}, - 'aro-hcp-dev-upgrade-westus3-slot': {}, 'aro-hcp-int-shard0-slot': {}, 'aro-hcp-prod-shard0-slot': {}, 'aro-hcp-prod-shard1-slot': {}, @@ -812,8 +811,6 @@ CONFIG['aro-hcp-dev-shard3-slot']['aro-hcp-dev-shard3-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1): CONFIG['aro-hcp-dev-hypershift-westus3-slot']['aro-hcp-dev-hypershift-westus3-slot-{i:0>2}'.format(i=i)] = 1 -for i in range(1): - CONFIG['aro-hcp-dev-upgrade-westus3-slot']['aro-hcp-dev-upgrade-westus3-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1): CONFIG['aro-hcp-int-shard0-slot']['aro-hcp-int-shard0-slot-{i:0>2}'.format(i=i)] = 1 for i in range(3): From 7a4c3c990a5446db1a31cdfb1f9c838ef647fed9 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 26 Jun 2026 13:11:00 +0200 Subject: [PATCH 03/19] ARO-HCP: fix git checkout steps when ref is a presubmit commit SHA --- .../checkout-head/aro-hcp-git-checkout-head-commands.sh | 8 +++++++- .../aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh index 992ad19d4efc2..93cd80682c7c0 100644 --- a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh @@ -10,8 +10,14 @@ if [[ -z "${ref}" ]]; then fi echo "Checking out PR head ${ref}" -git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +# PULL_PULL_SHA is already present from clonerefs; fetch is best-effort for branch refs. +git fetch --tags origin 2>/dev/null || true git fetch --unshallow origin 2>/dev/null || true +git fetch origin "${ref}" 2>/dev/null || true +git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null || { + echo "ERROR: ref ${ref} is not available locally after fetch" + exit 1 +} git checkout "${ref}" || { echo "ERROR: failed to checkout ${ref}" exit 1 diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh index 93339e4e513a9..90f142d46bd58 100644 --- a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh @@ -9,8 +9,14 @@ if [[ -z "${GIT_REF:-}" ]]; then fi echo "Checking out ${ref}" -git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +# Ref may already be present from clonerefs; fetch is best-effort for branch/tag refs. +git fetch --tags origin 2>/dev/null || true git fetch --unshallow origin 2>/dev/null || true +git fetch origin "${ref}" 2>/dev/null || true +git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null || { + echo "ERROR: ref ${ref} is not available locally after fetch" + exit 1 +} git checkout "${ref}" || { echo "ERROR: failed to checkout ${ref}" exit 1 From 780da6c434c466c59b5219fa4b3231a3d2c38001 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 26 Jun 2026 13:29:36 +0200 Subject: [PATCH 04/19] ARO-HCP: Remove printing of overrride config path and content --- .../test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index 7f889d40749a2..f731beda727ea 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -49,9 +49,6 @@ yq eval -n " ) " > "${OVERRIDE_CONFIG_FILE}" -echo "Created upgrade override at: ${OVERRIDE_CONFIG_FILE}" -cat "${OVERRIDE_CONFIG_FILE}" - cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml" echo "Hypershift operator image (in override, sourced from PR-head config/config.yaml):" From ee2804da95d18941b7568e858239ca368a35e736 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 2 Jul 2026 13:56:12 +0200 Subject: [PATCH 05/19] Update condition to trigger the job --- ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 2 +- .../jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml | 2 +- .../local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 4aee109534e89..1d31456d335ea 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -465,7 +465,7 @@ tests: timeout: 10h0m0s - as: e2e-parallel-hypershift-upgrade optional: true - run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|test/testdata/zz_fixture_TestMainListSuitesForEachSuite_upgrade_in_placeupgrade_in_place\.txt$) steps: env: ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 79167c213c548..82aa77a229244 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -713,7 +713,7 @@ presubmits: name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-hypershift-upgrade optional: true rerun_command: /test e2e-parallel-hypershift-upgrade - run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|test/testdata/zz_fixture_TestMainListSuitesForEachSuite_upgrade_in_placeupgrade_in_place\.txt$) spec: containers: - args: diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json index 0c5f11fcfc259..a20dc509a0355 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json @@ -16,4 +16,4 @@ "deads2k" ] } -} +} \ No newline at end of file From 6eb6c7e4bbe35ae7cbab7228acca1ce698a699a9 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 2 Jul 2026 17:44:33 +0200 Subject: [PATCH 06/19] ARO-HCP: Create OWNERS file for step-registry/aro-hcp/git directory --- ci-operator/step-registry/aro-hcp/git/OWNERS | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ci-operator/step-registry/aro-hcp/git/OWNERS diff --git a/ci-operator/step-registry/aro-hcp/git/OWNERS b/ci-operator/step-registry/aro-hcp/git/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k From f5de082d735a60884b526ab4aed8b36e5c27bc97 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 3 Jul 2026 11:30:25 +0200 Subject: [PATCH 07/19] Update workflow steps based on local-e2e workflow changes --- .../local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index aa0e6b877a6c1..116f4923d2cbe 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -14,6 +14,7 @@ workflow: - ref: aro-hcp-git-checkout-head - ref: aro-hcp-test-local-upgrade post: + - ref: aro-hcp-deprovision-tracked-resource-groups - ref: aro-hcp-gather-provision-failure - ref: aro-hcp-gather-visualization - ref: aro-hcp-gather-test-visualization From 6972c82e5fccd40460a7459331517f2f8bcd0182 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 3 Jul 2026 17:27:41 +0200 Subject: [PATCH 08/19] ARO-HCP: make hypershift upgrade presubmit manual-only Set run_if_changed to ^$ so e2e-parallel-hypershift-upgrade is triggered only via /test e2e-parallel-hypershift-upgrade. The upgrade/in-place suite and workflow are still being validated; the job provisions real dev regional infra, so we should not auto-run it on every PR until path-based triggers are re-enabled. Drop the explicit 12h job timeout to match e2e-parallel; the job inherits the global Prow default of 8h. Co-authored-by: Cursor --- ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 3 +-- .../jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 1d31456d335ea..518d4ddf08021 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -465,7 +465,7 @@ tests: timeout: 10h0m0s - as: e2e-parallel-hypershift-upgrade optional: true - run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|test/testdata/zz_fixture_TestMainListSuitesForEachSuite_upgrade_in_placeupgrade_in_place\.txt$) + run_if_changed: ^$ steps: env: ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 @@ -473,7 +473,6 @@ tests: ARO_HCP_DEPLOY_ENV: ci01 MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-local-e2e-upgrade - timeout: 12h0m0s - as: global-pipeline-postsubmit max_concurrency: 1 postsubmit: true diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 82aa77a229244..817d2244fb62f 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -706,14 +706,13 @@ presubmits: - sessiongate/Dockerfile - test/Containerfile.e2e - tooling/aro-hcp-exporter/Dockerfile - timeout: 12h0m0s labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-hypershift-upgrade optional: true rerun_command: /test e2e-parallel-hypershift-upgrade - run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|test/testdata/zz_fixture_TestMainListSuitesForEachSuite_upgrade_in_placeupgrade_in_place\.txt$) + run_if_changed: ^$ spec: containers: - args: From 10018b02e43e025f641e67149352717d99222d43 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Tue, 7 Jul 2026 13:49:31 +0200 Subject: [PATCH 09/19] ARO-HCP: fix upgrade baseline checkout across step pods Inline PROVISION_GIT_REF checkout into write-config and provision; drop standalone git-checkout steps that did not carry over between pods. Co-authored-by: Cursor --- .../Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 1 + ci-operator/step-registry/aro-hcp/git/OWNERS | 8 ------ .../aro-hcp/git/checkout-head/OWNERS | 8 ------ .../aro-hcp-git-checkout-head-commands.sh | 27 ------------------- ...ro-hcp-git-checkout-head-ref.metadata.json | 15 ----------- .../aro-hcp-git-checkout-head-ref.yaml | 16 ----------- .../step-registry/aro-hcp/git/checkout/OWNERS | 8 ------ .../checkout/aro-hcp-git-checkout-commands.sh | 26 ------------------ .../aro-hcp-git-checkout-ref.metadata.json | 15 ----------- .../checkout/aro-hcp-git-checkout-ref.yaml | 19 ------------- .../aro-hcp-local-e2e-upgrade-workflow.yaml | 11 ++++---- .../aro-hcp-provision-environment-commands.sh | 27 +++++++++++++++++++ .../aro-hcp-provision-environment-ref.yaml | 5 ++++ .../aro-hcp-test-local-upgrade-ref.yaml | 6 ++--- .../aro-hcp-write-config-commands.sh | 27 +++++++++++++++++++ .../aro-hcp-write-config-ref.yaml | 8 +++++- 16 files changed, 75 insertions(+), 152 deletions(-) delete mode 100644 ci-operator/step-registry/aro-hcp/git/OWNERS delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/OWNERS delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json delete mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 518d4ddf08021..e79c1c47fa8b7 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -472,6 +472,7 @@ tests: ARO_HCP_CLOUD: dev ARO_HCP_DEPLOY_ENV: ci01 MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 + PROVISION_GIT_REF: main workflow: aro-hcp-local-e2e-upgrade - as: global-pipeline-postsubmit max_concurrency: 1 diff --git a/ci-operator/step-registry/aro-hcp/git/OWNERS b/ci-operator/step-registry/aro-hcp/git/OWNERS deleted file mode 100644 index 8b5c8784266a3..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -approvers: -- aro-hcp-sl-approvers -- geoberle -- deads2k -reviewers: -- aro-hcp-sl-reviewers -- geoberle -- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS deleted file mode 100644 index 8b5c8784266a3..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -approvers: -- aro-hcp-sl-approvers -- geoberle -- deads2k -reviewers: -- aro-hcp-sl-reviewers -- geoberle -- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh deleted file mode 100644 index 93cd80682c7c0..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -ref="${GIT_REF:-${PULL_PULL_SHA:-}}" -if [[ -z "${ref}" ]]; then - echo "ERROR: PR head ref unknown; set GIT_REF or run on a presubmit with PULL_PULL_SHA" - exit 1 -fi - -echo "Checking out PR head ${ref}" -# PULL_PULL_SHA is already present from clonerefs; fetch is best-effort for branch refs. -git fetch --tags origin 2>/dev/null || true -git fetch --unshallow origin 2>/dev/null || true -git fetch origin "${ref}" 2>/dev/null || true -git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null || { - echo "ERROR: ref ${ref} is not available locally after fetch" - exit 1 -} -git checkout "${ref}" || { - echo "ERROR: failed to checkout ${ref}" - exit 1 -} -git rev-parse HEAD -echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" -git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json deleted file mode 100644 index ed31afc6706d1..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "path": "aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml", - "owners": { - "approvers": [ - "aro-hcp-sl-approvers", - "geoberle", - "deads2k" - ], - "reviewers": [ - "aro-hcp-sl-reviewers", - "geoberle", - "deads2k" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml deleted file mode 100644 index 0608db20bc365..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml +++ /dev/null @@ -1,16 +0,0 @@ -ref: - as: aro-hcp-git-checkout-head - from: aro-hcp-e2e-tools - commands: aro-hcp-git-checkout-head-commands.sh - grace_period: 15s - resources: - requests: - cpu: 100m - memory: 256Mi - env: - - name: GIT_REF - default: "" - documentation: |- - Override for the PR head. When empty, uses PULL_PULL_SHA from the Prow job. - documentation: |- - Checkout the PR head revision after baseline tests and before in-place operator upgrade. diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS deleted file mode 100644 index 8b5c8784266a3..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -approvers: -- aro-hcp-sl-approvers -- geoberle -- deads2k -reviewers: -- aro-hcp-sl-reviewers -- geoberle -- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh deleted file mode 100644 index 90f142d46bd58..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -ref="${GIT_REF:-main}" -if [[ -z "${GIT_REF:-}" ]]; then - echo "GIT_REF unset; using default ref=${ref}" -fi - -echo "Checking out ${ref}" -# Ref may already be present from clonerefs; fetch is best-effort for branch/tag refs. -git fetch --tags origin 2>/dev/null || true -git fetch --unshallow origin 2>/dev/null || true -git fetch origin "${ref}" 2>/dev/null || true -git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null || { - echo "ERROR: ref ${ref} is not available locally after fetch" - exit 1 -} -git checkout "${ref}" || { - echo "ERROR: failed to checkout ${ref}" - exit 1 -} -git rev-parse HEAD -echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" -git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json deleted file mode 100644 index f82f672536c8a..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "path": "aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml", - "owners": { - "approvers": [ - "aro-hcp-sl-approvers", - "geoberle", - "deads2k" - ], - "reviewers": [ - "aro-hcp-sl-reviewers", - "geoberle", - "deads2k" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml deleted file mode 100644 index 2b954beaf8125..0000000000000 --- a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml +++ /dev/null @@ -1,19 +0,0 @@ -ref: - as: aro-hcp-git-checkout - from: aro-hcp-e2e-tools - commands: aro-hcp-git-checkout-commands.sh - grace_period: 15s - resources: - requests: - cpu: 100m - memory: 256Mi - env: - - name: GIT_REF - default: "main" - documentation: |- - Git ref to checkout (branch, tag, or SHA). Defaults to main for the base - provision phase. Override via job env when a different base ref is needed. - documentation: |- - Fetch and checkout the base git ref before regional provision. Defaults to main - so infrastructure is bootstrapped from the stable branch; later steps checkout - the PR head for upgrade validation. diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index 116f4923d2cbe..9ed8bdac27052 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -7,11 +7,9 @@ workflow: resource_type: aro-hcp-msi-mock-cs-sp-dev pre: - ref: aro-hcp-lease-acquire - - ref: aro-hcp-git-checkout - ref: aro-hcp-write-config - ref: aro-hcp-provision-environment test: - - ref: aro-hcp-git-checkout-head - ref: aro-hcp-test-local-upgrade post: - ref: aro-hcp-deprovision-tracked-resource-groups @@ -24,7 +22,8 @@ workflow: - ref: aro-hcp-deprovision-environment - ref: aro-hcp-lease-release documentation: |- - Acquire runtime leases, checkout main (or GIT_REF override), provision regional - infrastructure, checkout the PR head, build a hypershift override from PR - config/config.yaml, then run upgrade/in-place: create hosted cluster, run - pipeline/RP.HypershiftOperator in the test, validate node pool hash change, cleanup. + Acquire runtime leases, render and provision regional infrastructure from main + (via PROVISION_GIT_REF), then run upgrade/in-place against the PR head clone: + build a hypershift override from PR config/config.yaml, create hosted cluster, + run pipeline/RP.HypershiftOperator in the test, validate node pool hash change, + cleanup. diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh index fc49166390553..0495bfbc9b4b8 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh @@ -3,6 +3,33 @@ set -o errexit set -o nounset set -o pipefail +# Each ci-operator step runs in its own pod; git state does not carry over. +aro_hcp_git_checkout() { + local ref="$1" + if [[ -z "${ref}" ]]; then + echo "ERROR: git checkout ref must not be empty" + return 1 + fi + + echo "Checking out ${ref}" + git fetch --tags origin 2>/dev/null || true + git fetch --unshallow origin 2>/dev/null || true + git fetch origin "${ref}" 2>/dev/null || true + if ! git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then + echo "ERROR: ref ${ref} is not available locally after fetch" + return 1 + fi + if ! git checkout "${ref}"; then + echo "ERROR: failed to checkout ${ref}" + return 1 + fi + git rev-parse HEAD +} + +if [[ -n "${PROVISION_GIT_REF:-}" ]]; then + aro_hcp_git_checkout "${PROVISION_GIT_REF}" +fi + env_file="${SHARED_DIR}/aro-hcp-slot.env" if [[ -f "${env_file}" ]]; then # shellcheck disable=SC1090 diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml index 8afd67e98ebe2..45f6042a5d51b 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml @@ -36,6 +36,11 @@ ref: - name: COMPRESS_TIMING_METADATA default: "true" documentation: Whether to compress timing metadata files with gzip. + - name: PROVISION_GIT_REF + default: "" + documentation: |- + When set, checkout this ref before provisioning (e.g. main for upgrade jobs). + Each ci-operator step runs in its own pod; git state does not carry over. dependencies: - name: "pipeline:aro-hcp-backend" env: BACKEND_IMAGE diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml index fffdfd3adb6b7..863e08a641850 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml @@ -39,6 +39,6 @@ ref: default: "true" documentation: Whether to compress timing metadata files with gzip. documentation: |- - After PR-head checkout, builds a hypershift override from config/config.yaml, fetches - svc kubeconfig for frontend discovery and mgmt kubeconfig for pipeline/RP.HypershiftOperator, - then runs the upgrade/in-place suite. Cluster state stays in-process. + Uses the PR head config/config.yaml to build a hypershift override, + fetches svc kubeconfig for frontend discovery and mgmt kubeconfig for + pipeline/RP.HypershiftOperator, then runs the upgrade/in-place suite. diff --git a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh index cf44662723385..f1dc66c59bede 100755 --- a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh +++ b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh @@ -3,6 +3,33 @@ set -o errexit set -o nounset set -o pipefail +# Each ci-operator step runs in its own pod; git state does not carry over. +aro_hcp_git_checkout() { + local ref="$1" + if [[ -z "${ref}" ]]; then + echo "ERROR: git checkout ref must not be empty" + return 1 + fi + + echo "Checking out ${ref}" + git fetch --tags origin 2>/dev/null || true + git fetch --unshallow origin 2>/dev/null || true + git fetch origin "${ref}" 2>/dev/null || true + if ! git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then + echo "ERROR: ref ${ref} is not available locally after fetch" + return 1 + fi + if ! git checkout "${ref}"; then + echo "ERROR: failed to checkout ${ref}" + return 1 + fi + git rev-parse HEAD +} + +if [[ -n "${PROVISION_GIT_REF:-}" ]]; then + aro_hcp_git_checkout "${PROVISION_GIT_REF}" +fi + if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_LOCATION:-}" ]]; then export LOCATION="${MULTISTAGE_PARAM_OVERRIDE_LOCATION}" fi diff --git a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml index ebda4b9aa62f2..ec164e78b487e 100644 --- a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml @@ -19,9 +19,15 @@ ref: - name: ARO_HCP_DEPLOY_ENV default: "" documentation: Config environment name (ci00, ci01, int, stg, prod). + - name: PROVISION_GIT_REF + default: "" + documentation: |- + When set, checkout this ref before rendering (e.g. main for upgrade jobs). + Keeps the partial config aligned with the ref used for provisioning. timeout: 5m documentation: |- Render config.yaml to SHARED_DIR using LOCATION, optionally overridden by MULTISTAGE_PARAM_OVERRIDE_LOCATION, or the runtime slot SELECTED_LOCATION - when the slot export file is present. + when the slot export file is present. When PROVISION_GIT_REF is set, renders + from that ref so the stub matches what provision deploys. Note: The rendered config is only partially accurate and won't always reflect the full production configuration. From 7965a3dc6c0c70eaff6e8eeb735c85aefc64f469 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 9 Jul 2026 17:17:09 +0200 Subject: [PATCH 10/19] ARO-HCP: fix hypershift upgrade override and skip cert verification Read hypershift images from .defaults.hypershift in PR config/config.yaml and override only operator and sharedIngress image coordinates for pipeline/RP.HypershiftOperator. The previous env-scoped path does not exist in source config and caused the upgrade step to fail. Set SKIP_CERT_VERIFICATION=true on run-suite, matching local e2e. --- .../aro-hcp-test-local-upgrade-commands.sh | 32 +++++++++++++++---- .../aro-hcp-test-local-upgrade-ref.yaml | 7 ++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index f731beda727ea..7c7c3958fcbed 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -36,17 +36,37 @@ export DETECT_DIRTY_GIT_WORKTREE=0 az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none -if ! yq -e ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift" config/config.yaml >/dev/null; then - echo "ERROR: hypershift defaults missing in config/config.yaml for DEPLOY_ENV=${DEPLOY_ENV}" >&2 +# Override only hypershift image coordinates from PR config; templatize keeps all +# other hypershift defaults from the base config (what provision deployed). +if ! yq -e '.defaults.hypershift.image.registry' config/config.yaml >/dev/null 2>&1 \ + || ! yq -e '.defaults.hypershift.image.repository' config/config.yaml >/dev/null 2>&1 \ + || ! yq -e '.defaults.hypershift.image.digest' config/config.yaml >/dev/null 2>&1; then + echo "ERROR: hypershift operator image missing in config/config.yaml (.defaults.hypershift.image)" >&2 exit 1 fi +if ! yq -e '.defaults.hypershift.sharedIngressImage.registry' config/config.yaml >/dev/null 2>&1 \ + || ! yq -e '.defaults.hypershift.sharedIngressImage.repository' config/config.yaml >/dev/null 2>&1 \ + || ! yq -e '.defaults.hypershift.sharedIngressImage.digest' config/config.yaml >/dev/null 2>&1; then + echo "ERROR: hypershift sharedIngressImage missing in config/config.yaml (.defaults.hypershift.sharedIngressImage)" >&2 + exit 1 +fi + +HO_IMAGE_REGISTRY=$(yq '.defaults.hypershift.image.registry' config/config.yaml) +HO_IMAGE_REPOSITORY=$(yq '.defaults.hypershift.image.repository' config/config.yaml) +HO_IMAGE_DIGEST=$(yq '.defaults.hypershift.image.digest' config/config.yaml) +HO_SHARED_INGRESS_REGISTRY=$(yq '.defaults.hypershift.sharedIngressImage.registry' config/config.yaml) +HO_SHARED_INGRESS_REPOSITORY=$(yq '.defaults.hypershift.sharedIngressImage.repository' config/config.yaml) +HO_SHARED_INGRESS_DIGEST=$(yq '.defaults.hypershift.sharedIngressImage.digest' config/config.yaml) export OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" yq eval -n " - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift = ( - load(\"config/config.yaml\") | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift - ) + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.registry = \"${HO_IMAGE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.repository = \"${HO_IMAGE_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.digest = \"${HO_IMAGE_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.registry = \"${HO_SHARED_INGRESS_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.repository = \"${HO_SHARED_INGRESS_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.digest = \"${HO_SHARED_INGRESS_DIGEST}\" " > "${OVERRIDE_CONFIG_FILE}" cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml" @@ -74,7 +94,7 @@ az account set --subscription "${CUSTOMER_SUBSCRIPTION}" make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" # Single suite: create cluster, run pipeline/RP.HypershiftOperator, hash watch, cleanup. -./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ +SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ --junit-path="${ARTIFACT_DIR}/junit.xml" \ --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ --max-concurrency 100 diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml index 863e08a641850..645bd48ca6118 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml @@ -39,6 +39,7 @@ ref: default: "true" documentation: Whether to compress timing metadata files with gzip. documentation: |- - Uses the PR head config/config.yaml to build a hypershift override, - fetches svc kubeconfig for frontend discovery and mgmt kubeconfig for - pipeline/RP.HypershiftOperator, then runs the upgrade/in-place suite. + Uses PR head config/config.yaml to build hypershift operator and shared ingress + image overrides, fetches svc kubeconfig for frontend discovery and mgmt + kubeconfig for pipeline/RP.HypershiftOperator, then runs the upgrade/in-place + suite. From 99698dd5460d69429fb0df06907481fd0431c3ca Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Mon, 13 Jul 2026 17:08:44 +0200 Subject: [PATCH 11/19] ARO-HCP: provision main baseline and apply PR images in upgrade test Use PROVISION_GIT_REF to skip PR pipeline image overrides during provision so upgrade jobs deploy from main config, then build a full PR pipeline and hypershift override in the upgrade test step for in-place svc/mgmt upgrades. --- .../aro-hcp-local-e2e-upgrade-workflow.yaml | 9 +- .../aro-hcp-provision-environment-commands.sh | 171 +++++++++--------- .../aro-hcp-provision-environment-ref.yaml | 4 +- .../aro-hcp-test-local-upgrade-commands.sh | 84 ++++++++- .../aro-hcp-test-local-upgrade-ref.yaml | 31 +++- 5 files changed, 194 insertions(+), 105 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index 9ed8bdac27052..d20d998a05513 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -22,8 +22,7 @@ workflow: - ref: aro-hcp-deprovision-environment - ref: aro-hcp-lease-release documentation: |- - Acquire runtime leases, render and provision regional infrastructure from main - (via PROVISION_GIT_REF), then run upgrade/in-place against the PR head clone: - build a hypershift override from PR config/config.yaml, create hosted cluster, - run pipeline/RP.HypershiftOperator in the test, validate node pool hash change, - cleanup. + Acquire runtime leases, render and provision regional infrastructure from + PROVISION_GIT_REF (main baseline images), then run the upgrade suite on the PR + head clone with OVERRIDE_CONFIG_FILE built from PR pipeline images and + hypershift config for in-place svc and mgmt component upgrades. diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh index 0495bfbc9b4b8..d524158c7e281 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh @@ -54,91 +54,96 @@ az account set --subscription "${INFRA_SUBSCRIPTION_ID}" oc version kubelogin --version -BACKEND_DIGEST=$(echo ${BACKEND_IMAGE} | cut -d'@' -f2) -BACKEND_REPOSITORY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -BACKEND_SOURCE_REGISTRY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${BACKEND_SOURCE_REGISTRY} and repo ${BACKEND_REPOSITORY} for Backend Image" - -FRONTEND_DIGEST=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f2) -FRONTEND_REPOSITORY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -FRONTEND_SOURCE_REGISTRY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${FRONTEND_SOURCE_REGISTRY} and repo ${FRONTEND_REPOSITORY} for Frontend Image" - -ADMIN_API_DIGEST=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f2) -ADMIN_API_REPOSITORY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -ADMIN_API_SOURCE_REGISTRY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${ADMIN_API_SOURCE_REGISTRY} and repo ${ADMIN_API_REPOSITORY} for Admin API Image" - -SESSIONGATE_DIGEST=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f2) -SESSIONGATE_REPOSITORY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -SESSIONGATE_SOURCE_REGISTRY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${SESSIONGATE_SOURCE_REGISTRY} and repo ${SESSIONGATE_REPOSITORY} for SessionGate Image" - -HCP_RECOVERY_DIGEST=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f2) -HCP_RECOVERY_REPOSITORY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -HCP_RECOVERY_SOURCE_REGISTRY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${HCP_RECOVERY_SOURCE_REGISTRY} and repo ${HCP_RECOVERY_REPOSITORY} for HCP Recovery Image" - -FLEET_DIGEST=$(echo ${FLEET_IMAGE} | cut -d'@' -f2) -FLEET_REPOSITORY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -FLEET_SOURCE_REGISTRY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${FLEET_SOURCE_REGISTRY} and repo ${FLEET_REPOSITORY} for Fleet Image" - -MGMT_AGENT_DIGEST=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f2) -MGMT_AGENT_REPOSITORY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -MGMT_AGENT_SOURCE_REGISTRY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${MGMT_AGENT_SOURCE_REGISTRY} and repo ${MGMT_AGENT_REPOSITORY} for Mgmt Agent Image" - -KUBE_APPLIER_DIGEST=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f2) -KUBE_APPLIER_REPOSITORY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -KUBE_APPLIER_SOURCE_REGISTRY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${KUBE_APPLIER_SOURCE_REGISTRY} and repo ${KUBE_APPLIER_REPOSITORY} for Kube Applier Image" - -EXPORTER_DIGEST=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f2) -EXPORTER_REPOSITORY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) -EXPORTER_SOURCE_REGISTRY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) -echo "source registry set to ${EXPORTER_SOURCE_REGISTRY} and repo ${EXPORTER_REPOSITORY} for Exporter Image" - -# Set up registries that require oc login - append backend and frontend registries -if [[ -n "${USE_OC_LOGIN_REGISTRIES}" ]]; then - USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" +if [[ -n "${PROVISION_GIT_REF:-}" ]]; then + echo "PROVISION_GIT_REF=${PROVISION_GIT_REF}; using config images from checked-out ref, skipping PR pipeline overrides" + yq eval -n "." > "${OVERRIDE_CONFIG_FILE}" else - USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" + BACKEND_DIGEST=$(echo ${BACKEND_IMAGE} | cut -d'@' -f2) + BACKEND_REPOSITORY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + BACKEND_SOURCE_REGISTRY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${BACKEND_SOURCE_REGISTRY} and repo ${BACKEND_REPOSITORY} for Backend Image" + + FRONTEND_DIGEST=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f2) + FRONTEND_REPOSITORY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + FRONTEND_SOURCE_REGISTRY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${FRONTEND_SOURCE_REGISTRY} and repo ${FRONTEND_REPOSITORY} for Frontend Image" + + ADMIN_API_DIGEST=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f2) + ADMIN_API_REPOSITORY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + ADMIN_API_SOURCE_REGISTRY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${ADMIN_API_SOURCE_REGISTRY} and repo ${ADMIN_API_REPOSITORY} for Admin API Image" + + SESSIONGATE_DIGEST=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f2) + SESSIONGATE_REPOSITORY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + SESSIONGATE_SOURCE_REGISTRY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${SESSIONGATE_SOURCE_REGISTRY} and repo ${SESSIONGATE_REPOSITORY} for SessionGate Image" + + HCP_RECOVERY_DIGEST=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f2) + HCP_RECOVERY_REPOSITORY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + HCP_RECOVERY_SOURCE_REGISTRY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${HCP_RECOVERY_SOURCE_REGISTRY} and repo ${HCP_RECOVERY_REPOSITORY} for HCP Recovery Image" + + FLEET_DIGEST=$(echo ${FLEET_IMAGE} | cut -d'@' -f2) + FLEET_REPOSITORY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + FLEET_SOURCE_REGISTRY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${FLEET_SOURCE_REGISTRY} and repo ${FLEET_REPOSITORY} for Fleet Image" + + MGMT_AGENT_DIGEST=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f2) + MGMT_AGENT_REPOSITORY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + MGMT_AGENT_SOURCE_REGISTRY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${MGMT_AGENT_SOURCE_REGISTRY} and repo ${MGMT_AGENT_REPOSITORY} for Mgmt Agent Image" + + KUBE_APPLIER_DIGEST=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f2) + KUBE_APPLIER_REPOSITORY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + KUBE_APPLIER_SOURCE_REGISTRY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${KUBE_APPLIER_SOURCE_REGISTRY} and repo ${KUBE_APPLIER_REPOSITORY} for Kube Applier Image" + + EXPORTER_DIGEST=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f2) + EXPORTER_REPOSITORY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) + EXPORTER_SOURCE_REGISTRY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) + echo "source registry set to ${EXPORTER_SOURCE_REGISTRY} and repo ${EXPORTER_REPOSITORY} for Exporter Image" + + # Set up registries that require oc login - append backend and frontend registries + if [[ -n "${USE_OC_LOGIN_REGISTRIES}" ]]; then + USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" + else + USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" + fi + echo "USE_OC_LOGIN_REGISTRIES set to: ${USE_OC_LOGIN_REGISTRIES}" + + OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override.yaml" + + # Image overrides + yq eval -n " + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.registry = \"${EXPORTER_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.repository = \"${EXPORTER_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.digest = \"${EXPORTER_DIGEST}\" + " > "${OVERRIDE_CONFIG_FILE}" fi -echo "USE_OC_LOGIN_REGISTRIES set to: ${USE_OC_LOGIN_REGISTRIES}" - -OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override.yaml" - -# Image overrides -yq eval -n " - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.registry = \"${EXPORTER_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.repository = \"${EXPORTER_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.digest = \"${EXPORTER_DIGEST}\" -" > "${OVERRIDE_CONFIG_FILE}" # MSI mock SP overrides (if provided) if [[ -n "${LEASED_MSI_MOCK_SP:-}" ]]; then diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml index 45f6042a5d51b..2eabef3b2ea91 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml @@ -39,8 +39,8 @@ ref: - name: PROVISION_GIT_REF default: "" documentation: |- - When set, checkout this ref before provisioning (e.g. main for upgrade jobs). - Each ci-operator step runs in its own pod; git state does not carry over. + When set, checkout this ref before provisioning (e.g. main for upgrade jobs) + and use image refs from that config instead of PR pipeline builds. dependencies: - name: "pipeline:aro-hcp-backend" env: BACKEND_IMAGE diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index 7c7c3958fcbed..946a3f4267271 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -4,6 +4,14 @@ set -o nounset set -o pipefail : "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" +: "${BACKEND_IMAGE:?BACKEND_IMAGE must be set}" +: "${FRONTEND_IMAGE:?FRONTEND_IMAGE must be set}" +: "${ADMIN_API_IMAGE:?ADMIN_API_IMAGE must be set}" +: "${SESSIONGATE_IMAGE:?SESSIONGATE_IMAGE must be set}" +: "${HCP_RECOVERY_IMAGE:?HCP_RECOVERY_IMAGE must be set}" +: "${FLEET_IMAGE:?FLEET_IMAGE must be set}" +: "${MGMT_AGENT_IMAGE:?MGMT_AGENT_IMAGE must be set}" +: "${KUBE_APPLIER_IMAGE:?KUBE_APPLIER_IMAGE must be set}" if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-environment first" @@ -36,8 +44,7 @@ export DETECT_DIRTY_GIT_WORKTREE=0 az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none -# Override only hypershift image coordinates from PR config; templatize keeps all -# other hypershift defaults from the base config (what provision deployed). +# Hypershift images from PR config; regional service images from PR pipeline builds. if ! yq -e '.defaults.hypershift.image.registry' config/config.yaml >/dev/null 2>&1 \ || ! yq -e '.defaults.hypershift.image.repository' config/config.yaml >/dev/null 2>&1 \ || ! yq -e '.defaults.hypershift.image.digest' config/config.yaml >/dev/null 2>&1; then @@ -58,9 +65,72 @@ HO_SHARED_INGRESS_REGISTRY=$(yq '.defaults.hypershift.sharedIngressImage.registr HO_SHARED_INGRESS_REPOSITORY=$(yq '.defaults.hypershift.sharedIngressImage.repository' config/config.yaml) HO_SHARED_INGRESS_DIGEST=$(yq '.defaults.hypershift.sharedIngressImage.digest' config/config.yaml) +BACKEND_DIGEST=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f2) +BACKEND_REPOSITORY=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +BACKEND_SOURCE_REGISTRY=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +FRONTEND_DIGEST=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f2) +FRONTEND_REPOSITORY=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +FRONTEND_SOURCE_REGISTRY=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +ADMIN_API_DIGEST=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f2) +ADMIN_API_REPOSITORY=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +ADMIN_API_SOURCE_REGISTRY=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +SESSIONGATE_DIGEST=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f2) +SESSIONGATE_REPOSITORY=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +SESSIONGATE_SOURCE_REGISTRY=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +HCP_RECOVERY_DIGEST=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f2) +HCP_RECOVERY_REPOSITORY=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +HCP_RECOVERY_SOURCE_REGISTRY=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +FLEET_DIGEST=$(echo "${FLEET_IMAGE}" | cut -d'@' -f2) +FLEET_REPOSITORY=$(echo "${FLEET_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +FLEET_SOURCE_REGISTRY=$(echo "${FLEET_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +MGMT_AGENT_DIGEST=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f2) +MGMT_AGENT_REPOSITORY=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +MGMT_AGENT_SOURCE_REGISTRY=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +KUBE_APPLIER_DIGEST=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f2) +KUBE_APPLIER_REPOSITORY=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-) +KUBE_APPLIER_SOURCE_REGISTRY=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1) + +if [[ -n "${USE_OC_LOGIN_REGISTRIES:-}" ]]; then + USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY}" +else + USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY}" +fi +export USE_OC_LOGIN_REGISTRIES + export OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" yq eval -n " + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.registry = \"${HO_IMAGE_REGISTRY}\" | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.repository = \"${HO_IMAGE_REPOSITORY}\" | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.digest = \"${HO_IMAGE_DIGEST}\" | @@ -71,10 +141,8 @@ yq eval -n " cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml" -echo "Hypershift operator image (in override, sourced from PR-head config/config.yaml):" -yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image" "${OVERRIDE_CONFIG_FILE}" -echo "Hypershift shared ingress image (in override, sourced from PR-head config/config.yaml):" -yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage" "${OVERRIDE_CONFIG_FILE}" +echo "Created upgrade override config at: ${OVERRIDE_CONFIG_FILE}" +cat "${OVERRIDE_CONFIG_FILE}" unset GOFLAGS @@ -85,15 +153,13 @@ export KUBECONFIG=kubeconfig FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" -# HypershiftOperator runs on the management cluster; upgrade/in-place invokes -# make pipeline/RP.HypershiftOperator and requires mgmt kubeconfig in KUBECONFIG. make -C dev-infrastructure/ mgmt.aks.kubeconfig MGMT_KUBECONFIG_FILE=../mgmt-kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" export KUBECONFIG=mgmt-kubeconfig az account set --subscription "${CUSTOMER_SUBSCRIPTION}" make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" -# Single suite: create cluster, run pipeline/RP.HypershiftOperator, hash watch, cleanup. +# Suite upgrades svc and mgmt regional components via pipeline/RP.* using OVERRIDE_CONFIG_FILE. SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ --junit-path="${ARTIFACT_DIR}/junit.xml" \ --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml index 645bd48ca6118..96d83662d6faf 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml @@ -30,16 +30,35 @@ ref: - name: ARO_HCP_DEPLOY_ENV default: "ci01" documentation: Config environment name. + - name: USE_OC_LOGIN_REGISTRIES + default: "" + documentation: Space-separated list of registries that require oc registry login for authentication - name: ARO_HCP_SUITE_NAME default: "upgrade/in-place" documentation: |- - One-shot suite: create hosted cluster, run pipeline/RP.HypershiftOperator with - OVERRIDE_CONFIG_FILE, validate node pool hash change, cleanup. + Upgrade suite: expects OVERRIDE_CONFIG_FILE and USE_OC_LOGIN_REGISTRIES in the + environment. Suite make targets fetch svc/mgmt kubeconfigs as needed. - name: COMPRESS_TIMING_METADATA default: "true" documentation: Whether to compress timing metadata files with gzip. + dependencies: + - name: "pipeline:aro-hcp-backend" + env: BACKEND_IMAGE + - name: "pipeline:aro-hcp-frontend" + env: FRONTEND_IMAGE + - name: "pipeline:aro-hcp-admin-api" + env: ADMIN_API_IMAGE + - name: "pipeline:aro-hcp-sessiongate" + env: SESSIONGATE_IMAGE + - name: "pipeline:aro-hcp-hcp-recovery" + env: HCP_RECOVERY_IMAGE + - name: "pipeline:aro-hcp-fleet" + env: FLEET_IMAGE + - name: "pipeline:aro-hcp-mgmt-agent" + env: MGMT_AGENT_IMAGE + - name: "pipeline:aro-hcp-kube-applier" + env: KUBE_APPLIER_IMAGE documentation: |- - Uses PR head config/config.yaml to build hypershift operator and shared ingress - image overrides, fetches svc kubeconfig for frontend discovery and mgmt - kubeconfig for pipeline/RP.HypershiftOperator, then runs the upgrade/in-place - suite. + Builds PR pipeline and hypershift image overrides into OVERRIDE_CONFIG_FILE, + then runs the upgrade suite. Kubeconfig fetch and pipeline/RP.* upgrades are + handled inside the suite via make. From 6eaa60020ecb3d498c7e5811995fd3d62b48c57b Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 16 Jul 2026 09:54:46 +0200 Subject: [PATCH 12/19] ARO-HCP: hardcode upgrade/in-place and export UPGRADE_SPEC_COUNT Drop ARO_HCP_SUITE_NAME from the local-upgrade step since this workflow only runs upgrade/in-place. Compute UPGRADE_SPEC_COUNT from the test binary so UpgradeBarrier stays in sync with the suite spec count. --- .../local-upgrade/aro-hcp-test-local-upgrade-commands.sh | 9 ++++++--- .../local-upgrade/aro-hcp-test-local-upgrade-ref.yaml | 9 ++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index 946a3f4267271..7bd86189c089c 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -3,7 +3,6 @@ set -o errexit set -o nounset set -o pipefail -: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" : "${BACKEND_IMAGE:?BACKEND_IMAGE must be set}" : "${FRONTEND_IMAGE:?FRONTEND_IMAGE must be set}" : "${ADMIN_API_IMAGE:?ADMIN_API_IMAGE must be set}" @@ -159,8 +158,12 @@ export KUBECONFIG=mgmt-kubeconfig az account set --subscription "${CUSTOMER_SUBSCRIPTION}" make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" -# Suite upgrades svc and mgmt regional components via pipeline/RP.* using OVERRIDE_CONFIG_FILE. -SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ +# UpgradeBarrier needs the spec count so it can wait for all participants before +# electing a runner to execute "make entrypoint/Region". +UPGRADE_SPEC_COUNT=$(./test/aro-hcp-tests list tests --suite upgrade/in-place --output names | grep -c .) +export UPGRADE_SPEC_COUNT + +SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite upgrade/in-place \ --junit-path="${ARTIFACT_DIR}/junit.xml" \ --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ --max-concurrency 100 diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml index 96d83662d6faf..c19579f27a5c7 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml @@ -33,11 +33,6 @@ ref: - name: USE_OC_LOGIN_REGISTRIES default: "" documentation: Space-separated list of registries that require oc registry login for authentication - - name: ARO_HCP_SUITE_NAME - default: "upgrade/in-place" - documentation: |- - Upgrade suite: expects OVERRIDE_CONFIG_FILE and USE_OC_LOGIN_REGISTRIES in the - environment. Suite make targets fetch svc/mgmt kubeconfigs as needed. - name: COMPRESS_TIMING_METADATA default: "true" documentation: Whether to compress timing metadata files with gzip. @@ -60,5 +55,5 @@ ref: env: KUBE_APPLIER_IMAGE documentation: |- Builds PR pipeline and hypershift image overrides into OVERRIDE_CONFIG_FILE, - then runs the upgrade suite. Kubeconfig fetch and pipeline/RP.* upgrades are - handled inside the suite via make. + prepares svc/mgmt access for the test harness, then runs the upgrade/in-place + suite. Pipeline/RP.* upgrades are handled inside the suite via make. From d2cc763a0bd73840b11d065e7fd6236e3b1f406f Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 16 Jul 2026 13:27:53 +0200 Subject: [PATCH 13/19] ARO-HCP: rename hypershift upgrade job to e2e-parallel-inplace-upgrade The job upgrades the full regional infrastructure in place, not only Hypershift, so drop the hypershift-specific name. --- .../config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 2 +- .../Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index e79c1c47fa8b7..946d2cfab395f 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -463,7 +463,7 @@ tests: MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-upgrade-e2e timeout: 10h0m0s -- as: e2e-parallel-hypershift-upgrade +- as: e2e-parallel-inplace-upgrade optional: true run_if_changed: ^$ steps: diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 817d2244fb62f..33bfa1d95da0e 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -691,7 +691,7 @@ presubmits: - ^main$ - ^main- cluster: build05 - context: ci/prow/e2e-parallel-hypershift-upgrade + context: ci/prow/e2e-parallel-inplace-upgrade decorate: true decoration_config: sparse_checkout_files: @@ -709,9 +709,9 @@ presubmits: labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-hypershift-upgrade + name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-inplace-upgrade optional: true - rerun_command: /test e2e-parallel-hypershift-upgrade + rerun_command: /test e2e-parallel-inplace-upgrade run_if_changed: ^$ spec: containers: @@ -720,7 +720,7 @@ presubmits: - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - - --target=e2e-parallel-hypershift-upgrade + - --target=e2e-parallel-inplace-upgrade command: - ci-operator env: @@ -770,7 +770,7 @@ presubmits: - name: result-aggregator secret: secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-parallel-hypershift-upgrade,?($|\s.*) + trigger: (?m)^/test( | .* )e2e-parallel-inplace-upgrade,?($|\s.*) - agent: kubernetes always_run: true branches: From 34f0903ca1ddd3c058f688ac55ed9bfbde487154 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 17 Jul 2026 08:30:21 +0200 Subject: [PATCH 14/19] ARO-HCP: use provision-from-main and run local e2e after inplace upgrade Replace the PROVISION_GIT_REF checkout path with aro-hcp-provision-from-main for the main baseline, then run the regular local suite after upgrade/in-place to validate the upgraded environment. Co-authored-by: Cursor --- .../Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 2 +- ...o-hcp-gather-provision-failure-commands.sh | 2 +- .../aro-hcp-local-e2e-upgrade-workflow.yaml | 12 +- .../aro-hcp-provision-environment-commands.sh | 198 ++++++++---------- .../aro-hcp-provision-environment-ref.yaml | 5 - .../aro-hcp-test-local-upgrade-commands.sh | 2 +- .../aro-hcp-write-config-commands.sh | 27 --- .../aro-hcp-write-config-ref.yaml | 8 +- 8 files changed, 94 insertions(+), 162 deletions(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 946d2cfab395f..c909e4f6ab797 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -471,8 +471,8 @@ tests: ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 ARO_HCP_CLOUD: dev ARO_HCP_DEPLOY_ENV: ci01 + ARO_HCP_SUITE_PARALLELISM: "55" MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 - PROVISION_GIT_REF: main workflow: aro-hcp-local-e2e-upgrade - as: global-pipeline-postsubmit max_concurrency: 1 diff --git a/ci-operator/step-registry/aro-hcp/gather/provision-failure/aro-hcp-gather-provision-failure-commands.sh b/ci-operator/step-registry/aro-hcp/gather/provision-failure/aro-hcp-gather-provision-failure-commands.sh index ce281ed957c26..fbbb31f64b50b 100644 --- a/ci-operator/step-registry/aro-hcp/gather/provision-failure/aro-hcp-gather-provision-failure-commands.sh +++ b/ci-operator/step-registry/aro-hcp/gather/provision-failure/aro-hcp-gather-provision-failure-commands.sh @@ -21,7 +21,7 @@ echo "MGMT_RESOURCEGROUP: ${MGMT_RESOURCEGROUP}" echo "REGIONAL_RESOURCEGROUP: ${REGIONAL_RESOURCEGROUP}" # Check if provisioning completed successfully -if [[ -f "${SHARED_DIR}/provision-complete" ]]; then +if [[ -f "${SHARED_DIR}/provision-complete" || -f "${SHARED_DIR}/provision-from-main-complete" ]]; then echo "Provisioning completed successfully, skipping failure data gathering" exit 0 fi diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index d20d998a05513..e19fd88fdeeb9 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -8,9 +8,10 @@ workflow: pre: - ref: aro-hcp-lease-acquire - ref: aro-hcp-write-config - - ref: aro-hcp-provision-environment + - ref: aro-hcp-provision-from-main test: - ref: aro-hcp-test-local-upgrade + - ref: aro-hcp-test-local post: - ref: aro-hcp-deprovision-tracked-resource-groups - ref: aro-hcp-gather-provision-failure @@ -22,7 +23,8 @@ workflow: - ref: aro-hcp-deprovision-environment - ref: aro-hcp-lease-release documentation: |- - Acquire runtime leases, render and provision regional infrastructure from - PROVISION_GIT_REF (main baseline images), then run the upgrade suite on the PR - head clone with OVERRIDE_CONFIG_FILE built from PR pipeline images and - hypershift config for in-place svc and mgmt component upgrades. + Acquire runtime leases, provision a main-branch baseline via + aro-hcp-provision-from-main (ACR images from images-push), run + upgrade/in-place on the PR head with OVERRIDE_CONFIG_FILE built from PR + pipeline images and hypershift config, then run the regular local e2e + suite against the upgraded environment. diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh index d524158c7e281..fc49166390553 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-commands.sh @@ -3,33 +3,6 @@ set -o errexit set -o nounset set -o pipefail -# Each ci-operator step runs in its own pod; git state does not carry over. -aro_hcp_git_checkout() { - local ref="$1" - if [[ -z "${ref}" ]]; then - echo "ERROR: git checkout ref must not be empty" - return 1 - fi - - echo "Checking out ${ref}" - git fetch --tags origin 2>/dev/null || true - git fetch --unshallow origin 2>/dev/null || true - git fetch origin "${ref}" 2>/dev/null || true - if ! git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then - echo "ERROR: ref ${ref} is not available locally after fetch" - return 1 - fi - if ! git checkout "${ref}"; then - echo "ERROR: failed to checkout ${ref}" - return 1 - fi - git rev-parse HEAD -} - -if [[ -n "${PROVISION_GIT_REF:-}" ]]; then - aro_hcp_git_checkout "${PROVISION_GIT_REF}" -fi - env_file="${SHARED_DIR}/aro-hcp-slot.env" if [[ -f "${env_file}" ]]; then # shellcheck disable=SC1090 @@ -54,96 +27,91 @@ az account set --subscription "${INFRA_SUBSCRIPTION_ID}" oc version kubelogin --version -if [[ -n "${PROVISION_GIT_REF:-}" ]]; then - echo "PROVISION_GIT_REF=${PROVISION_GIT_REF}; using config images from checked-out ref, skipping PR pipeline overrides" - yq eval -n "." > "${OVERRIDE_CONFIG_FILE}" +BACKEND_DIGEST=$(echo ${BACKEND_IMAGE} | cut -d'@' -f2) +BACKEND_REPOSITORY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +BACKEND_SOURCE_REGISTRY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${BACKEND_SOURCE_REGISTRY} and repo ${BACKEND_REPOSITORY} for Backend Image" + +FRONTEND_DIGEST=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f2) +FRONTEND_REPOSITORY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +FRONTEND_SOURCE_REGISTRY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${FRONTEND_SOURCE_REGISTRY} and repo ${FRONTEND_REPOSITORY} for Frontend Image" + +ADMIN_API_DIGEST=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f2) +ADMIN_API_REPOSITORY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +ADMIN_API_SOURCE_REGISTRY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${ADMIN_API_SOURCE_REGISTRY} and repo ${ADMIN_API_REPOSITORY} for Admin API Image" + +SESSIONGATE_DIGEST=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f2) +SESSIONGATE_REPOSITORY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +SESSIONGATE_SOURCE_REGISTRY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${SESSIONGATE_SOURCE_REGISTRY} and repo ${SESSIONGATE_REPOSITORY} for SessionGate Image" + +HCP_RECOVERY_DIGEST=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f2) +HCP_RECOVERY_REPOSITORY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +HCP_RECOVERY_SOURCE_REGISTRY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${HCP_RECOVERY_SOURCE_REGISTRY} and repo ${HCP_RECOVERY_REPOSITORY} for HCP Recovery Image" + +FLEET_DIGEST=$(echo ${FLEET_IMAGE} | cut -d'@' -f2) +FLEET_REPOSITORY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +FLEET_SOURCE_REGISTRY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${FLEET_SOURCE_REGISTRY} and repo ${FLEET_REPOSITORY} for Fleet Image" + +MGMT_AGENT_DIGEST=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f2) +MGMT_AGENT_REPOSITORY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +MGMT_AGENT_SOURCE_REGISTRY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${MGMT_AGENT_SOURCE_REGISTRY} and repo ${MGMT_AGENT_REPOSITORY} for Mgmt Agent Image" + +KUBE_APPLIER_DIGEST=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f2) +KUBE_APPLIER_REPOSITORY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +KUBE_APPLIER_SOURCE_REGISTRY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${KUBE_APPLIER_SOURCE_REGISTRY} and repo ${KUBE_APPLIER_REPOSITORY} for Kube Applier Image" + +EXPORTER_DIGEST=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f2) +EXPORTER_REPOSITORY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) +EXPORTER_SOURCE_REGISTRY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) +echo "source registry set to ${EXPORTER_SOURCE_REGISTRY} and repo ${EXPORTER_REPOSITORY} for Exporter Image" + +# Set up registries that require oc login - append backend and frontend registries +if [[ -n "${USE_OC_LOGIN_REGISTRIES}" ]]; then + USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" else - BACKEND_DIGEST=$(echo ${BACKEND_IMAGE} | cut -d'@' -f2) - BACKEND_REPOSITORY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - BACKEND_SOURCE_REGISTRY=$(echo ${BACKEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${BACKEND_SOURCE_REGISTRY} and repo ${BACKEND_REPOSITORY} for Backend Image" - - FRONTEND_DIGEST=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f2) - FRONTEND_REPOSITORY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - FRONTEND_SOURCE_REGISTRY=$(echo ${FRONTEND_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${FRONTEND_SOURCE_REGISTRY} and repo ${FRONTEND_REPOSITORY} for Frontend Image" - - ADMIN_API_DIGEST=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f2) - ADMIN_API_REPOSITORY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - ADMIN_API_SOURCE_REGISTRY=$(echo ${ADMIN_API_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${ADMIN_API_SOURCE_REGISTRY} and repo ${ADMIN_API_REPOSITORY} for Admin API Image" - - SESSIONGATE_DIGEST=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f2) - SESSIONGATE_REPOSITORY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - SESSIONGATE_SOURCE_REGISTRY=$(echo ${SESSIONGATE_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${SESSIONGATE_SOURCE_REGISTRY} and repo ${SESSIONGATE_REPOSITORY} for SessionGate Image" - - HCP_RECOVERY_DIGEST=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f2) - HCP_RECOVERY_REPOSITORY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - HCP_RECOVERY_SOURCE_REGISTRY=$(echo ${HCP_RECOVERY_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${HCP_RECOVERY_SOURCE_REGISTRY} and repo ${HCP_RECOVERY_REPOSITORY} for HCP Recovery Image" - - FLEET_DIGEST=$(echo ${FLEET_IMAGE} | cut -d'@' -f2) - FLEET_REPOSITORY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - FLEET_SOURCE_REGISTRY=$(echo ${FLEET_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${FLEET_SOURCE_REGISTRY} and repo ${FLEET_REPOSITORY} for Fleet Image" - - MGMT_AGENT_DIGEST=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f2) - MGMT_AGENT_REPOSITORY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - MGMT_AGENT_SOURCE_REGISTRY=$(echo ${MGMT_AGENT_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${MGMT_AGENT_SOURCE_REGISTRY} and repo ${MGMT_AGENT_REPOSITORY} for Mgmt Agent Image" - - KUBE_APPLIER_DIGEST=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f2) - KUBE_APPLIER_REPOSITORY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - KUBE_APPLIER_SOURCE_REGISTRY=$(echo ${KUBE_APPLIER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${KUBE_APPLIER_SOURCE_REGISTRY} and repo ${KUBE_APPLIER_REPOSITORY} for Kube Applier Image" - - EXPORTER_DIGEST=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f2) - EXPORTER_REPOSITORY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f2-) - EXPORTER_SOURCE_REGISTRY=$(echo ${EXPORTER_IMAGE} | cut -d'@' -f1 | cut -d '/' -f1) - echo "source registry set to ${EXPORTER_SOURCE_REGISTRY} and repo ${EXPORTER_REPOSITORY} for Exporter Image" - - # Set up registries that require oc login - append backend and frontend registries - if [[ -n "${USE_OC_LOGIN_REGISTRIES}" ]]; then - USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" - else - USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" - fi - echo "USE_OC_LOGIN_REGISTRIES set to: ${USE_OC_LOGIN_REGISTRIES}" - - OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override.yaml" - - # Image overrides - yq eval -n " - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.registry = \"${EXPORTER_SOURCE_REGISTRY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.repository = \"${EXPORTER_REPOSITORY}\" | - .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.digest = \"${EXPORTER_DIGEST}\" - " > "${OVERRIDE_CONFIG_FILE}" + USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY} ${EXPORTER_SOURCE_REGISTRY}" fi +echo "USE_OC_LOGIN_REGISTRIES set to: ${USE_OC_LOGIN_REGISTRIES}" + +OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override.yaml" + +# Image overrides +yq eval -n " + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.registry = \"${EXPORTER_SOURCE_REGISTRY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.repository = \"${EXPORTER_REPOSITORY}\" | + .clouds.dev.environments.${DEPLOY_ENV}.defaults.customExporter.image.digest = \"${EXPORTER_DIGEST}\" +" > "${OVERRIDE_CONFIG_FILE}" # MSI mock SP overrides (if provided) if [[ -n "${LEASED_MSI_MOCK_SP:-}" ]]; then diff --git a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml index 2eabef3b2ea91..8afd67e98ebe2 100644 --- a/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/provision/environment/aro-hcp-provision-environment-ref.yaml @@ -36,11 +36,6 @@ ref: - name: COMPRESS_TIMING_METADATA default: "true" documentation: Whether to compress timing metadata files with gzip. - - name: PROVISION_GIT_REF - default: "" - documentation: |- - When set, checkout this ref before provisioning (e.g. main for upgrade jobs) - and use image refs from that config instead of PR pipeline builds. dependencies: - name: "pipeline:aro-hcp-backend" env: BACKEND_IMAGE diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index 7bd86189c089c..c10f9f3a5c9ef 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -13,7 +13,7 @@ set -o pipefail : "${KUBE_APPLIER_IMAGE:?KUBE_APPLIER_IMAGE must be set}" if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then - echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-environment first" + echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-from-main first" exit 1 fi diff --git a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh index f1dc66c59bede..cf44662723385 100755 --- a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh +++ b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-commands.sh @@ -3,33 +3,6 @@ set -o errexit set -o nounset set -o pipefail -# Each ci-operator step runs in its own pod; git state does not carry over. -aro_hcp_git_checkout() { - local ref="$1" - if [[ -z "${ref}" ]]; then - echo "ERROR: git checkout ref must not be empty" - return 1 - fi - - echo "Checking out ${ref}" - git fetch --tags origin 2>/dev/null || true - git fetch --unshallow origin 2>/dev/null || true - git fetch origin "${ref}" 2>/dev/null || true - if ! git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then - echo "ERROR: ref ${ref} is not available locally after fetch" - return 1 - fi - if ! git checkout "${ref}"; then - echo "ERROR: failed to checkout ${ref}" - return 1 - fi - git rev-parse HEAD -} - -if [[ -n "${PROVISION_GIT_REF:-}" ]]; then - aro_hcp_git_checkout "${PROVISION_GIT_REF}" -fi - if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_LOCATION:-}" ]]; then export LOCATION="${MULTISTAGE_PARAM_OVERRIDE_LOCATION}" fi diff --git a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml index ec164e78b487e..ebda4b9aa62f2 100644 --- a/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/write-config/aro-hcp-write-config-ref.yaml @@ -19,15 +19,9 @@ ref: - name: ARO_HCP_DEPLOY_ENV default: "" documentation: Config environment name (ci00, ci01, int, stg, prod). - - name: PROVISION_GIT_REF - default: "" - documentation: |- - When set, checkout this ref before rendering (e.g. main for upgrade jobs). - Keeps the partial config aligned with the ref used for provisioning. timeout: 5m documentation: |- Render config.yaml to SHARED_DIR using LOCATION, optionally overridden by MULTISTAGE_PARAM_OVERRIDE_LOCATION, or the runtime slot SELECTED_LOCATION - when the slot export file is present. When PROVISION_GIT_REF is set, renders - from that ref so the stub matches what provision deploys. + when the slot export file is present. Note: The rendered config is only partially accurate and won't always reflect the full production configuration. From 8cdd174228bf5a56950c7b8d854e19aa01677166 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 17 Jul 2026 13:37:47 +0200 Subject: [PATCH 15/19] ARO-HCP: stop exporting UPGRADE_SPEC_COUNT from local-upgrade step The upgrade suite now derives the barrier participant count itself, so CI no longer needs to list specs and export UPGRADE_SPEC_COUNT. --- .../local-upgrade/aro-hcp-test-local-upgrade-commands.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index c10f9f3a5c9ef..9d261628aaca6 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -158,11 +158,6 @@ export KUBECONFIG=mgmt-kubeconfig az account set --subscription "${CUSTOMER_SUBSCRIPTION}" make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" -# UpgradeBarrier needs the spec count so it can wait for all participants before -# electing a runner to execute "make entrypoint/Region". -UPGRADE_SPEC_COUNT=$(./test/aro-hcp-tests list tests --suite upgrade/in-place --output names | grep -c .) -export UPGRADE_SPEC_COUNT - SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite upgrade/in-place \ --junit-path="${ARTIFACT_DIR}/junit.xml" \ --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ From 934d47108f242a6edb948575d7300100a99f7a7e Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Wed, 22 Jul 2026 12:59:48 +0200 Subject: [PATCH 16/19] ARO-HCP: pass FRONTEND_ADDRESS and subscription name to upgrade suite Resolve CUSTOMER_SUBSCRIPTION to the Azure display name after az account set, then pass FRONTEND_ADDRESS and CUSTOMER_SUBSCRIPTION into run-suite so the harness gets the same env as e2e-local. --- .../local-upgrade/aro-hcp-test-local-upgrade-commands.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh index 9d261628aaca6..250148d29fd73 100755 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh @@ -28,6 +28,7 @@ source "${env_file}" export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" : "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" +: "${CUSTOMER_SUBSCRIPTION:?CUSTOMER_SUBSCRIPTION must be provided by the runtime slot export file}" export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" @@ -156,9 +157,13 @@ make -C dev-infrastructure/ mgmt.aks.kubeconfig MGMT_KUBECONFIG_FILE=../mgmt-kub export KUBECONFIG=mgmt-kubeconfig az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +CUSTOMER_SUBSCRIPTION="$(az account show --output tsv --query 'name')" make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" -SKIP_CERT_VERIFICATION=true ./test/aro-hcp-tests run-suite upgrade/in-place \ +SKIP_CERT_VERIFICATION=true \ +FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" \ +CUSTOMER_SUBSCRIPTION="${CUSTOMER_SUBSCRIPTION}" \ + ./test/aro-hcp-tests run-suite upgrade/in-place \ --junit-path="${ARTIFACT_DIR}/junit.xml" \ --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ --max-concurrency 100 From 202fafb10410d4b0191d9f7d082933949e6734d5 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Mon, 27 Jul 2026 09:35:31 +0200 Subject: [PATCH 17/19] ARO-HCP: broaden inplace-upgrade customer subscription scope --- ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index c909e4f6ab797..fd5547fdc9bde 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -468,7 +468,9 @@ tests: run_if_changed: ^$ steps: env: - ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 + ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters (EA Subscription),ARO HCP + E2E Hosted Clusters 2 (EA Subscription),ARO HCP E2E Hosted Clusters - Dev + - 02,ARO HCP E2E Hosted Clusters - Dev - 03 ARO_HCP_CLOUD: dev ARO_HCP_DEPLOY_ENV: ci01 ARO_HCP_SUITE_PARALLELISM: "55" From d08520604f7d7af5a33a70810b04957e6820d5e9 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 30 Jul 2026 15:06:16 +0200 Subject: [PATCH 18/19] ARO-HCP: align inplace-upgrade OWNERS with aliases and disable always_run --- ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 5 ++--- .../jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml | 1 - .../step-registry/aro-hcp/local-e2e-upgrade/OWNERS | 8 ++------ .../aro-hcp-local-e2e-upgrade-workflow.metadata.json | 8 ++------ .../step-registry/aro-hcp/test/local-upgrade/OWNERS | 8 ++------ .../aro-hcp-test-local-upgrade-ref.metadata.json | 8 ++------ 6 files changed, 10 insertions(+), 28 deletions(-) diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index fd5547fdc9bde..5b35d6beeebad 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -463,9 +463,9 @@ tests: MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-upgrade-e2e timeout: 10h0m0s -- as: e2e-parallel-inplace-upgrade +- always_run: false + as: e2e-parallel-inplace-upgrade optional: true - run_if_changed: ^$ steps: env: ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters (EA Subscription),ARO HCP @@ -473,7 +473,6 @@ tests: - 02,ARO HCP E2E Hosted Clusters - Dev - 03 ARO_HCP_CLOUD: dev ARO_HCP_DEPLOY_ENV: ci01 - ARO_HCP_SUITE_PARALLELISM: "55" MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-local-e2e-upgrade - as: global-pipeline-postsubmit diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 33bfa1d95da0e..588ff0332c6fd 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -712,7 +712,6 @@ presubmits: name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-inplace-upgrade optional: true rerun_command: /test e2e-parallel-inplace-upgrade - run_if_changed: ^$ spec: containers: - args: diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS index 3ff2089d3c778..8b5c8784266a3 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS @@ -1,12 +1,8 @@ approvers: +- aro-hcp-sl-approvers - geoberle -- mmazur -- roivaz -- venkateshsredhat - deads2k reviewers: +- aro-hcp-sl-reviewers - geoberle -- mmazur -- roivaz -- venkateshsredhat - deads2k diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json index b02830fefd6bf..419fb88503aa4 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json @@ -2,17 +2,13 @@ "path": "aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml", "owners": { "approvers": [ + "aro-hcp-sl-approvers", "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", "deads2k" ], "reviewers": [ + "aro-hcp-sl-reviewers", "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", "deads2k" ] } diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS index 3ff2089d3c778..8b5c8784266a3 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS @@ -1,12 +1,8 @@ approvers: +- aro-hcp-sl-approvers - geoberle -- mmazur -- roivaz -- venkateshsredhat - deads2k reviewers: +- aro-hcp-sl-reviewers - geoberle -- mmazur -- roivaz -- venkateshsredhat - deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json index a20dc509a0355..ec7138049fc38 100644 --- a/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json +++ b/ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json @@ -2,17 +2,13 @@ "path": "aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml", "owners": { "approvers": [ + "aro-hcp-sl-approvers", "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", "deads2k" ], "reviewers": [ + "aro-hcp-sl-reviewers", "geoberle", - "mmazur", - "roivaz", - "venkateshsredhat", "deads2k" ] } From 8fe91a8601581714492a238e4db133641f2bb1d4 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Thu, 30 Jul 2026 15:06:16 +0200 Subject: [PATCH 19/19] ARO-HCP: drop post-upgrade local e2e from inplace-upgrade workflow --- .../local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml index e19fd88fdeeb9..7add67bfce809 100644 --- a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -11,7 +11,6 @@ workflow: - ref: aro-hcp-provision-from-main test: - ref: aro-hcp-test-local-upgrade - - ref: aro-hcp-test-local post: - ref: aro-hcp-deprovision-tracked-resource-groups - ref: aro-hcp-gather-provision-failure @@ -26,5 +25,4 @@ workflow: Acquire runtime leases, provision a main-branch baseline via aro-hcp-provision-from-main (ACR images from images-push), run upgrade/in-place on the PR head with OVERRIDE_CONFIG_FILE built from PR - pipeline images and hypershift config, then run the regular local e2e - suite against the upgraded environment. + pipeline images and hypershift config.