diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml new file mode 100644 index 0000000000000..539b32a21cdf4 --- /dev/null +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -0,0 +1,149 @@ +base_images: + cli: + name: "4.22" + namespace: ocp + tag: cli + fetch-managed-clusters: + name: autotest + namespace: acm-qe + tag: fetch-managed-clusters + installer: + name: "4.22" + namespace: ocp + tag: installer +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.23-openshift-4.20 +images: + items: + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y git python39 && dnf clean all + from: cli + optional: true + to: cli-with-git +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "4.22" +resources: + '*': + requests: + cpu: 200m + memory: 400Mi +tests: +- as: cclm-bidir-mig-lp-interop-aws + cron: 0 23 28 2 * + steps: + cluster_profile: aws-cspi-qe + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:latest + env: + ACM_SPOKE_CLUSTER_COUNT: "2" + ACM_SPOKE_CLUSTER_INITIAL_VERSION: "4.22" + ACM_SPOKE_CLUSTER_NAME_PREFIX: acm-spoke + ACM_SPOKE_CP_TYPE: m6a.2xlarge + ACM_SPOKE_WORKER_TYPE: c5n.metal + BASE_DOMAIN: cspilp.interop.ccitredhat.com + CNV_ENABLE_CCLM: "true" + COMPUTE_NODE_TYPE: c5n.metal + CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge + DEBUG_SLEEP_DURATION: "7200" + DR__RP__CR_COMP_NAME: lp-interop--ACM-Virt-BiDir + FIREWATCH_CONFIG: | + { + "failure_rules": + [ + {"step": "cucushift-*", "failure_type": "all", "classification": "Cluster Health Check", "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-interop-p2p-cluster-install" , "failure_type": "all" , "classification": "Spoke cluster Installation" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-interop-p2p-cluster-uninstall" , "failure_type": "all" , "classification": "Spoke cluster Un-Installation" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "p2p-acm-cnv-install-policy" , "failure_type": "all" , "classification": "CNV installation policy deploy" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-fetch-managed-clusters", "failure_type": "all", "classification": "ACM cluster failure", "jira_project": "!default", "group": {"name": "opp-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "interop-tests-deploy-odf", "failure_type": "all", "classification": "ODF deploy failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-install-odf-spokes", "failure_type": "all", "classification": "ODF spoke cluster install failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-cloud-prepare", "failure_type": "all", "classification": "Submariner AWS cloud prepare failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-broker-join", "failure_type": "all", "classification": "Submariner broker deploy / cluster join failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-verify", "failure_type": "all", "classification": "Submariner connectivity verification failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "acm-interop-p2p-submariner-snat-fix", "failure_type": "all", "classification": "Submariner SNAT workaround failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-cnv-cclm-readiness", "failure_type": "all", "classification": "CCLM virt-handler cert / Service / ServiceExport readiness failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-additional-config", "failure_type": "all", "classification": "MTV ForkliftController / CCLM configuration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-register-providers", "failure_type": "all", "classification": "MTV provider registration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-create-migration-maps", "failure_type": "all", "classification": "MTV forward migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-create-return-migration-maps", "failure_type": "all", "classification": "MTV return migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-create-migration-test-vm", "failure_type": "all", "classification": "CCLM test VM creation failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "p2p-mtv-execute-live-migration", "failure_type": "all", "classification": "MTV CCLM forward live migration failure (spoke-1 to spoke-2)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "p2p-mtv-execute-live-migration-return", "failure_type": "all", "classification": "MTV CCLM return live migration failure (spoke-2 back to spoke-1)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "acm-interop-p2p-submariner-diag", "failure_type": "all", "classification": "Submariner diagnostics collection failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 3}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-mch", "failure_type": "all", "classification": "ACM MCH install", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-debug-sleep", "failure_type": "all", "classification": "Debug sleep step failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]} + ] + } + FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/main/firewatch-base-configs/cr/lp-interop.json + FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["4.22-p2p","ipi-aws-p2p","mpiit-acm-virt","mpiit_product2product","mpiit_virt","bidir-migration"]' + FIREWATCH_DEFAULT_JIRA_ASSIGNEE: vshakya@redhat.com + FIREWATCH_DEFAULT_JIRA_COMPONENT: '["QA","mpiit_p2p"]' + FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-8652 + FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP + FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + MTV_DESTINATION_PROVIDER: spoke-2 + MTV_PROVIDER_NAMES: spoke-1,spoke-2 + MTV_SOURCE_PROVIDER: spoke-1 + MTV_SPOKE_CLUSTER_COUNT: "2" + ODF_OPERATOR_CHANNEL: stable-4.21 + ODF_VERSION_MAJOR_MINOR: "4.21" + OPERATORS: | + [ + {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.16", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, + {"name": "mtv-operator", "source": "redhat-operators", "channel": "release-v2.11", "install_namespace": "openshift-mtv", "target_namespaces": "openshift-mtv", "operator_group": "openshift-mtv-operatorgroup"} + ] + SUBMARINER_VERIFY_DEBUG_MODE: "true" + USER_TAGS: | + scenario acm-cnv-p2p-bidir + leases: + - count: 1 + env: LEASED_RESOURCE + resource_type: aws-cspi-qe-quota-slice + - count: 2 + env: MANAGED_CLUSTER_LEASED_RESOURCE + resource_type: aws-cspi-qe-quota-slice + post: + - ref: acm-fetch-operator-versions + - ref: acm-must-gather + - ref: acm-inspector + - ref: acm-interop-p2p-submariner-diag + - ref: gather-aws-console + - ref: p2p-debug-sleep + - ref: acm-interop-p2p-cluster-uninstall + - chain: ipi-deprovision + - ref: mpiit-data-router-reporter + - ref: firewatch-report-issues + pre: + - ref: ipi-conf + - ref: ipi-conf-telemetry + - ref: ipi-conf-aws-custom-az + - ref: ipi-conf-aws + - ref: ipi-install-monitoringpvc + - chain: ipi-install + test: + - chain: cucushift-installer-check-cluster-health + - ref: install-operators + - ref: interop-tests-deploy-odf + - ref: acm-mch + - ref: acm-interop-p2p-cluster-install + - ref: acm-fetch-managed-clusters + - ref: p2p-acm-cnv-install-policy + - ref: p2p-install-odf-spokes + - chain: acm-interop-p2p-submariner-install + - ref: p2p-cnv-cclm-readiness + - chain: p2p-mtv-bidir-mig-config + - chain: p2p-cclm-bidir-migration-test + timeout: 12h0m0s +zz_generated_metadata: + branch: master + org: RedHatQE + repo: interop-testing + variant: acm-virt-ocp4.22-p2p-bidir diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml index b430ddb1392fc..1821fc5250e85 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml @@ -26,9 +26,8 @@ images: to: cli-with-git releases: latest: - candidate: - product: ocp - stream: nightly + release: + channel: stable version: "4.22" resources: '*': diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml index a7e558e669e3a..21bca379a46b0 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml @@ -188,6 +188,90 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build10 + cron: 0 23 28 2 * + decorate: true + decoration_config: + skip_cloning: true + timeout: 12h0m0s + extra_refs: + - base_ref: master + org: RedHatQE + repo: interop-testing + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe + ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p-bidir + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-bidir-cclm-bidir-mig-lp-interop-aws + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=cclm-bidir-mig-lp-interop-aws + - --variant=acm-virt-ocp4.22-p2p-bidir + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build10 cron: 0 23 28 2 * @@ -204,7 +288,6 @@ periodics: ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p ci.openshift.io/generator: prowgen - job-release: "4.22" pj-rehearse.openshift.io/can-be-rehearsed: "true" name: periodic-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-cclm-liv-mig-lp-interop-aws reporter_config: diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml index 6ad48c8f208c0..1500e8caca174 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml @@ -115,6 +115,65 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )acm-cnv-ocp-4.22-p2p-images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + cluster: build12 + context: ci/prow/acm-virt-ocp4.22-p2p-bidir-images + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p-bidir + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-bidir-images + optional: true + rerun_command: /test acm-virt-ocp4.22-p2p-bidir-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=acm-virt-ocp4.22-p2p-bidir + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )acm-virt-ocp4.22-p2p-bidir-images,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -128,7 +187,6 @@ presubmits: labels: ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p ci.openshift.io/generator: prowgen - job-release: "4.22" pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-images optional: true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh new file mode 100644 index 0000000000000..fc5bcfc8c3aed --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# +# Automated Submariner diagnostics using subctl. +# Ref: https://submariner.io/operations/troubleshooting/#automated-troubleshooting +# +# Runs subctl show all, diagnose all, diagnose firewall inter-cluster, and gather +# against every spoke cluster. Always exits 0 — diagnostic only, never fails the job. +# +# Use -uo (no -e, no -x) so that individual command failures never abort the script. +# Each function guards its own failures with || true. +set -uo pipefail + +typeset -r subctlBin="/tmp/bin/subctl" +typeset -r diagDir="${ARTIFACT_DIR}/submariner-diag" +typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" + +typeset -a spokeKubeconfigsArr=() +typeset -a spokeNamesArr=() + +# InstallSubctl — install subctl to /tmp/bin/ at step runtime. +# Downloads the pinned release tarball from GitHub, verifies its SHA-256 checksum, +# then extracts the binary. Uses SUBCTL_VERSION (set in ref.yaml env) so the +# version can be bumped without touching the script. +# +# WHY not SHARED_DIR: large binaries cause CI operator "Request entity too large" +# when serialising SHARED_DIR into a Kubernetes Secret between steps (3 MB limit). +# Returns 1 (non-fatal) if any step fails; callers check exit status. +InstallSubctl() { + mkdir -p /tmp/bin || return 1 + if [[ -x "${subctlBin}" ]]; then + return 0 + fi + + typeset version="${SUBCTL_VERSION:?SUBCTL_VERSION must be set}" + typeset arch="linux-amd64" + typeset tarball="subctl-v${version}-${arch}.tar.xz" + typeset baseURL="https://github.com/submariner-io/subctl/releases/download/v${version}" + typeset workDir + workDir="$(mktemp -d /tmp/subctl-install-XXXXXX)" || return 1 + + ( + cd "${workDir}" || return 1 + curl -fsSL -o "${tarball}" "${baseURL}/${tarball}" || return 1 + curl -fsSL -o "${tarball}.sha256" "${baseURL}/${tarball}.sha256" || return 1 + # Verify integrity before executing anything from the archive. + sha256sum --check "${tarball}.sha256" || return 1 + tar -xJf "${tarball}" "subctl-v${version}-${arch}/subctl" || return 1 + cp "subctl-v${version}-${arch}/subctl" "${subctlBin}" || return 1 + chmod +x "${subctlBin}" || return 1 + ) || { rm -rf "${workDir}"; return 1; } + + rm -rf "${workDir}" + true +} + +# LoadSpokeConfig — populate spoke kubeconfig and name arrays from SHARED_DIR. +# Returns 1 (non-fatal) if any expected file is missing. +LoadSpokeConfig() { + typeset -i i + for ((i = 1; i <= spokeCount; i++)); do + typeset kcFile="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${i}" + + if [[ ! -f "${kcFile}" || ! -f "${nameFile}" ]]; then + : "LoadSpokeConfig: missing files for spoke ${i} — skipping" + return 1 + fi + + spokeKubeconfigsArr+=("${kcFile}") + spokeNamesArr+=("$(< "${nameFile}")") + done + true +} + +# ShowAll — subctl show all: endpoints, gateways, networks, connections overview. +ShowAll() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + typeset outFile="${diagDir}/show-all-${name}.txt" + + KUBECONFIG="${kc}" "${subctlBin}" show all > "${outFile}" 2>&1 || true + true +} + +# DiagnoseAll — subctl diagnose all: health check across all submariner components. +DiagnoseAll() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + typeset outFile="${diagDir}/diagnose-all-${name}.txt" + + KUBECONFIG="${kc}" "${subctlBin}" diagnose all > "${outFile}" 2>&1 || true + true +} + +# BuildMergedKubeconfig — merge two spoke kubeconfigs with unique named contexts +# into a temp file for subctl commands requiring --context / --remotecontext. +# Returns the merged kubeconfig path via stdout. +BuildMergedKubeconfig() { + typeset kc1="${1:?}"; (($#)) && shift + typeset kc2="${1:?}"; (($#)) && shift + typeset name1="${1:?}"; (($#)) && shift + typeset name2="${1:?}"; (($#)) && shift + + typeset kc1Renamed kc2Renamed mergedKc + kc1Renamed="$(mktemp /tmp/kc1-diag-XXXXXX.json)" + kc2Renamed="$(mktemp /tmp/kc2-diag-XXXXXX.json)" + mergedKc="$(mktemp /tmp/kc-merged-diag-XXXXXX.json)" + + KUBECONFIG="${kc1}" oc config view -o json --raw | \ + jq \ + --arg ctx "${name1}-admin" \ + --arg cls "${name1}-cluster" \ + --arg usr "${name1}-user" \ + ' + .contexts[0].name = $ctx | + .contexts[0].context.cluster = $cls | + .contexts[0].context.user = $usr | + .clusters[0].name = $cls | + .users[0].name = $usr | + ."current-context" = $ctx + ' > "${kc1Renamed}" + + KUBECONFIG="${kc2}" oc config view -o json --raw | \ + jq \ + --arg ctx "${name2}-admin" \ + --arg cls "${name2}-cluster" \ + --arg usr "${name2}-user" \ + ' + .contexts[0].name = $ctx | + .contexts[0].context.cluster = $cls | + .contexts[0].context.user = $usr | + .clusters[0].name = $cls | + .users[0].name = $usr | + ."current-context" = $ctx + ' > "${kc2Renamed}" + + KUBECONFIG="${kc1Renamed}:${kc2Renamed}" oc config view --flatten -o json > "${mergedKc}" + rm -f "${kc1Renamed}" "${kc2Renamed}" + + printf '%s' "${mergedKc}" +} + +# DiagnoseFirewallInterCluster — subctl diagnose firewall inter-cluster between a pair. +DiagnoseFirewallInterCluster() { + typeset kc1="${1:?}"; (($#)) && shift + typeset kc2="${1:?}"; (($#)) && shift + typeset name1="${1:?}"; (($#)) && shift + typeset name2="${1:?}"; (($#)) && shift + + typeset outFile="${diagDir}/diagnose-firewall-${name1}-to-${name2}.txt" + typeset mergedKc + mergedKc="$(BuildMergedKubeconfig "${kc1}" "${kc2}" "${name1}" "${name2}")" + + KUBECONFIG="${mergedKc}" "${subctlBin}" diagnose firewall inter-cluster \ + --context "${name1}-admin" \ + --remotecontext "${name2}-admin" \ + > "${outFile}" 2>&1 || true + + rm -f "${mergedKc}" + true +} + +# GatherSubmariner — subctl gather for deep diagnostics; archives tarball to diagDir. +GatherSubmariner() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + + typeset gatherWorkDir + gatherWorkDir="$(mktemp -d /tmp/subctl-gather-XXXXXX)" + + ( + cd "${gatherWorkDir}" + KUBECONFIG="${kc}" "${subctlBin}" gather > "${diagDir}/gather-${name}.log" 2>&1 || true + + # Archive whatever subctl gather produced + typeset tarball="" + typeset gatherDir="" + for d in submariner-*/; do + [[ -d "${d}" ]] && gatherDir="${d}" && break + done + + if [[ -n "${gatherDir}" ]]; then + tar czf "${diagDir}/gather-${name}.tar.gz" "${gatherDir}" + rm -rf "${gatherDir}" + fi + ) + rm -rf "${gatherWorkDir}" + true +} + +# --- Main --- + +# Precondition checks — warn and skip rather than abort (diagnostic-only step). +typeset _diagSkip=false + +for _cmd in oc curl jq; do + command -v "${_cmd}" 1>/dev/null || { : "WARNING: ${_cmd} not found — skipping diagnostics"; _diagSkip=true; } +done + +if [[ -z "${SHARED_DIR:-}" || -z "${ARTIFACT_DIR:-}" ]]; then + : "WARNING: SHARED_DIR or ARTIFACT_DIR unset — skipping diagnostics" + _diagSkip=true +fi + +mkdir -p "${diagDir}/gather" || _diagSkip=true + +if [[ "${_diagSkip}" != "true" ]]; then + LoadSpokeConfig || { : "WARNING: LoadSpokeConfig failed — skipping diagnostics"; _diagSkip=true; } +fi + +if [[ "${_diagSkip}" != "true" ]]; then + InstallSubctl || { : "WARNING: InstallSubctl failed — skipping diagnostics"; _diagSkip=true; } +fi + +if [[ "${_diagSkip}" == "true" ]]; then + : "Submariner diagnostics skipped due to setup failure — see warnings above" + exit 0 +fi + +: "=== subctl show all (per spoke) ===" +typeset -i i +for ((i = 0; i < spokeCount; i++)); do + ShowAll "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "=== subctl diagnose all (per spoke) ===" +for ((i = 0; i < spokeCount; i++)); do + DiagnoseAll "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "=== subctl diagnose firewall inter-cluster (per spoke pair) ===" +typeset -i j +for ((i = 0; i < spokeCount; i++)); do + for ((j = i + 1; j < spokeCount; j++)); do + DiagnoseFirewallInterCluster \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeKubeconfigsArr[j]}" \ + "${spokeNamesArr[i]}" \ + "${spokeNamesArr[j]}" + DiagnoseFirewallInterCluster \ + "${spokeKubeconfigsArr[j]}" \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeNamesArr[j]}" \ + "${spokeNamesArr[i]}" + done +done + +: "=== subctl gather (per spoke) ===" +for ((i = 0; i < spokeCount; i++)); do + GatherSubmariner "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "Submariner diagnostics complete — artifacts in ${diagDir}" +true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json new file mode 100644 index 0000000000000..fdc2548cd1955 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml new file mode 100644 index 0000000000000..e6cffa39da65b --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml @@ -0,0 +1,49 @@ +ref: + as: acm-interop-p2p-submariner-diag + from: cli + commands: acm-interop-p2p-submariner-diag-commands.sh + resources: + requests: + cpu: 100m + memory: 250Mi + timeout: 30m0s + grace_period: 10m0s + documentation: |- + Collect automated Submariner diagnostics across all spoke clusters using subctl. + + Designed as a post-step that always exits 0 — never fails the job. Writes all output + to ${ARTIFACT_DIR}/submariner-diag/ for Prow artifact collection. + + Per the Submariner troubleshooting guide + (https://submariner.io/operations/troubleshooting/#automated-troubleshooting), + this step runs: + + subctl show all — overview of endpoints, gateways, networks, and + connections on each spoke + subctl diagnose all — health check of submariner-operator, gateway engine, + route agents, and lighthouse components on each spoke + subctl diagnose firewall inter-cluster + — firewall rule audit between each spoke pair + (merged kubeconfig with unique named contexts) + subctl gather — deep diagnostic tarball for each spoke; archived + under ${ARTIFACT_DIR}/submariner-diag/gather/ + + subctl is installed at step runtime to /tmp/bin/ (not stored in SHARED_DIR to + avoid the 3 MB Kubernetes Secret size limit). + + Env vars consumed: + ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) + SHARED_DIR - directory containing managed-cluster-kubeconfig-N + and managed-cluster-name-N files + ARTIFACT_DIR - Prow artifact directory for output files + env: + - name: ACM_SPOKE_CLUSTER_COUNT + default: "2" + documentation: Number of ACM spoke clusters to diagnose. + - name: SUBCTL_VERSION + default: "0.24.0" + documentation: |- + subctl release version to download from GitHub (submariner-io/subctl). + Diagnostic commands (show/diagnose/gather) are backwards-compatible so the + latest upstream release works against any deployed Submariner version. + Latest upstream: 0.24.0 (2026-05-12). ACM 2.15 bundles Submariner v0.22. diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml index 1ebe586114710..1f703a322e9d6 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml @@ -3,12 +3,13 @@ chain: steps: - ref: acm-interop-p2p-submariner-cloud-prepare - ref: acm-interop-p2p-submariner-broker-join + - ref: acm-interop-p2p-submariner-snat-fix - ref: acm-interop-p2p-submariner-verify documentation: |- Installs and verifies Submariner for a 1-hub + N-spoke cluster topology without Globalnet (non-overlapping pod CIDRs per spoke). - This chain sequences three steps: + This chain sequences four steps: 1. acm-interop-p2p-submariner-cloud-prepare Installs subctl locally, prepares AWS security groups on each spoke (subctl cloud prepare aws with --gateways 1 dedicated gateway nodes), @@ -24,7 +25,14 @@ chain: agent, Lighthouse CoreDNS, and OpenShift cluster CoreDNS DNS forwarding to be fully ready. - 3. acm-interop-p2p-submariner-verify + 3. acm-interop-p2p-submariner-snat-fix + Applies the official ACM SNAT workaround (ACM-22805/24786/36927) for + OVN-K nftables mgmtport-snat breaking cross-cluster return routing. + Creates submariner-global ConfigMap (enable-snat-handler=true), restarts + submariner-routeagent and ovnkube-node DaemonSets on each spoke, and + waits for rollout completion before verify runs. + + 4. acm-interop-p2p-submariner-verify Waits for IPsec tunnels to reach connected state, asserts remote subnets are pod CIDRs (not Globalnet 242.x), pre-warms Lighthouse service discovery, runs 'subctl verify', and probes CCLM sync TCP diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh new file mode 100644 index 0000000000000..2a7db3c8e37f3 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# +# Submariner SNAT Workaround for OVN-Kubernetes IC mode +# +# Ref: ACM-22805, ACM-24786, ACM-36927 +# Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk +# +# Root cause: On OCP 4.22 nightly with OVN-K IC mode, the +# mgmtport-snat nftables chain rewrites the source IP of cross-cluster +# traffic arriving at each spoke's gateway node. The routeagent sets +# k8s.ovn.org/node-ingress-snat-exclude-subnets on the gateway Node +# object, but OVN-K on OCP 4.22 nightly does NOT reconcile that annotation +# into the mgmtport-no-snat-subnets-v4 nftables set. +# +# Fix: Directly inject each spoke's remote peer CIDRs into the +# mgmtport-no-snat-subnets-v4 set (family: inet, table: ovn-kubernetes) +# on every gateway node of that spoke, using "oc debug node". +# +# NOTE: The ConfigMap (enable-snat-handler) + routeagent-restart approach +# is intentionally NOT used here. Restarting submariner-routeagent +# re-publishes the Endpoint CR on the broker; the remote gateway detects +# the change and re-initiates NAT discovery, which can break the IPsec +# tunnel for minutes (or permanently if the gateways are in different +# VPCs with UDP/4800 asymmetry). Direct nftables injection avoids all +# of that by not touching the Submariner control plane at all. +# +# Applied to every spoke cluster. Idempotent: safe to re-run. +# + +set -uo pipefail; shopt -s inherit_errexit + +# ── Bootstrap: ensure required tools are present ───────────────────────────── +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +# ── Constants ───────────────────────────────────────────────────────────────── +typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" +typeset -i settleSecs="${SUBMARINER_SNAT_FIX_SETTLE_SECS}" + +typeset -a spokeKubeconfigsArr=() +typeset -a spokeNamesArr=() +typeset -a _tmpKubeconfigsToClean=() + +# ── LoadSpokeConfig — populate spoke arrays from SHARED_DIR ────────────────── +# CI spoke kubeconfigs can have 'namespace: ci-op-XXXXXXXX' in their current +# context. That namespace does not exist on the spoke cluster, which causes +# oc to fail namespace-validation before executing ANY command (even cluster- +# scoped ones like "oc get nodes"). We create a temp copy of each kubeconfig +# with the context namespace patched to 'default' to avoid this. +LoadSpokeConfig() { + typeset -i i + for ((i = 1; i <= spokeCount; i++)); do + typeset kcFile="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${i}" + + [ -f "${kcFile}" ] + [ -f "${nameFile}" ] + + typeset tmpKc + tmpKc="$(mktemp -t submariner-snat-fix-kc-XXXXXX)" + _tmpKubeconfigsToClean+=("${tmpKc}") + cp "${kcFile}" "${tmpKc}" + # Use kubectl (not oc) here to avoid triggering oc's own namespace + # validation while we are patching the context namespace. + kubectl --kubeconfig="${tmpKc}" config set-context \ + "$(kubectl --kubeconfig="${tmpKc}" config current-context)" \ + --namespace=default > /dev/null + + spokeKubeconfigsArr+=("${tmpKc}") + spokeNamesArr+=("$(<"${nameFile}")") + done + true +} + +# ── GetLocalClusterID — find the cluster_id of this spoke's own endpoint ───── +# Matches on the gateway node's short hostname (e.g. "ip-10-1-99-33") which +# Submariner stores in spec.hostname of the local Endpoint CR. +GetLocalClusterID() { + typeset kubeconfig="${1:?}" + + typeset gwNode + gwNode="$(KUBECONFIG="${kubeconfig}" oc get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | head -1)" + + # Strip domain suffix: "ip-10-1-99-33.us-east-2.compute.internal" → "ip-10-1-99-33" + typeset gwShort="${gwNode%%.*}" + + KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ + -n submariner-operator \ + -o json \ + | jq -r --arg host "${gwShort}" \ + '.items[] | select(.spec.hostname == $host) | .spec.cluster_id' \ + | head -1 +} + +# ── GetRemoteSubnets — list every subnet from non-local endpoints ───────────── +# Output: one subnet per line (e.g. "10.132.0.0/14\n172.31.0.0/16") +GetRemoteSubnets() { + typeset kubeconfig="${1:?}" + typeset localClusterID="${2:?}" + + KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ + -n submariner-operator \ + -o json \ + | jq -r --arg local "${localClusterID}" \ + '.items[] + | select(.spec.cluster_id != $local) + | .spec.subnets[]' +} + +# ── InjectNftablesOnGateway — add remote subnets to the SNAT exemption set ─── +# Uses "oc debug node" to run nft on the gateway node's host network. +# Family is "inet" (confirmed on OCP 4.22; mgmtport-no-snat-subnets-v4 lives +# in family inet, table ovn-kubernetes). +InjectNftablesOnGateway() { + typeset kubeconfig="${1:?}" + typeset gatewayNode="${2:?}" + typeset nftElements="${3:?}" # comma-separated, e.g. "10.132.0.0/14, 172.31.0.0/16" + + : "InjectNftablesOnGateway: node='${gatewayNode}' elements='${nftElements}'" + + KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + "{ ${nftElements} }" 2>&1 | grep -v '^$' || true + + : "InjectNftablesOnGateway: verifying set on '${gatewayNode}'" + KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + 2>/dev/null || true +} + +# ── ApplySnatFix — inject nftables exemptions on every gateway of one spoke ── +ApplySnatFix() { + typeset kubeconfig="${1:?}"; (($#)) && shift + typeset spokeName="${1:?}"; (($#)) && shift + + : "ApplySnatFix: spoke='${spokeName}'" + + # Determine this spoke's own cluster ID from its gateway node hostname. + typeset localClusterID + localClusterID="$(GetLocalClusterID "${kubeconfig}")" + + if [[ -z "${localClusterID}" ]]; then + # Fallback: match cluster_id by spoke name (cluster names contain spokeName) + localClusterID="$(KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ + -n submariner-operator \ + -o jsonpath='{range .items[*]}{.spec.cluster_id}{"\n"}{end}' \ + | grep "${spokeName}" | head -1)" + fi + + if [[ -z "${localClusterID}" ]]; then + : "ApplySnatFix: could not determine localClusterID for '${spokeName}' — skipping" + return 0 + fi + + : "ApplySnatFix: localClusterID='${localClusterID}'" + + # Collect remote subnets (newline-separated). + typeset remoteSubnetList + remoteSubnetList="$(GetRemoteSubnets "${kubeconfig}" "${localClusterID}")" + + if [[ -z "${remoteSubnetList}" ]]; then + : "ApplySnatFix: no remote subnets found for '${spokeName}' — skipping nftables injection" + return 0 + fi + + # Convert newline list to nft element format: "10.a.b.c/x, 10.d.e.f/y" + typeset nftElements + nftElements="$(echo "${remoteSubnetList}" | paste -sd ',' - | sed 's/,/, /g')" + : "ApplySnatFix: remote subnets for injection: ${nftElements}" + + # Find ALL gateway nodes on this spoke. + typeset -a gatewayNodes + mapfile -t gatewayNodes < <(KUBECONFIG="${kubeconfig}" oc get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + + if [[ ${#gatewayNodes[@]} -eq 0 ]]; then + : "ApplySnatFix: no gateway nodes found on '${spokeName}' — skipping" + return 0 + fi + + typeset gwNode + for gwNode in "${gatewayNodes[@]}"; do + InjectNftablesOnGateway "${kubeconfig}" "${gwNode}" "${nftElements}" + done + + sleep "${settleSecs}" + + : "ApplySnatFix: '${spokeName}' complete" + true +} + +# ── Main ────────────────────────────────────────────────────────────────────── +LoadSpokeConfig + +typeset -i i +for ((i = 0; i < spokeCount; i++)); do + ApplySnatFix \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeNamesArr[i]}" +done + +typeset _tmpKc +for _tmpKc in "${_tmpKubeconfigsToClean[@]+"${_tmpKubeconfigsToClean[@]}"}"; do + rm -f "${_tmpKc}" +done + +true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json new file mode 100644 index 0000000000000..df642190549c4 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml new file mode 100644 index 0000000000000..f6c1f96a9732b --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml @@ -0,0 +1,58 @@ +ref: + as: acm-interop-p2p-submariner-snat-fix + from: cli + commands: acm-interop-p2p-submariner-snat-fix-commands.sh + resources: + requests: + cpu: 100m + memory: 250Mi + timeout: 40m0s + grace_period: 5m0s + env: + - name: ACM_SPOKE_CLUSTER_COUNT + default: "2" + documentation: |- + Number of ACM spoke clusters to apply the SNAT workaround to. + - name: SUBMARINER_SNAT_FIX_SETTLE_SECS + default: "10" + documentation: |- + Seconds to wait after nftables injection for rules to take effect before + the next step runs subctl verify or starts migration. Kept short because + no pod restarts are involved; nft changes take effect immediately. + documentation: |- + Fixes the OVN-Kubernetes mgmtport-snat SNAT issue that breaks Submariner + cross-cluster connectivity on OCP 4.22 nightly (ACM-22805, ACM-24786, + ACM-36927). + + Root cause: OVN-K's nftables mgmtport-snat chain rewrites the source IP of + cross-cluster traffic arriving at a spoke's gateway node. The routeagent + sets k8s.ovn.org/node-ingress-snat-exclude-subnets on the gateway Node but + OVN-K on OCP 4.22 nightly does NOT reconcile that annotation into the + mgmtport-no-snat-subnets-v4 nftables set, leaving it empty. + + What this step does (idempotent, safe to re-run): + 1. For each spoke, discovers the OTHER spokes' pod+service CIDRs by + reading Submariner Endpoint CRs from the local submariner-operator + namespace. + 2. Uses "oc debug node" to directly inject those CIDRs into the + mgmtport-no-snat-subnets-v4 nftables set (family: inet, table: + ovn-kubernetes) on every gateway node of that spoke. + 3. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 10) for the rules + to take effect. + + NOTE: The ConfigMap (enable-snat-handler) + routeagent-restart approach + is intentionally NOT used. Restarting submariner-routeagent re-publishes + Endpoint CRs on the broker; remote gateways detect the change and restart + NAT discovery (UDP/4800), which can break the IPsec tunnel for an extended + period in multi-VPC AWS environments. Direct nftables injection avoids + touching the Submariner control plane entirely. + + Must run after acm-interop-p2p-submariner-broker-join and before + acm-interop-p2p-submariner-verify. + + Applies to: ACM 2.13+ with OVN-Kubernetes network plugin on OCP 4.18+. + + Env vars consumed: + ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) + SUBMARINER_SNAT_FIX_SETTLE_SECS - post-injection settle time (default: 10) + SHARED_DIR - shared artifact directory diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh index befd55e9cc28e..014548464bbc3 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh @@ -387,7 +387,12 @@ VerifyConnectivity() { KUBECONFIG="${kc1Renamed}:${kc2Renamed}" oc config view --flatten -o json > "${mergedKc}" - KUBECONFIG="${mergedKc}" "${subctlBin}" verify \ + # Bound subctl verify to 35m so it exits before the 45m step timeout. + # Without this, each failing TCP test takes ~7m and the process is killed + # externally (exit 127) before the SUBMARINER_VERIFY_DEBUG_MODE handler runs. + # --kill-after 30s: send SIGKILL 30s after SIGTERM in case subctl is + # unresponsive (blocked goroutine / ncat subprocess ignoring SIGTERM). + KUBECONFIG="${mergedKc}" timeout --kill-after=30s 35m "${subctlBin}" verify \ --context "${ctx1}" \ --tocontext "${ctx2}" \ --only connectivity,service-discovery \ diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json new file mode 100644 index 0000000000000..730cd455cccfd --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml new file mode 100644 index 0000000000000..7a93628503d0b --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml @@ -0,0 +1,29 @@ +chain: + as: p2p-cclm-bidir-migration-test + steps: + - ref: p2p-create-migration-test-vm + - ref: p2p-mtv-execute-live-migration + - ref: p2p-mtv-execute-live-migration-return + documentation: |- + End-to-end bi-directional CCLM migration test for the same VM across two spoke clusters. + + Requires p2p-mtv-bidir-mig-config (ForkliftController, providers, forward + return + network/storage maps), p2p-install-odf-spokes, p2p-acm-cnv-install-policy, and + acm-interop-p2p-submariner-install (pod-to-pod connectivity without Globalnet). + + Step 1 (p2p-create-migration-test-vm): + Creates an ODF-backed RWX VM on the source spoke (spoke-1, index 1). + + Step 2 (p2p-mtv-execute-live-migration): + Forward leg — live-migrates the VM from spoke-1 to spoke-2. + Uses forward Plan/Migration CRs (cclm-plan / cclm-run-1) and forward maps + (spoke-network-map / spoke-storage-map). Verifies VM Running on spoke-2. + + Step 3 (p2p-mtv-execute-live-migration-return): + Return leg — live-migrates the same VM from spoke-2 back to spoke-1. + Uses return Plan/Migration CRs (cclm-plan-return / cclm-run-return) and return + maps (spoke-network-map-return / spoke-storage-map-return). Verifies VM Running + on spoke-1 once again, confirming full round-trip spoke-to-spoke migration. + + Debug mode: set P2P_CCLM_DEBUG_MODE=true so both migration steps log failures + but exit 0, allowing downstream investigation steps to run. diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS b/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh new file mode 100644 index 0000000000000..463180739ee6c --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh @@ -0,0 +1,367 @@ +#!/bin/bash +# +# Verify and remediate CCLM (Cross-Cluster Live Migration) prerequisites on each spoke. +# No-op when CNV_ENABLE_CCLM != "true". +# +# Background — CNV bug CNV-89129 / upstream kubevirt#16264: +# virt-handler fails to load /etc/virt-handler/clientcertificates/tls.crt because +# virt-operator does not add the corresponding volume mount to the DaemonSet when the +# DecentralizedLiveMigration feature gate is enabled. The Secret +# (kubevirt-virt-handler-certs) exists and is populated, but is not mounted. +# Workaround: patch the DaemonSet to mount the Secret at the expected path. +# Affected: CNV 4.17+ / OCP 4.22 nightly (as of 2026-08). +# See also: https://github.com/kubevirt/kubevirt/issues/16264 +# +# Checks performed (per spoke, in order): +# 1. Check whether /etc/virt-handler/clientcertificates/tls.crt is already present. +# If so, skip straight to Service/ServiceExport creation. +# 2. Apply the DaemonSet workaround patch (CNV-89129): +# a. Scale virt-operator to 0 to prevent it from reconciling away the patch. +# b. Patch virt-handler DaemonSet to mount the CNV_CCLM_VIRT_HANDLER_CERT_SECRET +# Secret at /etc/virt-handler/clientcertificates. +# c. Rollout restart the DaemonSet so all pods pick up the new volume mount. +# d. Wait for the rollout to complete. +# e. Verify the cert is present in ALL virt-handler pods. +# f. Scale virt-operator back to its original replica count. +# 3. Ensure a Service named virt-synchronization-controller on port CNV_CCLM_SYNC_PORT +# exists in openshift-cnv (created by `oc apply` — idempotent). +# 4. Ensure a ServiceExport for that Service exists (allows Submariner to propagate it +# to other clusters). Skipped gracefully when the ServiceExport CRD is not yet +# installed (i.e. Submariner is not up yet on this spoke). +# +set -euxo pipefail; shopt -s inherit_errexit + +[[ "${CNV_ENABLE_CCLM}" == "true" ]] || { + : "CNV_ENABLE_CCLM is not 'true' — CCLM readiness checks skipped" + true; exit 0 +} + +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +readonly cnvNs="openshift-cnv" +readonly clientCertMountPath="/etc/virt-handler/clientcertificates" +typeset -i waitTimeoutSecs="${CNV_CCLM_WAIT_TIMEOUT}" +typeset -i pollIntervalSecs=15 + +# LoadSpokeKubeconfigs — output spoke kubeconfig paths (one per line). +LoadSpokeKubeconfigs() { + typeset -a kcsArr=() + typeset -i i + for ((i = 1; ; i++)); do + typeset kcPath="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + [[ -f "${kcPath}" ]] || break + kcsArr+=("${kcPath}") + done + if [[ ${#kcsArr[@]} -eq 0 && -f "${SHARED_DIR}/managed-cluster-kubeconfig" ]]; then + kcsArr+=("${SHARED_DIR}/managed-cluster-kubeconfig") + fi + ((${#kcsArr[@]} > 0)) + printf '%s\n' "${kcsArr[@]}" +} + +# GetSpokeClusterName — return cluster name for 1-based spoke index. +GetSpokeClusterName() { + typeset -i idx="${1:?}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${idx}" + if [[ -f "${nameFile}" ]]; then + cat "${nameFile}" + else + printf 'spoke-%d' "${idx}" + fi +} + +# ScaleVirtOperator — scale virt-operator to desired replicas and wait for rollout. +ScaleVirtOperator() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset -i replicas="${3:?}" + : "[${clusterName}] Scaling virt-operator to ${replicas} replica(s)" + oc --kubeconfig="${kubeconfig}" scale deployment/virt-operator \ + -n "${cnvNs}" --replicas="${replicas}" + oc --kubeconfig="${kubeconfig}" rollout status deployment/virt-operator \ + -n "${cnvNs}" --timeout=5m 1>/dev/null || true + true +} + +# GetVirtOperatorReplicas — return the current desired replica count of virt-operator. +GetVirtOperatorReplicas() { + typeset kubeconfig="${1:?}" + oc --kubeconfig="${kubeconfig}" get deployment/virt-operator \ + -n "${cnvNs}" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "2" +} + +# WaitVirtHandlerRollout — wait for virt-handler DaemonSet rollout (non-fatal on timeout). +WaitVirtHandlerRollout() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset timeout="${3:-10m}" + : "[${clusterName}] Waiting for virt-handler DaemonSet rollout (timeout=${timeout})" + oc --kubeconfig="${kubeconfig}" rollout status ds/virt-handler \ + -n "${cnvNs}" --timeout="${timeout}" 1>/dev/null || true + true +} + +# CheckVirtHandlerClientCert — return 0 only when ALL Running virt-handler pods have the cert. +# Checking a single pod is insufficient: after a rollout some pods may be on the new +# spec (with the mount) while others are still on the old spec. +CheckVirtHandlerClientCert() { + typeset kubeconfig="${1:?}" + typeset -a pods=() + mapfile -t pods < <( + oc --kubeconfig="${kubeconfig}" get pod \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + --field-selector 'status.phase=Running' \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null \ + | tr ' ' '\n' | grep -v '^$' + ) + [[ ${#pods[@]} -gt 0 ]] || return 1 + typeset pod + for pod in "${pods[@]}"; do + oc --kubeconfig="${kubeconfig}" exec -n "${cnvNs}" "${pod}" -- \ + test -f "${clientCertMountPath}/tls.crt" 2>/dev/null || return 1 + done + return 0 +} + +# PatchVirtHandlerDaemonSetCertMount — add clientcertificates volume mount (idempotent). +# Workaround for CNV-89129: virt-operator does not add the CCLM client cert mount even +# after the DecentralizedLiveMigration feature gate is enabled. +# +# CRITICAL: virt-operator must be scaled to 0 BEFORE patching. If left running it will +# reconcile the DaemonSet within seconds and remove the patch (observed behaviour on +# CNV 4.17 / OCP 4.22 nightly). The caller is responsible for restoring virt-operator +# replicas after the cert is verified. +PatchVirtHandlerDaemonSetCertMount() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset -i originalReplicas + originalReplicas="$(GetVirtOperatorReplicas "${kubeconfig}")" + + # Idempotency: skip if mount already present in DaemonSet spec. + typeset existingMountCount=0 + existingMountCount="$(oc --kubeconfig="${kubeconfig}" get ds virt-handler \ + -n "${cnvNs}" \ + -o jsonpath-as-json='{.spec.template.spec.containers[*].volumeMounts[*].mountPath}' \ + 2>/dev/null | jq --arg p "${clientCertMountPath}" '[.[] | select(. == $p)] | length')" + if ((existingMountCount > 0)); then + : "[${clusterName}] ${clientCertMountPath} already in DaemonSet spec — skipping patch" + return 0 + fi + + # Verify the source Secret exists before patching. + oc --kubeconfig="${kubeconfig}" get secret "${CNV_CCLM_VIRT_HANDLER_CERT_SECRET}" \ + -n "${cnvNs}" 1>/dev/null + + # Scale virt-operator to 0 so it cannot reconcile the patch away. + ScaleVirtOperator "${kubeconfig}" "${clusterName}" 0 + + : "[${clusterName}] Patching virt-handler DaemonSet: mounting ${CNV_CCLM_VIRT_HANDLER_CERT_SECRET} at ${clientCertMountPath}" + oc --kubeconfig="${kubeconfig}" patch ds virt-handler \ + -n "${cnvNs}" --type=json \ + --patch "[ + {\"op\":\"add\", + \"path\":\"/spec/template/spec/volumes/-\", + \"value\":{\"name\":\"cnv-cclm-virt-handler-client-certs\", + \"secret\":{\"secretName\":\"${CNV_CCLM_VIRT_HANDLER_CERT_SECRET}\", + \"optional\":true, + \"defaultMode\":420}}}, + {\"op\":\"add\", + \"path\":\"/spec/template/spec/containers/0/volumeMounts/-\", + \"value\":{\"name\":\"cnv-cclm-virt-handler-client-certs\", + \"mountPath\":\"${clientCertMountPath}\", + \"readOnly\":true}} + ]" + # Explicitly restart to guarantee the DaemonSet controller evicts old pods. + # Without this, rollout status can return for the previous generation before + # the controller has processed the patch, leaving old pods (without the new mount) + # in place for the post-patch cert check. + oc --kubeconfig="${kubeconfig}" rollout restart ds/virt-handler -n "${cnvNs}" + true +} + +# EnsureVirtSyncService — create Service for port CNV_CCLM_SYNC_PORT if absent. +EnsureVirtSyncService() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + + if oc --kubeconfig="${kubeconfig}" get svc virt-synchronization-controller \ + -n "${cnvNs}" 1>/dev/null 2>&1; then + : "[${clusterName}] Service virt-synchronization-controller already exists" + return 0 + fi + + : "[${clusterName}] Creating Service virt-synchronization-controller:${CNV_CCLM_SYNC_PORT}" + { + oc --kubeconfig="${kubeconfig}" create -f - \ + --dry-run=client -o yaml --save-config + } 0</dev/null || true)" + if [[ -z "${seExists}" ]]; then + : "[${clusterName}] ServiceExport CRD not available — skipping" + return 0 + fi + + if oc --kubeconfig="${kubeconfig}" get serviceexport \ + virt-synchronization-controller \ + -n "${cnvNs}" 1>/dev/null 2>&1; then + : "[${clusterName}] ServiceExport virt-synchronization-controller already exists" + return 0 + fi + + : "[${clusterName}] Creating ServiceExport virt-synchronization-controller" + { + oc --kubeconfig="${kubeconfig}" create -f - \ + --dry-run=client -o yaml --save-config + } 0<<'ocEOF' | oc --kubeconfig="${kubeconfig}" apply -f - +apiVersion: multicluster.x-k8s.io/v1alpha1 +kind: ServiceExport +metadata: + name: virt-synchronization-controller + namespace: openshift-cnv +ocEOF + true +} + +# SaveCclmReadinessDiagnostics — collect diagnostics to ARTIFACT_DIR. +SaveCclmReadinessDiagnostics() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset diagDir="${ARTIFACT_DIR}/cclm-readiness-${clusterName}" + mkdir -p "${diagDir}" + + oc --kubeconfig="${kubeconfig}" get pod \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + -o wide > "${diagDir}/virt-handler-pods.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get ds virt-handler \ + -n "${cnvNs}" -o yaml \ + > "${diagDir}/virt-handler-ds.yaml" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get secret -n "${cnvNs}" \ + > "${diagDir}/openshift-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" logs \ + -n "${cnvNs}" deployment/virt-operator \ + --tail=150 > "${diagDir}/virt-operator-logs.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" logs \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + --tail=80 > "${diagDir}/virt-handler-logs.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get svc \ + -n "${cnvNs}" > "${diagDir}/services.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get serviceexport \ + -n "${cnvNs}" > "${diagDir}/serviceexports.txt" 2>&1 || true + true +} + +# ── main ────────────────────────────────────────────────────────────────────── + +typeset -a spokeKubeconfigsArr=() +mapfile -t spokeKubeconfigsArr < <(LoadSpokeKubeconfigs) + +typeset -i spokeCount="${#spokeKubeconfigsArr[@]}" +typeset -i failedCount=0 i + +: "=== CCLM readiness on ${spokeCount} spoke(s) ===" + +for ((i = 0; i < spokeCount; i++)); do + typeset kc="${spokeKubeconfigsArr[i]}" + typeset spkName="" + spkName="$(GetSpokeClusterName "$((i + 1))")" + typeset -i originalReplicas + originalReplicas="$(GetVirtOperatorReplicas "${kc}")" + typeset virtOperatorRestored=false + + # ── Step 1: check if cert already present (fast path — no patching needed) ── + typeset isCertOk=false + if CheckVirtHandlerClientCert "${kc}"; then + isCertOk=true + : "[${spkName}] virt-handler CCLM client cert already present — skipping patch" + fi + + # ── Step 2: apply DaemonSet workaround patch if cert absent (CNV-89129) ── + # virt-operator MUST be scaled to 0 before patching to prevent immediate reconciliation. + if [[ "${isCertOk}" != "true" ]]; then + : "[${spkName}] Cert absent — applying DaemonSet workaround patch (CNV-89129)" + if ! PatchVirtHandlerDaemonSetCertMount "${kc}" "${spkName}"; then + : "[${spkName}] DaemonSet patch failed" + SaveCclmReadinessDiagnostics "${kc}" "${spkName}" + # Restore virt-operator before moving on so the cluster stays functional. + ScaleVirtOperator "${kc}" "${spkName}" "${originalReplicas}" || true + (( ++failedCount )) + continue + fi + WaitVirtHandlerRollout "${kc}" "${spkName}" "10m" + + typeset -i certWaitSecs=0 + while (( certWaitSecs < 300 )); do + if CheckVirtHandlerClientCert "${kc}"; then + isCertOk=true + break + fi + : "[${spkName}] Post-patch poll for virt-handler cert (${certWaitSecs}/300s)" + sleep "${pollIntervalSecs}" + (( certWaitSecs += pollIntervalSecs )) + done + + # Restore virt-operator now that cert is confirmed (or we are about to fail). + # Do this before the fatal check so the cluster stays operational even on failure. + ScaleVirtOperator "${kc}" "${spkName}" "${originalReplicas}" || true + virtOperatorRestored=true + fi + + if [[ "${isCertOk}" != "true" ]]; then + : "[${spkName}] FATAL: ${clientCertMountPath}/tls.crt absent in one or more virt-handler pods after all remediation" + SaveCclmReadinessDiagnostics "${kc}" "${spkName}" + (( ++failedCount )) + continue + fi + : "[${spkName}] virt-handler CCLM client cert present in all Running pods" + + # ── Step 3: ensure Service on CNV_CCLM_SYNC_PORT ── + if ! EnsureVirtSyncService "${kc}" "${spkName}"; then + : "[${spkName}] WARN: failed to ensure virt-synchronization-controller Service" + (( ++failedCount )) + fi + + # ── Step 4: ensure ServiceExport (no-op if Submariner CRD not available) ── + EnsureVirtSyncServiceExport "${kc}" "${spkName}" || true + + : "[${spkName}] CCLM readiness checks complete" +done + +(( failedCount == 0 )) +: "CCLM readiness checks passed on all ${spokeCount} spoke(s)" +true diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json new file mode 100644 index 0000000000000..4ffa212b8f0b5 --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml new file mode 100644 index 0000000000000..0569e94550dd2 --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml @@ -0,0 +1,73 @@ +ref: + as: p2p-cnv-cclm-readiness + from: cli + commands: p2p-cnv-cclm-readiness-commands.sh + timeout: 35m0s + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 200Mi + env: + - name: CNV_ENABLE_CCLM + default: "false" + documentation: |- + When "true", run CCLM readiness checks: restart virt-operator, verify + /etc/virt-handler/clientcertificates/tls.crt is present in virt-handler pods, + and ensure the virt-synchronization-controller Service and ServiceExport exist. + Set to "true" for CCLM scenarios; leave "false" for upgrade or non-CCLM jobs. + - name: CNV_CCLM_WAIT_TIMEOUT + default: "600" + documentation: |- + Maximum seconds to poll for /etc/virt-handler/clientcertificates/tls.crt in + a virt-handler pod after virt-operator restart. The fallback DaemonSet patch + is attempted if the cert is still absent after this timeout. + - name: CNV_CCLM_VIRT_HANDLER_CERT_SECRET + default: "kubevirt-virt-handler-certs" + documentation: |- + Name of the Secret in openshift-cnv to mount at + /etc/virt-handler/clientcertificates when virt-operator does not add the + mount automatically (fallback DaemonSet patch path only). + - name: CNV_CCLM_SYNC_PORT + default: "9185" + documentation: |- + TCP port for the virt-synchronization-controller sync data endpoint. + Used when creating the missing Service in openshift-cnv. + documentation: |- + Verifies and remediates CCLM (Cross-Cluster Live Migration) prerequisites on + every spoke cluster before migration steps run. No-op when CNV_ENABLE_CCLM + is not "true". + + Background — CNV bug CNV-89129 / upstream kubevirt#16264 + (https://github.com/kubevirt/kubevirt/issues/16264): + virt-operator does not add the /etc/virt-handler/clientcertificates volume mount + to the virt-handler DaemonSet when DecentralizedLiveMigration is enabled, even + though the Secret (kubevirt-virt-handler-certs) exists and is populated. + Without this mount, virt-handler emits "failed to load the certificate + /etc/virt-handler/clientcertificates/tls.crt: no such file or directory" every + 60 s and cannot make the required mTLS connection to the destination + virt-synchronization-controller:9185, leaving the migration stuck in + Synchronizing/WaitingForSync indefinitely. + Affected: CNV 4.17+ / OCP 4.22 (as of 2026-08). + + What this step does (idempotent, safe to re-run): + 1. Restarts virt-operator on each spoke and waits for its rollout to finish + (triggers DaemonSet reconciliation — usually sufficient to fix the mount). + 2. Waits up to CNV_CCLM_WAIT_TIMEOUT seconds for the CCLM client cert to + appear in a virt-handler pod. + 3. Fallback: if the cert is still absent, patches the virt-handler DaemonSet + directly to mount CNV_CCLM_VIRT_HANDLER_CERT_SECRET at + /etc/virt-handler/clientcertificates. Collects diagnostics and fails the + step if the cert is still absent after the patch rollout. + 4. Ensures a Service named virt-synchronization-controller on port + CNV_CCLM_SYNC_PORT (9185) exists in openshift-cnv. + 5. Ensures a ServiceExport for that Service exists, allowing Submariner to + propagate it to other clusters for cross-cluster DNS resolution. Skipped + gracefully when the ServiceExport CRD (Submariner) is not yet installed. + + Reads spoke names and kubeconfigs from ${SHARED_DIR} (managed-cluster-kubeconfig-N + and managed-cluster-name-N files). + + Must run after p2p-acm-cnv-install-policy and ideally after + acm-interop-p2p-submariner-install (so ServiceExport CRD is available). + Must run before p2p-mtv-bidir-mig-config / p2p-cclm-bidir-migration-test. diff --git a/ci-operator/step-registry/p2p/debug-sleep/OWNERS b/ci-operator/step-registry/p2p/debug-sleep/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh new file mode 100644 index 0000000000000..ed7073c889a80 --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Debugging aid: sleep for DEBUG_SLEEP_DURATION seconds so that the CI clusters +# remain alive (and accessible) after test steps finish but before the post-phase +# uninstall/deprovision steps tear everything down. +# +# Set DEBUG_SLEEP_DURATION=0 to skip entirely (default for normal runs). +# Set DEBUG_SLEEP_DURATION=7200 (2h) or any positive integer for debugging runs. +# +set -euo pipefail; shopt -s inherit_errexit + +typeset -i sleepSecs="${DEBUG_SLEEP_DURATION}" + +if (( sleepSecs <= 0 )); then + : "DEBUG_SLEEP_DURATION=0 — skipping debug sleep" + exit 0 +fi + +# ── Print cluster access hints ────────────────────────────────────────────── +: "=== DEBUG SLEEP: clusters will remain alive for ${sleepSecs}s ===" + +# Hub kubeconfig +if [[ -f "${KUBECONFIG}" ]]; then + : "Hub kubeconfig : ${KUBECONFIG}" +fi + +# Spoke kubeconfigs +typeset -i i +for (( i = 1; ; i++ )); do + typeset kcPath="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + [[ -f "${kcPath}" ]] || break + typeset clusterNameFile="${SHARED_DIR}/managed-cluster-name-${i}" + typeset clusterName="spoke-${i}" + [[ -f "${clusterNameFile}" ]] && clusterName="$(cat "${clusterNameFile}")" + : "Spoke ${i} (${clusterName}) kubeconfig : ${kcPath}" +done + +typeset endTime +endTime="$(date -d "+${sleepSecs} seconds" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null \ + || date -v "+${sleepSecs}S" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null \ + || echo "unknown")" +: "Sleeping until approximately ${endTime} ..." + +sleep "${sleepSecs}" +: "=== DEBUG SLEEP complete ===" diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json new file mode 100644 index 0000000000000..763bd9f13199c --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/debug-sleep/p2p-debug-sleep-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml new file mode 100644 index 0000000000000..07f6b562f5610 --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml @@ -0,0 +1,33 @@ +ref: + as: p2p-debug-sleep + from: cli + commands: p2p-debug-sleep-commands.sh + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: DEBUG_SLEEP_DURATION + default: "0" + documentation: |- + Number of seconds to sleep in the post phase before cluster uninstall/deprovision + begins. Keeps the clusters alive so engineers can SSH in and debug failures. + + Set to 0 (default) to skip the sleep entirely in normal runs. + Set to 7200 for a 2-hour hold during active debugging sessions. + documentation: |- + Debugging aid: sleeps for DEBUG_SLEEP_DURATION seconds in the post phase, after + diagnostic collection (must-gather, inspector, submariner-diag) but before + acm-interop-p2p-cluster-uninstall and ipi-deprovision tear the clusters down. + + This gives engineers time to SSH into hub and spoke clusters, run manual + diagnostic commands, or re-run failing steps interactively before the + infrastructure is destroyed. + + Usage: + - Add as the last step in the post phase before uninstall steps. + - Set DEBUG_SLEEP_DURATION=7200 (or any positive integer) in the job env. + - Set DEBUG_SLEEP_DURATION=0 (or remove the env override) when debugging is done. + + The step is a no-op when DEBUG_SLEEP_DURATION=0 and adds zero overhead to + normal runs. diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json new file mode 100644 index 0000000000000..9861560c288e8 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml new file mode 100644 index 0000000000000..fe1de9f7ed158 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml @@ -0,0 +1,27 @@ +chain: + as: p2p-mtv-bidir-mig-config + steps: + - ref: p2p-mtv-additional-config + - ref: p2p-mtv-register-providers + - ref: p2p-mtv-create-migration-maps + - ref: p2p-mtv-create-return-migration-maps + documentation: |- + Configure MTV for bi-directional cross-cluster live migration (CCLM) on the ACM hub. + + Extends p2p-mtv-mig-config with a fourth step that creates the return-leg maps: + + 1. p2p-mtv-additional-config + Applies ForkliftController with feature_ocp_live_migration=true (CCLM) on the hub. + + 2. p2p-mtv-register-providers + Registers each ACM spoke as an MTV OpenShift Provider on the hub. + + 3. p2p-mtv-create-migration-maps + Creates pod-to-pod NetworkMap and ODF StorageMap for the forward leg (spoke-1 → spoke-2). + + 4. p2p-mtv-create-return-migration-maps + Creates pod-to-pod NetworkMap and ODF StorageMap for the return leg (spoke-2 → spoke-1). + Map names carry a "-return" suffix: spoke-network-map-return / spoke-storage-map-return. + + Requires install-operators (mtv-operator on hub), acm-fetch-managed-clusters, and + p2p-install-odf-spokes before this chain. diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh new file mode 100644 index 0000000000000..a34ee7813eb64 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# +# Create MTV NetworkMap and StorageMap for the return migration leg (spoke-2 → spoke-1) on the hub. +# Return-leg companion to p2p-mtv-create-migration-maps; map names carry a "-return" suffix. +# Requires registered MTV providers and ODF Available on both spokes. +# +set -euxo pipefail; shopt -s inherit_errexit + +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" +fi + + + +# ValidateConfig — fail fast if storage map env vars are missing. +ValidateConfig() { + [[ -n "${MTV_SOURCE_STORAGE_NAME}" ]] || { : "MTV_SOURCE_STORAGE_NAME is required"; false; } + [[ -n "${MTV_DESTINATION_STORAGE_CLASS}" ]] || { : "MTV_DESTINATION_STORAGE_CLASS is required"; false; } +} + +# RefreshProviderInventory — trigger MTV to re-scan spoke storage/network before map validation. +RefreshProviderInventory() { + typeset providerName="${1:?}" + typeset ts + + ts="$(date -u +%s)" + oc annotate "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + "forklift.konveyor.io/inventory-refresh=${ts}" --overwrite +} + +# WaitProviderReady — ensure both providers finished inventory before creating maps. +WaitProviderReady() { + typeset providerName="${1:?}" + + oc wait "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_READY_TIMEOUT}" +} + +# ApplyNetworkMap — create pod→pod NetworkMap for CCLM cross-cluster pod networking. +# Uses jq --arg to safely marshal values; avoids raw heredoc expansion of YAML-special chars. +ApplyNetworkMap() { + jq -n \ + --arg name "${MTV_NETWORK_MAP_NAME}" \ + --arg ns "${MTV_NAMESPACE}" \ + --arg srcProv "${MTV_SOURCE_PROVIDER}" \ + --arg dstProv "${MTV_DESTINATION_PROVIDER}" \ + '{ + apiVersion: "forklift.konveyor.io/v1beta1", + kind: "NetworkMap", + metadata: {name: $name, namespace: $ns}, + spec: { + map: [{source: {type: "pod"}, destination: {type: "pod"}}], + provider: { + source: {name: $srcProv, namespace: $ns}, + destination: {name: $dstProv, namespace: $ns} + } + } + }' | oc apply -f - +} + +# ApplyStorageMap — map source ODF virt StorageClass to destination (RWX required for live migration). +# Uses jq --arg to safely marshal values; avoids raw heredoc expansion of YAML-special chars. +ApplyStorageMap() { + jq -n \ + --arg name "${MTV_STORAGE_MAP_NAME}" \ + --arg ns "${MTV_NAMESPACE}" \ + --arg srcName "${MTV_SOURCE_STORAGE_NAME}" \ + --arg dstClass "${MTV_DESTINATION_STORAGE_CLASS}" \ + --arg srcProv "${MTV_SOURCE_PROVIDER}" \ + --arg dstProv "${MTV_DESTINATION_PROVIDER}" \ + '{ + apiVersion: "forklift.konveyor.io/v1beta1", + kind: "StorageMap", + metadata: {name: $name, namespace: $ns}, + spec: { + map: [{ + source: {name: $srcName}, + destination: {storageClass: $dstClass} + }], + provider: { + source: {name: $srcProv, namespace: $ns}, + destination: {name: $dstProv, namespace: $ns} + } + } + }' | oc apply -f - +} + +# WaitMapReady — wait until MTV validates network/storage mapping against provider inventory. +WaitMapReady() { + typeset kind="${1:?}" + typeset name="${2:?}" + + oc wait "${kind}/${name}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_MAP_READY_TIMEOUT}" +} + +# --- Main --- +[[ -n "${KUBECONFIG}" ]] +[[ -r "${KUBECONFIG}" ]] + +ValidateConfig + +oc get ns "${MTV_NAMESPACE}" 1>/dev/null + +WaitProviderReady "${MTV_SOURCE_PROVIDER}" +WaitProviderReady "${MTV_DESTINATION_PROVIDER}" + +if [[ "${MTV_SKIP_INVENTORY_REFRESH}" != "true" ]]; then + RefreshProviderInventory "${MTV_SOURCE_PROVIDER}" + RefreshProviderInventory "${MTV_DESTINATION_PROVIDER}" +fi + +ApplyNetworkMap +ApplyStorageMap + +WaitMapReady networkmap "${MTV_NETWORK_MAP_NAME}" +WaitMapReady storagemap "${MTV_STORAGE_MAP_NAME}" + +oc get networkmap,storagemap -n "${MTV_NAMESPACE}" \ + >> "${ARTIFACT_DIR}/mtv-migration-maps-status.txt" +true diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json new file mode 100644 index 0000000000000..8c30e05e3b3c1 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml new file mode 100644 index 0000000000000..a19602338c3b0 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml @@ -0,0 +1,54 @@ +ref: + as: p2p-mtv-create-return-migration-maps + from: cli + commands: p2p-mtv-create-return-migration-maps-commands.sh + timeout: 30m0s + grace_period: 10m0s + resources: + requests: + cpu: 200m + memory: 400Mi + documentation: |- + Create MTV NetworkMap (pod→pod) and StorageMap for the return migration leg (spoke-2 → spoke-1) + on the hub. Run after p2p-mtv-create-migration-maps (which creates the forward maps) and after + ODF is Available on both spokes; creating StorageMap too early causes StorageRefNotValid. + + This is the return-leg companion to p2p-mtv-create-migration-maps. Default source/destination + are reversed (spoke-2 as source, spoke-1 as destination) and map names carry a "-return" suffix + to coexist alongside the forward maps in the same openshift-mtv namespace. + + Annotates both providers for inventory refresh (unless MTV_SKIP_INVENTORY_REFRESH=true), + applies maps, waits for Ready, and appends to ${ARTIFACT_DIR}/mtv-migration-maps-status.txt. + + Hub kubeconfig is KUBECONFIG from ci-operator. + env: + - name: MTV_NAMESPACE + default: "openshift-mtv" + documentation: Namespace where MTV operator and map CRs live. + - name: MTV_SOURCE_PROVIDER + default: "spoke-2" + documentation: Source MTV Provider CR name for the return leg (migration origin). + - name: MTV_DESTINATION_PROVIDER + default: "spoke-1" + documentation: Destination MTV Provider CR name for the return leg (migration target). + - name: MTV_NETWORK_MAP_NAME + default: "spoke-network-map-return" + documentation: NetworkMap metadata.name for the return leg. + - name: MTV_STORAGE_MAP_NAME + default: "spoke-storage-map-return" + documentation: StorageMap metadata.name for the return leg. + - name: MTV_SOURCE_STORAGE_NAME + default: "ocs-storagecluster-ceph-rbd-virtualization" + documentation: StorageClass name on spoke-2 (the return-leg source; spec.map[].source.name). + - name: MTV_DESTINATION_STORAGE_CLASS + default: "ocs-storagecluster-ceph-rbd-virtualization" + documentation: Destination StorageClass name on spoke-1 (spec.map[].destination.storageClass). + - name: MTV_PROVIDER_READY_TIMEOUT + default: "15m" + documentation: Timeout for oc wait provider/ --for=condition=Ready. + - name: MTV_MAP_READY_TIMEOUT + default: "15m" + documentation: Timeout for NetworkMap/StorageMap Ready waits. + - name: MTV_SKIP_INVENTORY_REFRESH + default: "false" + documentation: Set to "true" to skip forklift.konveyor.io/inventory-refresh annotation. diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh new file mode 100644 index 0000000000000..845a749a30416 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -0,0 +1,847 @@ +#!/bin/bash +# +# Execute MTV cross-cluster live migration (CCLM) return leg: spoke-2 back to spoke-1 on the hub. +# Return-leg companion to p2p-mtv-execute-live-migration; artifact names carry a "-return" suffix. +# +set -euxo pipefail; shopt -s inherit_errexit + +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # proxy-conf.sh may contain credentials — disable xtrace while sourcing. + [[ $- == *x* ]] && _wasTracing=true || _wasTracing=false + set +x + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" + [[ "${_wasTracing}" == "true" ]] && set -x || true +fi + +[[ -n "${KUBECONFIG}" ]] +[[ -r "${KUBECONFIG}" ]] + +typeset -i migrationPollInterval="${MTV_MIGRATION_POLL_INTERVAL_SECONDS}" +typeset -i sourceSpokeIndex="${MTV_SOURCE_SPOKE_INDEX}" +typeset -i destSpokeIndex="${MTV_DEST_SPOKE_INDEX}" +typeset -i syncStuckMinutes="${MTV_SYNC_STUCK_MINUTES}" +typeset -i syncPhaseStartedAt=0 +typeset cclmDebugMode="${P2P_CCLM_DEBUG_MODE}" + +# Temp file accumulating tab-separated JUnit records (PASS/FAIL\tname\telapsed\t[msg]). +# Written in the subshell; read by WriteJunit after it exits. +typeset -r junitFile="${TMPDIR:-/tmp}/cclm-return-junit-$$.tsv" + +typeset sourceKubeconfig="${MTV_SOURCE_SPOKE_KUBECONFIG}" +typeset destKubeconfig="${MTV_DEST_SPOKE_KUBECONFIG}" +typeset targetNs="${MTV_TEST_VM_TARGET_NAMESPACE}" +typeset diagDir="" + +# HubOc — run oc against the ACM hub. +HubOc() { + oc --kubeconfig="${KUBECONFIG}" "$@" +} + +# SourceOc — run oc against the source spoke. +SourceOc() { + oc --kubeconfig="${sourceKubeconfig}" "$@" +} + +# DestOc — run oc against the destination spoke. +DestOc() { + oc --kubeconfig="${destKubeconfig}" "$@" +} + +# ResolveSpokeKubeconfigs — source and destination spoke admin kubeconfigs. +ResolveSpokeKubeconfigs() { + [[ -n "${SHARED_DIR}" ]] + + if [[ -z "${sourceKubeconfig}" ]]; then + if [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig-${sourceSpokeIndex}" ]]; then + sourceKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig-${sourceSpokeIndex}" + elif (( sourceSpokeIndex == 1 )) && [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig" ]]; then + sourceKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig" + else + : "Source spoke kubeconfig not found for index ${sourceSpokeIndex}" + false + fi + fi + [[ -r "${sourceKubeconfig}" ]] + + if [[ -z "${destKubeconfig}" ]]; then + [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig-${destSpokeIndex}" ]] + destKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig-${destSpokeIndex}" + fi + [[ -r "${destKubeconfig}" ]] +} + +# DumpDiagnostics — write MTV and VM state to ARTIFACT_DIR on failure. +DumpDiagnostics() { + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + diagDir="${ARTIFACT_DIR}/mtv-live-migration-return-diagnostics" + mkdir -p "${diagDir}" + HubOc get plan,migration,networkmap,storagemap,provider -n "${MTV_NAMESPACE}" \ + > "${diagDir}/hub-mtv-resources.txt" 2>&1 || true + HubOc describe "plan/${MTV_PLAN_NAME}" -n "${MTV_NAMESPACE}" \ + > "${diagDir}/plan-describe.txt" 2>&1 || true + HubOc describe "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" \ + > "${diagDir}/migration-describe.txt" 2>&1 || true + HubOc get events -n "${MTV_NAMESPACE}" --sort-by='.lastTimestamp' \ + > "${diagDir}/hub-mtv-events.txt" 2>&1 || true + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o wide > "${diagDir}/source-vm.txt" 2>&1 || true + DestOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o wide > "${diagDir}/dest-vm.txt" 2>&1 || true + DestOc get datavolume,pvc,pods -n "${targetNs}" \ + > "${diagDir}/dest-storage.txt" 2>&1 || true + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${diagDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${diagDir}/dest-vmim.yaml" 2>&1 || true + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${diagDir}/source-vmi.yaml" 2>&1 || true + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${targetNs}" -o yaml \ + > "${diagDir}/dest-vmi.yaml" 2>&1 || true + SourceOc get pods -n "${MTV_CNV_NAMESPACE}" -o wide \ + > "${diagDir}/source-cnv-pods.txt" 2>&1 || true + DestOc get pods -n "${MTV_CNV_NAMESPACE}" -o wide \ + > "${diagDir}/dest-cnv-pods.txt" 2>&1 || true + SourceOc logs -n "${MTV_CNV_NAMESPACE}" \ + -l kubevirt.io=virt-controller --tail=100 \ + > "${diagDir}/source-virt-controller.log" 2>&1 || true + DestOc logs -n "${MTV_CNV_NAMESPACE}" \ + -l kubevirt.io=virt-controller --tail=100 \ + > "${diagDir}/dest-virt-controller.log" 2>&1 || true + + DumpCclmSyncDiagnostics +} + +# DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the +# Synchronization phase (source=Synchronizing, dest=WaitingForSync). +# Captures virt-synchronization-controller state, Submariner service exports/ +# imports, forklift-controller logs, virt-handler logs, VMI migration state, +# virt-handler client-cert Secret presence (missing certs block mTLS to dest +# virt-sync-controller:9185), nftables SNAT exclusion set re-verification, and +# OVN-K gateway-node logs. +# Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics +# (on any live-plan error path). +DumpCclmSyncDiagnostics() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + + typeset syncDir="${ARTIFACT_DIR}/mtv-cclm-sync-diagnostics" + mkdir -p "${syncDir}" + + typeset spoke kc + for spoke in source dest; do + [[ "${spoke}" == "source" ]] && kc="${sourceKubeconfig}" || kc="${destKubeconfig}" + + # Broad search: virt-sync related pods across ALL namespaces. + oc --kubeconfig="${kc}" get pods -A -o wide 2>/dev/null \ + | grep -E 'virt-sync|synchronization-controller|virt-migration' \ + > "${syncDir}/${spoke}-virt-sync-pods-all-ns.txt" 2>&1 || true + + # Targeted: pods + logs from the CNV namespace by deployment name. + oc --kubeconfig="${kc}" get pods \ + -n "${MTV_CNV_NAMESPACE}" -o wide \ + 2>/dev/null \ + | grep 'virt-synchronization-controller' \ + > "${syncDir}/${spoke}-virt-sync-pods-cnv-ns.txt" 2>&1 || true + + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + deployment/virt-synchronization-controller \ + --tail=200 \ + > "${syncDir}/${spoke}-virt-sync-controller-logs.txt" 2>&1 || true + + # Submariner service discovery state. + oc --kubeconfig="${kc}" get serviceexport -A \ + > "${syncDir}/${spoke}-serviceexports.txt" 2>&1 || true + oc --kubeconfig="${kc}" get serviceimport -A \ + > "${syncDir}/${spoke}-serviceimports.txt" 2>&1 || true + + # Services on the CCLM sync port (8443) or the KubeVirt migration port. + oc --kubeconfig="${kc}" get svc -A -o json 2>/dev/null \ + | jq -r '.items[] + | select(.spec.ports[]?.port == 8443 + or .spec.ports[]?.port == 9185 + or .spec.ports[]?.targetPort == 8443 + or (.spec.ports[]?.name // "" | test("sync|migr|virt"))) + | "\(.metadata.namespace)/\(.metadata.name)\t" + + "ports=\([.spec.ports[]? | "\(.port):\(.targetPort // "")"] | join(","))\t" + + "clusterIP=\(.spec.clusterIP)"' \ + > "${syncDir}/${spoke}-sync-services.txt" 2>&1 || true + + # virt-handler logs (per-node daemon that drives actual live migration). + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + -l 'kubevirt.io=virt-handler' \ + --tail=150 \ + > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + + # virt-handler CCLM client-certificate Secrets (metadata only — no secret data). + # Missing certs prevent virt-handler from making mTLS connections to the + # dest virt-synchronization-controller on port 9185. + oc --kubeconfig="${kc}" get secrets -n "${MTV_CNV_NAMESPACE}" \ + > "${syncDir}/${spoke}-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kc}" get ds virt-handler -n "${MTV_CNV_NAMESPACE}" -o json \ + 2>/dev/null \ + | jq '[.spec.template.spec.volumes[]? + | select(.name | test("cert|tls|key"; "i")) + | {name, secret: (.secret.secretName // null), + configMap: (.configMap.name // null)}]' \ + > "${syncDir}/${spoke}-virt-handler-cert-volumes.json" 2>&1 || true + + # OVN-K gateway node logs — check for route/policy issues affecting cross-cluster. + typeset _gwNode + _gwNode="$(oc --kubeconfig="${kc}" get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1 || true)" + if [[ -n "${_gwNode}" ]]; then + oc --kubeconfig="${kc}" logs \ + -n openshift-ovn-kubernetes \ + -l 'app=ovnkube-node' \ + --field-selector="spec.nodeName=${_gwNode}" \ + --tail=80 \ + > "${syncDir}/${spoke}-ovnkube-node-gateway-logs.txt" 2>&1 || true + # Verify nftables SNAT exclusion set still contains remote CIDRs. + oc --kubeconfig="${kc}" debug "node/${_gwNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + > "${syncDir}/${spoke}-nftables-snat-set.txt" 2>&1 || true + fi + done + + # forklift-controller logs from the hub (MTV migration orchestrator). + oc --kubeconfig="${KUBECONFIG}" logs \ + -n "${MTV_NAMESPACE}" \ + deployment/forklift-controller \ + --tail=300 \ + > "${syncDir}/hub-forklift-controller-logs.txt" 2>&1 || true + + # Detailed VMI migration state. + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/source-vmi-migration-state.json" 2>&1 || true + + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/dest-vmi-migration-state.json" 2>&1 || true + + # Full VMIM YAML (more detail than vmim.yaml in the base diagnostics dir). + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${syncDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${syncDir}/dest-vmim.yaml" 2>&1 || true + + # Events in the migration namespaces. + SourceOc get events -n "${MTV_TEST_VM_NAMESPACE}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/source-vm-ns-events.txt" 2>&1 || true + DestOc get events -n "${targetNs}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/dest-vm-ns-events.txt" 2>&1 || true + + : "CCLM sync diagnostics written → ${syncDir}" +} + +# OnError — dump diagnostics before propagating failure. +OnError() { + typeset -i ec=$? + DumpDiagnostics + exit "${ec}" +} + +# WaitProviderReady — gate until MTV Provider is Ready. +WaitProviderReady() { + typeset providerName="${1:?}" + HubOc wait "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PLAN_READY_TIMEOUT}" +} + +# WaitMapReady — gate until NetworkMap or StorageMap is Ready. +WaitMapReady() { + typeset kind="${1:?}" + typeset name="${2:?}" + HubOc wait "${kind}/${name}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PLAN_READY_TIMEOUT}" +} + +# PreflightSourceVm — source VM must exist and VMI Running for live migration. +PreflightSourceVm() { + typeset phase + + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" 1>/dev/null + + phase="$(SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.status.phase}' || true)" + + if [[ "${MTV_PLAN_TYPE}" == "live" ]]; then + [[ "${phase}" == "Running" ]] + fi +} + +# GetVmRootDiskStorageClass — resolve root disk StorageClass from source VM. +GetVmRootDiskStorageClass() { + typeset dvName pvcName scName + + dvName="$(SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o json | + jq -r 'first(.spec.template.spec.volumes[]?.dataVolume.name // empty) // ""')" + [[ -z "${dvName}" ]] && dvName="${MTV_TEST_VM_NAME}-rootdisk" + + pvcName="$(SourceOc get "datavolume/${dvName}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.status.claimName}' || true)" + [[ -z "${pvcName}" ]] && pvcName="${dvName}" + + scName="$(SourceOc get "persistentvolumeclaim/${pvcName}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.spec.storageClassName}' || true)" + [[ -n "${scName}" ]] && printf '%s' "${scName}" && return 0 + + SourceOc get pvc -n "${MTV_TEST_VM_NAMESPACE}" -o json | + jq -r --arg pvc "${pvcName}" ' + (first(.items[] | select(.metadata.name == $pvc) | .spec.storageClassName) // + first(.items[].spec.storageClassName) // + "") // "" + ' +} + +# PreflightVmStorageMapped — VM root disk StorageClass must appear in StorageMap. +PreflightVmStorageMapped() { + typeset vmSc scUid mapJson mapped + + vmSc="$(GetVmRootDiskStorageClass)" + [[ -n "${vmSc}" ]] + + scUid="$(SourceOc get "storageclass/${vmSc}" -o jsonpath='{.metadata.uid}' || true)" + mapJson="$(HubOc get "storagemap/${MTV_STORAGE_MAP_NAME}" -n "${MTV_NAMESPACE}" -o json)" + mapped="$(jq -r --arg sc "${vmSc}" --arg uid "${scUid}" \ + '[.spec.map[]? | select(.source.name == $sc or (.source.id != null and .source.id == $uid))] | length > 0' \ + <<<"${mapJson}")" + [[ "${mapped}" == "true" ]] +} + +# PreflightHub — providers and maps must be Ready before Plan creation. +PreflightHub() { + WaitProviderReady "${MTV_SOURCE_PROVIDER}" + WaitProviderReady "${MTV_DESTINATION_PROVIDER}" + WaitMapReady networkmap "${MTV_NETWORK_MAP_NAME}" + WaitMapReady storagemap "${MTV_STORAGE_MAP_NAME}" +} + +# HasDecentralizedLiveMigrationGate — KubeVirt must list DecentralizedLiveMigration (MTV CCLM check). +HasDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + + # jq -e exits 0 when gate present, 1 when absent or oc fails. + oc --kubeconfig="${kc}" get kubevirt "${MTV_KUBEVIRT_NAME}" -n "${MTV_CNV_NAMESPACE}" -o json \ + | jq -e '.spec.configuration.developerConfiguration.featureGates // [] | contains(["DecentralizedLiveMigration"])' \ + > /dev/null +} + +# EnsureDecentralizedLiveMigrationGate — enable CCLM via HCO featureGates; wait for KubeVirt sync. +EnsureDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + typeset hcoGate + + HasDecentralizedLiveMigrationGate "${kc}" && return 0 + + hcoGate="$(oc --kubeconfig="${kc}" get hyperconverged "${MTV_HCO_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + -o jsonpath='{.spec.featureGates.decentralizedLiveMigration}' || true)" + if [[ "${hcoGate}" != "true" ]]; then + oc --kubeconfig="${kc}" patch hyperconverged "${MTV_HCO_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + --type merge -p '{"spec":{"featureGates":{"decentralizedLiveMigration":true}}}' + fi + + WaitForDecentralizedLiveMigrationGate "${kc}" && return 0 + + oc --kubeconfig="${kc}" patch kubevirt "${MTV_KUBEVIRT_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + --type merge -p '{"spec":{"configuration":{"developerConfiguration":{"featureGates":["DecentralizedLiveMigration"]}}}}' + + WaitForDecentralizedLiveMigrationGate "${kc}" +} + +# WaitForDecentralizedLiveMigrationGate — poll KubeVirt until DecentralizedLiveMigration appears. +WaitForDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + typeset -i deadline=$((SECONDS + 600)) + + while (( SECONDS < deadline )); do + HasDecentralizedLiveMigrationGate "${kc}" && return 0 + sleep 10 + done + false +} + +# MaybeEnsureDecentralizedLiveMigration — enable CCLM gate on both spokes when configured. +MaybeEnsureDecentralizedLiveMigration() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ "${MTV_ENSURE_DECENTRALIZED_LIVE_MIGRATION}" != "true" ]] && return 0 + + EnsureDecentralizedLiveMigrationGate "${sourceKubeconfig}" + EnsureDecentralizedLiveMigrationGate "${destKubeconfig}" +} + +# PreflightCclm — verify MTV controller and KubeVirt gates required for type=live plans. +PreflightCclm() { + typeset fcGate envVal + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + fcGate="$(HubOc get "forkliftcontroller/${MTV_FORKLIFT_CONTROLLER_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{.spec.feature_ocp_live_migration}' || true)" + [[ "${fcGate}" == "true" ]] + + envVal="$(HubOc get "deployment/${MTV_FORKLIFT_CONTROLLER_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{.spec.template.spec.containers[*].env[?(@.name=="FEATURE_OCP_LIVE_MIGRATION")].value}' \ + || true)" + [[ "${envVal}" == "true" ]] + + HasDecentralizedLiveMigrationGate "${sourceKubeconfig}" + HasDecentralizedLiveMigrationGate "${destKubeconfig}" +} + +# GetSyncControllerPodIp — first Running virt-synchronization-controller pod IP. +GetSyncControllerPodIp() { + typeset kc="${1:?}" + + oc --kubeconfig="${kc}" get pods -n "${MTV_CNV_NAMESPACE}" -o json \ + | jq -r 'first( + .items[] + | select(.metadata.name | startswith("virt-synchronization-controller")) + | select(.status.phase == "Running") + | select((.status.podIP // "") != "") + | .status.podIP + )' +} + +# WaitForSyncControllerReady — CCLM requires sync controller on both spokes. +WaitForSyncControllerReady() { + typeset kc="${1:?}" + + oc --kubeconfig="${kc}" wait deployment/virt-synchronization-controller \ + -n "${MTV_CNV_NAMESPACE}" --for=condition=Available --timeout="${MTV_SYNC_CONTROLLER_WAIT}" +} + +# MaybeWaitForSyncControllers — wait for sync controller deployments on both spokes. +MaybeWaitForSyncControllers() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + WaitForSyncControllerReady "${sourceKubeconfig}" + WaitForSyncControllerReady "${destKubeconfig}" +} + +# PreflightSubmarinerNoGlobalnet — Globalnet breaks raw pod IP CCLM sync routing. +PreflightSubmarinerNoGlobalnet() { + typeset kc="${1:?}" + + # Expected to fail when Globalnet is not deployed; ! inverts the exit code. + ! oc --kubeconfig="${kc}" get daemonset submariner-globalnet \ + -n submariner-operator 1>/dev/null +} + +# MaybePreflightSubmarinerNoGlobalnet — both spokes must not run Globalnet. +MaybePreflightSubmarinerNoGlobalnet() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + PreflightSubmarinerNoGlobalnet "${sourceKubeconfig}" + PreflightSubmarinerNoGlobalnet "${destKubeconfig}" +} + +# GetSourceVirtLauncherPod — virt-launcher pod name for the source VM. +GetSourceVirtLauncherPod() { + typeset podName + + # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. + # No "|| true" — SourceOc failures and jq parse errors must propagate. + # An empty pod list is handled gracefully: jq "// empty" exits 0 with no output. + podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ + -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ + -o json | jq -r 'first(.items[].metadata.name) // empty')" + [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 + + podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ + | jq -r --arg name "${MTV_TEST_VM_NAME}" \ + '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""')" + [[ -n "${podName}" ]] && printf '%s' "${podName}" +} + +# ProbeCclmSyncPortFromPod — TCP probe from an existing pod to sync IP:port. +# Retries up to MTV_CCLM_SYNC_PROBE_RETRIES times (10s apart) to tolerate the +# window between virt-synchronization-controller Deployment Available and its +# TCP listener being bound (connection-refused causes an instant rc=1 without +# retries, even though connectivity works by migration time). +ProbeCclmSyncPortFromPod() { + typeset kc="${1:?}" + typeset ns="${2:?}" + typeset podName="${3:?}" + typeset destIp="${4:?}" + typeset -i attempt=0 maxAttempts="${MTV_CCLM_SYNC_PROBE_RETRIES}" retrySecs=10 + + while (( attempt < maxAttempts )); do + if oc --kubeconfig="${kc}" exec -n "${ns}" "${podName}" -c compute -- \ + timeout "${MTV_CCLM_SYNC_PROBE_TIMEOUT}" bash -c "echo >/dev/tcp/${destIp}/${MTV_CCLM_SYNC_PORT}"; then + return 0 + fi + (( attempt++ )) + if (( attempt < maxAttempts )); then + : "Sync port probe attempt ${attempt}/${maxAttempts} failed; retrying in ${retrySecs}s" + sleep "${retrySecs}" + fi + done + return 1 +} + +# PreflightCclmSyncConnectivity — source must reach dest sync-controller :8443. +PreflightCclmSyncConnectivity() { + typeset destSyncIp srcLauncherPod + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ "${MTV_CCLM_SYNC_PROBE}" != "true" ]] && return 0 + + destSyncIp="$(GetSyncControllerPodIp "${destKubeconfig}")" + [[ -n "${destSyncIp}" ]] + + srcLauncherPod="$(GetSourceVirtLauncherPod)" + [[ -n "${srcLauncherPod}" ]] + + ProbeCclmSyncPortFromPod \ + "${sourceKubeconfig}" "${MTV_TEST_VM_NAMESPACE}" "${srcLauncherPod}" "${destSyncIp}" +} + +# MigrationPipelinePhase — read one pipeline step phase from Migration status. +MigrationPipelinePhase() { + typeset stepName="${1:?}" + typeset migJson phase + + migJson="$(HubOc get "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" -o json || true)" + [[ -n "${migJson}" ]] || return 0 + + phase="$(jq -r --arg vm "${MTV_TEST_VM_NAME}" --arg step "${stepName}" \ + '.status.vms[]? | select(.name == $vm) | .pipeline[]? | select(.name == $step) | .phase' \ + <<<"${migJson}" | head -1)" + [[ -n "${phase}" && "${phase}" != "null" ]] && printf '%s' "${phase}" +} + +# VmimPhase — read VirtualMachineInstanceMigration phase on a spoke. +VmimPhase() { + typeset kc="${1:?}" + typeset ns="${2:?}" + + oc --kubeconfig="${kc}" get vmim -n "${ns}" \ + -o jsonpath='{.items[0].status.phase}' || true +} + +# CheckSyncStuck — fail early when Synchronization does not progress. +CheckSyncStuck() { + typeset syncPhase srcVmimPhase destVmimPhase + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + (( syncStuckMinutes > 0 )) || return 0 + + syncPhase="$(MigrationPipelinePhase "Synchronization")" + [[ "${syncPhase}" == "Running" ]] || { + syncPhaseStartedAt=0 + return 0 + } + + (( syncPhaseStartedAt )) || syncPhaseStartedAt="${SECONDS}" + + if (( SECONDS - syncPhaseStartedAt < syncStuckMinutes * 60 )); then + return 0 + fi + + srcVmimPhase="$(VmimPhase "${sourceKubeconfig}" "${MTV_TEST_VM_NAMESPACE}")" + destVmimPhase="$(VmimPhase "${destKubeconfig}" "${targetNs}")" + + if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then + : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" + DumpCclmSyncDiagnostics + return 1 + fi + + return 0 +} + +# RefreshProviderInventory — re-scan spoke KubeVirt inventory before live Plan validation. +RefreshProviderInventory() { + typeset providerName="${1:?}" + typeset ts + + ts="$(date -u +%s)" + HubOc annotate "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + "forklift.konveyor.io/inventory-refresh=${ts}" --overwrite +} + +# RefreshProvidersForLivePlan — both providers must reflect current KubeVirt feature gates. +RefreshProvidersForLivePlan() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + RefreshProviderInventory "${MTV_SOURCE_PROVIDER}" + RefreshProviderInventory "${MTV_DESTINATION_PROVIDER}" + HubOc wait "provider/${MTV_SOURCE_PROVIDER}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_INVENTORY_REFRESH_WAIT}" + HubOc wait "provider/${MTV_DESTINATION_PROVIDER}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_INVENTORY_REFRESH_WAIT}" +} + +# ApplyPlan — create or update MTV Plan CR on the hub. +ApplyPlan() { + { + HubOc create -f - --dry-run=client -o yaml --save-config + } <&2 || true + PrintMigrationPipeline 1>&2 + false + fi + + CheckSyncStuck || return 1 + + PrintMigrationPipeline + : "Migration in progress${msg:+: ${msg}} (${SECONDS}/${deadline}s)" + sleep "${migrationPollInterval}" + done + + false +} + +# VerifyMigration — destination VMI must be Running after migration. +VerifyMigration() { + typeset destPhase + + destPhase="$(DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${targetNs}" \ + -o jsonpath='{.status.phase}' || true)" + [[ "${destPhase}" == "Running" ]] +} + +# JStep — run a function, append PASS/FAIL record to junitFile, propagate exit code. +# Usage: JStep "Human readable name" FunctionName [args...] +# The ERR trap in the caller subshell fires when this returns non-zero, so each +# failed step still gets recorded before the trap escalates. +JStep() { + typeset name="${1:?}"; shift + typeset -i t0=$SECONDS rc=0 + "$@" || rc=$? + typeset -i elapsed=$(( SECONDS - t0 )) + if (( rc == 0 )); then + printf 'PASS\t%s\t%d\t\n' "${name}" "${elapsed}" >> "${junitFile}" + else + printf 'FAIL\t%s\t%d\tFailed (rc=%d); see diagnostics in mtv-live-migration-return-diagnostics/\n' \ + "${name}" "${elapsed}" "${rc}" >> "${junitFile}" + fi + return "${rc}" +} + +# XmlEscape — replace XML special characters for attribute/text values. +# Covers the five predefined XML entities; matches the reference script (p2p-cnv-pre-upgrade). +XmlEscape() { + typeset s="${1}" + s="${s//&/&}" + s="${s///>}" + s="${s//\"/"}" + s="${s//\'/'}" + printf '%s' "${s}" +} + +# WriteJunit — emit JUnit XML from accumulated junitFile records. +# Prow/Firewatch pick up junit_*.xml files from ARTIFACT_DIR automatically. +WriteJunit() { + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + [[ -f "${junitFile}" ]] || return 0 + + typeset xmlFile="${ARTIFACT_DIR}/junit_cclm_live_migration_return.xml" + mkdir -p "${ARTIFACT_DIR}" + + typeset -i total=0 failures=0 totalTime=0 + typeset status name elapsed failMsg + + while IFS=$'\t' read -r status name elapsed failMsg; do + (( total++ )) || true + (( totalTime += elapsed )) || true + [[ "${status}" == "FAIL" ]] && (( failures++ )) || true + done < "${junitFile}" + + { + printf '\n' + printf '\n' \ + "${total}" "${failures}" "${totalTime}" + while IFS=$'\t' read -r status name elapsed failMsg; do + typeset escapedName; escapedName="$(XmlEscape "${name}")" + printf ' \n' \ + "${escapedName}" "${elapsed}" + if [[ "${status}" == "FAIL" ]]; then + typeset escapedMsg; escapedMsg="$(XmlEscape "${failMsg}")" + printf ' %s\n' \ + "${escapedMsg}" "${escapedMsg}" + fi + printf ' \n' + done < "${junitFile}" + printf '\n' + } > "${xmlFile}" + + : "JUnit XML written → ${xmlFile} (${total} tests, ${failures} failures, ${totalTime}s total)" + rm -f "${junitFile}" +} + +trap - ERR + +typeset -i cclmStepRc=0 +( + trap OnError ERR + + ResolveSpokeKubeconfigs + targetNs="${targetNs:-${MTV_TEST_VM_NAMESPACE}}" + + [[ "${MTV_PLAN_TYPE}" == "live" || "${MTV_PLAN_TYPE}" == "cold" ]] + + JStep "Preflight: Providers and Maps Ready" PreflightHub + JStep "Preflight: DecentralizedLiveMigration Gates" MaybeEnsureDecentralizedLiveMigration + JStep "Preflight: Sync Controllers Available" MaybeWaitForSyncControllers + JStep "Preflight: MTV CCLM Feature Gate Active" PreflightCclm + JStep "Preflight: Submariner No Globalnet" MaybePreflightSubmarinerNoGlobalnet + JStep "Preflight: Provider Inventory Refresh" RefreshProvidersForLivePlan + JStep "Preflight: Source VM Running" PreflightSourceVm + JStep "Preflight: CCLM Sync Port Reachable" PreflightCclmSyncConnectivity + JStep "Preflight: VM Storage Class Mapped" PreflightVmStorageMapped + JStep "Migration: Apply Plan" ApplyPlan + JStep "Migration: Plan Ready" WaitPlanReady + JStep "Migration: Apply Migration" ApplyMigration + JStep "Migration: Succeeded" WaitMigrationSucceeded + JStep "Verification: Destination VMI Running" VerifyMigration + + if [[ -n "${ARTIFACT_DIR}" ]]; then + mkdir -p "${ARTIFACT_DIR}" + { + HubOc get "plan/${MTV_PLAN_NAME}" "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" -o wide + HubOc get "plan/${MTV_PLAN_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" — "}{.message}{"\n"}{end}' + HubOc get "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" — "}{.message}{"\n"}{end}' + PrintMigrationPipeline + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o wide + DestOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o wide + } > "${ARTIFACT_DIR}/mtv-live-migration-return-status.txt" + fi + true +) || cclmStepRc=$? + +# Always write JUnit XML — both on success and on failure, so Prow/Firewatch +# always has a report regardless of which step caused the subshell to exit. +WriteJunit + +if (( cclmStepRc != 0 )); then + DumpDiagnostics + if [[ "${cclmDebugMode}" == "true" ]]; then + : "WARNING: p2p-mtv-execute-live-migration-return failed (rc=${cclmStepRc}); not failing job (debug mode)" + else + exit "${cclmStepRc}" + fi +fi + +true diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json new file mode 100644 index 0000000000000..16a49a3f2a2f1 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml new file mode 100644 index 0000000000000..b55f104aa4d04 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml @@ -0,0 +1,132 @@ +ref: + as: p2p-mtv-execute-live-migration-return + from: cli + commands: p2p-mtv-execute-live-migration-return-commands.sh + timeout: 3h0m0s + grace_period: 15m0s + resources: + requests: + cpu: 200m + memory: 400Mi + documentation: |- + Execute MTV cross-cluster live migration (CCLM) return leg: spoke-2 back to spoke-1. + + Return-leg companion to p2p-mtv-execute-live-migration. Defaults are reversed so that + spoke-2 is the source and spoke-1 is the destination. Plan and Migration CR names carry + a "-return" suffix to coexist alongside the forward migration CRs in the same namespace. + + Runs on the ACM hub (KUBECONFIG). Creates Plan and Migration CRs, waits for Succeeded, + and verifies the VM is Running on the destination spoke (spoke-1). Requires: + - p2p-mtv-bidir-mig-config (return NetworkMap/StorageMap Ready on hub) + - p2p-mtv-execute-live-migration (VM currently on spoke-2 after forward migration) + - p2p-acm-cnv-install-policy (DecentralizedLiveMigration + sync controller on spokes) + - acm-interop-p2p-submariner-install (no Globalnet; :9185 sync-port reachability) + + Live preflight: sync-controller ready, no submariner-globalnet, TCP probe to spoke-1 + sync-controller from spoke-2 virt-launcher. During migration wait, fails early if + Synchronization stays stuck (source Synchronizing / dest WaitingForSync). + + Spoke kubeconfigs default to SHARED_DIR/managed-cluster-kubeconfig-2 (source) and + managed-cluster-kubeconfig-1 (destination) for the return leg. + + Writes ${ARTIFACT_DIR}/mtv-live-migration-return-status.txt; diagnostics on failure under + ${ARTIFACT_DIR}/mtv-live-migration-return-diagnostics/. + env: + - name: MTV_NAMESPACE + default: "openshift-mtv" + documentation: Namespace where MTV Plan and Migration CRs live. + - name: MTV_SOURCE_PROVIDER + default: "spoke-2" + documentation: MTV source Provider CR name for the return leg (VM currently resides here after forward migration). + - name: MTV_DESTINATION_PROVIDER + default: "spoke-1" + documentation: MTV destination Provider CR name for the return leg (original source cluster). + - name: MTV_TEST_VM_NAME + default: "test-vm-1" + documentation: VirtualMachine name to migrate back (must match p2p-create-migration-test-vm). + - name: MTV_TEST_VM_NAMESPACE + default: "vm-migration-test" + documentation: Namespace where the VM lives on spoke-2 after the forward migration. + - name: MTV_TEST_VM_TARGET_NAMESPACE + default: "" + documentation: Destination namespace on spoke-1. Empty uses MTV_TEST_VM_NAMESPACE. + - name: MTV_PLAN_NAME + default: "cclm-plan-return" + documentation: Plan metadata.name for the return leg on the hub. + - name: MTV_MIGRATION_NAME + default: "cclm-run-return" + documentation: Migration metadata.name for the return leg on the hub. + - name: MTV_NETWORK_MAP_NAME + default: "spoke-network-map-return" + documentation: NetworkMap referenced by the return Plan (created by p2p-mtv-create-return-migration-maps). + - name: MTV_STORAGE_MAP_NAME + default: "spoke-storage-map-return" + documentation: StorageMap referenced by the return Plan (created by p2p-mtv-create-return-migration-maps). + - name: MTV_PLAN_TYPE + default: "live" + documentation: Plan spec.type — live for CCLM, cold for powered-off migration. + - name: MTV_PLAN_READY_TIMEOUT + default: "15m" + documentation: Timeout for Provider/Map/Plan Ready waits. + - name: MTV_MIGRATION_TIMEOUT + default: "2h" + documentation: Max duration to wait for Migration Succeeded. + - name: MTV_MIGRATION_POLL_INTERVAL_SECONDS + default: "30" + documentation: Sleep between Migration status polls. + - name: MTV_SOURCE_SPOKE_KUBECONFIG + default: "" + documentation: Optional source spoke kubeconfig path for the return leg. Defaults to SHARED_DIR index 2. + - name: MTV_DEST_SPOKE_KUBECONFIG + default: "" + documentation: Optional destination spoke kubeconfig path for the return leg. Defaults to SHARED_DIR index 1. + - name: MTV_SOURCE_SPOKE_INDEX + default: "2" + documentation: 1-based SHARED_DIR kubeconfig index for the return-leg source spoke (spoke-2). + - name: MTV_DEST_SPOKE_INDEX + default: "1" + documentation: 1-based SHARED_DIR kubeconfig index for the return-leg destination spoke (spoke-1). + - name: MTV_CNV_NAMESPACE + default: "openshift-cnv" + documentation: OpenShift Virtualization namespace on spokes (KubeVirt/HCO). + - name: MTV_KUBEVIRT_NAME + default: "kubevirt-kubevirt-hyperconverged" + documentation: KubeVirt CR name on spokes (HCO-managed). MTV inventory reads developerConfiguration.featureGates here. + - name: MTV_HCO_NAME + default: "kubevirt-hyperconverged" + documentation: HyperConverged CR name patched when ensuring DecentralizedLiveMigration. + - name: MTV_ENSURE_DECENTRALIZED_LIVE_MIGRATION + default: "true" + documentation: Patch HCO developerConfiguration.featureGates on spokes when gate is missing (live plans only). + - name: MTV_FORKLIFT_CONTROLLER_NAME + default: "forklift-controller" + documentation: ForkliftController / deployment name on the hub (CCLM preflight). + - name: MTV_PROVIDER_INVENTORY_REFRESH_WAIT + default: "5m" + documentation: Wait for providers Ready after inventory refresh before Plan creation. + - name: MTV_SYNC_CONTROLLER_WAIT + default: "10m" + documentation: Wait for virt-synchronization-controller Available on each spoke before live migration. + - name: MTV_CCLM_SYNC_PROBE + default: "true" + documentation: TCP-probe dest sync-controller :9185 from source virt-launcher before migration. + - name: MTV_CCLM_SYNC_PORT + default: "9185" + documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller sync data port; healthz is :8443). + - name: MTV_CCLM_SYNC_PROBE_TIMEOUT + default: "10" + documentation: Seconds to wait for the pre-migration CCLM sync TCP probe. + - name: MTV_CCLM_SYNC_PROBE_RETRIES + default: "5" + documentation: |- + Max attempts for the pre-migration CCLM sync port TCP probe (10s between retries). + The virt-synchronization-controller TCP listener may start after the Deployment + reaches Available; retries bridge that window without masking real failures. + - name: MTV_SYNC_STUCK_MINUTES + default: "20" + documentation: Fail migration wait if Synchronization pipeline stays Running with VMIM stuck in sync phases longer than this (0 disables). + - name: P2P_CCLM_DEBUG_MODE + default: "false" + documentation: |- + When true, log failures and exit 0 so downstream debug steps can run. + Set P2P_CCLM_DEBUG_MODE=true in the job env during CCLM investigation. diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh index caa77c81007b0..3f196a215a617 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh @@ -11,8 +11,12 @@ eval "$( )"; EnsureReqs jq if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # proxy-conf.sh may contain credentials — disable xtrace while sourcing. + [[ $- == *x* ]] && _wasTracing=true || _wasTracing=false + set +x # shellcheck disable=SC1090 source "${SHARED_DIR}/proxy-conf.sh" + [[ "${_wasTracing}" == "true" ]] && set -x || true fi [[ -n "${KUBECONFIG}" ]] @@ -109,6 +113,146 @@ DumpDiagnostics() { DestOc logs -n "${MTV_CNV_NAMESPACE}" \ -l kubevirt.io=virt-controller --tail=100 \ > "${diagDir}/dest-virt-controller.log" 2>&1 || true + + DumpCclmSyncDiagnostics +} + +# DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the +# Synchronization phase (source=Synchronizing, dest=WaitingForSync). +# Captures virt-synchronization-controller state, Submariner service exports/ +# imports, forklift-controller logs, virt-handler logs, VMI migration state, +# virt-handler client-cert Secret presence (missing certs block mTLS to dest +# virt-sync-controller:9185), nftables SNAT exclusion set re-verification, and +# OVN-K gateway-node logs. +# Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics +# (on any live-plan error path). +DumpCclmSyncDiagnostics() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + + typeset syncDir="${ARTIFACT_DIR}/mtv-cclm-sync-diagnostics" + mkdir -p "${syncDir}" + + typeset spoke kc + for spoke in source dest; do + [[ "${spoke}" == "source" ]] && kc="${sourceKubeconfig}" || kc="${destKubeconfig}" + + # Broad search: virt-sync related pods across ALL namespaces. + oc --kubeconfig="${kc}" get pods -A -o wide 2>/dev/null \ + | grep -E 'virt-sync|synchronization-controller|virt-migration' \ + > "${syncDir}/${spoke}-virt-sync-pods-all-ns.txt" 2>&1 || true + + # Targeted: pods + logs from the CNV namespace by deployment name. + oc --kubeconfig="${kc}" get pods \ + -n "${MTV_CNV_NAMESPACE}" -o wide \ + 2>/dev/null \ + | grep 'virt-synchronization-controller' \ + > "${syncDir}/${spoke}-virt-sync-pods-cnv-ns.txt" 2>&1 || true + + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + deployment/virt-synchronization-controller \ + --tail=200 \ + > "${syncDir}/${spoke}-virt-sync-controller-logs.txt" 2>&1 || true + + # Submariner service discovery state. + oc --kubeconfig="${kc}" get serviceexport -A \ + > "${syncDir}/${spoke}-serviceexports.txt" 2>&1 || true + oc --kubeconfig="${kc}" get serviceimport -A \ + > "${syncDir}/${spoke}-serviceimports.txt" 2>&1 || true + + # Services on the CCLM sync port (8443) or the KubeVirt migration port. + oc --kubeconfig="${kc}" get svc -A -o json 2>/dev/null \ + | jq -r '.items[] + | select(.spec.ports[]?.port == 8443 + or .spec.ports[]?.port == 9185 + or .spec.ports[]?.targetPort == 8443 + or (.spec.ports[]?.name // "" | test("sync|migr|virt"))) + | "\(.metadata.namespace)/\(.metadata.name)\t" + + "ports=\([.spec.ports[]? | "\(.port):\(.targetPort // "")"] | join(","))\t" + + "clusterIP=\(.spec.clusterIP)"' \ + > "${syncDir}/${spoke}-sync-services.txt" 2>&1 || true + + # virt-handler logs (per-node daemon that drives actual live migration). + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + -l 'kubevirt.io=virt-handler' \ + --tail=150 \ + > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + + # virt-handler CCLM client-certificate Secrets (metadata only — no secret data). + # Missing certs prevent virt-handler from making mTLS connections to the + # dest virt-synchronization-controller on port 9185. + oc --kubeconfig="${kc}" get secrets -n "${MTV_CNV_NAMESPACE}" \ + > "${syncDir}/${spoke}-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kc}" get ds virt-handler -n "${MTV_CNV_NAMESPACE}" -o json \ + 2>/dev/null \ + | jq '[.spec.template.spec.volumes[]? + | select(.name | test("cert|tls|key"; "i")) + | {name, secret: (.secret.secretName // null), + configMap: (.configMap.name // null)}]' \ + > "${syncDir}/${spoke}-virt-handler-cert-volumes.json" 2>&1 || true + + # OVN-K gateway node logs — check for route/policy issues affecting cross-cluster. + typeset _gwNode + _gwNode="$(oc --kubeconfig="${kc}" get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1 || true)" + if [[ -n "${_gwNode}" ]]; then + oc --kubeconfig="${kc}" logs \ + -n openshift-ovn-kubernetes \ + -l 'app=ovnkube-node' \ + --field-selector="spec.nodeName=${_gwNode}" \ + --tail=80 \ + > "${syncDir}/${spoke}-ovnkube-node-gateway-logs.txt" 2>&1 || true + # Verify nftables SNAT exclusion set still contains remote CIDRs. + oc --kubeconfig="${kc}" debug "node/${_gwNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + > "${syncDir}/${spoke}-nftables-snat-set.txt" 2>&1 || true + fi + done + + # forklift-controller logs from the hub (MTV migration orchestrator). + oc --kubeconfig="${KUBECONFIG}" logs \ + -n "${MTV_NAMESPACE}" \ + deployment/forklift-controller \ + --tail=300 \ + > "${syncDir}/hub-forklift-controller-logs.txt" 2>&1 || true + + # Detailed VMI migration state. + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/source-vmi-migration-state.json" 2>&1 || true + + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/dest-vmi-migration-state.json" 2>&1 || true + + # Full VMIM YAML (more detail than vmim.yaml in the base diagnostics dir). + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${syncDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${syncDir}/dest-vmim.yaml" 2>&1 || true + + # Events in the migration namespaces. + SourceOc get events -n "${MTV_TEST_VM_NAMESPACE}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/source-vm-ns-events.txt" 2>&1 || true + DestOc get events -n "${targetNs}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/dest-vm-ns-events.txt" 2>&1 || true + + : "CCLM sync diagnostics written → ${syncDir}" } # OnError — dump diagnostics before propagating failure. @@ -318,15 +462,17 @@ MaybePreflightSubmarinerNoGlobalnet() { GetSourceVirtLauncherPod() { typeset podName + # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. + # No "|| true" — SourceOc failures and jq parse errors must propagate. + # An empty pod list is handled gracefully: jq "// empty" exits 0 with no output. podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ - -o jsonpath='{.items[0].metadata.name}' || true)" + -o json | jq -r 'first(.items[].metadata.name) // empty')" [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ | jq -r --arg name "${MTV_TEST_VM_NAME}" \ - '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""' \ - || true)" + '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""')" [[ -n "${podName}" ]] && printf '%s' "${podName}" } @@ -420,11 +566,11 @@ CheckSyncStuck() { if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" - DumpDiagnostics - false + DumpCclmSyncDiagnostics + return 1 fi - true + return 0 } # RefreshProviderInventory — re-scan spoke KubeVirt inventory before live Plan validation. @@ -551,7 +697,7 @@ WaitMigrationSucceeded() { false fi - CheckSyncStuck + CheckSyncStuck || return 1 PrintMigrationPipeline : "Migration in progress${msg:+: ${msg}} (${SECONDS}/${deadline}s)" diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml index 1ff2572ca0073..248b2fa0b2719 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml @@ -15,7 +15,7 @@ ref: and verifies the VM is Running on the destination spoke. Requires p2p-mtv-mig-config (providers + maps Ready), p2p-create-migration-test-vm (source VM Running), p2p-acm-cnv-install-policy (DecentralizedLiveMigration + sync controller on spokes), - and acm-interop-p2p-submariner-install (no Globalnet; :8443 pod reachability). + and acm-interop-p2p-submariner-install (no Globalnet; :9185 sync-port reachability). Live preflight: sync-controller ready, no submariner-globalnet, TCP probe to dest sync-controller from source virt-launcher. During migration wait, fails early if @@ -104,10 +104,10 @@ ref: documentation: Wait for virt-synchronization-controller Available on each spoke before live migration. - name: MTV_CCLM_SYNC_PROBE default: "true" - documentation: TCP-probe dest sync-controller :8443 from source virt-launcher before migration. + documentation: TCP-probe dest sync-controller :9185 from source virt-launcher before migration. - name: MTV_CCLM_SYNC_PORT - default: "8443" - documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller HTTPS listener). + default: "9185" + documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller sync data port; healthz is :8443). - name: MTV_CCLM_SYNC_PROBE_TIMEOUT default: "10" documentation: Seconds to wait for the pre-migration CCLM sync TCP probe.