From 47e0d572dfbde777f226d82659aa68b71d55967e Mon Sep 17 00:00:00 2001 From: shakyav Date: Fri, 31 Jul 2026 10:03:24 -0400 Subject: [PATCH 01/15] spoke to spoke bi-direction vm migration --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 142 ++++ ...atQE-interop-testing-master-periodics.yaml | 84 +++ ...tQE-interop-testing-master-presubmits.yaml | 59 ++ .../acm/interop-p2p/submariner-diag/OWNERS | 3 + ...cm-interop-p2p-submariner-diag-commands.sh | 211 ++++++ ...erop-p2p-submariner-diag-ref.metadata.json | 11 + .../acm-interop-p2p-submariner-diag-ref.yaml | 42 ++ .../p2p/cclm-bidir-migration-test/OWNERS | 3 + ...m-bidir-migration-test-chain.metadata.json | 11 + .../p2p-cclm-bidir-migration-test-chain.yaml | 29 + .../p2p/mtv-bidir-mig-config/OWNERS | 3 + ...p-mtv-bidir-mig-config-chain.metadata.json | 11 + .../p2p-mtv-bidir-mig-config-chain.yaml | 27 + .../mtv-create-return-migration-maps/OWNERS | 3 + ...v-create-return-migration-maps-commands.sh | 128 ++++ ...te-return-migration-maps-ref.metadata.json | 11 + ...-mtv-create-return-migration-maps-ref.yaml | 54 ++ .../mtv-execute-live-migration-return/OWNERS | 3 + ...-execute-live-migration-return-commands.sh | 701 ++++++++++++++++++ ...te-live-migration-return-ref.metadata.json | 11 + ...mtv-execute-live-migration-return-ref.yaml | 132 ++++ 21 files changed, 1679 insertions(+) create mode 100644 ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml create mode 100644 ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS create mode 100644 ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json create mode 100644 ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml create mode 100644 ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS create mode 100644 ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json create mode 100644 ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml create mode 100644 ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS create mode 100644 ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh create mode 100644 ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json create mode 100644 ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml create mode 100644 ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS create mode 100644 ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh create mode 100644 ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json create mode 100644 ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml new file mode 100644 index 0000000000000..c020445ad7e0f --- /dev/null +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -0,0 +1,142 @@ +base_images: + cli: + name: "4.22" + namespace: ocp + tag: cli + fetch-managed-clusters: + name: autotest + namespace: acm-qe + tag: fetch-managed-clusters + installer: + name: "4.22" + namespace: ocp + tag: installer +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.23-openshift-4.20 +images: + items: + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y git python39 && dnf clean all + from: cli + optional: true + to: cli-with-git +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "4.22" +resources: + '*': + requests: + cpu: 200m + memory: 400Mi +tests: +- as: cclm-bidir-mig-lp-interop-aws + cron: 0 23 28 2 * + steps: + cluster_profile: aws-cspi-qe + dependencies: + OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:latest + env: + ACM_SPOKE_CLUSTER_COUNT: "2" + ACM_SPOKE_CLUSTER_INITIAL_VERSION: "4.22" + ACM_SPOKE_CLUSTER_NAME_PREFIX: acm-spoke + ACM_SPOKE_CP_TYPE: m6a.2xlarge + ACM_SPOKE_WORKER_TYPE: c5n.metal + BASE_DOMAIN: cspilp.interop.ccitredhat.com + CNV_ENABLE_CCLM: "true" + COMPUTE_NODE_TYPE: c5n.metal + CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge + DR__RP__CR_COMP_NAME: lp-interop--ACM-Virt-BiDir + FIREWATCH_CONFIG: | + { + "failure_rules": + [ + {"step": "cucushift-*", "failure_type": "all", "classification": "Cluster Health Check", "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-interop-p2p-cluster-install" , "failure_type": "all" , "classification": "Spoke cluster Installation" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-interop-p2p-cluster-uninstall" , "failure_type": "all" , "classification": "Spoke cluster Un-Installation" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "p2p-acm-cnv-install-policy" , "failure_type": "all" , "classification": "CNV installation policy deploy" , "group": { "name" : "cluster" , "priority" : 2}, "jira_additional_labels": ["!default"] }, + {"step": "acm-fetch-managed-clusters", "failure_type": "all", "classification": "ACM cluster failure", "jira_project": "!default", "group": {"name": "opp-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "interop-tests-deploy-odf", "failure_type": "all", "classification": "ODF deploy failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-install-odf-spokes", "failure_type": "all", "classification": "ODF spoke cluster install failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-cloud-prepare", "failure_type": "all", "classification": "Submariner AWS cloud prepare failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-broker-join", "failure_type": "all", "classification": "Submariner broker deploy / cluster join failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-interop-p2p-submariner-verify", "failure_type": "all", "classification": "Submariner connectivity verification failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "p2p-mtv-additional-config", "failure_type": "all", "classification": "MTV ForkliftController / CCLM configuration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-register-providers", "failure_type": "all", "classification": "MTV provider registration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-create-migration-maps", "failure_type": "all", "classification": "MTV forward migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-mtv-create-return-migration-maps", "failure_type": "all", "classification": "MTV return migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-create-migration-test-vm", "failure_type": "all", "classification": "CCLM test VM creation failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "p2p-mtv-execute-live-migration", "failure_type": "all", "classification": "MTV CCLM forward live migration failure (spoke-1 to spoke-2)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "p2p-mtv-execute-live-migration-return", "failure_type": "all", "classification": "MTV CCLM return live migration failure (spoke-2 back to spoke-1)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "acm-interop-p2p-submariner-diag", "failure_type": "all", "classification": "Submariner diagnostics collection failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 3}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "acm-mch", "failure_type": "all", "classification": "ACM MCH install", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]} + ] + } + FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/main/firewatch-base-configs/cr/lp-interop.json + FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["4.22-p2p","ipi-aws-p2p","mpiit-acm-virt","mpiit_product2product","mpiit_virt","bidir-migration"]' + FIREWATCH_DEFAULT_JIRA_ASSIGNEE: vshakya@redhat.com + FIREWATCH_DEFAULT_JIRA_COMPONENT: '["QA","mpiit_p2p"]' + FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-8652 + FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP + FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + MTV_DESTINATION_PROVIDER: spoke-2 + MTV_PROVIDER_NAMES: spoke-1,spoke-2 + MTV_SOURCE_PROVIDER: spoke-1 + MTV_SPOKE_CLUSTER_COUNT: "2" + ODF_OPERATOR_CHANNEL: stable-4.21 + ODF_VERSION_MAJOR_MINOR: "4.21" + OPERATORS: | + [ + {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.16", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, + {"name": "mtv-operator", "source": "redhat-operators", "channel": "release-v2.11", "install_namespace": "openshift-mtv", "target_namespaces": "openshift-mtv", "operator_group": "openshift-mtv-operatorgroup"} + ] + USER_TAGS: | + scenario acm-cnv-p2p-bidir + leases: + - count: 1 + env: LEASED_RESOURCE + resource_type: aws-cspi-qe-quota-slice + - count: 2 + env: MANAGED_CLUSTER_LEASED_RESOURCE + resource_type: aws-cspi-qe-quota-slice + post: + - ref: acm-fetch-operator-versions + - ref: acm-must-gather + - ref: acm-inspector + - ref: acm-interop-p2p-submariner-diag + - ref: gather-aws-console + - ref: acm-interop-p2p-cluster-uninstall + - chain: ipi-deprovision + - ref: mpiit-data-router-reporter + - ref: firewatch-report-issues + pre: + - ref: ipi-conf + - ref: ipi-conf-telemetry + - ref: ipi-conf-aws-custom-az + - ref: ipi-conf-aws + - ref: ipi-install-monitoringpvc + - chain: ipi-install + test: + - chain: cucushift-installer-check-cluster-health + - ref: install-operators + - ref: interop-tests-deploy-odf + - ref: acm-mch + - ref: acm-interop-p2p-cluster-install + - ref: acm-fetch-managed-clusters + - ref: p2p-acm-cnv-install-policy + - ref: p2p-install-odf-spokes + - chain: acm-interop-p2p-submariner-install + - chain: p2p-mtv-bidir-mig-config + - chain: p2p-cclm-bidir-migration-test + timeout: 12h0m0s +zz_generated_metadata: + branch: master + org: RedHatQE + repo: interop-testing + variant: acm-virt-ocp4.22-p2p-bidir diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml index a7e558e669e3a..796c5b238331f 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml @@ -188,6 +188,90 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build10 + cron: 0 23 28 2 * + decorate: true + decoration_config: + skip_cloning: true + timeout: 12h0m0s + extra_refs: + - base_ref: master + org: RedHatQE + repo: interop-testing + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe + ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p-bidir + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-bidir-cclm-bidir-mig-lp-interop-aws + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=cclm-bidir-mig-lp-interop-aws + - --variant=acm-virt-ocp4.22-p2p-bidir + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build10 cron: 0 23 28 2 * diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml index 6ad48c8f208c0..8ada211971004 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml @@ -115,6 +115,65 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )acm-cnv-ocp-4.22-p2p-images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + cluster: build12 + context: ci/prow/acm-virt-ocp4.22-p2p-bidir-images + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p-bidir + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-bidir-images + optional: true + rerun_command: /test acm-virt-ocp4.22-p2p-bidir-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=acm-virt-ocp4.22-p2p-bidir + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )acm-virt-ocp4.22-p2p-bidir-images,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh new file mode 100644 index 0000000000000..cf4527ab9f0f4 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# +# Automated Submariner diagnostics using subctl. +# Ref: https://submariner.io/operations/troubleshooting/#automated-troubleshooting +# +# Runs subctl show all, diagnose all, diagnose firewall inter-cluster, and gather +# against every spoke cluster. Always exits 0 — diagnostic only, never fails the job. +# +set -euxo pipefail; shopt -s inherit_errexit + +typeset -r subctlBin="/tmp/bin/subctl" +typeset -r diagDir="${ARTIFACT_DIR}/submariner-diag" +typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" + +typeset -a spokeKubeconfigsArr=() +typeset -a spokeNamesArr=() + +# InstallSubctl — install subctl to /tmp/bin/ at step runtime. +# WHY not SHARED_DIR: large binaries cause CI operator "Request entity too large" +# when serialising SHARED_DIR into a Kubernetes Secret between steps (3 MB limit). +InstallSubctl() { + mkdir -p /tmp/bin + if [[ -x "${subctlBin}" ]]; then + return 0 + fi + curl -Ls https://get.submariner.io | bash + cp "${HOME}/.local/bin/subctl" "${subctlBin}" + chmod +x "${subctlBin}" + true +} + +# LoadSpokeConfig — populate spoke kubeconfig and name arrays from SHARED_DIR. +LoadSpokeConfig() { + typeset -i i + for ((i = 1; i <= spokeCount; i++)); do + typeset kcFile="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${i}" + + [ -f "${kcFile}" ] + [ -f "${nameFile}" ] + + spokeKubeconfigsArr+=("${kcFile}") + spokeNamesArr+=("$(< "${nameFile}")") + done + true +} + +# ShowAll — subctl show all: endpoints, gateways, networks, connections overview. +ShowAll() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + typeset outFile="${diagDir}/show-all-${name}.txt" + + KUBECONFIG="${kc}" "${subctlBin}" show all > "${outFile}" 2>&1 || true + true +} + +# DiagnoseAll — subctl diagnose all: health check across all submariner components. +DiagnoseAll() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + typeset outFile="${diagDir}/diagnose-all-${name}.txt" + + KUBECONFIG="${kc}" "${subctlBin}" diagnose all > "${outFile}" 2>&1 || true + true +} + +# BuildMergedKubeconfig — merge two spoke kubeconfigs with unique named contexts +# into a temp file for subctl commands requiring --context / --remotecontext. +# Returns the merged kubeconfig path via stdout. +BuildMergedKubeconfig() { + typeset kc1="${1:?}"; (($#)) && shift + typeset kc2="${1:?}"; (($#)) && shift + typeset name1="${1:?}"; (($#)) && shift + typeset name2="${1:?}"; (($#)) && shift + + typeset kc1Renamed kc2Renamed mergedKc + kc1Renamed="$(mktemp /tmp/kc1-diag-XXXXXX.json)" + kc2Renamed="$(mktemp /tmp/kc2-diag-XXXXXX.json)" + mergedKc="$(mktemp /tmp/kc-merged-diag-XXXXXX.json)" + + KUBECONFIG="${kc1}" oc config view -o json --raw | \ + jq \ + --arg ctx "${name1}-admin" \ + --arg cls "${name1}-cluster" \ + --arg usr "${name1}-user" \ + ' + .contexts[0].name = $ctx | + .contexts[0].context.cluster = $cls | + .contexts[0].context.user = $usr | + .clusters[0].name = $cls | + .users[0].name = $usr | + ."current-context" = $ctx + ' > "${kc1Renamed}" + + KUBECONFIG="${kc2}" oc config view -o json --raw | \ + jq \ + --arg ctx "${name2}-admin" \ + --arg cls "${name2}-cluster" \ + --arg usr "${name2}-user" \ + ' + .contexts[0].name = $ctx | + .contexts[0].context.cluster = $cls | + .contexts[0].context.user = $usr | + .clusters[0].name = $cls | + .users[0].name = $usr | + ."current-context" = $ctx + ' > "${kc2Renamed}" + + KUBECONFIG="${kc1Renamed}:${kc2Renamed}" oc config view --flatten -o json > "${mergedKc}" + rm -f "${kc1Renamed}" "${kc2Renamed}" + + printf '%s' "${mergedKc}" +} + +# DiagnoseFirewallInterCluster — subctl diagnose firewall inter-cluster between a pair. +DiagnoseFirewallInterCluster() { + typeset kc1="${1:?}"; (($#)) && shift + typeset kc2="${1:?}"; (($#)) && shift + typeset name1="${1:?}"; (($#)) && shift + typeset name2="${1:?}"; (($#)) && shift + + typeset outFile="${diagDir}/diagnose-firewall-${name1}-to-${name2}.txt" + typeset mergedKc + mergedKc="$(BuildMergedKubeconfig "${kc1}" "${kc2}" "${name1}" "${name2}")" + + KUBECONFIG="${mergedKc}" "${subctlBin}" diagnose firewall inter-cluster \ + --context "${name1}-admin" \ + --tocontext "${name2}-admin" \ + > "${outFile}" 2>&1 || true + + rm -f "${mergedKc}" + true +} + +# GatherSubmariner — subctl gather for deep diagnostics; archives tarball to diagDir. +GatherSubmariner() { + typeset kc="${1:?}"; (($#)) && shift + typeset name="${1:?}"; (($#)) && shift + + typeset gatherWorkDir + gatherWorkDir="$(mktemp -d /tmp/subctl-gather-XXXXXX)" + + ( + cd "${gatherWorkDir}" + KUBECONFIG="${kc}" "${subctlBin}" gather > "${diagDir}/gather-${name}.log" 2>&1 || true + + # Archive whatever subctl gather produced + typeset tarball="" + typeset gatherDir="" + for d in submariner-*/; do + [[ -d "${d}" ]] && gatherDir="${d}" && break + done + + if [[ -n "${gatherDir}" ]]; then + tar czf "${diagDir}/gather-${name}.tar.gz" "${gatherDir}" + rm -rf "${gatherDir}" + fi + ) + rm -rf "${gatherWorkDir}" + true +} + +# --- Main --- +command -v oc 1>/dev/null +command -v curl 1>/dev/null +command -v jq 1>/dev/null + +[[ -n "${SHARED_DIR}" ]] +[[ -n "${ARTIFACT_DIR}" ]] + +mkdir -p "${diagDir}/gather" + +LoadSpokeConfig +InstallSubctl + +: "=== subctl show all (per spoke) ===" +typeset -i i +for ((i = 0; i < spokeCount; i++)); do + ShowAll "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "=== subctl diagnose all (per spoke) ===" +for ((i = 0; i < spokeCount; i++)); do + DiagnoseAll "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "=== subctl diagnose firewall inter-cluster (per spoke pair) ===" +typeset -i j +for ((i = 0; i < spokeCount; i++)); do + for ((j = i + 1; j < spokeCount; j++)); do + DiagnoseFirewallInterCluster \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeKubeconfigsArr[j]}" \ + "${spokeNamesArr[i]}" \ + "${spokeNamesArr[j]}" + DiagnoseFirewallInterCluster \ + "${spokeKubeconfigsArr[j]}" \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeNamesArr[j]}" \ + "${spokeNamesArr[i]}" + done +done + +: "=== subctl gather (per spoke) ===" +for ((i = 0; i < spokeCount; i++)); do + GatherSubmariner "${spokeKubeconfigsArr[i]}" "${spokeNamesArr[i]}" +done + +: "Submariner diagnostics complete — artifacts in ${diagDir}" +true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json new file mode 100644 index 0000000000000..fdc2548cd1955 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml new file mode 100644 index 0000000000000..5416d01ef5561 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml @@ -0,0 +1,42 @@ +ref: + as: acm-interop-p2p-submariner-diag + from: cli + commands: acm-interop-p2p-submariner-diag-commands.sh + resources: + requests: + cpu: 100m + memory: 250Mi + timeout: 30m0s + grace_period: 10m0s + documentation: |- + Collect automated Submariner diagnostics across all spoke clusters using subctl. + + Designed as a post-step that always exits 0 — never fails the job. Writes all output + to ${ARTIFACT_DIR}/submariner-diag/ for Prow artifact collection. + + Per the Submariner troubleshooting guide + (https://submariner.io/operations/troubleshooting/#automated-troubleshooting), + this step runs: + + subctl show all — overview of endpoints, gateways, networks, and + connections on each spoke + subctl diagnose all — health check of submariner-operator, gateway engine, + route agents, and lighthouse components on each spoke + subctl diagnose firewall inter-cluster + — firewall rule audit between each spoke pair + (merged kubeconfig with unique named contexts) + subctl gather — deep diagnostic tarball for each spoke; archived + under ${ARTIFACT_DIR}/submariner-diag/gather/ + + subctl is installed at step runtime to /tmp/bin/ (not stored in SHARED_DIR to + avoid the 3 MB Kubernetes Secret size limit). + + Env vars consumed: + ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) + SHARED_DIR - directory containing managed-cluster-kubeconfig-N + and managed-cluster-name-N files + ARTIFACT_DIR - Prow artifact directory for output files + env: + - name: ACM_SPOKE_CLUSTER_COUNT + default: "2" + documentation: Number of ACM spoke clusters to diagnose. diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json new file mode 100644 index 0000000000000..730cd455cccfd --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml new file mode 100644 index 0000000000000..7a93628503d0b --- /dev/null +++ b/ci-operator/step-registry/p2p/cclm-bidir-migration-test/p2p-cclm-bidir-migration-test-chain.yaml @@ -0,0 +1,29 @@ +chain: + as: p2p-cclm-bidir-migration-test + steps: + - ref: p2p-create-migration-test-vm + - ref: p2p-mtv-execute-live-migration + - ref: p2p-mtv-execute-live-migration-return + documentation: |- + End-to-end bi-directional CCLM migration test for the same VM across two spoke clusters. + + Requires p2p-mtv-bidir-mig-config (ForkliftController, providers, forward + return + network/storage maps), p2p-install-odf-spokes, p2p-acm-cnv-install-policy, and + acm-interop-p2p-submariner-install (pod-to-pod connectivity without Globalnet). + + Step 1 (p2p-create-migration-test-vm): + Creates an ODF-backed RWX VM on the source spoke (spoke-1, index 1). + + Step 2 (p2p-mtv-execute-live-migration): + Forward leg — live-migrates the VM from spoke-1 to spoke-2. + Uses forward Plan/Migration CRs (cclm-plan / cclm-run-1) and forward maps + (spoke-network-map / spoke-storage-map). Verifies VM Running on spoke-2. + + Step 3 (p2p-mtv-execute-live-migration-return): + Return leg — live-migrates the same VM from spoke-2 back to spoke-1. + Uses return Plan/Migration CRs (cclm-plan-return / cclm-run-return) and return + maps (spoke-network-map-return / spoke-storage-map-return). Verifies VM Running + on spoke-1 once again, confirming full round-trip spoke-to-spoke migration. + + Debug mode: set P2P_CCLM_DEBUG_MODE=true so both migration steps log failures + but exit 0, allowing downstream investigation steps to run. diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json new file mode 100644 index 0000000000000..9861560c288e8 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml new file mode 100644 index 0000000000000..fe1de9f7ed158 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-bidir-mig-config/p2p-mtv-bidir-mig-config-chain.yaml @@ -0,0 +1,27 @@ +chain: + as: p2p-mtv-bidir-mig-config + steps: + - ref: p2p-mtv-additional-config + - ref: p2p-mtv-register-providers + - ref: p2p-mtv-create-migration-maps + - ref: p2p-mtv-create-return-migration-maps + documentation: |- + Configure MTV for bi-directional cross-cluster live migration (CCLM) on the ACM hub. + + Extends p2p-mtv-mig-config with a fourth step that creates the return-leg maps: + + 1. p2p-mtv-additional-config + Applies ForkliftController with feature_ocp_live_migration=true (CCLM) on the hub. + + 2. p2p-mtv-register-providers + Registers each ACM spoke as an MTV OpenShift Provider on the hub. + + 3. p2p-mtv-create-migration-maps + Creates pod-to-pod NetworkMap and ODF StorageMap for the forward leg (spoke-1 → spoke-2). + + 4. p2p-mtv-create-return-migration-maps + Creates pod-to-pod NetworkMap and ODF StorageMap for the return leg (spoke-2 → spoke-1). + Map names carry a "-return" suffix: spoke-network-map-return / spoke-storage-map-return. + + Requires install-operators (mtv-operator on hub), acm-fetch-managed-clusters, and + p2p-install-odf-spokes before this chain. diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh new file mode 100644 index 0000000000000..a34ee7813eb64 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-commands.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# +# Create MTV NetworkMap and StorageMap for the return migration leg (spoke-2 → spoke-1) on the hub. +# Return-leg companion to p2p-mtv-create-migration-maps; map names carry a "-return" suffix. +# Requires registered MTV providers and ODF Available on both spokes. +# +set -euxo pipefail; shopt -s inherit_errexit + +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" +fi + + + +# ValidateConfig — fail fast if storage map env vars are missing. +ValidateConfig() { + [[ -n "${MTV_SOURCE_STORAGE_NAME}" ]] || { : "MTV_SOURCE_STORAGE_NAME is required"; false; } + [[ -n "${MTV_DESTINATION_STORAGE_CLASS}" ]] || { : "MTV_DESTINATION_STORAGE_CLASS is required"; false; } +} + +# RefreshProviderInventory — trigger MTV to re-scan spoke storage/network before map validation. +RefreshProviderInventory() { + typeset providerName="${1:?}" + typeset ts + + ts="$(date -u +%s)" + oc annotate "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + "forklift.konveyor.io/inventory-refresh=${ts}" --overwrite +} + +# WaitProviderReady — ensure both providers finished inventory before creating maps. +WaitProviderReady() { + typeset providerName="${1:?}" + + oc wait "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_READY_TIMEOUT}" +} + +# ApplyNetworkMap — create pod→pod NetworkMap for CCLM cross-cluster pod networking. +# Uses jq --arg to safely marshal values; avoids raw heredoc expansion of YAML-special chars. +ApplyNetworkMap() { + jq -n \ + --arg name "${MTV_NETWORK_MAP_NAME}" \ + --arg ns "${MTV_NAMESPACE}" \ + --arg srcProv "${MTV_SOURCE_PROVIDER}" \ + --arg dstProv "${MTV_DESTINATION_PROVIDER}" \ + '{ + apiVersion: "forklift.konveyor.io/v1beta1", + kind: "NetworkMap", + metadata: {name: $name, namespace: $ns}, + spec: { + map: [{source: {type: "pod"}, destination: {type: "pod"}}], + provider: { + source: {name: $srcProv, namespace: $ns}, + destination: {name: $dstProv, namespace: $ns} + } + } + }' | oc apply -f - +} + +# ApplyStorageMap — map source ODF virt StorageClass to destination (RWX required for live migration). +# Uses jq --arg to safely marshal values; avoids raw heredoc expansion of YAML-special chars. +ApplyStorageMap() { + jq -n \ + --arg name "${MTV_STORAGE_MAP_NAME}" \ + --arg ns "${MTV_NAMESPACE}" \ + --arg srcName "${MTV_SOURCE_STORAGE_NAME}" \ + --arg dstClass "${MTV_DESTINATION_STORAGE_CLASS}" \ + --arg srcProv "${MTV_SOURCE_PROVIDER}" \ + --arg dstProv "${MTV_DESTINATION_PROVIDER}" \ + '{ + apiVersion: "forklift.konveyor.io/v1beta1", + kind: "StorageMap", + metadata: {name: $name, namespace: $ns}, + spec: { + map: [{ + source: {name: $srcName}, + destination: {storageClass: $dstClass} + }], + provider: { + source: {name: $srcProv, namespace: $ns}, + destination: {name: $dstProv, namespace: $ns} + } + } + }' | oc apply -f - +} + +# WaitMapReady — wait until MTV validates network/storage mapping against provider inventory. +WaitMapReady() { + typeset kind="${1:?}" + typeset name="${2:?}" + + oc wait "${kind}/${name}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_MAP_READY_TIMEOUT}" +} + +# --- Main --- +[[ -n "${KUBECONFIG}" ]] +[[ -r "${KUBECONFIG}" ]] + +ValidateConfig + +oc get ns "${MTV_NAMESPACE}" 1>/dev/null + +WaitProviderReady "${MTV_SOURCE_PROVIDER}" +WaitProviderReady "${MTV_DESTINATION_PROVIDER}" + +if [[ "${MTV_SKIP_INVENTORY_REFRESH}" != "true" ]]; then + RefreshProviderInventory "${MTV_SOURCE_PROVIDER}" + RefreshProviderInventory "${MTV_DESTINATION_PROVIDER}" +fi + +ApplyNetworkMap +ApplyStorageMap + +WaitMapReady networkmap "${MTV_NETWORK_MAP_NAME}" +WaitMapReady storagemap "${MTV_STORAGE_MAP_NAME}" + +oc get networkmap,storagemap -n "${MTV_NAMESPACE}" \ + >> "${ARTIFACT_DIR}/mtv-migration-maps-status.txt" +true diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json new file mode 100644 index 0000000000000..8c30e05e3b3c1 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml new file mode 100644 index 0000000000000..a19602338c3b0 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-create-return-migration-maps/p2p-mtv-create-return-migration-maps-ref.yaml @@ -0,0 +1,54 @@ +ref: + as: p2p-mtv-create-return-migration-maps + from: cli + commands: p2p-mtv-create-return-migration-maps-commands.sh + timeout: 30m0s + grace_period: 10m0s + resources: + requests: + cpu: 200m + memory: 400Mi + documentation: |- + Create MTV NetworkMap (pod→pod) and StorageMap for the return migration leg (spoke-2 → spoke-1) + on the hub. Run after p2p-mtv-create-migration-maps (which creates the forward maps) and after + ODF is Available on both spokes; creating StorageMap too early causes StorageRefNotValid. + + This is the return-leg companion to p2p-mtv-create-migration-maps. Default source/destination + are reversed (spoke-2 as source, spoke-1 as destination) and map names carry a "-return" suffix + to coexist alongside the forward maps in the same openshift-mtv namespace. + + Annotates both providers for inventory refresh (unless MTV_SKIP_INVENTORY_REFRESH=true), + applies maps, waits for Ready, and appends to ${ARTIFACT_DIR}/mtv-migration-maps-status.txt. + + Hub kubeconfig is KUBECONFIG from ci-operator. + env: + - name: MTV_NAMESPACE + default: "openshift-mtv" + documentation: Namespace where MTV operator and map CRs live. + - name: MTV_SOURCE_PROVIDER + default: "spoke-2" + documentation: Source MTV Provider CR name for the return leg (migration origin). + - name: MTV_DESTINATION_PROVIDER + default: "spoke-1" + documentation: Destination MTV Provider CR name for the return leg (migration target). + - name: MTV_NETWORK_MAP_NAME + default: "spoke-network-map-return" + documentation: NetworkMap metadata.name for the return leg. + - name: MTV_STORAGE_MAP_NAME + default: "spoke-storage-map-return" + documentation: StorageMap metadata.name for the return leg. + - name: MTV_SOURCE_STORAGE_NAME + default: "ocs-storagecluster-ceph-rbd-virtualization" + documentation: StorageClass name on spoke-2 (the return-leg source; spec.map[].source.name). + - name: MTV_DESTINATION_STORAGE_CLASS + default: "ocs-storagecluster-ceph-rbd-virtualization" + documentation: Destination StorageClass name on spoke-1 (spec.map[].destination.storageClass). + - name: MTV_PROVIDER_READY_TIMEOUT + default: "15m" + documentation: Timeout for oc wait provider/ --for=condition=Ready. + - name: MTV_MAP_READY_TIMEOUT + default: "15m" + documentation: Timeout for NetworkMap/StorageMap Ready waits. + - name: MTV_SKIP_INVENTORY_REFRESH + default: "false" + documentation: Set to "true" to skip forklift.konveyor.io/inventory-refresh annotation. diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh new file mode 100644 index 0000000000000..82e1965515bd6 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -0,0 +1,701 @@ +#!/bin/bash +# +# Execute MTV cross-cluster live migration (CCLM) return leg: spoke-2 back to spoke-1 on the hub. +# Return-leg companion to p2p-mtv-execute-live-migration; artifact names carry a "-return" suffix. +# +set -euxo pipefail; shopt -s inherit_errexit + +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + +if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" +fi + +[[ -n "${KUBECONFIG}" ]] +[[ -r "${KUBECONFIG}" ]] + +typeset -i migrationPollInterval="${MTV_MIGRATION_POLL_INTERVAL_SECONDS}" +typeset -i sourceSpokeIndex="${MTV_SOURCE_SPOKE_INDEX}" +typeset -i destSpokeIndex="${MTV_DEST_SPOKE_INDEX}" +typeset -i syncStuckMinutes="${MTV_SYNC_STUCK_MINUTES}" +typeset -i syncPhaseStartedAt=0 +typeset cclmDebugMode="${P2P_CCLM_DEBUG_MODE}" + +# Temp file accumulating tab-separated JUnit records (PASS/FAIL\tname\telapsed\t[msg]). +# Written in the subshell; read by WriteJunit after it exits. +typeset -r junitFile="${TMPDIR:-/tmp}/cclm-return-junit-$$.tsv" + +typeset sourceKubeconfig="${MTV_SOURCE_SPOKE_KUBECONFIG}" +typeset destKubeconfig="${MTV_DEST_SPOKE_KUBECONFIG}" +typeset targetNs="${MTV_TEST_VM_TARGET_NAMESPACE}" +typeset diagDir="" + +# HubOc — run oc against the ACM hub. +HubOc() { + oc --kubeconfig="${KUBECONFIG}" "$@" +} + +# SourceOc — run oc against the source spoke. +SourceOc() { + oc --kubeconfig="${sourceKubeconfig}" "$@" +} + +# DestOc — run oc against the destination spoke. +DestOc() { + oc --kubeconfig="${destKubeconfig}" "$@" +} + +# ResolveSpokeKubeconfigs — source and destination spoke admin kubeconfigs. +ResolveSpokeKubeconfigs() { + [[ -n "${SHARED_DIR}" ]] + + if [[ -z "${sourceKubeconfig}" ]]; then + if [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig-${sourceSpokeIndex}" ]]; then + sourceKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig-${sourceSpokeIndex}" + elif (( sourceSpokeIndex == 1 )) && [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig" ]]; then + sourceKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig" + else + : "Source spoke kubeconfig not found for index ${sourceSpokeIndex}" + false + fi + fi + [[ -r "${sourceKubeconfig}" ]] + + if [[ -z "${destKubeconfig}" ]]; then + [[ -r "${SHARED_DIR}/managed-cluster-kubeconfig-${destSpokeIndex}" ]] + destKubeconfig="${SHARED_DIR}/managed-cluster-kubeconfig-${destSpokeIndex}" + fi + [[ -r "${destKubeconfig}" ]] +} + +# DumpDiagnostics — write MTV and VM state to ARTIFACT_DIR on failure. +DumpDiagnostics() { + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + diagDir="${ARTIFACT_DIR}/mtv-live-migration-return-diagnostics" + mkdir -p "${diagDir}" + HubOc get plan,migration,networkmap,storagemap,provider -n "${MTV_NAMESPACE}" \ + > "${diagDir}/hub-mtv-resources.txt" 2>&1 || true + HubOc describe "plan/${MTV_PLAN_NAME}" -n "${MTV_NAMESPACE}" \ + > "${diagDir}/plan-describe.txt" 2>&1 || true + HubOc describe "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" \ + > "${diagDir}/migration-describe.txt" 2>&1 || true + HubOc get events -n "${MTV_NAMESPACE}" --sort-by='.lastTimestamp' \ + > "${diagDir}/hub-mtv-events.txt" 2>&1 || true + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o wide > "${diagDir}/source-vm.txt" 2>&1 || true + DestOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o wide > "${diagDir}/dest-vm.txt" 2>&1 || true + DestOc get datavolume,pvc,pods -n "${targetNs}" \ + > "${diagDir}/dest-storage.txt" 2>&1 || true + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${diagDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${diagDir}/dest-vmim.yaml" 2>&1 || true + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${diagDir}/source-vmi.yaml" 2>&1 || true + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${targetNs}" -o yaml \ + > "${diagDir}/dest-vmi.yaml" 2>&1 || true + SourceOc get pods -n "${MTV_CNV_NAMESPACE}" -o wide \ + > "${diagDir}/source-cnv-pods.txt" 2>&1 || true + DestOc get pods -n "${MTV_CNV_NAMESPACE}" -o wide \ + > "${diagDir}/dest-cnv-pods.txt" 2>&1 || true + SourceOc logs -n "${MTV_CNV_NAMESPACE}" \ + -l kubevirt.io=virt-controller --tail=100 \ + > "${diagDir}/source-virt-controller.log" 2>&1 || true + DestOc logs -n "${MTV_CNV_NAMESPACE}" \ + -l kubevirt.io=virt-controller --tail=100 \ + > "${diagDir}/dest-virt-controller.log" 2>&1 || true +} + +# OnError — dump diagnostics before propagating failure. +OnError() { + typeset -i ec=$? + DumpDiagnostics + exit "${ec}" +} + +# WaitProviderReady — gate until MTV Provider is Ready. +WaitProviderReady() { + typeset providerName="${1:?}" + HubOc wait "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PLAN_READY_TIMEOUT}" +} + +# WaitMapReady — gate until NetworkMap or StorageMap is Ready. +WaitMapReady() { + typeset kind="${1:?}" + typeset name="${2:?}" + HubOc wait "${kind}/${name}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PLAN_READY_TIMEOUT}" +} + +# PreflightSourceVm — source VM must exist and VMI Running for live migration. +PreflightSourceVm() { + typeset phase + + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" 1>/dev/null + + phase="$(SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.status.phase}' || true)" + + if [[ "${MTV_PLAN_TYPE}" == "live" ]]; then + [[ "${phase}" == "Running" ]] + fi +} + +# GetVmRootDiskStorageClass — resolve root disk StorageClass from source VM. +GetVmRootDiskStorageClass() { + typeset dvName pvcName scName + + dvName="$(SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o json | + jq -r 'first(.spec.template.spec.volumes[]?.dataVolume.name // empty) // ""')" + [[ -z "${dvName}" ]] && dvName="${MTV_TEST_VM_NAME}-rootdisk" + + pvcName="$(SourceOc get "datavolume/${dvName}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.status.claimName}' || true)" + [[ -z "${pvcName}" ]] && pvcName="${dvName}" + + scName="$(SourceOc get "persistentvolumeclaim/${pvcName}" -n "${MTV_TEST_VM_NAMESPACE}" \ + -o jsonpath='{.spec.storageClassName}' || true)" + [[ -n "${scName}" ]] && printf '%s' "${scName}" && return 0 + + SourceOc get pvc -n "${MTV_TEST_VM_NAMESPACE}" -o json | + jq -r --arg pvc "${pvcName}" ' + (first(.items[] | select(.metadata.name == $pvc) | .spec.storageClassName) // + first(.items[].spec.storageClassName) // + "") // "" + ' +} + +# PreflightVmStorageMapped — VM root disk StorageClass must appear in StorageMap. +PreflightVmStorageMapped() { + typeset vmSc scUid mapJson mapped + + vmSc="$(GetVmRootDiskStorageClass)" + [[ -n "${vmSc}" ]] + + scUid="$(SourceOc get "storageclass/${vmSc}" -o jsonpath='{.metadata.uid}' || true)" + mapJson="$(HubOc get "storagemap/${MTV_STORAGE_MAP_NAME}" -n "${MTV_NAMESPACE}" -o json)" + mapped="$(jq -r --arg sc "${vmSc}" --arg uid "${scUid}" \ + '[.spec.map[]? | select(.source.name == $sc or (.source.id != null and .source.id == $uid))] | length > 0' \ + <<<"${mapJson}")" + [[ "${mapped}" == "true" ]] +} + +# PreflightHub — providers and maps must be Ready before Plan creation. +PreflightHub() { + WaitProviderReady "${MTV_SOURCE_PROVIDER}" + WaitProviderReady "${MTV_DESTINATION_PROVIDER}" + WaitMapReady networkmap "${MTV_NETWORK_MAP_NAME}" + WaitMapReady storagemap "${MTV_STORAGE_MAP_NAME}" +} + +# HasDecentralizedLiveMigrationGate — KubeVirt must list DecentralizedLiveMigration (MTV CCLM check). +HasDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + + # jq -e exits 0 when gate present, 1 when absent or oc fails. + oc --kubeconfig="${kc}" get kubevirt "${MTV_KUBEVIRT_NAME}" -n "${MTV_CNV_NAMESPACE}" -o json \ + | jq -e '.spec.configuration.developerConfiguration.featureGates // [] | contains(["DecentralizedLiveMigration"])' \ + > /dev/null +} + +# EnsureDecentralizedLiveMigrationGate — enable CCLM via HCO featureGates; wait for KubeVirt sync. +EnsureDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + typeset hcoGate + + HasDecentralizedLiveMigrationGate "${kc}" && return 0 + + hcoGate="$(oc --kubeconfig="${kc}" get hyperconverged "${MTV_HCO_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + -o jsonpath='{.spec.featureGates.decentralizedLiveMigration}' || true)" + if [[ "${hcoGate}" != "true" ]]; then + oc --kubeconfig="${kc}" patch hyperconverged "${MTV_HCO_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + --type merge -p '{"spec":{"featureGates":{"decentralizedLiveMigration":true}}}' + fi + + WaitForDecentralizedLiveMigrationGate "${kc}" && return 0 + + oc --kubeconfig="${kc}" patch kubevirt "${MTV_KUBEVIRT_NAME}" -n "${MTV_CNV_NAMESPACE}" \ + --type merge -p '{"spec":{"configuration":{"developerConfiguration":{"featureGates":["DecentralizedLiveMigration"]}}}}' + + WaitForDecentralizedLiveMigrationGate "${kc}" +} + +# WaitForDecentralizedLiveMigrationGate — poll KubeVirt until DecentralizedLiveMigration appears. +WaitForDecentralizedLiveMigrationGate() { + typeset kc="${1:?}" + typeset -i deadline=$((SECONDS + 600)) + + while (( SECONDS < deadline )); do + HasDecentralizedLiveMigrationGate "${kc}" && return 0 + sleep 10 + done + false +} + +# MaybeEnsureDecentralizedLiveMigration — enable CCLM gate on both spokes when configured. +MaybeEnsureDecentralizedLiveMigration() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ "${MTV_ENSURE_DECENTRALIZED_LIVE_MIGRATION}" != "true" ]] && return 0 + + EnsureDecentralizedLiveMigrationGate "${sourceKubeconfig}" + EnsureDecentralizedLiveMigrationGate "${destKubeconfig}" +} + +# PreflightCclm — verify MTV controller and KubeVirt gates required for type=live plans. +PreflightCclm() { + typeset fcGate envVal + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + fcGate="$(HubOc get "forkliftcontroller/${MTV_FORKLIFT_CONTROLLER_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{.spec.feature_ocp_live_migration}' || true)" + [[ "${fcGate}" == "true" ]] + + envVal="$(HubOc get "deployment/${MTV_FORKLIFT_CONTROLLER_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{.spec.template.spec.containers[*].env[?(@.name=="FEATURE_OCP_LIVE_MIGRATION")].value}' \ + || true)" + [[ "${envVal}" == "true" ]] + + HasDecentralizedLiveMigrationGate "${sourceKubeconfig}" + HasDecentralizedLiveMigrationGate "${destKubeconfig}" +} + +# GetSyncControllerPodIp — first Running virt-synchronization-controller pod IP. +GetSyncControllerPodIp() { + typeset kc="${1:?}" + + oc --kubeconfig="${kc}" get pods -n "${MTV_CNV_NAMESPACE}" -o json \ + | jq -r 'first( + .items[] + | select(.metadata.name | startswith("virt-synchronization-controller")) + | select(.status.phase == "Running") + | select((.status.podIP // "") != "") + | .status.podIP + )' +} + +# WaitForSyncControllerReady — CCLM requires sync controller on both spokes. +WaitForSyncControllerReady() { + typeset kc="${1:?}" + + oc --kubeconfig="${kc}" wait deployment/virt-synchronization-controller \ + -n "${MTV_CNV_NAMESPACE}" --for=condition=Available --timeout="${MTV_SYNC_CONTROLLER_WAIT}" +} + +# MaybeWaitForSyncControllers — wait for sync controller deployments on both spokes. +MaybeWaitForSyncControllers() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + WaitForSyncControllerReady "${sourceKubeconfig}" + WaitForSyncControllerReady "${destKubeconfig}" +} + +# PreflightSubmarinerNoGlobalnet — Globalnet breaks raw pod IP CCLM sync routing. +PreflightSubmarinerNoGlobalnet() { + typeset kc="${1:?}" + + # Expected to fail when Globalnet is not deployed; ! inverts the exit code. + ! oc --kubeconfig="${kc}" get daemonset submariner-globalnet \ + -n submariner-operator 1>/dev/null +} + +# MaybePreflightSubmarinerNoGlobalnet — both spokes must not run Globalnet. +MaybePreflightSubmarinerNoGlobalnet() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + PreflightSubmarinerNoGlobalnet "${sourceKubeconfig}" + PreflightSubmarinerNoGlobalnet "${destKubeconfig}" +} + +# GetSourceVirtLauncherPod — virt-launcher pod name for the source VM. +GetSourceVirtLauncherPod() { + typeset podName + + podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ + -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ + -o jsonpath='{.items[0].metadata.name}' || true)" + [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 + + podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ + | jq -r --arg name "${MTV_TEST_VM_NAME}" \ + '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""' \ + || true)" + [[ -n "${podName}" ]] && printf '%s' "${podName}" +} + +# ProbeCclmSyncPortFromPod — TCP probe from an existing pod to sync IP:port. +# Retries up to MTV_CCLM_SYNC_PROBE_RETRIES times (10s apart) to tolerate the +# window between virt-synchronization-controller Deployment Available and its +# TCP listener being bound (connection-refused causes an instant rc=1 without +# retries, even though connectivity works by migration time). +ProbeCclmSyncPortFromPod() { + typeset kc="${1:?}" + typeset ns="${2:?}" + typeset podName="${3:?}" + typeset destIp="${4:?}" + typeset -i attempt=0 maxAttempts="${MTV_CCLM_SYNC_PROBE_RETRIES}" retrySecs=10 + + while (( attempt < maxAttempts )); do + if oc --kubeconfig="${kc}" exec -n "${ns}" "${podName}" -c compute -- \ + timeout "${MTV_CCLM_SYNC_PROBE_TIMEOUT}" bash -c "echo >/dev/tcp/${destIp}/${MTV_CCLM_SYNC_PORT}"; then + return 0 + fi + (( attempt++ )) + if (( attempt < maxAttempts )); then + : "Sync port probe attempt ${attempt}/${maxAttempts} failed; retrying in ${retrySecs}s" + sleep "${retrySecs}" + fi + done + return 1 +} + +# PreflightCclmSyncConnectivity — source must reach dest sync-controller :8443. +PreflightCclmSyncConnectivity() { + typeset destSyncIp srcLauncherPod + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ "${MTV_CCLM_SYNC_PROBE}" != "true" ]] && return 0 + + destSyncIp="$(GetSyncControllerPodIp "${destKubeconfig}")" + [[ -n "${destSyncIp}" ]] + + srcLauncherPod="$(GetSourceVirtLauncherPod)" + [[ -n "${srcLauncherPod}" ]] + + ProbeCclmSyncPortFromPod \ + "${sourceKubeconfig}" "${MTV_TEST_VM_NAMESPACE}" "${srcLauncherPod}" "${destSyncIp}" +} + +# MigrationPipelinePhase — read one pipeline step phase from Migration status. +MigrationPipelinePhase() { + typeset stepName="${1:?}" + typeset migJson phase + + migJson="$(HubOc get "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" -o json || true)" + [[ -n "${migJson}" ]] || return 0 + + phase="$(jq -r --arg vm "${MTV_TEST_VM_NAME}" --arg step "${stepName}" \ + '.status.vms[]? | select(.name == $vm) | .pipeline[]? | select(.name == $step) | .phase' \ + <<<"${migJson}" | head -1)" + [[ -n "${phase}" && "${phase}" != "null" ]] && printf '%s' "${phase}" +} + +# VmimPhase — read VirtualMachineInstanceMigration phase on a spoke. +VmimPhase() { + typeset kc="${1:?}" + typeset ns="${2:?}" + + oc --kubeconfig="${kc}" get vmim -n "${ns}" \ + -o jsonpath='{.items[0].status.phase}' || true +} + +# CheckSyncStuck — fail early when Synchronization does not progress. +CheckSyncStuck() { + typeset syncPhase srcVmimPhase destVmimPhase + + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + (( syncStuckMinutes > 0 )) || return 0 + + syncPhase="$(MigrationPipelinePhase "Synchronization")" + [[ "${syncPhase}" == "Running" ]] || { + syncPhaseStartedAt=0 + return 0 + } + + (( syncPhaseStartedAt )) || syncPhaseStartedAt="${SECONDS}" + + if (( SECONDS - syncPhaseStartedAt < syncStuckMinutes * 60 )); then + return 0 + fi + + srcVmimPhase="$(VmimPhase "${sourceKubeconfig}" "${MTV_TEST_VM_NAMESPACE}")" + destVmimPhase="$(VmimPhase "${destKubeconfig}" "${targetNs}")" + + if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then + : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" + DumpDiagnostics + false + fi + + true +} + +# RefreshProviderInventory — re-scan spoke KubeVirt inventory before live Plan validation. +RefreshProviderInventory() { + typeset providerName="${1:?}" + typeset ts + + ts="$(date -u +%s)" + HubOc annotate "provider/${providerName}" -n "${MTV_NAMESPACE}" \ + "forklift.konveyor.io/inventory-refresh=${ts}" --overwrite +} + +# RefreshProvidersForLivePlan — both providers must reflect current KubeVirt feature gates. +RefreshProvidersForLivePlan() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + + RefreshProviderInventory "${MTV_SOURCE_PROVIDER}" + RefreshProviderInventory "${MTV_DESTINATION_PROVIDER}" + HubOc wait "provider/${MTV_SOURCE_PROVIDER}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_INVENTORY_REFRESH_WAIT}" + HubOc wait "provider/${MTV_DESTINATION_PROVIDER}" -n "${MTV_NAMESPACE}" \ + --for=condition=Ready --timeout="${MTV_PROVIDER_INVENTORY_REFRESH_WAIT}" +} + +# ApplyPlan — create or update MTV Plan CR on the hub. +ApplyPlan() { + { + HubOc create -f - --dry-run=client -o yaml --save-config + } <&2 || true + PrintMigrationPipeline 1>&2 + false + fi + + CheckSyncStuck + + PrintMigrationPipeline + : "Migration in progress${msg:+: ${msg}} (${SECONDS}/${deadline}s)" + sleep "${migrationPollInterval}" + done + + false +} + +# VerifyMigration — destination VMI must be Running after migration. +VerifyMigration() { + typeset destPhase + + destPhase="$(DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" -n "${targetNs}" \ + -o jsonpath='{.status.phase}' || true)" + [[ "${destPhase}" == "Running" ]] +} + +# JStep — run a function, append PASS/FAIL record to junitFile, propagate exit code. +# Usage: JStep "Human readable name" FunctionName [args...] +# The ERR trap in the caller subshell fires when this returns non-zero, so each +# failed step still gets recorded before the trap escalates. +JStep() { + typeset name="${1:?}"; shift + typeset -i t0=$SECONDS rc=0 + "$@" || rc=$? + typeset -i elapsed=$(( SECONDS - t0 )) + if (( rc == 0 )); then + printf 'PASS\t%s\t%d\t\n' "${name}" "${elapsed}" >> "${junitFile}" + else + printf 'FAIL\t%s\t%d\tFailed (rc=%d); see diagnostics in mtv-live-migration-diagnostics/\n' \ + "${name}" "${elapsed}" "${rc}" >> "${junitFile}" + fi + return "${rc}" +} + +# XmlEscape — replace XML special characters for attribute/text values. +# Covers the five predefined XML entities; matches the reference script (p2p-cnv-pre-upgrade). +XmlEscape() { + typeset s="${1}" + s="${s//&/&}" + s="${s///>}" + s="${s//\"/"}" + s="${s//\'/'}" + printf '%s' "${s}" +} + +# WriteJunit — emit JUnit XML from accumulated junitFile records. +# Prow/Firewatch pick up junit_*.xml files from ARTIFACT_DIR automatically. +WriteJunit() { + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + [[ -f "${junitFile}" ]] || return 0 + + typeset xmlFile="${ARTIFACT_DIR}/junit_cclm_live_migration_return.xml" + mkdir -p "${ARTIFACT_DIR}" + + typeset -i total=0 failures=0 totalTime=0 + typeset status name elapsed failMsg + + while IFS=$'\t' read -r status name elapsed failMsg; do + (( total++ )) || true + (( totalTime += elapsed )) || true + [[ "${status}" == "FAIL" ]] && (( failures++ )) || true + done < "${junitFile}" + + { + printf '\n' + printf '\n' \ + "${total}" "${failures}" "${totalTime}" + while IFS=$'\t' read -r status name elapsed failMsg; do + typeset escapedName; escapedName="$(XmlEscape "${name}")" + printf ' \n' \ + "${escapedName}" "${elapsed}" + if [[ "${status}" == "FAIL" ]]; then + typeset escapedMsg; escapedMsg="$(XmlEscape "${failMsg}")" + printf ' %s\n' \ + "${escapedMsg}" "${escapedMsg}" + fi + printf ' \n' + done < "${junitFile}" + printf '\n' + } > "${xmlFile}" + + : "JUnit XML written → ${xmlFile} (${total} tests, ${failures} failures, ${totalTime}s total)" + rm -f "${junitFile}" +} + +trap - ERR + +typeset -i cclmStepRc=0 +( + trap OnError ERR + + ResolveSpokeKubeconfigs + targetNs="${targetNs:-${MTV_TEST_VM_NAMESPACE}}" + + [[ "${MTV_PLAN_TYPE}" == "live" || "${MTV_PLAN_TYPE}" == "cold" ]] + + JStep "Preflight: Providers and Maps Ready" PreflightHub + JStep "Preflight: DecentralizedLiveMigration Gates" MaybeEnsureDecentralizedLiveMigration + JStep "Preflight: Sync Controllers Available" MaybeWaitForSyncControllers + JStep "Preflight: MTV CCLM Feature Gate Active" PreflightCclm + JStep "Preflight: Submariner No Globalnet" MaybePreflightSubmarinerNoGlobalnet + JStep "Preflight: Provider Inventory Refresh" RefreshProvidersForLivePlan + JStep "Preflight: Source VM Running" PreflightSourceVm + JStep "Preflight: CCLM Sync Port Reachable" PreflightCclmSyncConnectivity + JStep "Preflight: VM Storage Class Mapped" PreflightVmStorageMapped + JStep "Migration: Apply Plan" ApplyPlan + JStep "Migration: Plan Ready" WaitPlanReady + JStep "Migration: Apply Migration" ApplyMigration + JStep "Migration: Succeeded" WaitMigrationSucceeded + JStep "Verification: Destination VMI Running" VerifyMigration + + if [[ -n "${ARTIFACT_DIR}" ]]; then + mkdir -p "${ARTIFACT_DIR}" + { + HubOc get "plan/${MTV_PLAN_NAME}" "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" -o wide + HubOc get "plan/${MTV_PLAN_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" — "}{.message}{"\n"}{end}' + HubOc get "migration/${MTV_MIGRATION_NAME}" -n "${MTV_NAMESPACE}" \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.status}{" — "}{.message}{"\n"}{end}' + PrintMigrationPipeline + SourceOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o wide + DestOc get "virtualmachine/${MTV_TEST_VM_NAME}" "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o wide + } > "${ARTIFACT_DIR}/mtv-live-migration-return-status.txt" + fi + true +) || cclmStepRc=$? + +# Always write JUnit XML — both on success and on failure, so Prow/Firewatch +# always has a report regardless of which step caused the subshell to exit. +WriteJunit + +if (( cclmStepRc != 0 )); then + DumpDiagnostics + if [[ "${cclmDebugMode}" == "true" ]]; then + : "WARNING: p2p-mtv-execute-live-migration failed (rc=${cclmStepRc}); not failing job (debug mode)" + else + exit "${cclmStepRc}" + fi +fi + +true diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json new file mode 100644 index 0000000000000..16a49a3f2a2f1 --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml new file mode 100644 index 0000000000000..c6ab49c92904b --- /dev/null +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml @@ -0,0 +1,132 @@ +ref: + as: p2p-mtv-execute-live-migration-return + from: cli + commands: p2p-mtv-execute-live-migration-return-commands.sh + timeout: 3h0m0s + grace_period: 15m0s + resources: + requests: + cpu: 200m + memory: 400Mi + documentation: |- + Execute MTV cross-cluster live migration (CCLM) return leg: spoke-2 back to spoke-1. + + Return-leg companion to p2p-mtv-execute-live-migration. Defaults are reversed so that + spoke-2 is the source and spoke-1 is the destination. Plan and Migration CR names carry + a "-return" suffix to coexist alongside the forward migration CRs in the same namespace. + + Runs on the ACM hub (KUBECONFIG). Creates Plan and Migration CRs, waits for Succeeded, + and verifies the VM is Running on the destination spoke (spoke-1). Requires: + - p2p-mtv-bidir-mig-config (return NetworkMap/StorageMap Ready on hub) + - p2p-mtv-execute-live-migration (VM currently on spoke-2 after forward migration) + - p2p-acm-cnv-install-policy (DecentralizedLiveMigration + sync controller on spokes) + - acm-interop-p2p-submariner-install (no Globalnet; :8443 pod reachability) + + Live preflight: sync-controller ready, no submariner-globalnet, TCP probe to spoke-1 + sync-controller from spoke-2 virt-launcher. During migration wait, fails early if + Synchronization stays stuck (source Synchronizing / dest WaitingForSync). + + Spoke kubeconfigs default to SHARED_DIR/managed-cluster-kubeconfig-2 (source) and + managed-cluster-kubeconfig-1 (destination) for the return leg. + + Writes ${ARTIFACT_DIR}/mtv-live-migration-return-status.txt; diagnostics on failure under + ${ARTIFACT_DIR}/mtv-live-migration-return-diagnostics/. + env: + - name: MTV_NAMESPACE + default: "openshift-mtv" + documentation: Namespace where MTV Plan and Migration CRs live. + - name: MTV_SOURCE_PROVIDER + default: "spoke-2" + documentation: MTV source Provider CR name for the return leg (VM currently resides here after forward migration). + - name: MTV_DESTINATION_PROVIDER + default: "spoke-1" + documentation: MTV destination Provider CR name for the return leg (original source cluster). + - name: MTV_TEST_VM_NAME + default: "test-vm-1" + documentation: VirtualMachine name to migrate back (must match p2p-create-migration-test-vm). + - name: MTV_TEST_VM_NAMESPACE + default: "vm-migration-test" + documentation: Namespace where the VM lives on spoke-2 after the forward migration. + - name: MTV_TEST_VM_TARGET_NAMESPACE + default: "" + documentation: Destination namespace on spoke-1. Empty uses MTV_TEST_VM_NAMESPACE. + - name: MTV_PLAN_NAME + default: "cclm-plan-return" + documentation: Plan metadata.name for the return leg on the hub. + - name: MTV_MIGRATION_NAME + default: "cclm-run-return" + documentation: Migration metadata.name for the return leg on the hub. + - name: MTV_NETWORK_MAP_NAME + default: "spoke-network-map-return" + documentation: NetworkMap referenced by the return Plan (created by p2p-mtv-create-return-migration-maps). + - name: MTV_STORAGE_MAP_NAME + default: "spoke-storage-map-return" + documentation: StorageMap referenced by the return Plan (created by p2p-mtv-create-return-migration-maps). + - name: MTV_PLAN_TYPE + default: "live" + documentation: Plan spec.type — live for CCLM, cold for powered-off migration. + - name: MTV_PLAN_READY_TIMEOUT + default: "15m" + documentation: Timeout for Provider/Map/Plan Ready waits. + - name: MTV_MIGRATION_TIMEOUT + default: "2h" + documentation: Max duration to wait for Migration Succeeded. + - name: MTV_MIGRATION_POLL_INTERVAL_SECONDS + default: "30" + documentation: Sleep between Migration status polls. + - name: MTV_SOURCE_SPOKE_KUBECONFIG + default: "" + documentation: Optional source spoke kubeconfig path for the return leg. Defaults to SHARED_DIR index 2. + - name: MTV_DEST_SPOKE_KUBECONFIG + default: "" + documentation: Optional destination spoke kubeconfig path for the return leg. Defaults to SHARED_DIR index 1. + - name: MTV_SOURCE_SPOKE_INDEX + default: "2" + documentation: 1-based SHARED_DIR kubeconfig index for the return-leg source spoke (spoke-2). + - name: MTV_DEST_SPOKE_INDEX + default: "1" + documentation: 1-based SHARED_DIR kubeconfig index for the return-leg destination spoke (spoke-1). + - name: MTV_CNV_NAMESPACE + default: "openshift-cnv" + documentation: OpenShift Virtualization namespace on spokes (KubeVirt/HCO). + - name: MTV_KUBEVIRT_NAME + default: "kubevirt-kubevirt-hyperconverged" + documentation: KubeVirt CR name on spokes (HCO-managed). MTV inventory reads developerConfiguration.featureGates here. + - name: MTV_HCO_NAME + default: "kubevirt-hyperconverged" + documentation: HyperConverged CR name patched when ensuring DecentralizedLiveMigration. + - name: MTV_ENSURE_DECENTRALIZED_LIVE_MIGRATION + default: "true" + documentation: Patch HCO developerConfiguration.featureGates on spokes when gate is missing (live plans only). + - name: MTV_FORKLIFT_CONTROLLER_NAME + default: "forklift-controller" + documentation: ForkliftController / deployment name on the hub (CCLM preflight). + - name: MTV_PROVIDER_INVENTORY_REFRESH_WAIT + default: "5m" + documentation: Wait for providers Ready after inventory refresh before Plan creation. + - name: MTV_SYNC_CONTROLLER_WAIT + default: "10m" + documentation: Wait for virt-synchronization-controller Available on each spoke before live migration. + - name: MTV_CCLM_SYNC_PROBE + default: "true" + documentation: TCP-probe dest sync-controller :8443 from source virt-launcher before migration. + - name: MTV_CCLM_SYNC_PORT + default: "8443" + documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller HTTPS listener). + - name: MTV_CCLM_SYNC_PROBE_TIMEOUT + default: "10" + documentation: Seconds to wait for the pre-migration CCLM sync TCP probe. + - name: MTV_CCLM_SYNC_PROBE_RETRIES + default: "5" + documentation: |- + Max attempts for the pre-migration CCLM sync port TCP probe (10s between retries). + The virt-synchronization-controller TCP listener may start after the Deployment + reaches Available; retries bridge that window without masking real failures. + - name: MTV_SYNC_STUCK_MINUTES + default: "20" + documentation: Fail migration wait if Synchronization pipeline stays Running with VMIM stuck in sync phases longer than this (0 disables). + - name: P2P_CCLM_DEBUG_MODE + default: "false" + documentation: |- + When true, log failures and exit 0 so downstream debug steps can run. + Set P2P_CCLM_DEBUG_MODE=true in the job env during CCLM investigation. From a3f85299d81ff281336964c196bbd4feea1d57c1 Mon Sep 17 00:00:00 2001 From: shakyav Date: Fri, 31 Jul 2026 14:29:42 -0400 Subject: [PATCH 02/15] add snat fix to preserve submariner routes --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 2 + .../interop-p2p/submariner-snat-fix/OWNERS | 3 + ...nterop-p2p-submariner-snat-fix-commands.sh | 103 ++++++++++++++++++ ...-p2p-submariner-snat-fix-ref.metadata.json | 11 ++ ...m-interop-p2p-submariner-snat-fix-ref.yaml | 55 ++++++++++ 5 files changed, 174 insertions(+) create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json create mode 100644 ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml index c020445ad7e0f..641deb4f3473e 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -67,6 +67,7 @@ tests: {"step": "acm-interop-p2p-submariner-cloud-prepare", "failure_type": "all", "classification": "Submariner AWS cloud prepare failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "acm-interop-p2p-submariner-broker-join", "failure_type": "all", "classification": "Submariner broker deploy / cluster join failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "acm-interop-p2p-submariner-verify", "failure_type": "all", "classification": "Submariner connectivity verification failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, + {"step": "acm-interop-p2p-submariner-snat-fix", "failure_type": "all", "classification": "Submariner SNAT workaround failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-additional-config", "failure_type": "all", "classification": "MTV ForkliftController / CCLM configuration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-register-providers", "failure_type": "all", "classification": "MTV provider registration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-create-migration-maps", "failure_type": "all", "classification": "MTV forward migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, @@ -132,6 +133,7 @@ tests: - ref: p2p-acm-cnv-install-policy - ref: p2p-install-odf-spokes - chain: acm-interop-p2p-submariner-install + - ref: acm-interop-p2p-submariner-snat-fix - chain: p2p-mtv-bidir-mig-config - chain: p2p-cclm-bidir-migration-test timeout: 12h0m0s diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh new file mode 100644 index 0000000000000..fce427fbfef05 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Submariner SNAT Workaround for OVN-Kubernetes IC mode +# +# Ref: ACM-22805, ACM-24786, ACM-36927 +# Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk +# +# Root cause: OVN-K nftables mgmtport-snat chain rewrites the source IP of +# cross-cluster traffic, breaking return routing on non-gateway nodes. +# +# Workaround: +# 1. Create submariner-global ConfigMap with enable-snat-handler=true +# Tells Submariner's route agent to manage SNAT itself (nftables rules) +# instead of relying on OVN-K. +# 2. Restart submariner-routeagent DaemonSet — only reads ConfigMap at startup. +# 3. Restart ovnkube-node DaemonSet — forces OVN to re-read Submariner route +# annotations on each node, repopulating non-gateway node routing. +# 4. Wait for both DaemonSets to finish rolling out, then settle. +# +# Applied to every spoke cluster. Idempotent: safe to re-run. +# + +set -euxo pipefail; shopt -s inherit_errexit + +# ── Constants ───────────────────────────────────────────────────────────────── +typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" +typeset -i settleSecs="${SUBMARINER_SNAT_FIX_SETTLE_SECS}" + +typeset -a spokeKubeconfigsArr=() +typeset -a spokeNamesArr=() + +# ── LoadSpokeConfig — populate spoke arrays from SHARED_DIR ────────────────── +LoadSpokeConfig() { + typeset -i i + for ((i = 1; i <= spokeCount; i++)); do + typeset kcFile="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${i}" + + [ -f "${kcFile}" ] + [ -f "${nameFile}" ] + + spokeKubeconfigsArr+=("${kcFile}") + spokeNamesArr+=("$(<"${nameFile}")") + done + true +} + +# ── ApplySnatFix — apply SNAT workaround to one spoke ──────────────────────── +ApplySnatFix() { + typeset kubeconfig="${1:?}"; (($#)) && shift + typeset spokeName="${1:?}"; (($#)) && shift + + : "ApplySnatFix: spoke='${spokeName}'" + + # Step 1: submariner-global ConfigMap — idempotent via dry-run+apply + KUBECONFIG="${kubeconfig}" oc create configmap submariner-global \ + -n submariner-operator \ + --from-literal=enable-snat-handler=true \ + --dry-run=client -o yaml --save-config | \ + KUBECONFIG="${kubeconfig}" oc apply -f - + + # Step 2: restart routeagent — picks up enable-snat-handler at startup + KUBECONFIG="${kubeconfig}" oc delete pod \ + -n submariner-operator \ + -l app=submariner-routeagent \ + --wait=false + + # Step 3: restart ovnkube-node — forces OVN to re-read Submariner route + # annotations; repopulates non-gateway node OVN cluster router routes + KUBECONFIG="${kubeconfig}" oc delete pod \ + -n openshift-ovn-kubernetes \ + -l app=ovnkube-node \ + --wait=false + + # Step 4: wait for DaemonSet rollouts; non-fatal timeout so job continues + KUBECONFIG="${kubeconfig}" oc rollout status daemonset/submariner-routeagent \ + -n submariner-operator \ + --timeout=5m || \ + : "routeagent rollout on '${spokeName}': timed out — continuing" + + KUBECONFIG="${kubeconfig}" oc rollout status daemonset/ovnkube-node \ + -n openshift-ovn-kubernetes \ + --timeout=10m || \ + : "ovnkube-node rollout on '${spokeName}': timed out — continuing" + + # Step 5: settle — OVN needs time to converge routes across all nodes + sleep "${settleSecs}" + + : "ApplySnatFix: '${spokeName}' complete" + true +} + +# ── Main ────────────────────────────────────────────────────────────────────── +LoadSpokeConfig + +typeset -i i +for ((i = 0; i < spokeCount; i++)); do + ApplySnatFix \ + "${spokeKubeconfigsArr[i]}" \ + "${spokeNamesArr[i]}" +done + +true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json new file mode 100644 index 0000000000000..df642190549c4 --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml new file mode 100644 index 0000000000000..d6aa3638deaee --- /dev/null +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml @@ -0,0 +1,55 @@ +ref: + as: acm-interop-p2p-submariner-snat-fix + from: cli + commands: acm-interop-p2p-submariner-snat-fix-commands.sh + resources: + requests: + cpu: 100m + memory: 250Mi + timeout: 40m0s + grace_period: 5m0s + env: + - name: ACM_SPOKE_CLUSTER_COUNT + default: "2" + documentation: |- + Number of ACM spoke clusters to apply the SNAT workaround to. + - name: SUBMARINER_SNAT_FIX_SETTLE_SECS + default: "30" + documentation: |- + Seconds to wait after the DaemonSet rollouts complete for OVN to + converge routes across all nodes. Increase if subsequent subctl verify + still observes connectivity failures due to slow OVN reconvergence. + documentation: |- + Applies the official Submariner SNAT workaround for OVN-Kubernetes IC mode + to every spoke cluster (ACM-22805, ACM-24786, ACM-36927). + + Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk + + Root cause: OVN-K's nftables mgmtport-snat chain rewrites the source IP of + cross-cluster traffic, breaking return routing on non-gateway nodes. + Submariner's route agent is not managing SNAT by default, so OVN-K's rules + take precedence and corrupt the return path. + + What this step does (idempotent, safe to re-run): + 1. Creates submariner-global ConfigMap (enable-snat-handler=true) in + submariner-operator namespace on each spoke. This tells Submariner's + route agent to own SNAT rule management via nftables, preventing OVN-K + from overwriting them. + 2. Deletes all submariner-routeagent pods (--wait=false). Route-agent only + reads the ConfigMap at startup; the rolling restart picks it up. + 3. Deletes all ovnkube-node pods (--wait=false). Forces OVN to re-read + Submariner's NonGatewayRoute annotations on each node, repopulating + missing routes in the OVN cluster router for non-gateway nodes. + 4. Waits up to 5m for submariner-routeagent rollout and up to 10m for + ovnkube-node rollout (timeouts are non-fatal; job continues either way). + 5. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 30) for OVN convergence. + + Must run after acm-interop-p2p-submariner-install (broker-join + verify) + and before any MTV/CCLM migration steps. + + Applies to: Submariner v0.24 on OCP 4.22+ with OVN-Kubernetes network plugin. + + Env vars consumed: + ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) + SUBMARINER_SNAT_FIX_SETTLE_SECS - OVN convergence settle time (default: 30) + SHARED_DIR - shared artifact directory From 1ccf35d871767fb0dcee6d8d8dea398fa28911ca Mon Sep 17 00:00:00 2001 From: shakyav Date: Fri, 31 Jul 2026 19:03:13 -0400 Subject: [PATCH 03/15] apply the snat fix for submariner routes before verify --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 1 - ...cm-interop-p2p-submariner-diag-commands.sh | 54 +++++++++++++------ ...-interop-p2p-submariner-install-chain.yaml | 12 ++++- ...-execute-live-migration-return-commands.sh | 15 ++++-- ...p2p-mtv-execute-live-migration-commands.sh | 7 ++- 5 files changed, 64 insertions(+), 25 deletions(-) diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml index 641deb4f3473e..c675b35931c70 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -133,7 +133,6 @@ tests: - ref: p2p-acm-cnv-install-policy - ref: p2p-install-odf-spokes - chain: acm-interop-p2p-submariner-install - - ref: acm-interop-p2p-submariner-snat-fix - chain: p2p-mtv-bidir-mig-config - chain: p2p-cclm-bidir-migration-test timeout: 12h0m0s diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh index cf4527ab9f0f4..f0b1a4c572452 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh @@ -6,7 +6,9 @@ # Runs subctl show all, diagnose all, diagnose firewall inter-cluster, and gather # against every spoke cluster. Always exits 0 — diagnostic only, never fails the job. # -set -euxo pipefail; shopt -s inherit_errexit +# Use -uo (no -e, no -x) so that individual command failures never abort the script. +# Each function guards its own failures with || true. +set -uo pipefail typeset -r subctlBin="/tmp/bin/subctl" typeset -r diagDir="${ARTIFACT_DIR}/submariner-diag" @@ -18,26 +20,30 @@ typeset -a spokeNamesArr=() # InstallSubctl — install subctl to /tmp/bin/ at step runtime. # WHY not SHARED_DIR: large binaries cause CI operator "Request entity too large" # when serialising SHARED_DIR into a Kubernetes Secret between steps (3 MB limit). +# Returns 1 (non-fatal) if installation fails; callers check exit status. InstallSubctl() { - mkdir -p /tmp/bin + mkdir -p /tmp/bin || return 1 if [[ -x "${subctlBin}" ]]; then return 0 fi - curl -Ls https://get.submariner.io | bash - cp "${HOME}/.local/bin/subctl" "${subctlBin}" - chmod +x "${subctlBin}" + curl -Ls https://get.submariner.io | bash || return 1 + cp "${HOME}/.local/bin/subctl" "${subctlBin}" || return 1 + chmod +x "${subctlBin}" || return 1 true } # LoadSpokeConfig — populate spoke kubeconfig and name arrays from SHARED_DIR. +# Returns 1 (non-fatal) if any expected file is missing. LoadSpokeConfig() { typeset -i i for ((i = 1; i <= spokeCount; i++)); do typeset kcFile="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" typeset nameFile="${SHARED_DIR}/managed-cluster-name-${i}" - [ -f "${kcFile}" ] - [ -f "${nameFile}" ] + if [[ ! -f "${kcFile}" || ! -f "${nameFile}" ]]; then + : "LoadSpokeConfig: missing files for spoke ${i} — skipping" + return 1 + fi spokeKubeconfigsArr+=("${kcFile}") spokeNamesArr+=("$(< "${nameFile}")") @@ -126,7 +132,7 @@ DiagnoseFirewallInterCluster() { KUBECONFIG="${mergedKc}" "${subctlBin}" diagnose firewall inter-cluster \ --context "${name1}-admin" \ - --tocontext "${name2}-admin" \ + --remotecontext "${name2}-admin" \ > "${outFile}" 2>&1 || true rm -f "${mergedKc}" @@ -162,17 +168,33 @@ GatherSubmariner() { } # --- Main --- -command -v oc 1>/dev/null -command -v curl 1>/dev/null -command -v jq 1>/dev/null -[[ -n "${SHARED_DIR}" ]] -[[ -n "${ARTIFACT_DIR}" ]] +# Precondition checks — warn and skip rather than abort (diagnostic-only step). +typeset _diagSkip=false + +for _cmd in oc curl jq; do + command -v "${_cmd}" 1>/dev/null || { : "WARNING: ${_cmd} not found — skipping diagnostics"; _diagSkip=true; } +done + +if [[ -z "${SHARED_DIR:-}" || -z "${ARTIFACT_DIR:-}" ]]; then + : "WARNING: SHARED_DIR or ARTIFACT_DIR unset — skipping diagnostics" + _diagSkip=true +fi + +mkdir -p "${diagDir}/gather" || _diagSkip=true + +if [[ "${_diagSkip}" != "true" ]]; then + LoadSpokeConfig || { : "WARNING: LoadSpokeConfig failed — skipping diagnostics"; _diagSkip=true; } +fi -mkdir -p "${diagDir}/gather" +if [[ "${_diagSkip}" != "true" ]]; then + InstallSubctl || { : "WARNING: InstallSubctl failed — skipping diagnostics"; _diagSkip=true; } +fi -LoadSpokeConfig -InstallSubctl +if [[ "${_diagSkip}" == "true" ]]; then + : "Submariner diagnostics skipped due to setup failure — see warnings above" + exit 0 +fi : "=== subctl show all (per spoke) ===" typeset -i i diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml index 1ebe586114710..1f703a322e9d6 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-install/acm-interop-p2p-submariner-install-chain.yaml @@ -3,12 +3,13 @@ chain: steps: - ref: acm-interop-p2p-submariner-cloud-prepare - ref: acm-interop-p2p-submariner-broker-join + - ref: acm-interop-p2p-submariner-snat-fix - ref: acm-interop-p2p-submariner-verify documentation: |- Installs and verifies Submariner for a 1-hub + N-spoke cluster topology without Globalnet (non-overlapping pod CIDRs per spoke). - This chain sequences three steps: + This chain sequences four steps: 1. acm-interop-p2p-submariner-cloud-prepare Installs subctl locally, prepares AWS security groups on each spoke (subctl cloud prepare aws with --gateways 1 dedicated gateway nodes), @@ -24,7 +25,14 @@ chain: agent, Lighthouse CoreDNS, and OpenShift cluster CoreDNS DNS forwarding to be fully ready. - 3. acm-interop-p2p-submariner-verify + 3. acm-interop-p2p-submariner-snat-fix + Applies the official ACM SNAT workaround (ACM-22805/24786/36927) for + OVN-K nftables mgmtport-snat breaking cross-cluster return routing. + Creates submariner-global ConfigMap (enable-snat-handler=true), restarts + submariner-routeagent and ovnkube-node DaemonSets on each spoke, and + waits for rollout completion before verify runs. + + 4. acm-interop-p2p-submariner-verify Waits for IPsec tunnels to reach connected state, asserts remote subnets are pod CIDRs (not Globalnet 242.x), pre-warms Lighthouse service discovery, runs 'subctl verify', and probes CCLM sync TCP diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh index 82e1965515bd6..d0095bdd0f3c0 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -12,8 +12,12 @@ eval "$( )"; EnsureReqs jq if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # proxy-conf.sh may contain credentials — disable xtrace while sourcing. + [[ $- == *x* ]] && _wasTracing=true || _wasTracing=false + set +x # shellcheck disable=SC1090 source "${SHARED_DIR}/proxy-conf.sh" + [[ "${_wasTracing}" == "true" ]] && set -x || true fi [[ -n "${KUBECONFIG}" ]] @@ -319,9 +323,10 @@ MaybePreflightSubmarinerNoGlobalnet() { GetSourceVirtLauncherPod() { typeset podName + # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ - -o jsonpath='{.items[0].metadata.name}' || true)" + -o json | jq -r 'first(.items[].metadata.name) // empty' || true)" [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ @@ -583,7 +588,7 @@ JStep() { if (( rc == 0 )); then printf 'PASS\t%s\t%d\t\n' "${name}" "${elapsed}" >> "${junitFile}" else - printf 'FAIL\t%s\t%d\tFailed (rc=%d); see diagnostics in mtv-live-migration-diagnostics/\n' \ + printf 'FAIL\t%s\t%d\tFailed (rc=%d); see diagnostics in mtv-live-migration-return-diagnostics/\n' \ "${name}" "${elapsed}" "${rc}" >> "${junitFile}" fi return "${rc}" @@ -621,11 +626,11 @@ WriteJunit() { { printf '\n' - printf '\n' \ + printf '\n' \ "${total}" "${failures}" "${totalTime}" while IFS=$'\t' read -r status name elapsed failMsg; do typeset escapedName; escapedName="$(XmlEscape "${name}")" - printf ' \n' \ + printf ' \n' \ "${escapedName}" "${elapsed}" if [[ "${status}" == "FAIL" ]]; then typeset escapedMsg; escapedMsg="$(XmlEscape "${failMsg}")" @@ -692,7 +697,7 @@ WriteJunit if (( cclmStepRc != 0 )); then DumpDiagnostics if [[ "${cclmDebugMode}" == "true" ]]; then - : "WARNING: p2p-mtv-execute-live-migration failed (rc=${cclmStepRc}); not failing job (debug mode)" + : "WARNING: p2p-mtv-execute-live-migration-return failed (rc=${cclmStepRc}); not failing job (debug mode)" else exit "${cclmStepRc}" fi diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh index caa77c81007b0..cca835e3dd8f5 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh @@ -11,8 +11,12 @@ eval "$( )"; EnsureReqs jq if [[ -n "${SHARED_DIR}" && -s "${SHARED_DIR}/proxy-conf.sh" ]]; then + # proxy-conf.sh may contain credentials — disable xtrace while sourcing. + [[ $- == *x* ]] && _wasTracing=true || _wasTracing=false + set +x # shellcheck disable=SC1090 source "${SHARED_DIR}/proxy-conf.sh" + [[ "${_wasTracing}" == "true" ]] && set -x || true fi [[ -n "${KUBECONFIG}" ]] @@ -318,9 +322,10 @@ MaybePreflightSubmarinerNoGlobalnet() { GetSourceVirtLauncherPod() { typeset podName + # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ - -o jsonpath='{.items[0].metadata.name}' || true)" + -o json | jq -r 'first(.items[].metadata.name) // empty' || true)" [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ From ba59206ee120ae7ff3efc4295f024e859f3588b0 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sat, 1 Aug 2026 01:21:03 -0400 Subject: [PATCH 04/15] test the subamriner fix --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 1 + ...cm-interop-p2p-submariner-diag-commands.sh | 29 ++++++++++++-- .../acm-interop-p2p-submariner-diag-ref.yaml | 7 ++++ ...nterop-p2p-submariner-snat-fix-commands.sh | 33 ++++++---------- ...m-interop-p2p-submariner-snat-fix-ref.yaml | 39 ++++++++++--------- ...-execute-live-migration-return-commands.sh | 7 ++-- ...p2p-mtv-execute-live-migration-commands.sh | 7 ++-- 7 files changed, 73 insertions(+), 50 deletions(-) diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml index c675b35931c70..f7b1d51b44c99 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -97,6 +97,7 @@ tests: {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.16", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, {"name": "mtv-operator", "source": "redhat-operators", "channel": "release-v2.11", "install_namespace": "openshift-mtv", "target_namespaces": "openshift-mtv", "operator_group": "openshift-mtv-operatorgroup"} ] + SUBMARINER_VERIFY_DEBUG_MODE: "true" USER_TAGS: | scenario acm-cnv-p2p-bidir leases: diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh index f0b1a4c572452..fc5bcfc8c3aed 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-commands.sh @@ -18,17 +18,38 @@ typeset -a spokeKubeconfigsArr=() typeset -a spokeNamesArr=() # InstallSubctl — install subctl to /tmp/bin/ at step runtime. +# Downloads the pinned release tarball from GitHub, verifies its SHA-256 checksum, +# then extracts the binary. Uses SUBCTL_VERSION (set in ref.yaml env) so the +# version can be bumped without touching the script. +# # WHY not SHARED_DIR: large binaries cause CI operator "Request entity too large" # when serialising SHARED_DIR into a Kubernetes Secret between steps (3 MB limit). -# Returns 1 (non-fatal) if installation fails; callers check exit status. +# Returns 1 (non-fatal) if any step fails; callers check exit status. InstallSubctl() { mkdir -p /tmp/bin || return 1 if [[ -x "${subctlBin}" ]]; then return 0 fi - curl -Ls https://get.submariner.io | bash || return 1 - cp "${HOME}/.local/bin/subctl" "${subctlBin}" || return 1 - chmod +x "${subctlBin}" || return 1 + + typeset version="${SUBCTL_VERSION:?SUBCTL_VERSION must be set}" + typeset arch="linux-amd64" + typeset tarball="subctl-v${version}-${arch}.tar.xz" + typeset baseURL="https://github.com/submariner-io/subctl/releases/download/v${version}" + typeset workDir + workDir="$(mktemp -d /tmp/subctl-install-XXXXXX)" || return 1 + + ( + cd "${workDir}" || return 1 + curl -fsSL -o "${tarball}" "${baseURL}/${tarball}" || return 1 + curl -fsSL -o "${tarball}.sha256" "${baseURL}/${tarball}.sha256" || return 1 + # Verify integrity before executing anything from the archive. + sha256sum --check "${tarball}.sha256" || return 1 + tar -xJf "${tarball}" "subctl-v${version}-${arch}/subctl" || return 1 + cp "subctl-v${version}-${arch}/subctl" "${subctlBin}" || return 1 + chmod +x "${subctlBin}" || return 1 + ) || { rm -rf "${workDir}"; return 1; } + + rm -rf "${workDir}" true } diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml index 5416d01ef5561..e6cffa39da65b 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-diag/acm-interop-p2p-submariner-diag-ref.yaml @@ -40,3 +40,10 @@ ref: - name: ACM_SPOKE_CLUSTER_COUNT default: "2" documentation: Number of ACM spoke clusters to diagnose. + - name: SUBCTL_VERSION + default: "0.24.0" + documentation: |- + subctl release version to download from GitHub (submariner-io/subctl). + Diagnostic commands (show/diagnose/gather) are backwards-compatible so the + latest upstream release works against any deployed Submariner version. + Latest upstream: 0.24.0 (2026-05-12). ACM 2.15 bundles Submariner v0.22. diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh index fce427fbfef05..1f2c0f2925ab0 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -6,16 +6,19 @@ # Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk # # Root cause: OVN-K nftables mgmtport-snat chain rewrites the source IP of -# cross-cluster traffic, breaking return routing on non-gateway nodes. +# cross-cluster traffic, breaking return routing. # -# Workaround: +# Workaround (per official ACM docs): # 1. Create submariner-global ConfigMap with enable-snat-handler=true -# Tells Submariner's route agent to manage SNAT itself (nftables rules) -# instead of relying on OVN-K. +# Tells Submariner's route agent to install nftables rules that exempt +# cross-cluster traffic from OVN-K's SNAT. # 2. Restart submariner-routeagent DaemonSet — only reads ConfigMap at startup. -# 3. Restart ovnkube-node DaemonSet — forces OVN to re-read Submariner route -# annotations on each node, repopulating non-gateway node routing. -# 4. Wait for both DaemonSets to finish rolling out, then settle. +# 3. Wait for rollout and settle. +# +# NOTE: ovnkube-node is intentionally NOT restarted. The official workaround +# (ACM 2.13 docs) only requires route-agent restart. Restarting ovnkube-node +# after route-agent re-programs OVN flows and can overwrite the nftables +# exemption installed by the route-agent, negating the fix on OCP 4.22+. # # Applied to every spoke cluster. Idempotent: safe to re-run. # @@ -65,25 +68,13 @@ ApplySnatFix() { -l app=submariner-routeagent \ --wait=false - # Step 3: restart ovnkube-node — forces OVN to re-read Submariner route - # annotations; repopulates non-gateway node OVN cluster router routes - KUBECONFIG="${kubeconfig}" oc delete pod \ - -n openshift-ovn-kubernetes \ - -l app=ovnkube-node \ - --wait=false - - # Step 4: wait for DaemonSet rollouts; non-fatal timeout so job continues + # Step 3: wait for routeagent rollout; non-fatal timeout so job continues KUBECONFIG="${kubeconfig}" oc rollout status daemonset/submariner-routeagent \ -n submariner-operator \ --timeout=5m || \ : "routeagent rollout on '${spokeName}': timed out — continuing" - KUBECONFIG="${kubeconfig}" oc rollout status daemonset/ovnkube-node \ - -n openshift-ovn-kubernetes \ - --timeout=10m || \ - : "ovnkube-node rollout on '${spokeName}': timed out — continuing" - - # Step 5: settle — OVN needs time to converge routes across all nodes + # Step 4: settle — allow nftables rules to propagate across all nodes sleep "${settleSecs}" : "ApplySnatFix: '${spokeName}' complete" diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml index d6aa3638deaee..a190bcf9efdcd 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml @@ -14,11 +14,11 @@ ref: documentation: |- Number of ACM spoke clusters to apply the SNAT workaround to. - name: SUBMARINER_SNAT_FIX_SETTLE_SECS - default: "30" + default: "60" documentation: |- - Seconds to wait after the DaemonSet rollouts complete for OVN to - converge routes across all nodes. Increase if subsequent subctl verify - still observes connectivity failures due to slow OVN reconvergence. + Seconds to wait after routeagent rollout completes for its nftables + exemption rules to propagate across all nodes. 30s was insufficient on + OCP 4.22 nightly; raised to 60s as baseline. documentation: |- Applies the official Submariner SNAT workaround for OVN-Kubernetes IC mode to every spoke cluster (ACM-22805, ACM-24786, ACM-36927). @@ -26,30 +26,31 @@ ref: Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk Root cause: OVN-K's nftables mgmtport-snat chain rewrites the source IP of - cross-cluster traffic, breaking return routing on non-gateway nodes. - Submariner's route agent is not managing SNAT by default, so OVN-K's rules - take precedence and corrupt the return path. + cross-cluster traffic, breaking return routing. Submariner's route agent is + not managing SNAT by default, so OVN-K's rules corrupt the return path. What this step does (idempotent, safe to re-run): 1. Creates submariner-global ConfigMap (enable-snat-handler=true) in submariner-operator namespace on each spoke. This tells Submariner's - route agent to own SNAT rule management via nftables, preventing OVN-K - from overwriting them. + route agent to install nftables rules that exempt cross-cluster traffic + from OVN-K SNAT. 2. Deletes all submariner-routeagent pods (--wait=false). Route-agent only reads the ConfigMap at startup; the rolling restart picks it up. - 3. Deletes all ovnkube-node pods (--wait=false). Forces OVN to re-read - Submariner's NonGatewayRoute annotations on each node, repopulating - missing routes in the OVN cluster router for non-gateway nodes. - 4. Waits up to 5m for submariner-routeagent rollout and up to 10m for - ovnkube-node rollout (timeouts are non-fatal; job continues either way). - 5. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 30) for OVN convergence. + 3. Waits up to 5m for routeagent rollout (non-fatal timeout; job continues). + 4. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 60) for nftables rules + to propagate across all nodes. - Must run after acm-interop-p2p-submariner-install (broker-join + verify) - and before any MTV/CCLM migration steps. + NOTE: ovnkube-node is intentionally NOT restarted. The official ACM docs + only require route-agent restart. Restarting ovnkube-node re-programs all + OVN flows after routeagent and can overwrite the nftables exemption on + OCP 4.22+, negating the fix. - Applies to: Submariner v0.24 on OCP 4.22+ with OVN-Kubernetes network plugin. + Must run after acm-interop-p2p-submariner-broker-join and before + acm-interop-p2p-submariner-verify. + + Applies to: ACM 2.13+ with OVN-Kubernetes network plugin. Env vars consumed: ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) - SUBMARINER_SNAT_FIX_SETTLE_SECS - OVN convergence settle time (default: 30) + SUBMARINER_SNAT_FIX_SETTLE_SECS - nftables propagation settle time (default: 60) SHARED_DIR - shared artifact directory diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh index d0095bdd0f3c0..9591ca85039ab 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -324,15 +324,16 @@ GetSourceVirtLauncherPod() { typeset podName # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. + # No "|| true" — SourceOc failures and jq parse errors must propagate. + # An empty pod list is handled gracefully: jq "// empty" exits 0 with no output. podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ - -o json | jq -r 'first(.items[].metadata.name) // empty' || true)" + -o json | jq -r 'first(.items[].metadata.name) // empty')" [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ | jq -r --arg name "${MTV_TEST_VM_NAME}" \ - '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""' \ - || true)" + '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""')" [[ -n "${podName}" ]] && printf '%s' "${podName}" } diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh index cca835e3dd8f5..5e8b58831544e 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh @@ -323,15 +323,16 @@ GetSourceVirtLauncherPod() { typeset podName # Use jq to avoid "array index out of bounds" stderr from oc jsonpath on empty lists. + # No "|| true" — SourceOc failures and jq parse errors must propagate. + # An empty pod list is handled gracefully: jq "// empty" exits 0 with no output. podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" \ -l "kubevirt.io=virt-launcher,kubevirt.io/domain=${MTV_TEST_VM_NAME}" \ - -o json | jq -r 'first(.items[].metadata.name) // empty' || true)" + -o json | jq -r 'first(.items[].metadata.name) // empty')" [[ -n "${podName}" ]] && printf '%s' "${podName}" && return 0 podName="$(SourceOc get pods -n "${MTV_TEST_VM_NAMESPACE}" -o json \ | jq -r --arg name "${MTV_TEST_VM_NAME}" \ - '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""' \ - || true)" + '[.items[].metadata.name | select(startswith("virt-launcher-" + $name))] | first // ""')" [[ -n "${podName}" ]] && printf '%s' "${podName}" } From 25a45fad47c8d13db287f9d78d4793f4e9bfac3a Mon Sep 17 00:00:00 2001 From: shakyav Date: Sat, 1 Aug 2026 08:25:19 -0400 Subject: [PATCH 05/15] add timout for subctl verify --- .../acm-interop-p2p-submariner-verify-commands.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh index befd55e9cc28e..dbf51f4496159 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh @@ -387,7 +387,10 @@ VerifyConnectivity() { KUBECONFIG="${kc1Renamed}:${kc2Renamed}" oc config view --flatten -o json > "${mergedKc}" - KUBECONFIG="${mergedKc}" "${subctlBin}" verify \ + # Bound subctl verify to 35m so it exits before the 45m step timeout. + # Without this, each failing TCP test takes ~7m and the process is killed + # externally (exit 127) before the SUBMARINER_VERIFY_DEBUG_MODE handler runs. + KUBECONFIG="${mergedKc}" timeout 35m "${subctlBin}" verify \ --context "${ctx1}" \ --tocontext "${ctx2}" \ --only connectivity,service-discovery \ From efc4b98aeea1a52b3a9417be94a4cc78f1de0435 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sat, 1 Aug 2026 08:36:16 -0400 Subject: [PATCH 06/15] add timout for subctl verify --- .../acm-interop-p2p-submariner-verify-commands.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh index dbf51f4496159..014548464bbc3 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-verify/acm-interop-p2p-submariner-verify-commands.sh @@ -390,7 +390,9 @@ VerifyConnectivity() { # Bound subctl verify to 35m so it exits before the 45m step timeout. # Without this, each failing TCP test takes ~7m and the process is killed # externally (exit 127) before the SUBMARINER_VERIFY_DEBUG_MODE handler runs. - KUBECONFIG="${mergedKc}" timeout 35m "${subctlBin}" verify \ + # --kill-after 30s: send SIGKILL 30s after SIGTERM in case subctl is + # unresponsive (blocked goroutine / ncat subprocess ignoring SIGTERM). + KUBECONFIG="${mergedKc}" timeout --kill-after=30s 35m "${subctlBin}" verify \ --context "${ctx1}" \ --tocontext "${ctx2}" \ --only connectivity,service-discovery \ From b1bb30a461b66c45fc913c7a6ee60a6b2ce21b07 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sat, 1 Aug 2026 15:00:20 -0400 Subject: [PATCH 07/15] add fix to debug cross cluster connectivity --- ...-testing-master__acm-virt-ocp4.22-p2p.yaml | 5 +- ...atQE-interop-testing-master-periodics.yaml | 1 - ...tQE-interop-testing-master-presubmits.yaml | 1 - ...nterop-p2p-submariner-snat-fix-commands.sh | 161 ++++++++++++++---- ...m-interop-p2p-submariner-snat-fix-ref.yaml | 52 +++--- 5 files changed, 158 insertions(+), 62 deletions(-) diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml index b430ddb1392fc..1821fc5250e85 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p.yaml @@ -26,9 +26,8 @@ images: to: cli-with-git releases: latest: - candidate: - product: ocp - stream: nightly + release: + channel: stable version: "4.22" resources: '*': diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml index 796c5b238331f..21bca379a46b0 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml @@ -288,7 +288,6 @@ periodics: ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p ci.openshift.io/generator: prowgen - job-release: "4.22" pj-rehearse.openshift.io/can-be-rehearsed: "true" name: periodic-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-cclm-liv-mig-lp-interop-aws reporter_config: diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml index 8ada211971004..1500e8caca174 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml @@ -187,7 +187,6 @@ presubmits: labels: ci-operator.openshift.io/variant: acm-virt-ocp4.22-p2p ci.openshift.io/generator: prowgen - job-release: "4.22" pj-rehearse.openshift.io/can-be-rehearsed: "true" name: pull-ci-RedHatQE-interop-testing-master-acm-virt-ocp4.22-p2p-images optional: true diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh index 1f2c0f2925ab0..679689a9e1499 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -5,25 +5,29 @@ # Ref: ACM-22805, ACM-24786, ACM-36927 # Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk # -# Root cause: OVN-K nftables mgmtport-snat chain rewrites the source IP of -# cross-cluster traffic, breaking return routing. +# Root cause: On OCP 4.22 nightly with OVN-K IC mode, the +# mgmtport-snat nftables chain rewrites the source IP of cross-cluster +# traffic arriving at each spoke's gateway node. The routeagent sets +# k8s.ovn.org/node-ingress-snat-exclude-subnets on the gateway Node +# object, but OVN-K on OCP 4.22 nightly does NOT reconcile that annotation +# into the mgmtport-no-snat-subnets-v4 nftables set. # -# Workaround (per official ACM docs): -# 1. Create submariner-global ConfigMap with enable-snat-handler=true -# Tells Submariner's route agent to install nftables rules that exempt -# cross-cluster traffic from OVN-K's SNAT. -# 2. Restart submariner-routeagent DaemonSet — only reads ConfigMap at startup. -# 3. Wait for rollout and settle. +# Fix: Directly inject each spoke's remote peer CIDRs into the +# mgmtport-no-snat-subnets-v4 set (family: inet, table: ovn-kubernetes) +# on every gateway node of that spoke, using "oc debug node". # -# NOTE: ovnkube-node is intentionally NOT restarted. The official workaround -# (ACM 2.13 docs) only requires route-agent restart. Restarting ovnkube-node -# after route-agent re-programs OVN flows and can overwrite the nftables -# exemption installed by the route-agent, negating the fix on OCP 4.22+. +# NOTE: The ConfigMap (enable-snat-handler) + routeagent-restart approach +# is intentionally NOT used here. Restarting submariner-routeagent +# re-publishes the Endpoint CR on the broker; the remote gateway detects +# the change and re-initiates NAT discovery, which can break the IPsec +# tunnel for minutes (or permanently if the gateways are in different +# VPCs with UDP/4800 asymmetry). Direct nftables injection avoids all +# of that by not touching the Submariner control plane at all. # # Applied to every spoke cluster. Idempotent: safe to re-run. # -set -euxo pipefail; shopt -s inherit_errexit +set -uo pipefail; shopt -s inherit_errexit # ── Constants ───────────────────────────────────────────────────────────────── typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" @@ -48,33 +52,126 @@ LoadSpokeConfig() { true } -# ── ApplySnatFix — apply SNAT workaround to one spoke ──────────────────────── +# ── GetRemoteSubnets — discover the other spokes' pod+service CIDRs ────────── +# Reads Submariner Endpoint CRs from the given spoke's submariner-operator +# namespace, filters out the local cluster, and prints each remote subnet on +# its own line. +GetRemoteSubnets() { + typeset kubeconfig="${1:?}" + typeset localClusterID="${2:?}" + + KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ + -n submariner-operator \ + -o json \ + | jq -r --arg local "${localClusterID}" \ + '.items[] + | select(.spec.cluster_id != $local) + | .spec.subnets[]' +} + +# ── GetLocalClusterID — read the cluster_id from the local Endpoint CR ─────── +GetLocalClusterID() { + typeset kubeconfig="${1:?}" + + KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ + -n submariner-operator \ + -o json \ + | jq -r --arg gw "$(KUBECONFIG="${kubeconfig}" \ + oc get nodes -l submariner.io/gateway=true \ + -o jsonpath='{.items[0].metadata.name}')" \ + '.items[] + | select(.spec.private_ip == ($gw | split(".") | map(tonumber) | @sh) + or (.metadata.name | test($gw | split(".")[0]))) + | .spec.cluster_id' \ + | head -1 +} + +# ── InjectNftablesOnGateway — add remote subnets to the SNAT exemption set ─── +# Uses "oc debug node" to run nft on the gateway node's host network. +# The nftables table family is "inet" (confirmed on OCP 4.22; the +# mgmtport-no-snat-subnets-v4 set lives in family inet, table ovn-kubernetes). +InjectNftablesOnGateway() { + typeset kubeconfig="${1:?}" + typeset gatewayNode="${2:?}" + typeset nftElements="${3:?}" # comma-separated, e.g. "10.132.0.0/14, 172.31.0.0/16" + + : "InjectNftablesOnGateway: node='${gatewayNode}' elements='${nftElements}'" + + KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ + --quiet -- chroot /host \ + nft add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + "{ ${nftElements} }" 2>&1 | grep -v '^$' || true + + : "InjectNftablesOnGateway: verifying set on '${gatewayNode}'" + KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ + --quiet -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + 2>/dev/null || true +} + +# ── ApplySnatFix — inject nftables exemptions on every gateway of one spoke ── ApplySnatFix() { typeset kubeconfig="${1:?}"; (($#)) && shift typeset spokeName="${1:?}"; (($#)) && shift : "ApplySnatFix: spoke='${spokeName}'" - # Step 1: submariner-global ConfigMap — idempotent via dry-run+apply - KUBECONFIG="${kubeconfig}" oc create configmap submariner-global \ - -n submariner-operator \ - --from-literal=enable-snat-handler=true \ - --dry-run=client -o yaml --save-config | \ - KUBECONFIG="${kubeconfig}" oc apply -f - - - # Step 2: restart routeagent — picks up enable-snat-handler at startup - KUBECONFIG="${kubeconfig}" oc delete pod \ + # Determine this spoke's own cluster ID from its local Endpoint CRs. + typeset localClusterID + localClusterID="$(KUBECONFIG="${kubeconfig}" \ + oc get endpoints.submariner.io \ -n submariner-operator \ - -l app=submariner-routeagent \ - --wait=false - - # Step 3: wait for routeagent rollout; non-fatal timeout so job continues - KUBECONFIG="${kubeconfig}" oc rollout status daemonset/submariner-routeagent \ - -n submariner-operator \ - --timeout=5m || \ - : "routeagent rollout on '${spokeName}': timed out — continuing" + -o json \ + | jq -r '.items[] | select(.spec.private_ip != null) + | .spec.cluster_id' \ + | sort -u \ + | grep "${spokeName}" \ + | head -1)" + + if [[ -z "${localClusterID}" ]]; then + # Fallback: derive from the endpoint name (always contains cluster ID) + localClusterID="$(KUBECONFIG="${kubeconfig}" \ + oc get endpoints.submariner.io \ + -n submariner-operator \ + -o jsonpath='{.items[0].spec.cluster_id}')" + fi + + : "ApplySnatFix: localClusterID='${localClusterID}'" + + # Collect remote subnets across all remote spokes (newline-separated). + typeset remoteSubnetList + remoteSubnetList="$(GetRemoteSubnets "${kubeconfig}" "${localClusterID}")" + + if [[ -z "${remoteSubnetList}" ]]; then + : "ApplySnatFix: no remote subnets found for '${spokeName}' — skipping nftables injection" + return 0 + fi + + # Convert newline list to nft element format: "10.a.b.c/x, 10.d.e.f/y" + typeset nftElements + nftElements="$(echo "${remoteSubnetList}" | paste -sd ',' - | sed 's/,/, /g')" + : "ApplySnatFix: remote subnets for injection: ${nftElements}" + + # Find ALL gateway nodes on this spoke (there may be more than one + # in a HA Submariner setup, though typically only one is active). + typeset gatewayNodes + mapfile -t gatewayNodes < <(KUBECONFIG="${kubeconfig}" \ + oc get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + + if [[ ${#gatewayNodes[@]} -eq 0 ]]; then + : "ApplySnatFix: no gateway nodes found on '${spokeName}' — skipping" + return 0 + fi + + typeset gwNode + for gwNode in "${gatewayNodes[@]}"; do + InjectNftablesOnGateway "${kubeconfig}" "${gwNode}" "${nftElements}" + done - # Step 4: settle — allow nftables rules to propagate across all nodes + # Brief settle so the nftables rules take effect before the caller + # runs subctl verify or starts migration. sleep "${settleSecs}" : "ApplySnatFix: '${spokeName}' complete" diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml index a190bcf9efdcd..f6c1f96a9732b 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-ref.yaml @@ -14,43 +14,45 @@ ref: documentation: |- Number of ACM spoke clusters to apply the SNAT workaround to. - name: SUBMARINER_SNAT_FIX_SETTLE_SECS - default: "60" + default: "10" documentation: |- - Seconds to wait after routeagent rollout completes for its nftables - exemption rules to propagate across all nodes. 30s was insufficient on - OCP 4.22 nightly; raised to 60s as baseline. + Seconds to wait after nftables injection for rules to take effect before + the next step runs subctl verify or starts migration. Kept short because + no pod restarts are involved; nft changes take effect immediately. documentation: |- - Applies the official Submariner SNAT workaround for OVN-Kubernetes IC mode - to every spoke cluster (ACM-22805, ACM-24786, ACM-36927). - - Ref: https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_management_for_kubernetes/2.13/html/release_notes/acm-release-notes#source-ip-ocp-ovnk + Fixes the OVN-Kubernetes mgmtport-snat SNAT issue that breaks Submariner + cross-cluster connectivity on OCP 4.22 nightly (ACM-22805, ACM-24786, + ACM-36927). Root cause: OVN-K's nftables mgmtport-snat chain rewrites the source IP of - cross-cluster traffic, breaking return routing. Submariner's route agent is - not managing SNAT by default, so OVN-K's rules corrupt the return path. + cross-cluster traffic arriving at a spoke's gateway node. The routeagent + sets k8s.ovn.org/node-ingress-snat-exclude-subnets on the gateway Node but + OVN-K on OCP 4.22 nightly does NOT reconcile that annotation into the + mgmtport-no-snat-subnets-v4 nftables set, leaving it empty. What this step does (idempotent, safe to re-run): - 1. Creates submariner-global ConfigMap (enable-snat-handler=true) in - submariner-operator namespace on each spoke. This tells Submariner's - route agent to install nftables rules that exempt cross-cluster traffic - from OVN-K SNAT. - 2. Deletes all submariner-routeagent pods (--wait=false). Route-agent only - reads the ConfigMap at startup; the rolling restart picks it up. - 3. Waits up to 5m for routeagent rollout (non-fatal timeout; job continues). - 4. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 60) for nftables rules - to propagate across all nodes. + 1. For each spoke, discovers the OTHER spokes' pod+service CIDRs by + reading Submariner Endpoint CRs from the local submariner-operator + namespace. + 2. Uses "oc debug node" to directly inject those CIDRs into the + mgmtport-no-snat-subnets-v4 nftables set (family: inet, table: + ovn-kubernetes) on every gateway node of that spoke. + 3. Sleeps SUBMARINER_SNAT_FIX_SETTLE_SECS (default 10) for the rules + to take effect. - NOTE: ovnkube-node is intentionally NOT restarted. The official ACM docs - only require route-agent restart. Restarting ovnkube-node re-programs all - OVN flows after routeagent and can overwrite the nftables exemption on - OCP 4.22+, negating the fix. + NOTE: The ConfigMap (enable-snat-handler) + routeagent-restart approach + is intentionally NOT used. Restarting submariner-routeagent re-publishes + Endpoint CRs on the broker; remote gateways detect the change and restart + NAT discovery (UDP/4800), which can break the IPsec tunnel for an extended + period in multi-VPC AWS environments. Direct nftables injection avoids + touching the Submariner control plane entirely. Must run after acm-interop-p2p-submariner-broker-join and before acm-interop-p2p-submariner-verify. - Applies to: ACM 2.13+ with OVN-Kubernetes network plugin. + Applies to: ACM 2.13+ with OVN-Kubernetes network plugin on OCP 4.18+. Env vars consumed: ACM_SPOKE_CLUSTER_COUNT - number of spoke clusters (default: 2) - SUBMARINER_SNAT_FIX_SETTLE_SECS - nftables propagation settle time (default: 60) + SUBMARINER_SNAT_FIX_SETTLE_SECS - post-injection settle time (default: 10) SHARED_DIR - shared artifact directory From 9f73ea995119d1fcfa695ca221ae96504107413e Mon Sep 17 00:00:00 2001 From: shakyav Date: Sat, 1 Aug 2026 19:48:08 -0400 Subject: [PATCH 08/15] fix the command --- ...nterop-p2p-submariner-snat-fix-commands.sh | 89 ++++++++++--------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh index 679689a9e1499..2b2688117058a 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -29,6 +29,13 @@ set -uo pipefail; shopt -s inherit_errexit +# ── Bootstrap: ensure required tools are present ───────────────────────────── +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + # ── Constants ───────────────────────────────────────────────────────────────── typeset -i spokeCount="${ACM_SPOKE_CLUSTER_COUNT}" typeset -i settleSecs="${SUBMARINER_SNAT_FIX_SETTLE_SECS}" @@ -52,44 +59,47 @@ LoadSpokeConfig() { true } -# ── GetRemoteSubnets — discover the other spokes' pod+service CIDRs ────────── -# Reads Submariner Endpoint CRs from the given spoke's submariner-operator -# namespace, filters out the local cluster, and prints each remote subnet on -# its own line. -GetRemoteSubnets() { +# ── GetLocalClusterID — find the cluster_id of this spoke's own endpoint ───── +# Matches on the gateway node's short hostname (e.g. "ip-10-1-99-33") which +# Submariner stores in spec.hostname of the local Endpoint CR. +GetLocalClusterID() { typeset kubeconfig="${1:?}" - typeset localClusterID="${2:?}" + + typeset gwNode + gwNode="$(KUBECONFIG="${kubeconfig}" oc get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{.items[0].metadata.name}')" + + # Strip domain suffix: "ip-10-1-99-33.us-east-2.compute.internal" → "ip-10-1-99-33" + typeset gwShort="${gwNode%%.*}" KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ -n submariner-operator \ -o json \ - | jq -r --arg local "${localClusterID}" \ - '.items[] - | select(.spec.cluster_id != $local) - | .spec.subnets[]' + | jq -r --arg host "${gwShort}" \ + '.items[] | select(.spec.hostname == $host) | .spec.cluster_id' \ + | head -1 } -# ── GetLocalClusterID — read the cluster_id from the local Endpoint CR ─────── -GetLocalClusterID() { +# ── GetRemoteSubnets — list every subnet from non-local endpoints ───────────── +# Output: one subnet per line (e.g. "10.132.0.0/14\n172.31.0.0/16") +GetRemoteSubnets() { typeset kubeconfig="${1:?}" + typeset localClusterID="${2:?}" KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ -n submariner-operator \ -o json \ - | jq -r --arg gw "$(KUBECONFIG="${kubeconfig}" \ - oc get nodes -l submariner.io/gateway=true \ - -o jsonpath='{.items[0].metadata.name}')" \ + | jq -r --arg local "${localClusterID}" \ '.items[] - | select(.spec.private_ip == ($gw | split(".") | map(tonumber) | @sh) - or (.metadata.name | test($gw | split(".")[0]))) - | .spec.cluster_id' \ - | head -1 + | select(.spec.cluster_id != $local) + | .spec.subnets[]' } # ── InjectNftablesOnGateway — add remote subnets to the SNAT exemption set ─── # Uses "oc debug node" to run nft on the gateway node's host network. -# The nftables table family is "inet" (confirmed on OCP 4.22; the -# mgmtport-no-snat-subnets-v4 set lives in family inet, table ovn-kubernetes). +# Family is "inet" (confirmed on OCP 4.22; mgmtport-no-snat-subnets-v4 lives +# in family inet, table ovn-kubernetes). InjectNftablesOnGateway() { typeset kubeconfig="${1:?}" typeset gatewayNode="${2:?}" @@ -116,29 +126,26 @@ ApplySnatFix() { : "ApplySnatFix: spoke='${spokeName}'" - # Determine this spoke's own cluster ID from its local Endpoint CRs. + # Determine this spoke's own cluster ID from its gateway node hostname. typeset localClusterID - localClusterID="$(KUBECONFIG="${kubeconfig}" \ - oc get endpoints.submariner.io \ - -n submariner-operator \ - -o json \ - | jq -r '.items[] | select(.spec.private_ip != null) - | .spec.cluster_id' \ - | sort -u \ - | grep "${spokeName}" \ - | head -1)" + localClusterID="$(GetLocalClusterID "${kubeconfig}")" if [[ -z "${localClusterID}" ]]; then - # Fallback: derive from the endpoint name (always contains cluster ID) - localClusterID="$(KUBECONFIG="${kubeconfig}" \ - oc get endpoints.submariner.io \ + # Fallback: match cluster_id by spoke name (cluster names contain spokeName) + localClusterID="$(KUBECONFIG="${kubeconfig}" oc get endpoints.submariner.io \ -n submariner-operator \ - -o jsonpath='{.items[0].spec.cluster_id}')" + -o jsonpath='{range .items[*]}{.spec.cluster_id}{"\n"}{end}' \ + | grep "${spokeName}" | head -1)" + fi + + if [[ -z "${localClusterID}" ]]; then + : "ApplySnatFix: could not determine localClusterID for '${spokeName}' — skipping" + return 0 fi : "ApplySnatFix: localClusterID='${localClusterID}'" - # Collect remote subnets across all remote spokes (newline-separated). + # Collect remote subnets (newline-separated). typeset remoteSubnetList remoteSubnetList="$(GetRemoteSubnets "${kubeconfig}" "${localClusterID}")" @@ -152,11 +159,9 @@ ApplySnatFix() { nftElements="$(echo "${remoteSubnetList}" | paste -sd ',' - | sed 's/,/, /g')" : "ApplySnatFix: remote subnets for injection: ${nftElements}" - # Find ALL gateway nodes on this spoke (there may be more than one - # in a HA Submariner setup, though typically only one is active). - typeset gatewayNodes - mapfile -t gatewayNodes < <(KUBECONFIG="${kubeconfig}" \ - oc get nodes \ + # Find ALL gateway nodes on this spoke. + typeset -a gatewayNodes + mapfile -t gatewayNodes < <(KUBECONFIG="${kubeconfig}" oc get nodes \ -l submariner.io/gateway=true \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') @@ -170,8 +175,6 @@ ApplySnatFix() { InjectNftablesOnGateway "${kubeconfig}" "${gwNode}" "${nftElements}" done - # Brief settle so the nftables rules take effect before the caller - # runs subctl verify or starts migration. sleep "${settleSecs}" : "ApplySnatFix: '${spokeName}' complete" From a5a681c1fcc444c8b2c405d2d4eec516cc207046 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sun, 2 Aug 2026 01:55:27 -0400 Subject: [PATCH 09/15] add debug commands --- ...nterop-p2p-submariner-snat-fix-commands.sh | 30 ++++- ...-execute-live-migration-return-commands.sh | 106 +++++++++++++++++- ...p2p-mtv-execute-live-migration-commands.sh | 106 +++++++++++++++++- 3 files changed, 236 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh index 2b2688117058a..2a7db3c8e37f3 100644 --- a/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh +++ b/ci-operator/step-registry/acm/interop-p2p/submariner-snat-fix/acm-interop-p2p-submariner-snat-fix-commands.sh @@ -42,8 +42,14 @@ typeset -i settleSecs="${SUBMARINER_SNAT_FIX_SETTLE_SECS}" typeset -a spokeKubeconfigsArr=() typeset -a spokeNamesArr=() +typeset -a _tmpKubeconfigsToClean=() # ── LoadSpokeConfig — populate spoke arrays from SHARED_DIR ────────────────── +# CI spoke kubeconfigs can have 'namespace: ci-op-XXXXXXXX' in their current +# context. That namespace does not exist on the spoke cluster, which causes +# oc to fail namespace-validation before executing ANY command (even cluster- +# scoped ones like "oc get nodes"). We create a temp copy of each kubeconfig +# with the context namespace patched to 'default' to avoid this. LoadSpokeConfig() { typeset -i i for ((i = 1; i <= spokeCount; i++)); do @@ -53,7 +59,17 @@ LoadSpokeConfig() { [ -f "${kcFile}" ] [ -f "${nameFile}" ] - spokeKubeconfigsArr+=("${kcFile}") + typeset tmpKc + tmpKc="$(mktemp -t submariner-snat-fix-kc-XXXXXX)" + _tmpKubeconfigsToClean+=("${tmpKc}") + cp "${kcFile}" "${tmpKc}" + # Use kubectl (not oc) here to avoid triggering oc's own namespace + # validation while we are patching the context namespace. + kubectl --kubeconfig="${tmpKc}" config set-context \ + "$(kubectl --kubeconfig="${tmpKc}" config current-context)" \ + --namespace=default > /dev/null + + spokeKubeconfigsArr+=("${tmpKc}") spokeNamesArr+=("$(<"${nameFile}")") done true @@ -68,7 +84,8 @@ GetLocalClusterID() { typeset gwNode gwNode="$(KUBECONFIG="${kubeconfig}" oc get nodes \ -l submariner.io/gateway=true \ - -o jsonpath='{.items[0].metadata.name}')" + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | head -1)" # Strip domain suffix: "ip-10-1-99-33.us-east-2.compute.internal" → "ip-10-1-99-33" typeset gwShort="${gwNode%%.*}" @@ -108,13 +125,13 @@ InjectNftablesOnGateway() { : "InjectNftablesOnGateway: node='${gatewayNode}' elements='${nftElements}'" KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ - --quiet -- chroot /host \ + --quiet --to-namespace=default -- chroot /host \ nft add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ "{ ${nftElements} }" 2>&1 | grep -v '^$' || true : "InjectNftablesOnGateway: verifying set on '${gatewayNode}'" KUBECONFIG="${kubeconfig}" oc debug "node/${gatewayNode}" \ - --quiet -- chroot /host \ + --quiet --to-namespace=default -- chroot /host \ nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ 2>/dev/null || true } @@ -191,4 +208,9 @@ for ((i = 0; i < spokeCount; i++)); do "${spokeNamesArr[i]}" done +typeset _tmpKc +for _tmpKc in "${_tmpKubeconfigsToClean[@]+"${_tmpKubeconfigsToClean[@]}"}"; do + rm -f "${_tmpKc}" +done + true diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh index 9591ca85039ab..6980d7ad99f50 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -114,6 +114,110 @@ DumpDiagnostics() { DestOc logs -n "${MTV_CNV_NAMESPACE}" \ -l kubevirt.io=virt-controller --tail=100 \ > "${diagDir}/dest-virt-controller.log" 2>&1 || true + + DumpCclmSyncDiagnostics +} + +# DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the +# Synchronization phase (source=Synchronizing, dest=WaitingForSync). +# Captures virt-synchronization-controller state, Submariner service exports/ +# imports, forklift-controller logs, virt-handler logs, and VMI migration state. +# Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics +# (on any live-plan error path). +DumpCclmSyncDiagnostics() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + + typeset syncDir="${ARTIFACT_DIR}/mtv-cclm-sync-diagnostics" + mkdir -p "${syncDir}" + + typeset spoke kc + for spoke in source dest; do + [[ "${spoke}" == "source" ]] && kc="${sourceKubeconfig}" || kc="${destKubeconfig}" + + # Broad search: virt-sync related pods across ALL namespaces. + oc --kubeconfig="${kc}" get pods -A -o wide 2>/dev/null \ + | grep -E 'virt-sync|synchronization-controller|virt-migration' \ + > "${syncDir}/${spoke}-virt-sync-pods-all-ns.txt" 2>&1 || true + + # Targeted: pods + logs from the CNV namespace by deployment name. + oc --kubeconfig="${kc}" get pods \ + -n "${MTV_CNV_NAMESPACE}" -o wide \ + 2>/dev/null \ + | grep 'virt-synchronization-controller' \ + > "${syncDir}/${spoke}-virt-sync-pods-cnv-ns.txt" 2>&1 || true + + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + deployment/virt-synchronization-controller \ + --tail=200 \ + > "${syncDir}/${spoke}-virt-sync-controller-logs.txt" 2>&1 || true + + # Submariner service discovery state. + oc --kubeconfig="${kc}" get serviceexport -A \ + > "${syncDir}/${spoke}-serviceexports.txt" 2>&1 || true + oc --kubeconfig="${kc}" get serviceimport -A \ + > "${syncDir}/${spoke}-serviceimports.txt" 2>&1 || true + + # Services on the CCLM sync port (8443) or the KubeVirt migration port. + oc --kubeconfig="${kc}" get svc -A -o json 2>/dev/null \ + | jq -r '.items[] + | select(.spec.ports[]?.port == 8443 + or .spec.ports[]?.port == 9185 + or .spec.ports[]?.targetPort == 8443 + or (.spec.ports[]?.name // "" | test("sync|migr|virt"))) + | "\(.metadata.namespace)/\(.metadata.name)\t" + + "ports=\([.spec.ports[]? | "\(.port):\(.targetPort // "")"] | join(","))\t" + + "clusterIP=\(.spec.clusterIP)"' \ + > "${syncDir}/${spoke}-sync-services.txt" 2>&1 || true + + # virt-handler logs (per-node daemon that drives actual live migration). + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + -l 'kubevirt.io=virt-handler' \ + --tail=150 \ + > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + done + + # forklift-controller logs from the hub (MTV migration orchestrator). + oc --kubeconfig="${KUBECONFIG}" logs \ + -n "${MTV_NAMESPACE}" \ + deployment/forklift-controller \ + --tail=300 \ + > "${syncDir}/hub-forklift-controller-logs.txt" 2>&1 || true + + # Detailed VMI migration state. + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/source-vmi-migration-state.json" 2>&1 || true + + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/dest-vmi-migration-state.json" 2>&1 || true + + # Full VMIM YAML (more detail than vmim.yaml in the base diagnostics dir). + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${syncDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${syncDir}/dest-vmim.yaml" 2>&1 || true + + # Events in the migration namespaces. + SourceOc get events -n "${MTV_TEST_VM_NAMESPACE}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/source-vm-ns-events.txt" 2>&1 || true + DestOc get events -n "${targetNs}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/dest-vm-ns-events.txt" 2>&1 || true + + : "CCLM sync diagnostics written → ${syncDir}" } # OnError — dump diagnostics before propagating failure. @@ -427,7 +531,7 @@ CheckSyncStuck() { if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" - DumpDiagnostics + DumpCclmSyncDiagnostics false fi diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh index 5e8b58831544e..4135ef9cc9146 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh @@ -113,6 +113,110 @@ DumpDiagnostics() { DestOc logs -n "${MTV_CNV_NAMESPACE}" \ -l kubevirt.io=virt-controller --tail=100 \ > "${diagDir}/dest-virt-controller.log" 2>&1 || true + + DumpCclmSyncDiagnostics +} + +# DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the +# Synchronization phase (source=Synchronizing, dest=WaitingForSync). +# Captures virt-synchronization-controller state, Submariner service exports/ +# imports, forklift-controller logs, virt-handler logs, and VMI migration state. +# Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics +# (on any live-plan error path). +DumpCclmSyncDiagnostics() { + [[ "${MTV_PLAN_TYPE}" != "live" ]] && return 0 + [[ -n "${ARTIFACT_DIR}" ]] || return 0 + + typeset syncDir="${ARTIFACT_DIR}/mtv-cclm-sync-diagnostics" + mkdir -p "${syncDir}" + + typeset spoke kc + for spoke in source dest; do + [[ "${spoke}" == "source" ]] && kc="${sourceKubeconfig}" || kc="${destKubeconfig}" + + # Broad search: virt-sync related pods across ALL namespaces. + oc --kubeconfig="${kc}" get pods -A -o wide 2>/dev/null \ + | grep -E 'virt-sync|synchronization-controller|virt-migration' \ + > "${syncDir}/${spoke}-virt-sync-pods-all-ns.txt" 2>&1 || true + + # Targeted: pods + logs from the CNV namespace by deployment name. + oc --kubeconfig="${kc}" get pods \ + -n "${MTV_CNV_NAMESPACE}" -o wide \ + 2>/dev/null \ + | grep 'virt-synchronization-controller' \ + > "${syncDir}/${spoke}-virt-sync-pods-cnv-ns.txt" 2>&1 || true + + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + deployment/virt-synchronization-controller \ + --tail=200 \ + > "${syncDir}/${spoke}-virt-sync-controller-logs.txt" 2>&1 || true + + # Submariner service discovery state. + oc --kubeconfig="${kc}" get serviceexport -A \ + > "${syncDir}/${spoke}-serviceexports.txt" 2>&1 || true + oc --kubeconfig="${kc}" get serviceimport -A \ + > "${syncDir}/${spoke}-serviceimports.txt" 2>&1 || true + + # Services on the CCLM sync port (8443) or the KubeVirt migration port. + oc --kubeconfig="${kc}" get svc -A -o json 2>/dev/null \ + | jq -r '.items[] + | select(.spec.ports[]?.port == 8443 + or .spec.ports[]?.port == 9185 + or .spec.ports[]?.targetPort == 8443 + or (.spec.ports[]?.name // "" | test("sync|migr|virt"))) + | "\(.metadata.namespace)/\(.metadata.name)\t" + + "ports=\([.spec.ports[]? | "\(.port):\(.targetPort // "")"] | join(","))\t" + + "clusterIP=\(.spec.clusterIP)"' \ + > "${syncDir}/${spoke}-sync-services.txt" 2>&1 || true + + # virt-handler logs (per-node daemon that drives actual live migration). + oc --kubeconfig="${kc}" logs \ + -n "${MTV_CNV_NAMESPACE}" \ + -l 'kubevirt.io=virt-handler' \ + --tail=150 \ + > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + done + + # forklift-controller logs from the hub (MTV migration orchestrator). + oc --kubeconfig="${KUBECONFIG}" logs \ + -n "${MTV_NAMESPACE}" \ + deployment/forklift-controller \ + --tail=300 \ + > "${syncDir}/hub-forklift-controller-logs.txt" 2>&1 || true + + # Detailed VMI migration state. + SourceOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${MTV_TEST_VM_NAMESPACE}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/source-vmi-migration-state.json" 2>&1 || true + + DestOc get "virtualmachineinstance/${MTV_TEST_VM_NAME}" \ + -n "${targetNs}" -o json 2>/dev/null \ + | jq '{name: .metadata.name, phase: .status.phase, + migrationState: .status.migrationState, + conditions: [.status.conditions[]? + | {type, status, reason, message}]}' \ + > "${syncDir}/dest-vmi-migration-state.json" 2>&1 || true + + # Full VMIM YAML (more detail than vmim.yaml in the base diagnostics dir). + SourceOc get vmim -n "${MTV_TEST_VM_NAMESPACE}" -o yaml \ + > "${syncDir}/source-vmim.yaml" 2>&1 || true + DestOc get vmim -n "${targetNs}" -o yaml \ + > "${syncDir}/dest-vmim.yaml" 2>&1 || true + + # Events in the migration namespaces. + SourceOc get events -n "${MTV_TEST_VM_NAMESPACE}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/source-vm-ns-events.txt" 2>&1 || true + DestOc get events -n "${targetNs}" \ + --sort-by='.lastTimestamp' \ + > "${syncDir}/dest-vm-ns-events.txt" 2>&1 || true + + : "CCLM sync diagnostics written → ${syncDir}" } # OnError — dump diagnostics before propagating failure. @@ -426,7 +530,7 @@ CheckSyncStuck() { if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" - DumpDiagnostics + DumpCclmSyncDiagnostics false fi From a1598f976f110309d20e12b910c77cd5e2ddad49 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sun, 2 Aug 2026 10:57:00 -0400 Subject: [PATCH 10/15] additional debug commands --- ...-execute-live-migration-return-commands.sh | 44 +++++++++++++++++-- ...mtv-execute-live-migration-return-ref.yaml | 8 ++-- ...p2p-mtv-execute-live-migration-commands.sh | 44 +++++++++++++++++-- .../p2p-mtv-execute-live-migration-ref.yaml | 8 ++-- 4 files changed, 88 insertions(+), 16 deletions(-) diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh index 6980d7ad99f50..845a749a30416 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-commands.sh @@ -121,7 +121,10 @@ DumpDiagnostics() { # DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the # Synchronization phase (source=Synchronizing, dest=WaitingForSync). # Captures virt-synchronization-controller state, Submariner service exports/ -# imports, forklift-controller logs, virt-handler logs, and VMI migration state. +# imports, forklift-controller logs, virt-handler logs, VMI migration state, +# virt-handler client-cert Secret presence (missing certs block mTLS to dest +# virt-sync-controller:9185), nftables SNAT exclusion set re-verification, and +# OVN-K gateway-node logs. # Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics # (on any live-plan error path). DumpCclmSyncDiagnostics() { @@ -177,6 +180,39 @@ DumpCclmSyncDiagnostics() { -l 'kubevirt.io=virt-handler' \ --tail=150 \ > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + + # virt-handler CCLM client-certificate Secrets (metadata only — no secret data). + # Missing certs prevent virt-handler from making mTLS connections to the + # dest virt-synchronization-controller on port 9185. + oc --kubeconfig="${kc}" get secrets -n "${MTV_CNV_NAMESPACE}" \ + > "${syncDir}/${spoke}-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kc}" get ds virt-handler -n "${MTV_CNV_NAMESPACE}" -o json \ + 2>/dev/null \ + | jq '[.spec.template.spec.volumes[]? + | select(.name | test("cert|tls|key"; "i")) + | {name, secret: (.secret.secretName // null), + configMap: (.configMap.name // null)}]' \ + > "${syncDir}/${spoke}-virt-handler-cert-volumes.json" 2>&1 || true + + # OVN-K gateway node logs — check for route/policy issues affecting cross-cluster. + typeset _gwNode + _gwNode="$(oc --kubeconfig="${kc}" get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1 || true)" + if [[ -n "${_gwNode}" ]]; then + oc --kubeconfig="${kc}" logs \ + -n openshift-ovn-kubernetes \ + -l 'app=ovnkube-node' \ + --field-selector="spec.nodeName=${_gwNode}" \ + --tail=80 \ + > "${syncDir}/${spoke}-ovnkube-node-gateway-logs.txt" 2>&1 || true + # Verify nftables SNAT exclusion set still contains remote CIDRs. + oc --kubeconfig="${kc}" debug "node/${_gwNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + > "${syncDir}/${spoke}-nftables-snat-set.txt" 2>&1 || true + fi done # forklift-controller logs from the hub (MTV migration orchestrator). @@ -532,10 +568,10 @@ CheckSyncStuck() { if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" DumpCclmSyncDiagnostics - false + return 1 fi - true + return 0 } # RefreshProviderInventory — re-scan spoke KubeVirt inventory before live Plan validation. @@ -662,7 +698,7 @@ WaitMigrationSucceeded() { false fi - CheckSyncStuck + CheckSyncStuck || return 1 PrintMigrationPipeline : "Migration in progress${msg:+: ${msg}} (${SECONDS}/${deadline}s)" diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml index c6ab49c92904b..b55f104aa4d04 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration-return/p2p-mtv-execute-live-migration-return-ref.yaml @@ -20,7 +20,7 @@ ref: - p2p-mtv-bidir-mig-config (return NetworkMap/StorageMap Ready on hub) - p2p-mtv-execute-live-migration (VM currently on spoke-2 after forward migration) - p2p-acm-cnv-install-policy (DecentralizedLiveMigration + sync controller on spokes) - - acm-interop-p2p-submariner-install (no Globalnet; :8443 pod reachability) + - acm-interop-p2p-submariner-install (no Globalnet; :9185 sync-port reachability) Live preflight: sync-controller ready, no submariner-globalnet, TCP probe to spoke-1 sync-controller from spoke-2 virt-launcher. During migration wait, fails early if @@ -109,10 +109,10 @@ ref: documentation: Wait for virt-synchronization-controller Available on each spoke before live migration. - name: MTV_CCLM_SYNC_PROBE default: "true" - documentation: TCP-probe dest sync-controller :8443 from source virt-launcher before migration. + documentation: TCP-probe dest sync-controller :9185 from source virt-launcher before migration. - name: MTV_CCLM_SYNC_PORT - default: "8443" - documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller HTTPS listener). + default: "9185" + documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller sync data port; healthz is :8443). - name: MTV_CCLM_SYNC_PROBE_TIMEOUT default: "10" documentation: Seconds to wait for the pre-migration CCLM sync TCP probe. diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh index 4135ef9cc9146..3f196a215a617 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-commands.sh @@ -120,7 +120,10 @@ DumpDiagnostics() { # DumpCclmSyncDiagnostics — targeted dump when the migration is stuck in the # Synchronization phase (source=Synchronizing, dest=WaitingForSync). # Captures virt-synchronization-controller state, Submariner service exports/ -# imports, forklift-controller logs, virt-handler logs, and VMI migration state. +# imports, forklift-controller logs, virt-handler logs, VMI migration state, +# virt-handler client-cert Secret presence (missing certs block mTLS to dest +# virt-sync-controller:9185), nftables SNAT exclusion set re-verification, and +# OVN-K gateway-node logs. # Called from CheckSyncStuck (immediately on detection) and from DumpDiagnostics # (on any live-plan error path). DumpCclmSyncDiagnostics() { @@ -176,6 +179,39 @@ DumpCclmSyncDiagnostics() { -l 'kubevirt.io=virt-handler' \ --tail=150 \ > "${syncDir}/${spoke}-virt-handler-logs.txt" 2>&1 || true + + # virt-handler CCLM client-certificate Secrets (metadata only — no secret data). + # Missing certs prevent virt-handler from making mTLS connections to the + # dest virt-synchronization-controller on port 9185. + oc --kubeconfig="${kc}" get secrets -n "${MTV_CNV_NAMESPACE}" \ + > "${syncDir}/${spoke}-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kc}" get ds virt-handler -n "${MTV_CNV_NAMESPACE}" -o json \ + 2>/dev/null \ + | jq '[.spec.template.spec.volumes[]? + | select(.name | test("cert|tls|key"; "i")) + | {name, secret: (.secret.secretName // null), + configMap: (.configMap.name // null)}]' \ + > "${syncDir}/${spoke}-virt-handler-cert-volumes.json" 2>&1 || true + + # OVN-K gateway node logs — check for route/policy issues affecting cross-cluster. + typeset _gwNode + _gwNode="$(oc --kubeconfig="${kc}" get nodes \ + -l submariner.io/gateway=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1 || true)" + if [[ -n "${_gwNode}" ]]; then + oc --kubeconfig="${kc}" logs \ + -n openshift-ovn-kubernetes \ + -l 'app=ovnkube-node' \ + --field-selector="spec.nodeName=${_gwNode}" \ + --tail=80 \ + > "${syncDir}/${spoke}-ovnkube-node-gateway-logs.txt" 2>&1 || true + # Verify nftables SNAT exclusion set still contains remote CIDRs. + oc --kubeconfig="${kc}" debug "node/${_gwNode}" \ + --quiet --to-namespace=default -- chroot /host \ + nft list set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 \ + > "${syncDir}/${spoke}-nftables-snat-set.txt" 2>&1 || true + fi done # forklift-controller logs from the hub (MTV migration orchestrator). @@ -531,10 +567,10 @@ CheckSyncStuck() { if [[ "${srcVmimPhase}" == "Synchronizing" && "${destVmimPhase}" == "WaitingForSync" ]]; then : "Synchronization stuck >${syncStuckMinutes}m (source=${srcVmimPhase}, dest=${destVmimPhase})" DumpCclmSyncDiagnostics - false + return 1 fi - true + return 0 } # RefreshProviderInventory — re-scan spoke KubeVirt inventory before live Plan validation. @@ -661,7 +697,7 @@ WaitMigrationSucceeded() { false fi - CheckSyncStuck + CheckSyncStuck || return 1 PrintMigrationPipeline : "Migration in progress${msg:+: ${msg}} (${SECONDS}/${deadline}s)" diff --git a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml index 1ff2572ca0073..248b2fa0b2719 100644 --- a/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml +++ b/ci-operator/step-registry/p2p/mtv-execute-live-migration/p2p-mtv-execute-live-migration-ref.yaml @@ -15,7 +15,7 @@ ref: and verifies the VM is Running on the destination spoke. Requires p2p-mtv-mig-config (providers + maps Ready), p2p-create-migration-test-vm (source VM Running), p2p-acm-cnv-install-policy (DecentralizedLiveMigration + sync controller on spokes), - and acm-interop-p2p-submariner-install (no Globalnet; :8443 pod reachability). + and acm-interop-p2p-submariner-install (no Globalnet; :9185 sync-port reachability). Live preflight: sync-controller ready, no submariner-globalnet, TCP probe to dest sync-controller from source virt-launcher. During migration wait, fails early if @@ -104,10 +104,10 @@ ref: documentation: Wait for virt-synchronization-controller Available on each spoke before live migration. - name: MTV_CCLM_SYNC_PROBE default: "true" - documentation: TCP-probe dest sync-controller :8443 from source virt-launcher before migration. + documentation: TCP-probe dest sync-controller :9185 from source virt-launcher before migration. - name: MTV_CCLM_SYNC_PORT - default: "8443" - documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller HTTPS listener). + default: "9185" + documentation: KubeVirt cross-cluster sync controller TCP port (virt-synchronization-controller sync data port; healthz is :8443). - name: MTV_CCLM_SYNC_PROBE_TIMEOUT default: "10" documentation: Seconds to wait for the pre-migration CCLM sync TCP probe. From 3bb132bdebb36138bad5aed4ac4efb7e156d0490 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sun, 2 Aug 2026 16:01:23 -0400 Subject: [PATCH 11/15] debug virt-operator issue --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 2 + .../p2p/cnv-cclm-readiness/OWNERS | 3 + .../p2p-cnv-cclm-readiness-commands.sh | 321 ++++++++++++++++++ .../p2p-cnv-cclm-readiness-ref.metadata.json | 11 + .../p2p-cnv-cclm-readiness-ref.yaml | 71 ++++ 5 files changed, 408 insertions(+) create mode 100644 ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS create mode 100644 ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh create mode 100644 ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json create mode 100644 ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml index f7b1d51b44c99..51cf7ba01668d 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -68,6 +68,7 @@ tests: {"step": "acm-interop-p2p-submariner-broker-join", "failure_type": "all", "classification": "Submariner broker deploy / cluster join failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "acm-interop-p2p-submariner-verify", "failure_type": "all", "classification": "Submariner connectivity verification failure", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 4}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, {"step": "acm-interop-p2p-submariner-snat-fix", "failure_type": "all", "classification": "Submariner SNAT workaround failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-cnv-cclm-readiness", "failure_type": "all", "classification": "CCLM virt-handler cert / Service / ServiceExport readiness failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-additional-config", "failure_type": "all", "classification": "MTV ForkliftController / CCLM configuration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-register-providers", "failure_type": "all", "classification": "MTV provider registration failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, {"step": "p2p-mtv-create-migration-maps", "failure_type": "all", "classification": "MTV forward migration maps failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, @@ -134,6 +135,7 @@ tests: - ref: p2p-acm-cnv-install-policy - ref: p2p-install-odf-spokes - chain: acm-interop-p2p-submariner-install + - ref: p2p-cnv-cclm-readiness - chain: p2p-mtv-bidir-mig-config - chain: p2p-cclm-bidir-migration-test timeout: 12h0m0s diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS b/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh new file mode 100644 index 0000000000000..6697d9554f924 --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh @@ -0,0 +1,321 @@ +#!/bin/bash +# +# Verify and remediate CCLM (Cross-Cluster Live Migration) prerequisites on each spoke. +# No-op when CNV_ENABLE_CCLM != "true". +# +# Checks performed (per spoke, in order): +# 1. Restart virt-operator to trigger DaemonSet reconciliation. Required when the +# DecentralizedLiveMigration feature gate does not cause virt-operator to add +# the /etc/virt-handler/clientcertificates volume mount at install time (observed +# on CNV 4.17+ / OCP 4.22 nightly — treated as an operator reconciliation lag). +# 2. Wait for virt-handler DaemonSet rollout to complete. +# 3. Poll until /etc/virt-handler/clientcertificates/tls.crt appears in a +# virt-handler pod (up to CNV_CCLM_WAIT_TIMEOUT seconds). +# 4. Fallback: if cert still absent, patch virt-handler DaemonSet to mount the +# CNV_CCLM_VIRT_HANDLER_CERT_SECRET Secret at /etc/virt-handler/clientcertificates. +# Collects diagnostics and fails the step if cert is still absent after the patch. +# 5. Ensure a Service named virt-synchronization-controller on port CNV_CCLM_SYNC_PORT +# exists in openshift-cnv (created by `oc apply` — idempotent). +# 6. Ensure a ServiceExport for that Service exists (allows Submariner to propagate it +# to other clusters). Skipped gracefully when the ServiceExport CRD is not yet +# installed (i.e. Submariner is not up yet on this spoke). +# +set -euxo pipefail; shopt -s inherit_errexit + +[[ "${CNV_ENABLE_CCLM}" == "true" ]] || { + : "CNV_ENABLE_CCLM is not 'true' — CCLM readiness checks skipped" + true; exit 0 +} + +readonly cnvNs="openshift-cnv" +readonly clientCertMountPath="/etc/virt-handler/clientcertificates" +typeset -i waitTimeoutSecs="${CNV_CCLM_WAIT_TIMEOUT}" +typeset -i pollIntervalSecs=15 + +# LoadSpokeKubeconfigs — output spoke kubeconfig paths (one per line). +LoadSpokeKubeconfigs() { + typeset -a kcsArr=() + typeset -i i + for ((i = 1; ; i++)); do + typeset kcPath="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + [[ -f "${kcPath}" ]] || break + kcsArr+=("${kcPath}") + done + if [[ ${#kcsArr[@]} -eq 0 && -f "${SHARED_DIR}/managed-cluster-kubeconfig" ]]; then + kcsArr+=("${SHARED_DIR}/managed-cluster-kubeconfig") + fi + ((${#kcsArr[@]} > 0)) + printf '%s\n' "${kcsArr[@]}" +} + +# GetSpokeClusterName — return cluster name for 1-based spoke index. +GetSpokeClusterName() { + typeset -i idx="${1:?}" + typeset nameFile="${SHARED_DIR}/managed-cluster-name-${idx}" + if [[ -f "${nameFile}" ]]; then + cat "${nameFile}" + else + printf 'spoke-%d' "${idx}" + fi +} + +# RestartVirtOperatorAndWait — restart virt-operator and wait for rollout. +RestartVirtOperatorAndWait() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + : "[${clusterName}] Restarting virt-operator to trigger DaemonSet reconciliation" + oc --kubeconfig="${kubeconfig}" rollout restart deployment/virt-operator \ + -n "${cnvNs}" + oc --kubeconfig="${kubeconfig}" rollout status deployment/virt-operator \ + -n "${cnvNs}" --timeout=5m 1>/dev/null + true +} + +# WaitVirtHandlerRollout — wait for virt-handler DaemonSet rollout (non-fatal on timeout). +WaitVirtHandlerRollout() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset timeout="${3:-10m}" + : "[${clusterName}] Waiting for virt-handler DaemonSet rollout (timeout=${timeout})" + oc --kubeconfig="${kubeconfig}" rollout status ds/virt-handler \ + -n "${cnvNs}" --timeout="${timeout}" 1>/dev/null || true + true +} + +# CheckVirtHandlerClientCert — return 0 if any virt-handler pod has the CCLM client cert. +CheckVirtHandlerClientCert() { + typeset kubeconfig="${1:?}" + typeset podName="" + podName="$(oc --kubeconfig="${kubeconfig}" get pod \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + -o jsonpath-as-json='{.items[*].metadata.name}' \ + 2>/dev/null | jq -re '.[0]' || true)" + [[ -n "${podName}" ]] || return 1 + oc --kubeconfig="${kubeconfig}" exec -n "${cnvNs}" "${podName}" -- \ + test -f "${clientCertMountPath}/tls.crt" 2>/dev/null +} + +# PatchVirtHandlerDaemonSetCertMount — add clientcertificates volume mount (idempotent). +# Workaround for virt-operator not reconciling the CCLM client cert mount when +# DecentralizedLiveMigration is enabled. Uses a unique volume name +# (cnv-cclm-virt-handler-client-certs) to avoid collisions with operator-managed volumes. +PatchVirtHandlerDaemonSetCertMount() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + + # Idempotency: skip if mount already present. + typeset existingMountCount=0 + existingMountCount="$(oc --kubeconfig="${kubeconfig}" get ds virt-handler \ + -n "${cnvNs}" \ + -o jsonpath-as-json='{.spec.template.spec.containers[*].volumeMounts[*].mountPath}' \ + 2>/dev/null | jq --arg p "${clientCertMountPath}" '[.[] | select(. == $p)] | length')" + if ((existingMountCount > 0)); then + : "[${clusterName}] ${clientCertMountPath} already mounted — skipping DaemonSet patch" + return 0 + fi + + # Verify the source Secret exists before patching. + oc --kubeconfig="${kubeconfig}" get secret "${CNV_CCLM_VIRT_HANDLER_CERT_SECRET}" \ + -n "${cnvNs}" 1>/dev/null + : "[${clusterName}] Patching virt-handler DaemonSet: mounting ${CNV_CCLM_VIRT_HANDLER_CERT_SECRET} at ${clientCertMountPath}" + oc --kubeconfig="${kubeconfig}" patch ds virt-handler \ + -n "${cnvNs}" --type=json \ + --patch "[ + {\"op\":\"add\", + \"path\":\"/spec/template/spec/volumes/-\", + \"value\":{\"name\":\"cnv-cclm-virt-handler-client-certs\", + \"secret\":{\"secretName\":\"${CNV_CCLM_VIRT_HANDLER_CERT_SECRET}\", + \"optional\":true, + \"defaultMode\":420}}}, + {\"op\":\"add\", + \"path\":\"/spec/template/spec/containers/0/volumeMounts/-\", + \"value\":{\"name\":\"cnv-cclm-virt-handler-client-certs\", + \"mountPath\":\"${clientCertMountPath}\", + \"readOnly\":true}} + ]" + true +} + +# EnsureVirtSyncService — create Service for port CNV_CCLM_SYNC_PORT if absent. +EnsureVirtSyncService() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + + if oc --kubeconfig="${kubeconfig}" get svc virt-synchronization-controller \ + -n "${cnvNs}" 1>/dev/null 2>&1; then + : "[${clusterName}] Service virt-synchronization-controller already exists" + return 0 + fi + + : "[${clusterName}] Creating Service virt-synchronization-controller:${CNV_CCLM_SYNC_PORT}" + { + oc --kubeconfig="${kubeconfig}" create -f - \ + --dry-run=client -o yaml --save-config + } 0</dev/null || true)" + if [[ -z "${seExists}" ]]; then + : "[${clusterName}] ServiceExport CRD not available — skipping" + return 0 + fi + + if oc --kubeconfig="${kubeconfig}" get serviceexport \ + virt-synchronization-controller \ + -n "${cnvNs}" 1>/dev/null 2>&1; then + : "[${clusterName}] ServiceExport virt-synchronization-controller already exists" + return 0 + fi + + : "[${clusterName}] Creating ServiceExport virt-synchronization-controller" + { + oc --kubeconfig="${kubeconfig}" create -f - \ + --dry-run=client -o yaml --save-config + } 0<<'ocEOF' | oc --kubeconfig="${kubeconfig}" apply -f - +apiVersion: multicluster.x-k8s.io/v1alpha1 +kind: ServiceExport +metadata: + name: virt-synchronization-controller + namespace: openshift-cnv +ocEOF + true +} + +# SaveCclmReadinessDiagnostics — collect diagnostics to ARTIFACT_DIR. +SaveCclmReadinessDiagnostics() { + typeset kubeconfig="${1:?}" + typeset clusterName="${2:?}" + typeset diagDir="${ARTIFACT_DIR}/cclm-readiness-${clusterName}" + mkdir -p "${diagDir}" + + oc --kubeconfig="${kubeconfig}" get pod \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + -o wide > "${diagDir}/virt-handler-pods.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get ds virt-handler \ + -n "${cnvNs}" -o yaml \ + > "${diagDir}/virt-handler-ds.yaml" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get secret -n "${cnvNs}" \ + > "${diagDir}/openshift-cnv-secrets-list.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" logs \ + -n "${cnvNs}" deployment/virt-operator \ + --tail=150 > "${diagDir}/virt-operator-logs.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" logs \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + --tail=80 > "${diagDir}/virt-handler-logs.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get svc \ + -n "${cnvNs}" > "${diagDir}/services.txt" 2>&1 || true + oc --kubeconfig="${kubeconfig}" get serviceexport \ + -n "${cnvNs}" > "${diagDir}/serviceexports.txt" 2>&1 || true + true +} + +# ── main ────────────────────────────────────────────────────────────────────── + +typeset -a spokeKubeconfigsArr=() +mapfile -t spokeKubeconfigsArr < <(LoadSpokeKubeconfigs) + +typeset -i spokeCount="${#spokeKubeconfigsArr[@]}" +typeset -i failedCount=0 i + +: "=== CCLM readiness on ${spokeCount} spoke(s) ===" + +for ((i = 0; i < spokeCount; i++)); do + typeset kc="${spokeKubeconfigsArr[i]}" + typeset spkName="" + spkName="$(GetSpokeClusterName "$((i + 1))")" + + # ── Step 1: restart virt-operator ── + RestartVirtOperatorAndWait "${kc}" "${spkName}" + + # ── Step 2: wait for virt-handler DaemonSet rollout ── + WaitVirtHandlerRollout "${kc}" "${spkName}" "10m" + + # ── Step 3: poll for CCLM client cert ── + typeset -i certWaitSecs=0 + typeset isCertOk=false + while (( certWaitSecs < waitTimeoutSecs )); do + if CheckVirtHandlerClientCert "${kc}"; then + isCertOk=true + break + fi + : "[${spkName}] Waiting for virt-handler ${clientCertMountPath}/tls.crt (${certWaitSecs}/${waitTimeoutSecs}s)" + sleep "${pollIntervalSecs}" + (( certWaitSecs += pollIntervalSecs )) + done + + # ── Step 4: fallback DaemonSet patch if cert still absent ── + if [[ "${isCertOk}" != "true" ]]; then + : "[${spkName}] Cert absent after virt-operator restart — applying DaemonSet workaround patch" + if ! PatchVirtHandlerDaemonSetCertMount "${kc}" "${spkName}"; then + : "[${spkName}] DaemonSet patch failed" + SaveCclmReadinessDiagnostics "${kc}" "${spkName}" + (( ++failedCount )) + continue + fi + WaitVirtHandlerRollout "${kc}" "${spkName}" "10m" + + certWaitSecs=0 + isCertOk=false + while (( certWaitSecs < 300 )); do + if CheckVirtHandlerClientCert "${kc}"; then + isCertOk=true + break + fi + : "[${spkName}] Post-patch poll for virt-handler cert (${certWaitSecs}/300s)" + sleep "${pollIntervalSecs}" + (( certWaitSecs += pollIntervalSecs )) + done + fi + + if [[ "${isCertOk}" != "true" ]]; then + : "[${spkName}] FATAL: ${clientCertMountPath}/tls.crt absent after all remediation" + SaveCclmReadinessDiagnostics "${kc}" "${spkName}" + (( ++failedCount )) + continue + fi + : "[${spkName}] virt-handler CCLM client cert present" + + # ── Step 5: ensure Service on CNV_CCLM_SYNC_PORT ── + if ! EnsureVirtSyncService "${kc}" "${spkName}"; then + : "[${spkName}] WARN: failed to ensure virt-synchronization-controller Service" + (( ++failedCount )) + fi + + # ── Step 6: ensure ServiceExport (no-op if Submariner CRD not available) ── + EnsureVirtSyncServiceExport "${kc}" "${spkName}" || true + + : "[${spkName}] CCLM readiness checks complete" +done + +(( failedCount == 0 )) +: "CCLM readiness checks passed on all ${spokeCount} spoke(s)" +true diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json new file mode 100644 index 0000000000000..4ffa212b8f0b5 --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml new file mode 100644 index 0000000000000..4bbcf100a1151 --- /dev/null +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml @@ -0,0 +1,71 @@ +ref: + as: p2p-cnv-cclm-readiness + from: cli + commands: p2p-cnv-cclm-readiness-commands.sh + timeout: 35m0s + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 200Mi + env: + - name: CNV_ENABLE_CCLM + default: "false" + documentation: |- + When "true", run CCLM readiness checks: restart virt-operator, verify + /etc/virt-handler/clientcertificates/tls.crt is present in virt-handler pods, + and ensure the virt-synchronization-controller Service and ServiceExport exist. + Set to "true" for CCLM scenarios; leave "false" for upgrade or non-CCLM jobs. + - name: CNV_CCLM_WAIT_TIMEOUT + default: "600" + documentation: |- + Maximum seconds to poll for /etc/virt-handler/clientcertificates/tls.crt in + a virt-handler pod after virt-operator restart. The fallback DaemonSet patch + is attempted if the cert is still absent after this timeout. + - name: CNV_CCLM_VIRT_HANDLER_CERT_SECRET + default: "kubevirt-virt-handler-certs" + documentation: |- + Name of the Secret in openshift-cnv to mount at + /etc/virt-handler/clientcertificates when virt-operator does not add the + mount automatically (fallback DaemonSet patch path only). + - name: CNV_CCLM_SYNC_PORT + default: "9185" + documentation: |- + TCP port for the virt-synchronization-controller sync data endpoint. + Used when creating the missing Service in openshift-cnv. + documentation: |- + Verifies and remediates CCLM (Cross-Cluster Live Migration) prerequisites on + every spoke cluster before migration steps run. No-op when CNV_ENABLE_CCLM + is not "true". + + Background — root cause observed on OCP 4.22 nightly with CNV 4.17+: + When decentralizedLiveMigration=true is set in HyperConverged featureGates at + install time, virt-operator may not reconcile the virt-handler DaemonSet to add + the /etc/virt-handler/clientcertificates volume mount in its first pass. Without + this mount, virt-handler emits "failed to load the certificate + /etc/virt-handler/clientcertificates/tls.crt: no such file or directory" every + 60 s and cannot make the required mTLS connection to the destination + virt-synchronization-controller:9185, leaving the migration stuck in + Synchronizing/WaitingForSync indefinitely. + + What this step does (idempotent, safe to re-run): + 1. Restarts virt-operator on each spoke and waits for its rollout to finish + (triggers DaemonSet reconciliation — usually sufficient to fix the mount). + 2. Waits up to CNV_CCLM_WAIT_TIMEOUT seconds for the CCLM client cert to + appear in a virt-handler pod. + 3. Fallback: if the cert is still absent, patches the virt-handler DaemonSet + directly to mount CNV_CCLM_VIRT_HANDLER_CERT_SECRET at + /etc/virt-handler/clientcertificates. Collects diagnostics and fails the + step if the cert is still absent after the patch rollout. + 4. Ensures a Service named virt-synchronization-controller on port + CNV_CCLM_SYNC_PORT (9185) exists in openshift-cnv. + 5. Ensures a ServiceExport for that Service exists, allowing Submariner to + propagate it to other clusters for cross-cluster DNS resolution. Skipped + gracefully when the ServiceExport CRD (Submariner) is not yet installed. + + Reads spoke names and kubeconfigs from ${SHARED_DIR} (managed-cluster-kubeconfig-N + and managed-cluster-name-N files). + + Must run after p2p-acm-cnv-install-policy and ideally after + acm-interop-p2p-submariner-install (so ServiceExport CRD is available). + Must run before p2p-mtv-bidir-mig-config / p2p-cclm-bidir-migration-test. From 3ffbde234d83af3a04dfbcc06ca0702f44fa1737 Mon Sep 17 00:00:00 2001 From: shakyav Date: Sun, 2 Aug 2026 23:33:53 -0400 Subject: [PATCH 12/15] add jq --- .../cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh index 6697d9554f924..cae8e5fbe9e97 100644 --- a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh @@ -27,6 +27,12 @@ set -euxo pipefail; shopt -s inherit_errexit true; exit 0 } +eval "$( + typeset -a _fURL=() + type -t wget 1>/dev/null && _fURL=(wget -nv -O-) || _fURL=(curl -fsSL) + "${_fURL[@]}" https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/common/EnsureReqs.sh +)"; EnsureReqs jq + readonly cnvNs="openshift-cnv" readonly clientCertMountPath="/etc/virt-handler/clientcertificates" typeset -i waitTimeoutSecs="${CNV_CCLM_WAIT_TIMEOUT}" From 2e32722688e9bea9d2d9d4ec78892eb5f49ee3ed Mon Sep 17 00:00:00 2001 From: shakyav Date: Mon, 3 Aug 2026 09:26:51 -0400 Subject: [PATCH 13/15] fix virt restart --- .../p2p-cnv-cclm-readiness-commands.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh index cae8e5fbe9e97..6bf7a66e36430 100644 --- a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh @@ -139,6 +139,16 @@ PatchVirtHandlerDaemonSetCertMount() { \"mountPath\":\"${clientCertMountPath}\", \"readOnly\":true}} ]" + # Explicitly restart to guarantee the DaemonSet controller evicts old pods. + # Without this, `rollout status` can return for the * # Without this, `rollout status` can return for the *previous* generation before + # the controller has processed the patch, leaving old pods (without the new mount) + # in place for the post-patch cert check. + oc --kubeconfig="${kubeconfig}" rollout restart ds/virt-handler -n "${cnvNs}" + true +}previous* generation before + # the controller has processed the patch, leaving old pods (without the new mount) + # in place for the post-patch cert check. + oc --kubeconfig="${kubeconfig}" rollout restart ds/virt-handler -n "${cnvNs}" true } From 4d4bb319bcafa6e339ef5f16f446143206b05991 Mon Sep 17 00:00:00 2001 From: shakyav Date: Mon, 3 Aug 2026 10:20:58 -0400 Subject: [PATCH 14/15] add debug step --- ...ng-master__acm-virt-ocp4.22-p2p-bidir.yaml | 5 ++- .../step-registry/p2p/debug-sleep/OWNERS | 3 ++ .../debug-sleep/p2p-debug-sleep-commands.sh | 45 +++++++++++++++++++ .../p2p-debug-sleep-ref.metadata.json | 11 +++++ .../p2p/debug-sleep/p2p-debug-sleep-ref.yaml | 33 ++++++++++++++ 5 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 ci-operator/step-registry/p2p/debug-sleep/OWNERS create mode 100644 ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh create mode 100644 ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json create mode 100644 ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml index 51cf7ba01668d..539b32a21cdf4 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-virt-ocp4.22-p2p-bidir.yaml @@ -52,6 +52,7 @@ tests: CNV_ENABLE_CCLM: "true" COMPUTE_NODE_TYPE: c5n.metal CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge + DEBUG_SLEEP_DURATION: "7200" DR__RP__CR_COMP_NAME: lp-interop--ACM-Virt-BiDir FIREWATCH_CONFIG: | { @@ -77,7 +78,8 @@ tests: {"step": "p2p-mtv-execute-live-migration", "failure_type": "all", "classification": "MTV CCLM forward live migration failure (spoke-1 to spoke-2)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, {"step": "p2p-mtv-execute-live-migration-return", "failure_type": "all", "classification": "MTV CCLM return live migration failure (spoke-2 back to spoke-1)", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 5}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default","interop-tests"]}, {"step": "acm-interop-p2p-submariner-diag", "failure_type": "all", "classification": "Submariner diagnostics collection failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 3}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, - {"step": "acm-mch", "failure_type": "all", "classification": "ACM MCH install", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]} + {"step": "acm-mch", "failure_type": "all", "classification": "ACM MCH install", "jira_project": "!default", "group": {"name": "interop-p2p-tests", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]}, + {"step": "p2p-debug-sleep", "failure_type": "all", "classification": "Debug sleep step failure", "jira_project": "!default", "group": {"name": "cluster", "priority": 2}, "jira_epic": "!default", "jira_assignee": "!default", "jira_additional_labels": ["!default"]} ] } FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/main/firewatch-base-configs/cr/lp-interop.json @@ -114,6 +116,7 @@ tests: - ref: acm-inspector - ref: acm-interop-p2p-submariner-diag - ref: gather-aws-console + - ref: p2p-debug-sleep - ref: acm-interop-p2p-cluster-uninstall - chain: ipi-deprovision - ref: mpiit-data-router-reporter diff --git a/ci-operator/step-registry/p2p/debug-sleep/OWNERS b/ci-operator/step-registry/p2p/debug-sleep/OWNERS new file mode 100644 index 0000000000000..a95e6bb67979e --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/OWNERS @@ -0,0 +1,3 @@ +approvers: &owners +- cspi-qe-ocp-lp +reviewers: *owners diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh new file mode 100644 index 0000000000000..ed7073c889a80 --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-commands.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Debugging aid: sleep for DEBUG_SLEEP_DURATION seconds so that the CI clusters +# remain alive (and accessible) after test steps finish but before the post-phase +# uninstall/deprovision steps tear everything down. +# +# Set DEBUG_SLEEP_DURATION=0 to skip entirely (default for normal runs). +# Set DEBUG_SLEEP_DURATION=7200 (2h) or any positive integer for debugging runs. +# +set -euo pipefail; shopt -s inherit_errexit + +typeset -i sleepSecs="${DEBUG_SLEEP_DURATION}" + +if (( sleepSecs <= 0 )); then + : "DEBUG_SLEEP_DURATION=0 — skipping debug sleep" + exit 0 +fi + +# ── Print cluster access hints ────────────────────────────────────────────── +: "=== DEBUG SLEEP: clusters will remain alive for ${sleepSecs}s ===" + +# Hub kubeconfig +if [[ -f "${KUBECONFIG}" ]]; then + : "Hub kubeconfig : ${KUBECONFIG}" +fi + +# Spoke kubeconfigs +typeset -i i +for (( i = 1; ; i++ )); do + typeset kcPath="${SHARED_DIR}/managed-cluster-kubeconfig-${i}" + [[ -f "${kcPath}" ]] || break + typeset clusterNameFile="${SHARED_DIR}/managed-cluster-name-${i}" + typeset clusterName="spoke-${i}" + [[ -f "${clusterNameFile}" ]] && clusterName="$(cat "${clusterNameFile}")" + : "Spoke ${i} (${clusterName}) kubeconfig : ${kcPath}" +done + +typeset endTime +endTime="$(date -d "+${sleepSecs} seconds" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null \ + || date -v "+${sleepSecs}S" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null \ + || echo "unknown")" +: "Sleeping until approximately ${endTime} ..." + +sleep "${sleepSecs}" +: "=== DEBUG SLEEP complete ===" diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json new file mode 100644 index 0000000000000..763bd9f13199c --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "p2p/debug-sleep/p2p-debug-sleep-ref.yaml", + "owners": { + "approvers": [ + "cspi-qe-ocp-lp" + ], + "reviewers": [ + "cspi-qe-ocp-lp" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml new file mode 100644 index 0000000000000..07f6b562f5610 --- /dev/null +++ b/ci-operator/step-registry/p2p/debug-sleep/p2p-debug-sleep-ref.yaml @@ -0,0 +1,33 @@ +ref: + as: p2p-debug-sleep + from: cli + commands: p2p-debug-sleep-commands.sh + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: DEBUG_SLEEP_DURATION + default: "0" + documentation: |- + Number of seconds to sleep in the post phase before cluster uninstall/deprovision + begins. Keeps the clusters alive so engineers can SSH in and debug failures. + + Set to 0 (default) to skip the sleep entirely in normal runs. + Set to 7200 for a 2-hour hold during active debugging sessions. + documentation: |- + Debugging aid: sleeps for DEBUG_SLEEP_DURATION seconds in the post phase, after + diagnostic collection (must-gather, inspector, submariner-diag) but before + acm-interop-p2p-cluster-uninstall and ipi-deprovision tear the clusters down. + + This gives engineers time to SSH into hub and spoke clusters, run manual + diagnostic commands, or re-run failing steps interactively before the + infrastructure is destroyed. + + Usage: + - Add as the last step in the post phase before uninstall steps. + - Set DEBUG_SLEEP_DURATION=7200 (or any positive integer) in the job env. + - Set DEBUG_SLEEP_DURATION=0 (or remove the env override) when debugging is done. + + The step is a no-op when DEBUG_SLEEP_DURATION=0 and adds zero overhead to + normal runs. From da1d65b3e2d473c42f689cc7da664b211d0c2a25 Mon Sep 17 00:00:00 2001 From: shakyav Date: Mon, 3 Aug 2026 19:41:17 -0400 Subject: [PATCH 15/15] improve virt handler probe script --- .../p2p-cnv-cclm-readiness-commands.sh | 156 +++++++++++------- .../p2p-cnv-cclm-readiness-ref.yaml | 12 +- 2 files changed, 100 insertions(+), 68 deletions(-) diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh index 6bf7a66e36430..463180739ee6c 100644 --- a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-commands.sh @@ -3,20 +3,29 @@ # Verify and remediate CCLM (Cross-Cluster Live Migration) prerequisites on each spoke. # No-op when CNV_ENABLE_CCLM != "true". # +# Background — CNV bug CNV-89129 / upstream kubevirt#16264: +# virt-handler fails to load /etc/virt-handler/clientcertificates/tls.crt because +# virt-operator does not add the corresponding volume mount to the DaemonSet when the +# DecentralizedLiveMigration feature gate is enabled. The Secret +# (kubevirt-virt-handler-certs) exists and is populated, but is not mounted. +# Workaround: patch the DaemonSet to mount the Secret at the expected path. +# Affected: CNV 4.17+ / OCP 4.22 nightly (as of 2026-08). +# See also: https://github.com/kubevirt/kubevirt/issues/16264 +# # Checks performed (per spoke, in order): -# 1. Restart virt-operator to trigger DaemonSet reconciliation. Required when the -# DecentralizedLiveMigration feature gate does not cause virt-operator to add -# the /etc/virt-handler/clientcertificates volume mount at install time (observed -# on CNV 4.17+ / OCP 4.22 nightly — treated as an operator reconciliation lag). -# 2. Wait for virt-handler DaemonSet rollout to complete. -# 3. Poll until /etc/virt-handler/clientcertificates/tls.crt appears in a -# virt-handler pod (up to CNV_CCLM_WAIT_TIMEOUT seconds). -# 4. Fallback: if cert still absent, patch virt-handler DaemonSet to mount the -# CNV_CCLM_VIRT_HANDLER_CERT_SECRET Secret at /etc/virt-handler/clientcertificates. -# Collects diagnostics and fails the step if cert is still absent after the patch. -# 5. Ensure a Service named virt-synchronization-controller on port CNV_CCLM_SYNC_PORT +# 1. Check whether /etc/virt-handler/clientcertificates/tls.crt is already present. +# If so, skip straight to Service/ServiceExport creation. +# 2. Apply the DaemonSet workaround patch (CNV-89129): +# a. Scale virt-operator to 0 to prevent it from reconciling away the patch. +# b. Patch virt-handler DaemonSet to mount the CNV_CCLM_VIRT_HANDLER_CERT_SECRET +# Secret at /etc/virt-handler/clientcertificates. +# c. Rollout restart the DaemonSet so all pods pick up the new volume mount. +# d. Wait for the rollout to complete. +# e. Verify the cert is present in ALL virt-handler pods. +# f. Scale virt-operator back to its original replica count. +# 3. Ensure a Service named virt-synchronization-controller on port CNV_CCLM_SYNC_PORT # exists in openshift-cnv (created by `oc apply` — idempotent). -# 6. Ensure a ServiceExport for that Service exists (allows Submariner to propagate it +# 4. Ensure a ServiceExport for that Service exists (allows Submariner to propagate it # to other clusters). Skipped gracefully when the ServiceExport CRD is not yet # installed (i.e. Submariner is not up yet on this spoke). # @@ -65,18 +74,27 @@ GetSpokeClusterName() { fi } -# RestartVirtOperatorAndWait — restart virt-operator and wait for rollout. -RestartVirtOperatorAndWait() { +# ScaleVirtOperator — scale virt-operator to desired replicas and wait for rollout. +ScaleVirtOperator() { typeset kubeconfig="${1:?}" typeset clusterName="${2:?}" - : "[${clusterName}] Restarting virt-operator to trigger DaemonSet reconciliation" - oc --kubeconfig="${kubeconfig}" rollout restart deployment/virt-operator \ - -n "${cnvNs}" + typeset -i replicas="${3:?}" + : "[${clusterName}] Scaling virt-operator to ${replicas} replica(s)" + oc --kubeconfig="${kubeconfig}" scale deployment/virt-operator \ + -n "${cnvNs}" --replicas="${replicas}" oc --kubeconfig="${kubeconfig}" rollout status deployment/virt-operator \ - -n "${cnvNs}" --timeout=5m 1>/dev/null + -n "${cnvNs}" --timeout=5m 1>/dev/null || true true } +# GetVirtOperatorReplicas — return the current desired replica count of virt-operator. +GetVirtOperatorReplicas() { + typeset kubeconfig="${1:?}" + oc --kubeconfig="${kubeconfig}" get deployment/virt-operator \ + -n "${cnvNs}" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "2" +} + # WaitVirtHandlerRollout — wait for virt-handler DaemonSet rollout (non-fatal on timeout). WaitVirtHandlerRollout() { typeset kubeconfig="${1:?}" @@ -88,41 +106,60 @@ WaitVirtHandlerRollout() { true } -# CheckVirtHandlerClientCert — return 0 if any virt-handler pod has the CCLM client cert. +# CheckVirtHandlerClientCert — return 0 only when ALL Running virt-handler pods have the cert. +# Checking a single pod is insufficient: after a rollout some pods may be on the new +# spec (with the mount) while others are still on the old spec. CheckVirtHandlerClientCert() { typeset kubeconfig="${1:?}" - typeset podName="" - podName="$(oc --kubeconfig="${kubeconfig}" get pod \ - -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ - -o jsonpath-as-json='{.items[*].metadata.name}' \ - 2>/dev/null | jq -re '.[0]' || true)" - [[ -n "${podName}" ]] || return 1 - oc --kubeconfig="${kubeconfig}" exec -n "${cnvNs}" "${podName}" -- \ - test -f "${clientCertMountPath}/tls.crt" 2>/dev/null + typeset -a pods=() + mapfile -t pods < <( + oc --kubeconfig="${kubeconfig}" get pod \ + -n "${cnvNs}" -l 'kubevirt.io=virt-handler' \ + --field-selector 'status.phase=Running' \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null \ + | tr ' ' '\n' | grep -v '^$' + ) + [[ ${#pods[@]} -gt 0 ]] || return 1 + typeset pod + for pod in "${pods[@]}"; do + oc --kubeconfig="${kubeconfig}" exec -n "${cnvNs}" "${pod}" -- \ + test -f "${clientCertMountPath}/tls.crt" 2>/dev/null || return 1 + done + return 0 } # PatchVirtHandlerDaemonSetCertMount — add clientcertificates volume mount (idempotent). -# Workaround for virt-operator not reconciling the CCLM client cert mount when -# DecentralizedLiveMigration is enabled. Uses a unique volume name -# (cnv-cclm-virt-handler-client-certs) to avoid collisions with operator-managed volumes. +# Workaround for CNV-89129: virt-operator does not add the CCLM client cert mount even +# after the DecentralizedLiveMigration feature gate is enabled. +# +# CRITICAL: virt-operator must be scaled to 0 BEFORE patching. If left running it will +# reconcile the DaemonSet within seconds and remove the patch (observed behaviour on +# CNV 4.17 / OCP 4.22 nightly). The caller is responsible for restoring virt-operator +# replicas after the cert is verified. PatchVirtHandlerDaemonSetCertMount() { typeset kubeconfig="${1:?}" typeset clusterName="${2:?}" + typeset -i originalReplicas + originalReplicas="$(GetVirtOperatorReplicas "${kubeconfig}")" - # Idempotency: skip if mount already present. + # Idempotency: skip if mount already present in DaemonSet spec. typeset existingMountCount=0 existingMountCount="$(oc --kubeconfig="${kubeconfig}" get ds virt-handler \ -n "${cnvNs}" \ -o jsonpath-as-json='{.spec.template.spec.containers[*].volumeMounts[*].mountPath}' \ 2>/dev/null | jq --arg p "${clientCertMountPath}" '[.[] | select(. == $p)] | length')" if ((existingMountCount > 0)); then - : "[${clusterName}] ${clientCertMountPath} already mounted — skipping DaemonSet patch" + : "[${clusterName}] ${clientCertMountPath} already in DaemonSet spec — skipping patch" return 0 fi # Verify the source Secret exists before patching. oc --kubeconfig="${kubeconfig}" get secret "${CNV_CCLM_VIRT_HANDLER_CERT_SECRET}" \ -n "${cnvNs}" 1>/dev/null + + # Scale virt-operator to 0 so it cannot reconcile the patch away. + ScaleVirtOperator "${kubeconfig}" "${clusterName}" 0 + : "[${clusterName}] Patching virt-handler DaemonSet: mounting ${CNV_CCLM_VIRT_HANDLER_CERT_SECRET} at ${clientCertMountPath}" oc --kubeconfig="${kubeconfig}" patch ds virt-handler \ -n "${cnvNs}" --type=json \ @@ -140,12 +177,7 @@ PatchVirtHandlerDaemonSetCertMount() { \"readOnly\":true}} ]" # Explicitly restart to guarantee the DaemonSet controller evicts old pods. - # Without this, `rollout status` can return for the * # Without this, `rollout status` can return for the *previous* generation before - # the controller has processed the patch, leaving old pods (without the new mount) - # in place for the post-patch cert check. - oc --kubeconfig="${kubeconfig}" rollout restart ds/virt-handler -n "${cnvNs}" - true -}previous* generation before + # Without this, rollout status can return for the previous generation before # the controller has processed the patch, leaving old pods (without the new mount) # in place for the post-patch cert check. oc --kubeconfig="${kubeconfig}" rollout restart ds/virt-handler -n "${cnvNs}" @@ -268,39 +300,32 @@ for ((i = 0; i < spokeCount; i++)); do typeset kc="${spokeKubeconfigsArr[i]}" typeset spkName="" spkName="$(GetSpokeClusterName "$((i + 1))")" + typeset -i originalReplicas + originalReplicas="$(GetVirtOperatorReplicas "${kc}")" + typeset virtOperatorRestored=false - # ── Step 1: restart virt-operator ── - RestartVirtOperatorAndWait "${kc}" "${spkName}" - - # ── Step 2: wait for virt-handler DaemonSet rollout ── - WaitVirtHandlerRollout "${kc}" "${spkName}" "10m" - - # ── Step 3: poll for CCLM client cert ── - typeset -i certWaitSecs=0 + # ── Step 1: check if cert already present (fast path — no patching needed) ── typeset isCertOk=false - while (( certWaitSecs < waitTimeoutSecs )); do - if CheckVirtHandlerClientCert "${kc}"; then - isCertOk=true - break - fi - : "[${spkName}] Waiting for virt-handler ${clientCertMountPath}/tls.crt (${certWaitSecs}/${waitTimeoutSecs}s)" - sleep "${pollIntervalSecs}" - (( certWaitSecs += pollIntervalSecs )) - done + if CheckVirtHandlerClientCert "${kc}"; then + isCertOk=true + : "[${spkName}] virt-handler CCLM client cert already present — skipping patch" + fi - # ── Step 4: fallback DaemonSet patch if cert still absent ── + # ── Step 2: apply DaemonSet workaround patch if cert absent (CNV-89129) ── + # virt-operator MUST be scaled to 0 before patching to prevent immediate reconciliation. if [[ "${isCertOk}" != "true" ]]; then - : "[${spkName}] Cert absent after virt-operator restart — applying DaemonSet workaround patch" + : "[${spkName}] Cert absent — applying DaemonSet workaround patch (CNV-89129)" if ! PatchVirtHandlerDaemonSetCertMount "${kc}" "${spkName}"; then : "[${spkName}] DaemonSet patch failed" SaveCclmReadinessDiagnostics "${kc}" "${spkName}" + # Restore virt-operator before moving on so the cluster stays functional. + ScaleVirtOperator "${kc}" "${spkName}" "${originalReplicas}" || true (( ++failedCount )) continue fi WaitVirtHandlerRollout "${kc}" "${spkName}" "10m" - certWaitSecs=0 - isCertOk=false + typeset -i certWaitSecs=0 while (( certWaitSecs < 300 )); do if CheckVirtHandlerClientCert "${kc}"; then isCertOk=true @@ -310,23 +335,28 @@ for ((i = 0; i < spokeCount; i++)); do sleep "${pollIntervalSecs}" (( certWaitSecs += pollIntervalSecs )) done + + # Restore virt-operator now that cert is confirmed (or we are about to fail). + # Do this before the fatal check so the cluster stays operational even on failure. + ScaleVirtOperator "${kc}" "${spkName}" "${originalReplicas}" || true + virtOperatorRestored=true fi if [[ "${isCertOk}" != "true" ]]; then - : "[${spkName}] FATAL: ${clientCertMountPath}/tls.crt absent after all remediation" + : "[${spkName}] FATAL: ${clientCertMountPath}/tls.crt absent in one or more virt-handler pods after all remediation" SaveCclmReadinessDiagnostics "${kc}" "${spkName}" (( ++failedCount )) continue fi - : "[${spkName}] virt-handler CCLM client cert present" + : "[${spkName}] virt-handler CCLM client cert present in all Running pods" - # ── Step 5: ensure Service on CNV_CCLM_SYNC_PORT ── + # ── Step 3: ensure Service on CNV_CCLM_SYNC_PORT ── if ! EnsureVirtSyncService "${kc}" "${spkName}"; then : "[${spkName}] WARN: failed to ensure virt-synchronization-controller Service" (( ++failedCount )) fi - # ── Step 6: ensure ServiceExport (no-op if Submariner CRD not available) ── + # ── Step 4: ensure ServiceExport (no-op if Submariner CRD not available) ── EnsureVirtSyncServiceExport "${kc}" "${spkName}" || true : "[${spkName}] CCLM readiness checks complete" diff --git a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml index 4bbcf100a1151..0569e94550dd2 100644 --- a/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml +++ b/ci-operator/step-registry/p2p/cnv-cclm-readiness/p2p-cnv-cclm-readiness-ref.yaml @@ -38,15 +38,17 @@ ref: every spoke cluster before migration steps run. No-op when CNV_ENABLE_CCLM is not "true". - Background — root cause observed on OCP 4.22 nightly with CNV 4.17+: - When decentralizedLiveMigration=true is set in HyperConverged featureGates at - install time, virt-operator may not reconcile the virt-handler DaemonSet to add - the /etc/virt-handler/clientcertificates volume mount in its first pass. Without - this mount, virt-handler emits "failed to load the certificate + Background — CNV bug CNV-89129 / upstream kubevirt#16264 + (https://github.com/kubevirt/kubevirt/issues/16264): + virt-operator does not add the /etc/virt-handler/clientcertificates volume mount + to the virt-handler DaemonSet when DecentralizedLiveMigration is enabled, even + though the Secret (kubevirt-virt-handler-certs) exists and is populated. + Without this mount, virt-handler emits "failed to load the certificate /etc/virt-handler/clientcertificates/tls.crt: no such file or directory" every 60 s and cannot make the required mTLS connection to the destination virt-synchronization-controller:9185, leaving the migration stuck in Synchronizing/WaitingForSync indefinitely. + Affected: CNV 4.17+ / OCP 4.22 (as of 2026-08). What this step does (idempotent, safe to re-run): 1. Restarts virt-operator on each spoke and waits for its rollout to finish