diff --git a/ci-operator/config/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main__ocm-fvt-osdfm-integration.yaml b/ci-operator/config/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main__ocm-fvt-osdfm-integration.yaml index 6478cf28759bb..11c25e14a6501 100644 --- a/ci-operator/config/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main__ocm-fvt-osdfm-integration.yaml +++ b/ci-operator/config/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main__ocm-fvt-osdfm-integration.yaml @@ -33,6 +33,32 @@ resources: cpu: 100m memory: 200Mi tests: +- as: ocm-fvt-gating-osdfm-integration-canary + capabilities: + - intranet + - nested-podman + cluster: build05 + cron: 0 0 31 2 * + nested_podman: true + reporter_config: + channel: '#rosa-prow-info' + job_states_to_report: + - success + - failure + - error + restrict_network_access: false + steps: + env: + OCM_FVT_BACKPLANE_CREDENTIALS_DIR: /usr/local/rosa-clusters-service-sandbox + OCM_FVT_EXTRA_ENVS: SECTOR=canary + OCM_FVT_JOB_NAME: osdfm-basic-integration + OCM_FVT_OCM_ENV: integration + OCM_FVT_REPORT_JIRA: "false" + OCM_FVT_SERVICE: osdfm + OCM_FVT_USE_BACKPLANE: "true" + test: + - ref: rosa-e2e-ocm-fvt-longrunning + timeout: 4h0m0s - as: ocm-fvt-periodic-osdfm-basic-periodic-integration capabilities: - intranet diff --git a/ci-operator/jobs/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main-periodics.yaml b/ci-operator/jobs/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main-periodics.yaml index 7a2abca493201..a8aa7b43a447e 100644 --- a/ci-operator/jobs/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main-periodics.yaml +++ b/ci-operator/jobs/openshift-online/rosa-e2e/openshift-online-rosa-e2e-main-periodics.yaml @@ -497,6 +497,98 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 0 0 31 2 * + decorate: true + decoration_config: + skip_cloning: true + timeout: 4h0m0s + extra_refs: + - base_ref: main + org: openshift-online + repo: rosa-e2e + labels: + capability/intranet: intranet + capability/nested-podman: nested-podman + ci-operator.openshift.io/cluster: build05 + ci-operator.openshift.io/variant: ocm-fvt-osdfm-integration + ci.openshift.io/generator: prowgen + job-release: "4.22" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-online-rosa-e2e-main-ocm-fvt-osdfm-integration-ocm-fvt-gating-osdfm-integration-canary + reporter_config: + slack: + channel: '#rosa-prow-info' + job_states_to_report: + - success + - failure + - error + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=ocm-fvt-gating-osdfm-integration-canary + - --variant=ocm-fvt-osdfm-integration + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build10 cron: 0 9 * * 1 diff --git a/ci-operator/step-registry/rosa/e2e/ocm-fvt/longrunning/rosa-e2e-ocm-fvt-longrunning-ref.yaml b/ci-operator/step-registry/rosa/e2e/ocm-fvt/longrunning/rosa-e2e-ocm-fvt-longrunning-ref.yaml index ec1cb143a65ca..dfe52cd84f03b 100644 --- a/ci-operator/step-registry/rosa/e2e/ocm-fvt/longrunning/rosa-e2e-ocm-fvt-longrunning-ref.yaml +++ b/ci-operator/step-registry/rosa/e2e/ocm-fvt/longrunning/rosa-e2e-ocm-fvt-longrunning-ref.yaml @@ -12,6 +12,9 @@ ref: - namespace: ci name: osdfm-qe-credentials mount_path: /usr/local/osdfm-qe-credentials + - namespace: ci + name: rosa-clusters-service-sandbox + mount_path: /usr/local/rosa-clusters-service-sandbox resources: requests: cpu: 100m @@ -50,6 +53,38 @@ ref: Optional. Set to "true" to mount the GCP credentials file (osd-ccs-admin.json) into the podman container at /home/ci-user/.gcp/osd-ccs-admin.json. Required for OSD GCP tests. + - name: OCM_FVT_USE_BACKPLANE + default: "false" + documentation: |- + Optional. Prow-only. Set to "true" to enable corp proxy (and optional + backplane login) so vault aws_account_operator_kubeconfig can reach + Hive. AAO secrets still use that vault consumer kubeconfig (Tekton + parity), not the elevated backplane kubeconfig. Leave "false" when + Hive is not needed. + - name: OCM_FVT_BACKPLANE_CLUSTER_ID + default: "1g268u7pp694gj152nj16me4sv615lpv" + documentation: |- + Optional. OCM cluster id passed to ocm-backplane login. Default is + hivei01ue1 (integration Hive). + - name: OCM_FVT_BACKPLANE_CREDENTIALS_DIR + default: "/usr/local/rosa-clusters-service-sandbox" + documentation: |- + Optional. Directory containing backplane_client_id and + backplane_client_secret. + - name: OCM_FVT_BACKPLANE_OCM_URL + default: "https://api.openshift.com" + documentation: |- + Optional. OCM API URL used for ocm login before backplane login. + Default is production API (hivei01ue1 is registered there). + - name: OCM_FVT_BACKPLANE_PROXY_URL + default: "http://squid.corp.redhat.com:3128" + documentation: |- + Optional. HTTP(S) proxy for backplane/corp access from the build + farm. Default is the Red Hat corp squid proxy. + - name: OCM_FVT_BACKPLANE_ELEVATE_REASON + default: "https://issues.redhat.com/browse/ROSAENG-62717" + documentation: |- + Optional. Reason/URL passed to ocm-backplane elevate for audit. documentation: |- Runs an OCM FVT (Functional Verification Test) using ocmtest inside a nested podman container. Same as rosa-e2e-ocm-fvt but with a 12h diff --git a/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-commands.sh b/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-commands.sh index 97688688407da..99e69b0d8ef7d 100755 --- a/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-commands.sh +++ b/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-commands.sh @@ -15,10 +15,106 @@ else JOB_LINK="${JOB_LINK}logs/${JOB_NAME}/${BUILD_ID}" fi +# Prow-only: Hive via ocm-backplane when OCM_FVT_USE_BACKPLANE=true (Jenkins/Tekton skip this). +hive_kubeconfig="" +hive_kubeconfig_src="" +backplane_bin_dir="" +backplane_proxy_url="" +if [[ "${OCM_FVT_USE_BACKPLANE:-false}" == "true" ]]; then + echo "=== OCM backplane login (Hive) ===" + # (BACKPLANE_CLIENT_ID/SECRET env, or backplane_client_{id,secret} files). + cred_dir="${OCM_FVT_BACKPLANE_CREDENTIALS_DIR:-/usr/local/rosa-clusters-service-sandbox}" + # Disable tracing while reading backplane client credentials. + [[ $- == *x* ]] && WAS_TRACING_BP=true || WAS_TRACING_BP=false + set +x + backplane_client_id="${BACKPLANE_CLIENT_ID:-}" + backplane_client_secret="${BACKPLANE_CLIENT_SECRET:-}" + if [[ -z "${backplane_client_id}" && -f "${cred_dir}/backplane_client_id" ]]; then + backplane_client_id="$(cat "${cred_dir}/backplane_client_id")" + fi + if [[ -z "${backplane_client_secret}" && -f "${cred_dir}/backplane_client_secret" ]]; then + backplane_client_secret="$(cat "${cred_dir}/backplane_client_secret")" + fi + $WAS_TRACING_BP && set -x + if [[ -z "${backplane_client_id}" || -z "${backplane_client_secret}" ]]; then + echo "ERROR: OCM_FVT_USE_BACKPLANE=true but backplane client credentials are missing" >&2 + echo "Expected BACKPLANE_CLIENT_ID/SECRET env or ${cred_dir}/backplane_client_{id,secret}" >&2 + echo "(CS mounts these from ci/rosa-clusters-service-sandbox)" >&2 + exit 1 + fi + echo "Using backplane credentials from ${cred_dir} (or env)" + + # Defaults match rosa-e2e-ocm-fvt*-ref.yaml; override via job/workflow env if needed. + backplane_cluster_id="${OCM_FVT_BACKPLANE_CLUSTER_ID:-1g268u7pp694gj152nj16me4sv615lpv}" + backplane_ocm_url="${OCM_FVT_BACKPLANE_OCM_URL:-https://api.openshift.com}" + backplane_proxy_url="${OCM_FVT_BACKPLANE_PROXY_URL:-http://squid.corp.redhat.com:3128}" + backplane_elevate_reason="${OCM_FVT_BACKPLANE_ELEVATE_REASON:-https://issues.redhat.com/browse/ROSAENG-62717}" + + backplane_bin_dir="$(mktemp -d /tmp/ocm-backplane-bin.XXXXXX)" + export PATH="${backplane_bin_dir}:${PATH}" + + echo "Installing ocm CLI into ${backplane_bin_dir}" + curl -sSL -o "${backplane_bin_dir}/ocm" \ + "https://github.com/openshift-online/ocm-cli/releases/download/v1.0.15/ocm-linux-amd64" + chmod 0755 "${backplane_bin_dir}/ocm" + + echo "Installing ocm-backplane CLI into ${backplane_bin_dir}" + bp_ver="0.11.0" + bp_tar="$(mktemp /tmp/ocm-backplane.XXXXXX.tar.gz)" + curl -sSL -o "${bp_tar}" \ + "https://github.com/openshift/backplane-cli/releases/download/v${bp_ver}/ocm-backplane_${bp_ver}_Linux_x86_64.tar.gz" + tar -xzf "${bp_tar}" -C "${backplane_bin_dir}" ocm-backplane + chmod 0755 "${backplane_bin_dir}/ocm-backplane" + rm -f "${bp_tar}" + + # nested-podman image has no oc; ocm-backplane login/elevate invoke it. + echo "Installing oc CLI into ${backplane_bin_dir}" + oc_tar="$(mktemp /tmp/openshift-client.XXXXXX.tar.gz)" + curl -sSL -o "${oc_tar}" \ + "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz" + tar -xzf "${oc_tar}" -C "${backplane_bin_dir}" oc + chmod 0755 "${backplane_bin_dir}/oc" + rm -f "${oc_tar}" + + mkdir -p "${HOME}/.config/backplane" + printf '{"proxy-url":"%s"}\n' "${backplane_proxy_url}" > "${HOME}/.config/backplane/config.json" + + # Pin kubeconfig path — Prow may set KUBECONFIG elsewhere; sed must read the same file login writes. + hive_kubeconfig_src="$(mktemp /tmp/backplane-kubeconfig.XXXXXX)" + rm -f "${hive_kubeconfig_src}" + export KUBECONFIG="${hive_kubeconfig_src}" + + # Disable tracing due to client-secret handling on ocm login. + [[ $- == *x* ]] && WAS_TRACING_BP=true || WAS_TRACING_BP=false + set +x + ocm login \ + --client-id="${backplane_client_id}" \ + --client-secret="${backplane_client_secret}" \ + --url="${backplane_ocm_url}" + ocm-backplane login "${backplane_cluster_id}" + $WAS_TRACING_BP && set -x + ocm-backplane elevate "${backplane_elevate_reason}" -- whoami + + if [[ ! -f "${KUBECONFIG}" ]]; then + echo "ERROR: backplane login did not write kubeconfig at ${KUBECONFIG}" >&2 + exit 1 + fi + + hive_kubeconfig="$(mktemp /tmp/hive-kubeconfig.XXXXXX)" + chmod 0600 "${hive_kubeconfig}" + # Rewrite exec plugin command to the path mounted inside the ocmci container. + sed -E \ + -e 's|command:[[:space:]]*ocm-backplane([[:space:]]*$)|command: /usr/local/backplane-bin/ocm-backplane\1|' \ + -e 's|command:[[:space:]]*ocm([[:space:]]*$)|command: /usr/local/backplane-bin/ocm\1|' \ + "${KUBECONFIG}" > "${hive_kubeconfig}" + echo "Backplane kubeconfig ready for cluster ${backplane_cluster_id}" + echo "================================" +fi + old_umask=$(umask) umask 077 podman_env_file="$(mktemp /tmp/podman.env.XXXXXX)" -trap 'rm -f "${podman_env_file}"' EXIT +trap 'rm -f "${podman_env_file}"; rm -f "${hive_kubeconfig:-}" "${hive_kubeconfig_src:-}"' EXIT umask "${old_umask}" { @@ -29,6 +125,16 @@ umask "${old_umask}" echo "CONSOLE_CLIENT_SECRET=$(cat /usr/local/cs-qe-credentials/console_client_secret)" } > "${podman_env_file}" +if [[ -n "${hive_kubeconfig}" ]]; then + # Proxy for Hive reachability; AAO identity comes from vault, not elevate. + echo "PATH=/usr/local/backplane-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "${podman_env_file}" + echo "HOME=/home/ci-user" >> "${podman_env_file}" + echo "HTTPS_PROXY=${backplane_proxy_url}" >> "${podman_env_file}" + echo "HTTP_PROXY=${backplane_proxy_url}" >> "${podman_env_file}" + echo "https_proxy=${backplane_proxy_url}" >> "${podman_env_file}" + echo "http_proxy=${backplane_proxy_url}" >> "${podman_env_file}" +fi + if [[ "${OCM_FVT_REPORT_JIRA:-true}" == "true" ]]; then echo "ENABLE_JIRA_REPORTING=true" >> "${podman_env_file}" fi @@ -46,11 +152,16 @@ fi osdfm_qe_creds_dir=/usr/local/osdfm-qe-credentials aao_kubeconfig_env=() +# Vault consumer kubeconfig (Tekton parity): live per-MC/SC AAO secrets, all regions. if [[ -f "${osdfm_qe_creds_dir}/aws_account_operator_kubeconfig" ]]; then [[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false set +x aao_kubeconfig_env=("-e" "AWS_ACCOUNT_OPERATOR_KUBECONFIG=$(<"${osdfm_qe_creds_dir}/aws_account_operator_kubeconfig")") $WAS_TRACING && set -x +elif [[ "${OCM_FVT_SERVICE:-}" == "osdfm" && -n "${hive_kubeconfig}" ]]; then + echo "ERROR: osdfm AAO tests need ${osdfm_qe_creds_dir}/aws_account_operator_kubeconfig" >&2 + echo "Elevated backplane kubeconfig cannot get secrets in osd-fleet-manager-aao." >&2 + exit 1 fi # DR_AWS_CREDENTIALS lets DR-account validation tests (e.g. OSDFM disaster_recovery_test.go, @@ -80,6 +191,14 @@ podman_args=( "-v" "/usr/local/cs-qe-credentials:/credentials:ro,z" ) +if [[ -n "${hive_kubeconfig}" ]]; then + podman_args+=( + "-v" "${hive_kubeconfig}:/credentials-hive/kubeconfig:ro,z" + "-v" "${backplane_bin_dir}:/usr/local/backplane-bin:ro,z" + "-v" "${HOME}/.config:/home/ci-user/.config:ro,z" + ) +fi + if [[ "${OCM_FVT_GCP_CREDS:-false}" == "true" ]]; then podman_args+=( "-v" "/usr/local/cs-qe-credentials/osd-ccs-admin.json:/home/ci-user/.gcp/osd-ccs-admin.json:ro,z" @@ -107,6 +226,47 @@ podman inspect \ || echo "WARNING: failed to get ocmci image digest" echo "==========================" +# Temporary AAO/Hive reachability probe (remove after ROSAENG-62716 diagnosis). +# Uses the same nested container env as ocmtest. Keep xtrace off while aao +# kubeconfig is on argv; never cat/echo the kubeconfig or copy it to ARTIFACT_DIR. +# Prefer process substitution over a named kubeconfig file so the SA token is +# not left on disk (tests already materialize it via SetupHiveAAOKubeconfig). +if [[ "${OCM_FVT_SERVICE:-}" == "osdfm" && ${#aao_kubeconfig_env[@]} -gt 0 ]]; then + echo "=== AAO Hive reachability probe ===" + [[ $- == *x* ]] && WAS_TRACING_PROBE=true || WAS_TRACING_PROBE=false + set +x + podman run --rm \ + "${podman_args[@]}" \ + "${aao_kubeconfig_env[@]}" \ + quay.io/redhat-services-prod/rosa-tenant/rosa-backend-tests/rosa-backend-tests:latest \ + bash -lc ' + echo "--- proxy env ---" + env | grep -iE "^(HTTPS?_PROXY|https?_proxy|NO_PROXY|no_proxy)=" || echo "(no proxy env)" + echo "--- DNS ---" + getent hosts api.hivei01ue1.f7i5.p1.openshiftapps.com || true + echo "--- curl via HTTPS_PROXY ---" + if [[ -n "${HTTPS_PROXY:-}" ]]; then + curl -v -x "${HTTPS_PROXY}" --connect-timeout 10 \ + "https://api.hivei01ue1.f7i5.p1.openshiftapps.com:6443/" || true + else + echo "HTTPS_PROXY unset; skipping curl" + fi + echo "--- oc with consumer kubeconfig ---" + if [[ -z "${AWS_ACCOUNT_OPERATOR_KUBECONFIG:-}" ]]; then + echo "AWS_ACCOUNT_OPERATOR_KUBECONFIG unset; skipping oc" + elif ! command -v oc >/dev/null 2>&1; then + echo "oc not in PATH; skipping oc" + else + # /dev/fd process substitution — no named file under /tmp + oc --kubeconfig <(printf "%s" "$AWS_ACCOUNT_OPERATOR_KUBECONFIG") \ + --request-timeout=15s \ + get secrets -n osd-fleet-manager-aao 2>&1 | head -20 || true + fi + ' || echo "WARNING: AAO Hive reachability probe failed" + $WAS_TRACING_PROBE && set -x + echo "=== end probe ===" +fi + echo "Running ocmtest: ${ocmtest_args[*]}" exit_code=0 # aao_kubeconfig_env / dr_aws_creds_env may hold raw secret contents as literal @@ -132,4 +292,4 @@ done # whether it is safe to act on this run instead of always running. echo "${exit_code}" > "${SHARED_DIR}/ocm-fvt-exit-code" 2>/dev/null || true -exit "${exit_code}" \ No newline at end of file +exit "${exit_code}" diff --git a/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-ref.yaml b/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-ref.yaml index 574e2e44a29db..49820bed636f4 100644 --- a/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-ref.yaml +++ b/ci-operator/step-registry/rosa/e2e/ocm-fvt/rosa-e2e-ocm-fvt-ref.yaml @@ -47,6 +47,39 @@ ref: Optional. Set to "true" to mount the GCP credentials file (osd-ccs-admin.json) into the podman container at /home/ci-user/.gcp/osd-ccs-admin.json. Required for OSD GCP tests. + - name: OCM_FVT_USE_BACKPLANE + default: "false" + documentation: |- + Optional. Prow-only. Set to "true" to enable corp proxy (and optional + backplane login) so vault aws_account_operator_kubeconfig can reach + Hive. AAO secrets still use that vault consumer kubeconfig (Tekton + parity), not the elevated backplane kubeconfig. Leave "false" when + Hive is not needed. + - name: OCM_FVT_BACKPLANE_CLUSTER_ID + default: "1g268u7pp694gj152nj16me4sv615lpv" + documentation: |- + Optional. OCM cluster id passed to ocm-backplane login. Default is + hivei01ue1 (integration Hive). + - name: OCM_FVT_BACKPLANE_CREDENTIALS_DIR + default: "/usr/local/rosa-clusters-service-sandbox" + documentation: |- + Optional. Directory containing backplane_client_id and + backplane_client_secret.Mount that secret on the + ref when OCM_FVT_USE_BACKPLANE=true (see longrunning ref). + - name: OCM_FVT_BACKPLANE_OCM_URL + default: "https://api.openshift.com" + documentation: |- + Optional. OCM API URL used for ocm login before backplane login. + Default is production API (hivei01ue1 is registered there). + - name: OCM_FVT_BACKPLANE_PROXY_URL + default: "http://squid.corp.redhat.com:3128" + documentation: |- + Optional. HTTP(S) proxy for backplane/corp access from the build + farm. Default is the Red Hat corp squid proxy. + - name: OCM_FVT_BACKPLANE_ELEVATE_REASON + default: "https://issues.redhat.com/browse/ROSAENG-62717" + documentation: |- + Optional. Reason/URL passed to ocm-backplane elevate for audit. documentation: |- Runs an OCM FVT (Functional Verification Test) using ocmtest inside a nested podman container. The ocmtest tool is pulled from