diff --git a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml index 94aca7b66dab1..17b97b00fe48e 100644 --- a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml +++ b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml @@ -121,6 +121,7 @@ tests: steps: cluster_profile: metal-dpf-doca8 test: + - ref: dpf-hypervisor-load-kubeconfig - ref: dpf-hypervisor-network-tests timeout: 2h0m0s - always_run: false diff --git a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml index 041860266b718..3f3915d993072 100644 --- a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml +++ b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml @@ -98,6 +98,7 @@ tests: steps: cluster_profile: metal-dpf-doca8 test: + - ref: dpf-hypervisor-load-kubeconfig - ref: dpf-hypervisor-network-tests timeout: 2h0m0s - always_run: false diff --git a/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh b/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh index 6af2880058a62..1184b47ef8371 100755 --- a/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh @@ -46,6 +46,7 @@ fi # Copy the cluster kubeconfig from the last install dir on the hypervisor echo "=== Copying kubeconfig from ${LAST_OPENSHIFT_DPF} on hypervisor ===" scp ${SSH_OPTS} root@${REMOTE_HOST}:${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME} /tmp/kubeconfig.${CLUSTER_NAME} +scp ${SSH_OPTS} root@${REMOTE_HOST}:${LAST_OPENSHIFT_DPF}/.env "${SHARED_DIR}/.env" # The kubeconfig's API server is addressed by an internal hostname that is # not resolvable from the CI cluster's network. Resolve it from the diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh index ca0278a8ba5ee..700beca2b3152 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh @@ -1,159 +1,19 @@ #!/bin/bash set -euo pipefail -# Configuration -REMOTE_HOST="${REMOTE_HOST:-10.6.135.45}" -CLUSTER_NAME=$(cat "${CLUSTER_PROFILE_DIR}/cluster-name") +cp "${SHARED_DIR}/.env" . -echo "Setting up SSH access to DPF hypervisor: ${REMOTE_HOST}" +echo "=== Run DPF Kubernetes Traffic Flow Tests ===" -# Prepare SSH key from Vault (add trailing newline if missing) -echo "Configuring SSH private key..." -cat /var/run/dpf-ci/private-key | base64 -d > /tmp/id_rsa -echo "" >> /tmp/id_rsa -chmod 600 /tmp/id_rsa +echo "Verifying cluster access..." +oc get nodes -# Define SSH command with explicit options (don't rely on ~/.ssh/config) -SSH_OPTS="-i /tmp/id_rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=30 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes" - -# Test SSH connection -echo "Testing SSH connection to ${REMOTE_HOST}..." -if ssh ${SSH_OPTS} root@${REMOTE_HOST} echo 'SSH connection successful'; then - echo "SSH setup complete and tested successfully" -else - echo "ERROR: Failed to connect to hypervisor ${REMOTE_HOST}" - echo "Debug information:" - echo "- Checking if SSH key exists:" - ls -la /tmp/id_rsa - echo "- Testing SSH connectivity with verbose output:" - ssh -v ${SSH_OPTS} root@${REMOTE_HOST} echo 'test' || true - exit 1 -fi - -# Export SSH settings for subsequent steps -echo "REMOTE_HOST=${REMOTE_HOST}" >> ${SHARED_DIR}/dpf-env -echo "SSH_OPTS=${SSH_OPTS}" >> ${SHARED_DIR}/dpf-env -echo "SSH setup completed successfully for ${REMOTE_HOST}" - -echo "Remote host: ${REMOTE_HOST}" - -datetime_string=$(date +"%Y-%m-%d_%H-%M-%S") -NETWORK_TESTS_RESULT="" - -# Run dpf make target checks test -REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION="/root/${CLUSTER_NAME}/ci/last-openshift-dpf-dir.sh" - -echo "=== DPF Make Target checks on Existing Cluster ===" -echo "Using openshift-dpf dir from last cluster-deploy: '${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}'" - -echo "Updating VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variable values to true, 4 and 3 respectively in .env file" -# Using delimiter '|' since we have '/' in the patterns -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -e; \ - test -f ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd ; \ - set -e; \ - test -f .env ; \ - cat .env | grep VERIFY ; \ - cp .env .env_orig ; \ - sed -i 's|VERIFY_DEPLOYMENT=.*|VERIFY_DEPLOYMENT=true|' .env ; \ - sed -i 's|VERIFY_MAX_RETRIES=.*|VERIFY_MAX_RETRIES=4|' .env ; \ - sed -i 's|VERIFY_SLEEP_SECONDS=.*|VERIFY_SLEEP_SECONDS=3|' .env ; \ - cat .env | grep VERIFY"; then - echo "VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variables updated successfully in .env file" -else - echo "ERROR: Failed to update VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variables in .env file" - exit 1 -fi - - -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -a; \ - pwd; \ - ls -ltr; \ - env; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd; \ - set -e;\ - export KUBECONFIG=\${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME}; \ - oc get co; \ - oc get nodes; \ - oc get dpu -A; \ - oc get application -A; \ - echo \${KUBECONFIG}; \ - ls -ltr ; \ - make verify-workers; \ - make verify-dpu-nodes; \ - make verify-deployment; \ - make verify-dpudeployment; \ - echo \$? > verification-result"; then - - echo "DPF spot check tests Passed"; - -else - echo "DPF spot checks tests Failed"; - exit 1 -fi - - -echo "=== Run DPF Kubernetes Traffic Flow Tests on Existing Cluster ===" - -# Need to export the dpu-workers after they are renamed, on hypervisor: -# export TFT_SERVER_NODE=worker-303ea713ea90 -# export TFT_CLIENT_NODE=worker-303ea713ea94 - -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -euo pipefail; \ - ls -ltr; \ - env; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd; \ - ls -ltra; \ - set -e;\ - export KUBECONFIG=\${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME}; \ - oc get nodes; \ - export TFT_SERVER_NODE=\$(oc get nodes | grep worker-dpu | awk 'NR==1 {print \$1}'); \ - echo \${TFT_SERVER_NODE} ;\ - export TFT_CLIENT_NODE=\$(oc get nodes | grep worker-dpu | awk 'NR==2 {print \$1}'); \ - echo \${TFT_CLIENT_NODE} ; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - make run-traffic-flow-tests 2>&1 | tee log-traffic-flow-tests-${datetime_string}"; then - - echo "Kubernetes Network Traffic Flow Iperf Tests Passed"; - NETWORK_TESTS_RESULT="PASS" - -else - echo "Kubernetes Network Traffic Flow Iperf Tests Failed"; - -fi - -echo "====== Output DPF Kubernetes Traffic Flow Tests Log file:" -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - cat log-traffic-flow-tests-${datetime_string}"; then - - echo "Successfully output Kubernetes Network Traffic Flow Iperf Tests logs"; - -else - echo "Failed to output DPF kubernetes Traffic Flow Iperf Tests logs"; - -fi - -# Parse the log files, may need to scp to container running ssh cmds and process the -# output file and exit accordingly - -if [ "${NETWORK_TESTS_RESULT}" == "PASS" ]; then - exit 0 -fi - -exit 1 +export TFT_KUBECONFIG="${SHARED_DIR}/kubeconfig" +TFT_SERVER_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==1 {print $1}') +export TFT_SERVER_NODE +echo "TFT_SERVER_NODE: ${TFT_SERVER_NODE}" +TFT_CLIENT_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==2 {print $1}') +export TFT_CLIENT_NODE +echo "TFT_CLIENT_NODE: ${TFT_CLIENT_NODE}" +make run-traffic-flow-tests diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml index 5c9358eceb792..577c8ae86a589 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml @@ -1,29 +1,16 @@ ref: as: dpf-hypervisor-network-tests commands: dpf-hypervisor-network-tests-commands.sh - credentials: - - mount_path: /var/run/dpf-ci - name: cluster-secrets-metal-dpf-doca8 - namespace: ci documentation: |- - Run Kubernetes Network Flow tests on an EXISTING cluster without provisioning. + Run Kubernetes Network Traffic Flow tests on an existing DPF cluster. - This step: - 1. Sets up SSH access to hypervisor - 2. Finds the last DPF cluster install openshift-dpf dir - 3. Runs 'make run-traffic-flow-tests' on hypervisor - 4. Collect and output logs from hypervisor + This step expects a kubeconfig at ${SHARED_DIR}/kubeconfig (provided by + dpf-hypervisor-load-kubeconfig) and runs 'make run-traffic-flow-tests' + from the dpf-ci image. - Use this for quick validation on doca8 or other existing clusters. Does NOT create or destroy any cluster resources. - Required Vault secrets: - - private-key: SSH key for hypervisor - Expected duration: 60-120 minutes - env: - - default: "10.6.135.45" - name: REMOTE_HOST from: dpf-ci resources: requests: