From cbafa98026084bef70e4090327cdce800dbf2faa Mon Sep 17 00:00:00 2001 From: Jose Castillo Lema Date: Fri, 31 Jul 2026 18:12:37 +0200 Subject: [PATCH 1/4] DPF: Run network tests directly from Prow Signed-off-by: Jose Castillo Lema gitleaks.version: 8.30.0 gitleaks.check-secrets: ENABLED --- .../rh-ecosystem-edge-openshift-dpf-main.yaml | 1 + ...ystem-edge-openshift-dpf-release-4.22.yaml | 1 + .../dpf-hypervisor-network-tests-commands.sh | 161 +----------------- .../dpf-hypervisor-network-tests-ref.yaml | 21 +-- 4 files changed, 14 insertions(+), 170 deletions(-) diff --git a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml index 94aca7b66dab1..17b97b00fe48e 100644 --- a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml +++ b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-main.yaml @@ -121,6 +121,7 @@ tests: steps: cluster_profile: metal-dpf-doca8 test: + - ref: dpf-hypervisor-load-kubeconfig - ref: dpf-hypervisor-network-tests timeout: 2h0m0s - always_run: false diff --git a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml index 041860266b718..3f3915d993072 100644 --- a/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml +++ b/ci-operator/config/rh-ecosystem-edge/openshift-dpf/rh-ecosystem-edge-openshift-dpf-release-4.22.yaml @@ -98,6 +98,7 @@ tests: steps: cluster_profile: metal-dpf-doca8 test: + - ref: dpf-hypervisor-load-kubeconfig - ref: dpf-hypervisor-network-tests timeout: 2h0m0s - always_run: false diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh index ca0278a8ba5ee..c50e6befd817c 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh @@ -1,159 +1,14 @@ #!/bin/bash set -euo pipefail -# Configuration -REMOTE_HOST="${REMOTE_HOST:-10.6.135.45}" -CLUSTER_NAME=$(cat "${CLUSTER_PROFILE_DIR}/cluster-name") +echo "=== Run DPF Kubernetes Traffic Flow Tests ===" -echo "Setting up SSH access to DPF hypervisor: ${REMOTE_HOST}" +echo "Verifying cluster access..." +oc get nodes -# Prepare SSH key from Vault (add trailing newline if missing) -echo "Configuring SSH private key..." -cat /var/run/dpf-ci/private-key | base64 -d > /tmp/id_rsa -echo "" >> /tmp/id_rsa -chmod 600 /tmp/id_rsa - -# Define SSH command with explicit options (don't rely on ~/.ssh/config) -SSH_OPTS="-i /tmp/id_rsa -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=30 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes" - -# Test SSH connection -echo "Testing SSH connection to ${REMOTE_HOST}..." -if ssh ${SSH_OPTS} root@${REMOTE_HOST} echo 'SSH connection successful'; then - echo "SSH setup complete and tested successfully" -else - echo "ERROR: Failed to connect to hypervisor ${REMOTE_HOST}" - echo "Debug information:" - echo "- Checking if SSH key exists:" - ls -la /tmp/id_rsa - echo "- Testing SSH connectivity with verbose output:" - ssh -v ${SSH_OPTS} root@${REMOTE_HOST} echo 'test' || true - exit 1 -fi - -# Export SSH settings for subsequent steps -echo "REMOTE_HOST=${REMOTE_HOST}" >> ${SHARED_DIR}/dpf-env -echo "SSH_OPTS=${SSH_OPTS}" >> ${SHARED_DIR}/dpf-env -echo "SSH setup completed successfully for ${REMOTE_HOST}" - -echo "Remote host: ${REMOTE_HOST}" - -datetime_string=$(date +"%Y-%m-%d_%H-%M-%S") -NETWORK_TESTS_RESULT="" - -# Run dpf make target checks test -REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION="/root/${CLUSTER_NAME}/ci/last-openshift-dpf-dir.sh" - -echo "=== DPF Make Target checks on Existing Cluster ===" -echo "Using openshift-dpf dir from last cluster-deploy: '${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}'" - -echo "Updating VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variable values to true, 4 and 3 respectively in .env file" -# Using delimiter '|' since we have '/' in the patterns -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -e; \ - test -f ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd ; \ - set -e; \ - test -f .env ; \ - cat .env | grep VERIFY ; \ - cp .env .env_orig ; \ - sed -i 's|VERIFY_DEPLOYMENT=.*|VERIFY_DEPLOYMENT=true|' .env ; \ - sed -i 's|VERIFY_MAX_RETRIES=.*|VERIFY_MAX_RETRIES=4|' .env ; \ - sed -i 's|VERIFY_SLEEP_SECONDS=.*|VERIFY_SLEEP_SECONDS=3|' .env ; \ - cat .env | grep VERIFY"; then - echo "VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variables updated successfully in .env file" -else - echo "ERROR: Failed to update VERIFY_DEPLOYMENT, VERIFY_MAX_RETRIES and VERIFY_SLEEP_SECONDS variables in .env file" - exit 1 -fi - - -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -a; \ - pwd; \ - ls -ltr; \ - env; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd; \ - set -e;\ - export KUBECONFIG=\${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME}; \ - oc get co; \ - oc get nodes; \ - oc get dpu -A; \ - oc get application -A; \ - echo \${KUBECONFIG}; \ - ls -ltr ; \ - make verify-workers; \ - make verify-dpu-nodes; \ - make verify-deployment; \ - make verify-dpudeployment; \ - echo \$? > verification-result"; then - - echo "DPF spot check tests Passed"; - -else - echo "DPF spot checks tests Failed"; - exit 1 -fi - - -echo "=== Run DPF Kubernetes Traffic Flow Tests on Existing Cluster ===" - -# Need to export the dpu-workers after they are renamed, on hypervisor: -# export TFT_SERVER_NODE=worker-303ea713ea90 -# export TFT_CLIENT_NODE=worker-303ea713ea94 - -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "set -euo pipefail; \ - ls -ltr; \ - env; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - pwd; \ - ls -ltra; \ - set -e;\ - export KUBECONFIG=\${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME}; \ - oc get nodes; \ - export TFT_SERVER_NODE=\$(oc get nodes | grep worker-dpu | awk 'NR==1 {print \$1}'); \ - echo \${TFT_SERVER_NODE} ;\ - export TFT_CLIENT_NODE=\$(oc get nodes | grep worker-dpu | awk 'NR==2 {print \$1}'); \ - echo \${TFT_CLIENT_NODE} ; \ - source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - make run-traffic-flow-tests 2>&1 | tee log-traffic-flow-tests-${datetime_string}"; then - - echo "Kubernetes Network Traffic Flow Iperf Tests Passed"; - NETWORK_TESTS_RESULT="PASS" - -else - echo "Kubernetes Network Traffic Flow Iperf Tests Failed"; - -fi - -echo "====== Output DPF Kubernetes Traffic Flow Tests Log file:" -if ssh ${SSH_OPTS} root@${REMOTE_HOST} "source ${REMOTE_LAST_OPENSHIFT_DPF_DIR_LOCATION}; \ - echo \${LAST_OPENSHIFT_DPF}; \ - env; \ - cd \${LAST_OPENSHIFT_DPF}; \ - cat log-traffic-flow-tests-${datetime_string}"; then - - echo "Successfully output Kubernetes Network Traffic Flow Iperf Tests logs"; - -else - echo "Failed to output DPF kubernetes Traffic Flow Iperf Tests logs"; - -fi - -# Parse the log files, may need to scp to container running ssh cmds and process the -# output file and exit accordingly - -if [ "${NETWORK_TESTS_RESULT}" == "PASS" ]; then - exit 0 -fi - -exit 1 +export TFT_SERVER_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==1 {print $1}') +echo "TFT_SERVER_NODE: ${TFT_SERVER_NODE}" +export TFT_CLIENT_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==2 {print $1}') +echo "TFT_CLIENT_NODE: ${TFT_CLIENT_NODE}" +make run-traffic-flow-tests diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml index 5c9358eceb792..577c8ae86a589 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-ref.yaml @@ -1,29 +1,16 @@ ref: as: dpf-hypervisor-network-tests commands: dpf-hypervisor-network-tests-commands.sh - credentials: - - mount_path: /var/run/dpf-ci - name: cluster-secrets-metal-dpf-doca8 - namespace: ci documentation: |- - Run Kubernetes Network Flow tests on an EXISTING cluster without provisioning. + Run Kubernetes Network Traffic Flow tests on an existing DPF cluster. - This step: - 1. Sets up SSH access to hypervisor - 2. Finds the last DPF cluster install openshift-dpf dir - 3. Runs 'make run-traffic-flow-tests' on hypervisor - 4. Collect and output logs from hypervisor + This step expects a kubeconfig at ${SHARED_DIR}/kubeconfig (provided by + dpf-hypervisor-load-kubeconfig) and runs 'make run-traffic-flow-tests' + from the dpf-ci image. - Use this for quick validation on doca8 or other existing clusters. Does NOT create or destroy any cluster resources. - Required Vault secrets: - - private-key: SSH key for hypervisor - Expected duration: 60-120 minutes - env: - - default: "10.6.135.45" - name: REMOTE_HOST from: dpf-ci resources: requests: From 7bf25fbaeffc4b62eb799a5bdcdace96e5e36a05 Mon Sep 17 00:00:00 2001 From: Jose Castillo Lema Date: Fri, 31 Jul 2026 18:28:23 +0200 Subject: [PATCH 2/4] Fix bash warnings Signed-off-by: Jose Castillo Lema gitleaks.version: 8.30.0 gitleaks.check-secrets: ENABLED --- .../network-tests/dpf-hypervisor-network-tests-commands.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh index c50e6befd817c..1a4384ab926e1 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh @@ -6,9 +6,11 @@ echo "=== Run DPF Kubernetes Traffic Flow Tests ===" echo "Verifying cluster access..." oc get nodes -export TFT_SERVER_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==1 {print $1}') +TFT_SERVER_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==1 {print $1}') +export TFT_SERVER_NODE echo "TFT_SERVER_NODE: ${TFT_SERVER_NODE}" -export TFT_CLIENT_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==2 {print $1}') +TFT_CLIENT_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==2 {print $1}') +export TFT_CLIENT_NODE echo "TFT_CLIENT_NODE: ${TFT_CLIENT_NODE}" make run-traffic-flow-tests From b147e385dc934476960e1d771373213de1b02d81 Mon Sep 17 00:00:00 2001 From: Jose Castillo Lema Date: Mon, 3 Aug 2026 07:40:08 +0200 Subject: [PATCH 3/4] Copy .env file in load-kubeconfig Signed-off-by: Jose Castillo Lema gitleaks.version: 8.30.0 gitleaks.check-secrets: ENABLED --- .../load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh | 1 + .../network-tests/dpf-hypervisor-network-tests-commands.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh b/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh index 6af2880058a62..1184b47ef8371 100755 --- a/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/load-kubeconfig/dpf-hypervisor-load-kubeconfig-commands.sh @@ -46,6 +46,7 @@ fi # Copy the cluster kubeconfig from the last install dir on the hypervisor echo "=== Copying kubeconfig from ${LAST_OPENSHIFT_DPF} on hypervisor ===" scp ${SSH_OPTS} root@${REMOTE_HOST}:${LAST_OPENSHIFT_DPF}/kubeconfig.${CLUSTER_NAME} /tmp/kubeconfig.${CLUSTER_NAME} +scp ${SSH_OPTS} root@${REMOTE_HOST}:${LAST_OPENSHIFT_DPF}/.env "${SHARED_DIR}/.env" # The kubeconfig's API server is addressed by an internal hostname that is # not resolvable from the CI cluster's network. Resolve it from the diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh index 1a4384ab926e1..822519e04a331 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh @@ -1,6 +1,8 @@ #!/bin/bash set -euo pipefail +cp "${SHARED_DIR}/.env" . + echo "=== Run DPF Kubernetes Traffic Flow Tests ===" echo "Verifying cluster access..." From e675975a9bef481113a099109db457f7afa4150c Mon Sep 17 00:00:00 2001 From: Jose Castillo Lema Date: Tue, 4 Aug 2026 10:12:00 +0200 Subject: [PATCH 4/4] Set TFT_KUBECONFIG Signed-off-by: Jose Castillo Lema gitleaks.version: 8.30.0 gitleaks.check-secrets: ENABLED --- .../network-tests/dpf-hypervisor-network-tests-commands.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh index 822519e04a331..700beca2b3152 100644 --- a/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh +++ b/ci-operator/step-registry/dpf/hypervisor/network-tests/dpf-hypervisor-network-tests-commands.sh @@ -8,6 +8,7 @@ echo "=== Run DPF Kubernetes Traffic Flow Tests ===" echo "Verifying cluster access..." oc get nodes +export TFT_KUBECONFIG="${SHARED_DIR}/kubeconfig" TFT_SERVER_NODE=$(oc get nodes --no-headers | grep worker-dpu | awk 'NR==1 {print $1}') export TFT_SERVER_NODE echo "TFT_SERVER_NODE: ${TFT_SERVER_NODE}"