diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index d7cca36ce9..0000000000 --- a/.cirrus.yml +++ /dev/null @@ -1,332 +0,0 @@ ---- - -# Main collection of env. vars to set for all tasks and scripts. -env: - #### - #### Global variables used for all tasks - #### - # Overrides default location (/tmp/cirrus) for repo clone - CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/container-libs" - # Shell used to execute all script commands - CIRRUS_SHELL: "/bin/bash" - # Automation script path relative to $CIRRUS_WORKING_DIR) - SCRIPT_BASE: "./storage/contrib/cirrus" - IMAGE_SCRIPT_BASE: "./image/contrib/cirrus" - # No need to go crazy, but grab enough to cover most PRs - CIRRUS_CLONE_DEPTH: 50 - DEST_BRANCH: "main" - GOPATH: "/var/tmp/go" - GOBIN: "${GOPATH}/bin" - GOCACHE: "${GOPATH}/cache" - SKOPEO_PATH: "/var/tmp/go/src/github.com/containers/skopeo" - IMAGE_WORKING_DIR: "/var/tmp/go/src/github.com/containers/image" - - - #### - #### Cache-image names to test with (double-quotes around names are critical) - ### - FEDORA_NAME: "fedora-42" - DEBIAN_NAME: "debian-13" - - # GCE project where images live - IMAGE_PROJECT: "libpod-218412" - # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c20260425t010036z-f43f42d14" - - FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" - DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" - FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}" - - - #### - #### Command variables to help avoid duplication - #### - # Command to prefix every output line with a timestamp - # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) - _TIMESTAMP: 'awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk' - _DFCMD: 'df -lhTx tmpfs' - _RAUDITCMD: 'cat /var/log/audit/audit.log' - _UAUDITCMD: 'cat /var/log/kern.log' - _JOURNALCMD: 'journalctl -b' - -gcp_credentials: ENCRYPTED[f448b0e5ed38a52ba93550305f9e38c9b0c03852b9fc3115930f231fc7124b7423e8c912b4d9e002cc34b3a7ac3c145b] - -# Default timeout for each task -timeout_in: 20m - -# Default VM to use unless set or modified by task -gce_instance: - image_project: "${IMAGE_PROJECT}" - zone: "us-central1-b" # Required by Cirrus for the time being - cpu: 4 - memory: "4Gb" - disk: 200 - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - - -storage_linux_testing: &linux_testing - gce_instance: # Only need to specify differences from defaults (above) - image_name: "${VM_IMAGE}" - - # Go module cache that is shared for all tasks - modules_cache: &modules_cache - fingerprint_script: cat **/go.sum - folder: $GOPATH/pkg/mod - # Note the cache is populated the first time it is used with the given deps. - # But because each task might not download all he same deps force make vendor - # which downloads all deps for all modules. This is only run once when the - # cache is created so it should not slow things down much. - populate_script: make vendor - - # Separate scripts for separate outputs, makes debugging easier. - setup_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}' - build_and_test_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/build_and_test.sh |& ${_TIMESTAMP}' - - always: - df_script: '${_DFCMD} || true' - rh_audit_log_script: '${_RAUDITCMD} || true' - debian_audit_log_script: '${_UAUDITCMD} || true' - journal_log_script: '${_JOURNALCMD} || true' - - -storage_fedora_testing_task: - <<: *linux_testing - alias: storage_fedora_testing - only_if: &only_if_storage >- - $CIRRUS_PR == '' || - changesInclude('.cirrus.yml', 'go.work', 'go.work.sum') || - changesInclude('storage/**') - name: &std_test_name "Storage: ${OS_NAME} ${TEST_DRIVER}" - env: - OS_NAME: "${FEDORA_NAME}" - VM_IMAGE: "${FEDORA_CACHE_IMAGE_NAME}" - # Not all $TEST_DRIVER combinations valid for all $VM_IMAGE types. - matrix: &test_matrix - - env: - TEST_DRIVER: "vfs" - - env: - TEST_DRIVER: "overlay" - - env: - TEST_DRIVER: "overlay-transient" - - env: - TEST_DRIVER: "fuse-overlay" - - env: - TEST_DRIVER: "fuse-overlay-whiteout" - - env: - TEST_DRIVER: "btrfs" - - -# aufs was dropped between 20.04 and 22.04, can't test it -storage_debian_testing_task: - <<: *linux_testing - alias: storage_debian_testing - name: *std_test_name - only_if: *only_if_storage - env: - OS_NAME: "${DEBIAN_NAME}" - VM_IMAGE: "${DEBIAN_CACHE_IMAGE_NAME}" - # /tmp is a tmpfs, and as of 2025-09-11 we are using Debian images with Linux 6.1, where tmpfs does not support extended attributes. - # That prevents testing various graph drivers; setting TMPDIR changes where graph driver tests place their roots. - TMPDIR: "/var/tmp" - # Not all $TEST_DRIVER combinations valid for all $VM_IMAGE types. - matrix: - - env: - TEST_DRIVER: "vfs" - - env: - TEST_DRIVER: "overlay" - - env: - TEST_DRIVER: "fuse-overlay" - - env: - TEST_DRIVER: "fuse-overlay-whiteout" - - env: - TEST_DRIVER: "btrfs" - -# Update metadata on VM images referenced by this repository state -images_timestamp_update_task: - alias: images_timestamp_update - name: "VM Images Timestamp Update" - - container: - image: "quay.io/libpod/imgts:latest" - cpu: 1 - memory: 1 - - env: - # Space-separated list of images used by this repository state - IMGNAMES: |- - ${FEDORA_CACHE_IMAGE_NAME} - ${DEBIAN_CACHE_IMAGE_NAME} - BUILDID: "${CIRRUS_BUILD_ID}" - REPOREF: "${CIRRUS_CHANGE_IN_REPO}" - GCPJSON: ENCRYPTED[84d8f13981b019af7dfffbe13129ed035aa09ac167c55f2b4ebaccc4f91e8fca0b9c805e2bcf8f18da5b964b35c68203] - GCPNAME: ENCRYPTED[abc17b496b2a96dbf232a97e95a3696e71241f6addc82815ee230fcb67ce2908d4a572f13203cc5797ec24176ea40301] - GCPPROJECT: ENCRYPTED[dc578f8953d01e33fe6b329886b8b088c7a909246e169e640d621ac3f7f0ac251ea4f41353e162b1e5e6586cfb01520d] - CIRRUS_CLONE_DEPTH: 1 # source not used - - script: '/usr/local/bin/entrypoint.sh |& ${_TIMESTAMP}' - - -image_cross_task: - alias: image_cross - name: "Image: Cross" - only_if: &only_if_image >- - $CIRRUS_PR == '' || - changesInclude('.cirrus.yml', 'go.work', 'go.work.sum') || - changesInclude('storage/**', 'image/**') - gce_instance: &fedora_vm - image_project: libpod-218412 - zone: "us-central1-f" - cpu: 2 - memory: "4Gb" - # Required to be 200gig, do not modify - has i/o performance impact - # according to gcloud CLI tool warning messages. - disk: 200 - # Eventually, hard-code FEDORA_CACHE_IMAGE_NAME here again and remove the - # VM_IMAGE_NAME parameter. - image_name: ${VM_IMAGE_NAME} - modules_cache: *modules_cache - env: - HOME: "/root" # default unset, needed by golangci-lint. - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - script: | - cd image - make cross - - -image_test_task: - alias: image_test - only_if: *only_if_image - gce_instance: *fedora_vm - modules_cache: *modules_cache - env: - HOME: "/root" # default unset, needed by golangci-lint. - GOSRC: "${CIRRUS_WORKING_DIR}/image" - GOPATH: "/var/tmp/go" - # Required for consistency with containers/skopeo CI - matrix: - - name: "Image: Test" - env: - BUILDTAGS: '' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Image: Test w/ opengpg" - env: - BUILDTAGS: &withopengpg 'containers_image_openpgp' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Test w/ Sequoia" - env: - BUILDTAGS: &withsequoia 'containers_image_sequoia' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - script: | - cd image - ls -l ${CIRRUS_WORKING_DIR} - echo "${GOPATH}" - ${CIRRUS_WORKING_DIR}/${IMAGE_SCRIPT_BASE}/runner.sh image_tests - - -##### -##### NOTE: This task is subtantially duplicated in the containers/skopeo -##### repository's `.cirrus.yml`. Changes made here should be fully merged -##### prior to being manually duplicated and maintained in containers/skopeo. -##### -image_test_skopeo_task: - alias: image_test_skopeo - only_if: *only_if_image - gce_instance: *fedora_vm - env: - HOME: "/root" # default unset, needed by golangci-lint. - GOSRC: "${CIRRUS_WORKING_DIR}" - GOPATH: "/var/tmp/go" - SKOPEO_CI_BRANCH: "main" - SCRIPT_BASE: "./contrib/cirrus" - # Built along with the standard PR-based workflow in c/automation_images - SKOPEO_CIDEV_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:${IMAGE_SUFFIX}" - modules_cache: *modules_cache - matrix: - - name: "Image: Skopeo Test" - env: - BUILDTAGS: '' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Image: Skopeo Test w/ opengpg" - env: - BUILDTAGS: *withopengpg - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Skopeo Test w/ Sequoia" - env: - BUILDTAGS: *withsequoia - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - setup_script: >- - "${CIRRUS_WORKING_DIR}/${IMAGE_SCRIPT_BASE}/runner.sh" setup - vendor_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" vendor - build_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" build - unit_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" unit - integration_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" integration - system_script: > - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" system - - - -common_testing_task: - alias: common_testing - name: "Common: Test" - modules_cache: *modules_cache - env: - NETAVARK_BINARY: "/usr/local/libexec/podman/netavark" - test_script: | - cd common - make build - make build-cross - make test - make test-integration - -storage_cross_task: - only_if: *only_if_storage - alias: storage_cross - name: "Storage: Cross" - container: - image: golang:1.25 - modules_cache: *modules_cache - build_script: | - cd storage - make cross - -storage_gofix_task: - only_if: *only_if_storage - alias: storage_gofix - name: "Storage: gofix" - container: - image: golang:1.25 - build_script: | - cd storage - go fix ./... - test_script: | - # The git diff complains about broken symlinks in some test files. - # We therefore filter-out these files, since they are not changed by - # gofix anyway. - git diff --diff-filter=M --exit-code -- . ":(exclude)*.crt" ":(exclude)*.key" ":(exclude)*.cert" - -# Status aggregator for all tests. This task simply ensures a defined -# set of tasks all passed, and allows confirming that based on the status -# of this task. -success_task: - alias: success - # N/B: The prow merge-bot (tide) is sensitized to this exact name, DO NOT CHANGE IT. - # Ref: https://github.com/openshift/release/pull/49820 - name: "Total Success" - depends_on: - - storage_fedora_testing - - storage_debian_testing - - images_timestamp_update - - storage_cross - - storage_gofix - - common_testing - - image_cross - - image_test - - image_test_skopeo - container: - image: golang:1.25 - clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"' # Source code not needed - script: /bin/true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..d2ab74911a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,237 @@ +name: ci + +on: + push: + branches: + - main + - podman-* + pull_request: + branches: + - main + - podman-* + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: "1.26.x" + +jobs: + path-filter: + runs-on: ubuntu-24.04 + permissions: + pull-requests: read + outputs: + storage: ${{ steps.filter.outputs.storage }} + image: ${{ steps.filter.outputs.image }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + storage: + - '.github/workflows/ci.yml' + - 'contrib/ci/**' + - 'go.work' + - 'go.work.sum' + - 'storage/**' + image: + - '.github/workflows/ci.yml' + - 'contrib/ci/**' + - 'go.work' + - 'go.work.sum' + - 'storage/**' + - 'image/**' + + storage-test: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.storage == 'true' + runs-on: cncf-ubuntu-8-32-x86 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + distro: [fedora-current, debian-current] + driver: [vfs, overlay, overlay-transient, fuse-overlay, fuse-overlay-whiteout, btrfs] + exclude: + - distro: debian-current + driver: overlay-transient + name: "Storage: ${{ matrix.distro }} ${{ matrix.driver }}" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: lima-vm/lima-actions/setup@v1 + with: + version: v2.1.1 + - name: Run tests + run: ./contrib/ci/ci.sh storage ${{ matrix.distro }} ${{ matrix.driver }} + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: "journal-storage-${{ matrix.distro }}-${{ matrix.driver }}.log" + path: "./contrib/ci/journal.log" + if-no-files-found: ignore + + storage-cross: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.storage == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + name: "Storage: Cross" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: "**/go.sum" + - name: Cross-compile + working-directory: ./storage + run: make cross + + storage-gofix: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.storage == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + name: "Storage: gofix" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: "**/go.sum" + - name: Install dependencies + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y libbtrfs-dev libgpgme-dev libdevmapper-dev + - name: Run go fix + working-directory: ./storage + run: go fix ./... + - name: Verify no changes + run: | + git diff --diff-filter=M --exit-code -- . ":(exclude)*.crt" ":(exclude)*.key" ":(exclude)*.cert" + + image-cross: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.image == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + name: "Image: Cross" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: "**/go.sum" + - name: Install dependencies + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev + - name: Cross-compile + working-directory: ./image + run: make cross + + image-test: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.image == 'true' + runs-on: cncf-ubuntu-8-32-x86 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + variant: [default, openpgp, sequoia] + name: "Image: Test ${{ matrix.variant }}" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: lima-vm/lima-actions/setup@v1 + with: + version: v2.1.1 + - name: Run tests + run: ./contrib/ci/ci.sh image fedora-current ${{ matrix.variant }} + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: "journal-image-${{ matrix.variant }}.log" + path: "./contrib/ci/journal.log" + if-no-files-found: ignore + + image-test-skopeo: + needs: path-filter + if: github.event_name == 'push' || needs.path-filter.outputs.image == 'true' + runs-on: cncf-ubuntu-8-32-x86 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + variant: [default, openpgp, sequoia] + name: "Image: Skopeo ${{ matrix.variant }}" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: lima-vm/lima-actions/setup@v1 + with: + version: v2.1.1 + - name: Run tests + run: ./contrib/ci/ci.sh image-skopeo fedora-current ${{ matrix.variant }} + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: "journal-image-skopeo-${{ matrix.variant }}.log" + path: "./contrib/ci/journal.log" + if-no-files-found: ignore + + common-test: + runs-on: cncf-ubuntu-8-32-x86 + timeout-minutes: 45 + name: "Common: Test" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: lima-vm/lima-actions/setup@v1 + with: + version: v2.1.1 + - name: Run tests + run: ./contrib/ci/ci.sh common fedora-current + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: "journal-common.log" + path: "./contrib/ci/journal.log" + if-no-files-found: ignore + + # N/B: The prow merge-bot (tide) is sensitized to this exact name, DO NOT CHANGE IT. + # Ref: https://github.com/openshift/release/pull/49820 + success: + name: "Total Success" + if: always() + needs: + - storage-test + - storage-cross + - storage-gofix + - image-cross + - image-test + - image-test-skopeo + - common-test + runs-on: ubuntu-24.04 + steps: + - name: Check all required jobs + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]] || \ + [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more required jobs failed or were cancelled" + exit 1 + fi + echo "All required jobs passed or were skipped" diff --git a/contrib/ci/ci.sh b/contrib/ci/ci.sh new file mode 100755 index 0000000000..d43b8495d6 --- /dev/null +++ b/contrib/ci/ci.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" && pwd ) + +source "$SCRIPT_DIR/lib.sh" + +AUTOMATION_RELEASE="20260520t200858z" +LIMA_VM_NAME=container-libs-ci + +MODULE=${1:?must give module as first argument} + +REPO_DIR="$SCRIPT_DIR/../.." + +parse_args "$@" + +IMAGE="$DISTRO_NAME.x86_64.qcow2.zst" + +IMAGE_URL="https://objectstorage.us-ashburn-1.oraclecloud.com/n/id0lmbbwgcdv/b/podman-ci-vm-images/o/releases/$AUTOMATION_RELEASE/$IMAGE" + +trap "limactl delete --force $LIMA_VM_NAME" EXIT + +limactl --yes start --plain --name=$LIMA_VM_NAME --cpus $(nproc) --memory 8 --nested-virt \ + --set ".images=[{\"location\":\"$IMAGE_URL\", \"arch\": \"x86_64\"}]" \ + "$SCRIPT_DIR/template.lima.yml" + +limactl copy "$REPO_DIR" $LIMA_VM_NAME:/var/tmp/container-libs + +set +e + +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME ./contrib/ci/runner.sh "${@}" +rc=$? + +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME sudo contrib/ci/logcollector.sh journal &> "$SCRIPT_DIR/journal.log" + +exit $rc diff --git a/contrib/ci/lib.sh b/contrib/ci/lib.sh new file mode 100644 index 0000000000..7c9b0e4994 --- /dev/null +++ b/contrib/ci/lib.sh @@ -0,0 +1,70 @@ +# This must be sourced from other scripts to work. + +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')" +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +OS_REL_VER="$OS_RELEASE_ID-$OS_RELEASE_VER" + +function die() { + echo "$1" >&2 + exit 1 +} + +function parse_args() { + # module name: storage, image, image-skopeo, common + MODULE=${1:?must give module as first argument} + # distro: fedora-current, debian-current + DISTRO_NAME=${2:?must give distro as second argument} + # variant: driver for storage, buildtag for image, unused for common + VARIANT=${3:-} + + validate_module "$MODULE" + validate_distro "$DISTRO_NAME" + validate_variant "$MODULE" "$VARIANT" +} + +function validate_module() { + case "$1" in + "storage"|"image"|"image-skopeo"|"common") + ;; + *) + die "Unknown MODULE '$1', expected: storage, image, image-skopeo, common" + ;; + esac +} + +function validate_distro() { + case "$1" in + "fedora-current"|"debian-current") + ;; + *) + die "Unknown DISTRO_NAME '$1', expected: fedora-current, debian-current" + ;; + esac +} + +function validate_variant() { + local module="$1" + local variant="$2" + case "$module" in + storage) + case "$variant" in + "vfs"|"overlay"|"overlay-transient"|"fuse-overlay"|"fuse-overlay-whiteout"|"btrfs") + ;; + *) + die "Unknown storage variant '$variant', expected: vfs, overlay, overlay-transient, fuse-overlay, fuse-overlay-whiteout, btrfs" + ;; + esac + ;; + image|image-skopeo) + case "$variant" in + ""|"default"|"openpgp"|"sequoia") + ;; + *) + die "Unknown image variant '$variant', expected: default, openpgp, sequoia" + ;; + esac + ;; + common) + ;; + esac +} diff --git a/contrib/ci/logcollector.sh b/contrib/ci/logcollector.sh new file mode 100755 index 0000000000..5aca898032 --- /dev/null +++ b/contrib/ci/logcollector.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -e + +# shellcheck source=contrib/ci/lib.sh +source $(dirname $0)/lib.sh + +# Assume there are other log collection commands to follow - Don't +# let one break another that may be useful, but also keep any +# actual script-problems fatal so they are noticed right away. +showrun() { + echo '+ '$(printf " %q" "$@") + set +e + echo '------------------------------------------------------------' + "$@" + local status=$? + [[ $status -eq 0 ]] || \ + echo "[ rc = $status -- proceeding anyway ]" + echo '------------------------------------------------------------' + set -e +} + +bad_os_id_ver() { + die "Unknown OS '$OS_RELEASE_ID'" +} + +case $1 in + audit) + case $OS_RELEASE_ID in + debian) showrun cat /var/log/kern.log ;; + fedora) showrun cat /var/log/audit/audit.log ;; + *) bad_os_id_ver ;; + esac + ;; + df) showrun df -lhTx tmpfs ;; + journal) showrun journalctl -b ;; + packages) + PKG_NAMES=(\ + golang + podman + skopeo + btrfs-progs + fuse-overlayfs + ) + case $OS_RELEASE_ID in + fedora) + cat /etc/fedora-release + PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n' + PKG_NAMES+=(\ + gpgme-devel + device-mapper-devel + libseccomp-devel + ) + ;; + debian) + cat /etc/issue + PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n' + PKG_NAMES+=(\ + libgpgme-dev + libdevmapper-dev + libseccomp-dev + ) + ;; + *) bad_os_id_ver ;; + esac + echo "Kernel: " $(uname -r) + echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup) + # Any not-present packages will be listed as such + $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u + ;; + ip) showrun sh -c "ip addr && ip route && ip -6 route" ;; + *) die "Warning, $(basename $0) doesn't know how to handle the parameter '$1'" +esac diff --git a/contrib/ci/runner.sh b/contrib/ci/runner.sh new file mode 100755 index 0000000000..352e79a2c8 --- /dev/null +++ b/contrib/ci/runner.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash + +# This script is only intended to be run inside the Lima VM to configure it and start the tests. +# Do not run locally. + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" && pwd ) + +source "$SCRIPT_DIR/lib.sh" + +MODULE=${1:?must give module as first argument} + +parse_args "$@" + +############################################################################### +# Dependency installation +############################################################################### + +install_deps_storage() { + case $OS_RELEASE_ID in + fedora) + dnf install -y \ + git golang bats \ + btrfs-progs btrfs-progs-devel \ + gpgme-devel libassuan-devel device-mapper-devel \ + fuse-overlayfs fuse3 \ + gcc make kmod util-linux e2fsprogs + dnf remove -y gcc-go || true + ;; + debian) + apt-get update + apt-get install -y \ + git golang bats \ + btrfs-progs libbtrfs-dev \ + libgpgme-dev libassuan-dev libdevmapper-dev \ + fuse-overlayfs fuse3 \ + gcc make pkg-config kmod e2fsprogs util-linux bzip2 + ;; + *) die "Unsupported OS for storage: $OS_RELEASE_ID" ;; + esac +} + +install_deps_image() { + case $OS_RELEASE_ID in + fedora) + dnf install -y \ + git golang \ + gpgme-devel libassuan-devel btrfs-progs-devel device-mapper-devel \ + subid-devel \ + podman openssh-server \ + gcc make + dnf remove -y gcc-go || true + ;; + debian) + apt-get update + apt-get install -y \ + git golang \ + libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev \ + libsubid-dev \ + podman openssh-server \ + gcc make pkg-config + ;; + *) die "Unsupported OS for image: $OS_RELEASE_ID" ;; + esac + printf 'unqualified-search-registries = ["docker.io"]\n' | sudo tee /etc/containers/registries.conf + + if [[ "$VARIANT" == "sequoia" ]]; then + case $OS_RELEASE_ID in + fedora) dnf install -y openssl-devel capnproto ;; + debian) apt-get install -y libssl-dev capnproto ;; + esac + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source "$HOME/.cargo/env" + git clone --depth=1 https://github.com/ueno/podman-sequoia.git /tmp/podman-sequoia + make -C /tmp/podman-sequoia + case $OS_RELEASE_ID in + fedora) sudo cp /tmp/podman-sequoia/target/release/libpodman_sequoia.so /usr/lib64/ ;; + debian) sudo cp /tmp/podman-sequoia/target/release/libpodman_sequoia.so /usr/lib/x86_64-linux-gnu/ ;; + esac + sudo ldconfig + fi +} + +install_deps_image_skopeo() { + install_deps_image + case $OS_RELEASE_ID in + fedora) + dnf install -y bats go-md2man docker-distribution httpd-tools + ;; + debian) + apt-get install -y bats go-md2man docker-registry apache2-utils + ;; + esac + echo "root:100000:65536" | sudo tee -a /etc/subuid + echo "root:100000:65536" | sudo tee -a /etc/subgid + sudo ln -sf /usr/bin/docker-registry /usr/local/bin/registry 2>/dev/null || true +} + +install_deps_common() { + case $OS_RELEASE_ID in + fedora) + dnf install -y \ + git golang \ + libseccomp-devel gpgme-devel btrfs-progs-devel device-mapper-devel \ + subid-devel \ + podman protobuf-compiler protobuf-devel \ + gcc make + dnf remove -y gcc-go || true + ;; + debian) + apt-get update + apt-get install -y \ + git golang \ + libseccomp-dev libgpgme-dev libbtrfs-dev libdevmapper-dev \ + libsubid-dev \ + podman protobuf-compiler libprotobuf-dev \ + gcc make pkg-config + ;; + *) die "Unsupported OS for common: $OS_RELEASE_ID" ;; + esac + printf 'unqualified-search-registries = ["docker.io"]\n' | sudo tee /etc/containers/registries.conf + + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source "$HOME/.cargo/env" + git clone --depth=1 https://github.com/containers/netavark.git /tmp/netavark-src + make -C /tmp/netavark-src build + sudo mkdir -p /usr/local/libexec/podman + sudo install -m 755 /tmp/netavark-src/bin/netavark /usr/local/libexec/podman/netavark +} + +############################################################################### +# Environment preparation +############################################################################### + +prepare_storage_env() { + truncate -s 10G /var/tmp/test-fs.img + mkfs.ext4 -q /var/tmp/test-fs.img + mount -o loop /var/tmp/test-fs.img /tmp + + for i in $(seq 0 1023); do + [ -e /dev/loop$i ] || mknod /dev/loop$i b 7 $i 2>/dev/null || true + done +} + +prepare_image_env() { + ROOTLESS_USER="testuser$$" + rootless_uid=$((RANDOM+1000)) + rootless_gid=$((RANDOM+1000)) + groupadd -g $rootless_gid $ROOTLESS_USER + useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER + + chown -R $ROOTLESS_USER:$ROOTLESS_USER "$(go env GOPATH)" + chown -R $ROOTLESS_USER:$ROOTLESS_USER "$(pwd)" + + mkdir -p "/run/user/$rootless_uid" + chown $ROOTLESS_USER:$ROOTLESS_USER "/run/user/$rootless_uid" + + mkdir -p /root/.ssh "/home/$ROOTLESS_USER/.ssh" + ssh-keygen -t ed25519 -P "" -f /root/.ssh/id_ed25519 + cat /root/.ssh/*.pub >> "/home/$ROOTLESS_USER/.ssh/authorized_keys" + chmod -R 700 /root/.ssh "/home/$ROOTLESS_USER/.ssh" + chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh" + + systemctl start sshd || systemctl start ssh + ssh-keyscan localhost > /root/.ssh/known_hosts + + export ROOTLESS_USER rootless_uid +} + +############################################################################### +# Test runners +############################################################################### + +run_storage() { + cd storage + make local-binary + + case "$VARIANT" in + overlay) + make STORAGE_DRIVER=overlay local-test-integration local-test-unit + ;; + overlay-transient) + make STORAGE_DRIVER=overlay STORAGE_TRANSIENT=1 local-test-integration local-test-unit + ;; + fuse-overlay) + make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit + ;; + fuse-overlay-whiteout) + FUSE_OVERLAYFS_DISABLE_OVL_WHITEOUT=1 make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit + ;; + vfs) + make STORAGE_DRIVER=vfs local-test-integration local-test-unit + ;; + btrfs) + if [[ "$(./hack/btrfs_tag.sh)" =~ exclude_graphdriver_btrfs ]]; then + echo "Built without btrfs, so we can't test it" + exit 1 + fi + if ! grep -q " btrfs$" /proc/filesystems; then + modprobe btrfs || true + if ! grep -q " btrfs$" /proc/filesystems; then + echo "Kernel does not support btrfs" + exit 1 + fi + fi + if ! command -v mkfs.btrfs &> /dev/null; then + echo "mkfs.btrfs not installed" + exit 1 + fi + tmpdir=$(mktemp -d) + trap "umount -l $tmpdir; rm -f btrfs.img" EXIT + truncate -s 0 btrfs.img + fallocate -l 1G btrfs.img + mkfs.btrfs btrfs.img + mount -o loop btrfs.img $tmpdir + TMPDIR="$tmpdir" make STORAGE_DRIVER=btrfs local-test-integration local-test-unit + ;; + *) + die "Unknown storage variant: $VARIANT" + ;; + esac +} + +run_image() { + cd image + + local BUILDTAGS="" + case "$VARIANT" in + default|"") BUILDTAGS="" ;; + openpgp) BUILDTAGS="containers_image_openpgp" ;; + sequoia) BUILDTAGS="containers_image_sequoia" ;; + esac + + GOPATH_DIR="$(go env GOPATH)" + GOROOT_DIR="$(go env GOROOT)" + GOSRC="$(cd .. && pwd)" + + # Run root tests for storage-dependent tests + test_filter=$(git grep -h --show-function ensureTestCanCreateImages ../storage | + sed -n 's/func \(Test[[:alnum:]]*\)(.*/^\1$/p' | + paste -sd "|" -) + if [ -n "$test_filter" ]; then + make test "BUILDTAGS=$BUILDTAGS" "TESTFLAGS=-v -run '$test_filter'" TEST_PACKAGES=./storage + fi + + # Run rootless tests + cleanup() { + ssh -o StrictHostKeyChecking=no -i /root/.ssh/id_ed25519 \ + $ROOTLESS_USER@localhost \ + "export XDG_RUNTIME_DIR=/run/user/$rootless_uid && export PATH=$GOROOT_DIR/bin:\$PATH && bash $GOSRC/image/signature/sigstore/rekor/testdata/start-rekor.sh ci remove" || true + chown -R $(id -u):$(id -g) "$GOPATH_DIR" "$GOSRC" + } + trap cleanup EXIT + + ssh -o StrictHostKeyChecking=no -i /root/.ssh/id_ed25519 \ + $ROOTLESS_USER@localhost \ + "export XDG_RUNTIME_DIR=/run/user/$rootless_uid && export PATH=$GOROOT_DIR/bin:\$PATH && export GOPATH=$GOPATH_DIR && bash $GOSRC/image/signature/sigstore/rekor/testdata/start-rekor.sh ci" + + ssh -o StrictHostKeyChecking=no -i /root/.ssh/id_ed25519 \ + $ROOTLESS_USER@localhost \ + "export XDG_RUNTIME_DIR=/run/user/$rootless_uid && export PATH=$GOROOT_DIR/bin:\$PATH && export GOPATH=$GOPATH_DIR && cd $GOSRC/image && make test BUILDTAGS='$BUILDTAGS' TESTFLAGS=-v REKOR_SERVER_URL='http://127.0.0.1:3000'" +} + +run_image_skopeo() { + local BUILDTAGS="" + case "$VARIANT" in + default|"") BUILDTAGS="" ;; + openpgp) BUILDTAGS="containers_image_openpgp" ;; + sequoia) BUILDTAGS="containers_image_sequoia" ;; + esac + + GOSRC="$(pwd)" + SKOPEO_PATH="/var/tmp/skopeo" + SKOPEO_CIDEV_CONTAINER_FQIN="quay.io/libpod/skopeo_cidev:latest" + + sudo podman pull --quiet "$SKOPEO_CIDEV_CONTAINER_FQIN" + mnt=$(sudo podman mount $(sudo podman create "$SKOPEO_CIDEV_CONTAINER_FQIN")) + sudo cp -a "$mnt/usr/local/bin/." /usr/local/bin/ + sudo mkdir -p /registry + sudo cp -a "$mnt/atomic-registry-config.yml" / + sudo podman umount --latest + sudo podman rm --latest + + git clone -b main https://github.com/containers/skopeo.git "$SKOPEO_PATH" + cd "$SKOPEO_PATH" + go mod edit -replace "go.podman.io/storage=$GOSRC/storage" + go mod edit -replace "go.podman.io/image/v5=$GOSRC/image" + go mod edit -replace "go.podman.io/common=$GOSRC/common" + make vendor + + make bin/skopeo "BUILDTAGS=$BUILDTAGS" + sudo make install PREFIX=/usr/local "BUILDTAGS=$BUILDTAGS" + + make test-unit-local "BUILDTAGS=$BUILDTAGS" + + sudo podman system reset --force + export SKOPEO_CONTAINER_TESTS=1 + sudo -E env "PATH=/usr/local/bin:$PATH" "GOPATH=$(go env GOPATH)" "SKOPEO_CONTAINER_TESTS=$SKOPEO_CONTAINER_TESTS" \ + make test-integration-local "BUILDTAGS=$BUILDTAGS" + + sudo podman system reset --force + sudo -E env "PATH=/usr/local/bin:$PATH" "GOPATH=$(go env GOPATH)" "SKOPEO_CONTAINER_TESTS=$SKOPEO_CONTAINER_TESTS" \ + make test-system-local "BUILDTAGS=$BUILDTAGS" +} + +run_common() { + cd common + NETAVARK_BINARY=/usr/local/libexec/podman/netavark + export NETAVARK_BINARY + + make build + make build-cross + + sudo -E env "PATH=$PATH" "GOPATH=$(go env GOPATH)" "HOME=$HOME" \ + make test + sudo -E env "PATH=$PATH" "GOPATH=$(go env GOPATH)" "HOME=$HOME" \ + make test-integration +} + +############################################################################### +# Main dispatch +############################################################################### + +echo +echo "#################" +echo "Installing dependencies for $MODULE" +echo "#################" + +# Normalize module name for function dispatch (image-skopeo -> image_skopeo) +MODULE_FUNC="${MODULE//-/_}" + +install_deps_${MODULE_FUNC} + +if type -t prepare_${MODULE_FUNC}_env &>/dev/null; then + echo + echo "#################" + echo "Preparing environment for $MODULE" + echo "#################" + prepare_${MODULE_FUNC}_env +fi + +echo +echo "#################" +echo "Logging system info" +echo "#################" + +"$SCRIPT_DIR/logcollector.sh" packages +"$SCRIPT_DIR/logcollector.sh" ip + +echo +echo "#################" +echo "Starting tests: $MODULE $VARIANT" +echo "#################" + +run_${MODULE_FUNC} diff --git a/contrib/ci/template.lima.yml b/contrib/ci/template.lima.yml new file mode 100644 index 0000000000..3b82607142 --- /dev/null +++ b/contrib/ci/template.lima.yml @@ -0,0 +1,4 @@ + +# images is empty and should be override on the cli +# --set '.images=[{"location":"...", "arch": "x86_64"}]' +images: diff --git a/image/contrib/cirrus/runner.sh b/image/contrib/cirrus/runner.sh deleted file mode 100755 index 2b802f64f9..0000000000 --- a/image/contrib/cirrus/runner.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# This script is intended to be executed by automation or humans -# under a hack/get_ci_vm.sh context. Use under any other circumstances -# is unlikely to function. - -set -e - -if [[ -r "/etc/automation_environment" ]]; then - source /etc/automation_environment - source $AUTOMATION_LIB_PATH/common_lib.sh -else - ( - echo "WARNING: It does not appear that containers/automation was installed." - echo " Functionality of most of ${BASH_SOURCE[0]} will be negatively" - echo " impacted." - ) > /dev/stderr -fi - -export "PATH=$PATH:$GOPATH/bin" - -_run_setup() { - req_env_vars SKOPEO_PATH SKOPEO_CI_BRANCH GOSRC - - rm -rf "${SKOPEO_PATH}" - git clone -b ${SKOPEO_CI_BRANCH} \ - https://github.com/containers/skopeo.git ${SKOPEO_PATH} - - cd "${SKOPEO_PATH}" - if [[ -n "$SKOPEO_PR" ]] && [[ $SKOPEO_PR -gt 1000 ]]; then - warn "Fetching and checking out code from skopeo pull-request #$SKOPEO_PR" - git fetch origin "+refs/pull/$SKOPEO_PR/head" - git checkout FETCH_HEAD - fi - - go mod edit -replace go.podman.io/storage="$GOSRC/storage" - go mod edit -replace go.podman.io/image/v5="$GOSRC/image" - go mod edit -replace go.podman.io/common="$GOSRC/common" - - "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" setup -} - -_run_image_tests() { - req_env_vars GOPATH GOSRC - - # Hacky solution to find test that must be run as root. - # This looks for the ensureTestCanCreateImages() test function call and gets the - # function name where it is called via git grep, - # then trims the line to only show the actual function name and add "^$" around it - # since go test commands only accepts a single regex. - # Then join all names with "|" with paste to again build up a single regex string - # that matches all these names. - # With that we don't have to run everything twice and can just run the ones that - # actually need to be root. - # Note we must run git before we switch/chown to the user because it will error - # out otherwise since the file ownership doesn't match. - test_filter=$(git grep -h --show-function ensureTestCanCreateImages ./storage | - sed -n 's/func \(Test[[:alnum:]]*\)(.*/^\1\$\$/p' | - paste -sd "|" -) - showrun make test "BUILDTAGS='$BUILDTAGS'" "TESTFLAGS=-v -run '$test_filter'" TEST_PACKAGES=./storage - - # Most tests in this repo are intended to run as a regular user. - ROOTLESS_USER="testuser$RANDOM" - msg "Setting up rootless user '$ROOTLESS_USER'" - cd $GOSRC || exit 1 - # Guarantee independence from specific values - rootless_uid=$((RANDOM+1000)) - rootless_gid=$((RANDOM+1000)) - msg "Creating $rootless_uid:$rootless_gid $ROOTLESS_USER user" - groupadd -g $rootless_gid $ROOTLESS_USER - useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER - - msg "Setting ownership of $GOPATH and $GOSRC" - chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC" - - msg "Creating ssh key pairs" - mkdir -p "/root/.ssh" "/home/$ROOTLESS_USER/.ssh" - ssh-keygen -t ed25519 -P "" -f "/root/.ssh/id_ed25519" - - msg "Setup authorized_keys" - cat /root/.ssh/*.pub >> /home/$ROOTLESS_USER/.ssh/authorized_keys - - msg "Configure ssh file permissions" - chmod -R 700 "/root/.ssh" - chmod -R 700 "/home/$ROOTLESS_USER/.ssh" - chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh" - - msg "Ensure the ssh daemon is up and running within 5 minutes" - systemctl is-active sshd || \ - systemctl start sshd - - msg "Setup known_hosts for root" - ssh-keyscan localhost > /root/.ssh/known_hosts \ - - msg "Start rekor server as $ROOTLESS_USER" - showrun ssh $ROOTLESS_USER@localhost $GOSRC/signature/sigstore/rekor/testdata/start-rekor.sh ci - # remove rekor server on function exit - trap "ssh $ROOTLESS_USER@localhost $GOSRC/signature/sigstore/rekor/testdata/start-rekor.sh ci remove" RETURN - - msg "Executing tests as $ROOTLESS_USER" - showrun ssh $ROOTLESS_USER@localhost make -C $GOSRC test "BUILDTAGS='$BUILDTAGS'" "TESTFLAGS=-v" "REKOR_SERVER_URL='http://127.0.0.1:3000'" -} - -req_env_vars GOSRC - -handler="_run_${1}" -if [ "$(type -t $handler)" != "function" ]; then - die "Unknown/Unsupported command-line argument '$1'" -fi - -msg "************************************************************" -msg "Runner executing $1 on $OS_REL_VER" -msg "************************************************************" - -cd "$GOSRC" -$handler diff --git a/storage/contrib/cirrus/build_and_test.sh b/storage/contrib/cirrus/build_and_test.sh deleted file mode 100755 index a1742facc4..0000000000 --- a/storage/contrib/cirrus/build_and_test.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -cd $GOSRC/storage -pwd -ls -l -showrun make local-binary - -case $TEST_DRIVER in - overlay) - showrun make STORAGE_DRIVER=overlay local-test-integration local-test-unit - ;; - overlay-transient) - showrun make STORAGE_DRIVER=overlay STORAGE_TRANSIENT=1 local-test-integration local-test-unit - ;; - fuse-overlay) - showrun make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit - ;; - fuse-overlay-whiteout) - showrun make STORAGE_DRIVER=overlay FUSE_OVERLAYFS_DISABLE_OVL_WHITEOUT=1 STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit - ;; - vfs) - showrun make STORAGE_DRIVER=vfs local-test-integration local-test-unit - ;; - btrfs) - # Fedora: Needs btrfs-progs, btrfs-progs-devel - # Debian: Needs btrfs-progs, libbtrfs-dev - if [[ "$(./hack/btrfs_tag.sh)" =~ exclude_graphdriver_btrfs ]]; then - die "Built without btrfs, so we can't test it" - fi - if ! check_filesystem_supported $TEST_DRIVER ; then - die "This CI VM does not support $TEST_DRIVER in its kernel" - fi - if test -z "$(which mkfs.btrfs 2> /dev/null)" ; then - die "This CI VM does not have mkfs.btrfs installed" - fi - tmpdir=$(mktemp -d) - if [ -z "$tmpdir" ]; then - die "Error creating temporary directory" - fi - trap "umount -l $tmpdir; rm -f $GOSRC/$TEST_DRIVER.img" EXIT - truncate -s 0 $GOSRC/$TEST_DRIVER.img - fallocate -l 1G $GOSRC/$TEST_DRIVER.img - mkfs.btrfs $GOSRC/$TEST_DRIVER.img - mount -o loop $GOSRC/$TEST_DRIVER.img $tmpdir - TMPDIR="$tmpdir" showrun make STORAGE_DRIVER=$TEST_DRIVER local-test-integration local-test-unit - ;; - zfs) - # Debian: Needs zfsutils - if ! check_filesystem_supported $TEST_DRIVER ; then - die "This CI VM does not support $TEST_DRIVER in its kernel" - fi - if test -z "$(which zpool 2> /dev/null)" ; then - die "This CI VM does not have zpool installed" - fi - if test -z "$(which zfs 2> /dev/null)" ; then - die "This CI VM does not have zfs installed" - fi - tmpfile=$(mktemp -p $GOSRC) - truncate -s 0 $tmpfile - fallocate -l 1G $tmpfile - zpool=$(basename $tmpfile) - zpool create $zpool $tmpfile - trap "zfs destroy -Rf $zpool/tmp; zpool destroy -f $zpool; rm -f $tmpfile" EXIT - zfs create $zpool/tmp - TMPDIR="/$zpool/tmp" showrun make STORAGE_DRIVER=$TEST_DRIVER local-test-integration local-test-unit - # Ensure no datasets are held open prior to `zfs destroy` trap. - datasets=$(lsns -J -t mnt --output-all | jq '.namespaces[]|select(.command=="sleep 1000s").pid') - if [[ -n "$datasets" ]]; then - kill $datasets - fi - ;; - *) - die "Unknown/Unsupported \$TEST_DRIVER=$TEST_DRIVER (see .cirrus.yml and $(basename $0))" - ;; -esac diff --git a/storage/contrib/cirrus/lib.sh b/storage/contrib/cirrus/lib.sh deleted file mode 100755 index 1dabb6aaf1..0000000000 --- a/storage/contrib/cirrus/lib.sh +++ /dev/null @@ -1,106 +0,0 @@ - - -# Library of common, shared utility functions. This file is intended -# to be sourced by other scripts, not called directly. - -# BEGIN Global export of all variables -set -a - -# Due to differences across platforms and runtime execution environments, -# handling of the (otherwise) default shell setup is non-uniform. Rather -# than attempt to workaround differences, simply force-load/set required -# items every time this library is utilized. -USER="$(whoami)" -HOME="$(getent passwd $USER | cut -d : -f 6)" -# Some platforms set and make this read-only -[[ -n "$UID" ]] || \ - UID=$(getent passwd $USER | cut -d : -f 3) - -# Automation library installed at image-build time, -# defining $AUTOMATION_LIB_PATH in this file. -if [[ -r "/etc/automation_environment" ]]; then - source /etc/automation_environment -fi -# shellcheck disable=SC2154 -if [[ -n "$AUTOMATION_LIB_PATH" ]]; then - # shellcheck source=/usr/share/automation/lib/common_lib.sh - source $AUTOMATION_LIB_PATH/common_lib.sh -else - ( - echo "WARNING: It does not appear that containers/automation was installed." - echo " Functionality of most of this library will be negatively impacted" - echo " This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}" - ) > /dev/stderr -fi - -# Essential default paths, many are overridden when executing under Cirrus-CI -# others are duplicated here, to assist in debugging. -GOPATH="${GOPATH:-/var/tmp/go}" -GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}" -# called processes like `make` and other tools need these vars. -eval "$(go env)" -CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/storage}" -GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" -PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" -SCRIPT_BASE=${GOSRC}/contrib/cirrus - -CI="${CI:-false}" -CIRRUS_CI="${CIRRUS_CI:-false}" -DEST_BRANCH="${DEST_BRANCH:-main}" -CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" -CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-storage} -# Cirrus only sets $CIRRUS_BASE_SHA properly for PRs, but $EPOCH_TEST_COMMIT -# needs to be set from this value in order for `make validate` to run properly. -# When running get_ci_vm.sh, most $CIRRUS_xyz variables are empty. Attempt -# to accommodate both branch and get_ci_vm.sh testing by discovering the base -# branch SHA value. -if [[ -z "$CIRRUS_BASE_SHA" ]] && [[ -z "$CIRRUS_TAG" ]] -then # Operating on a branch, or under `get_ci_vm.sh` - CIRRUS_BASE_SHA=$(git rev-parse ${UPSTREAM_REMOTE:-origin}/$DEST_BRANCH) -elif [[ -z "$CIRRUS_BASE_SHA" ]] -then # Operating on a tag - CIRRUS_BASE_SHA=$(git rev-parse HEAD) -fi -# The starting place for linting and code validation -EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA" - -# Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' - -# Working with dnf + timeout/retry -SHORT_DNFY='lilto dnf -y' -LONG_DNFY='bigto dnf -y' -# Working with apt under Debian/Ubuntu automation is a PITA, make it easy -# Avoid some ways of getting stuck waiting for user input -DEBIAN_FRONTEND=noninteractive -# Short-cut for frequently used base command -SUDOAPTGET='sudo -E apt-get -q --yes' -# Short list of packages or quick-running command -SHORT_APTGET="lilto $SUDOAPTGET" -# Long list / long-running command -LONG_APTGET="bigto $SUDOAPTGET" - -# Packages in generic VM images that conflict with containers/storage testing -RPMS_CONFLICTING="gcc-go" -DEBS_CONFLICTING="" - -# END Global export of all variables -set +a - -bad_os_id_ver() { - die "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" -} - -lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max -bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max - -check_filesystem_supported(){ - if ! grep -q " $1\$" /proc/filesystems ; then - modprobe $1 > /dev/null 2> /dev/null || :en - if ! grep -q " $1\$" /proc/filesystems ; then - echo "This CI VM does not support $TEST_DRIVER in its kernel" - false - fi - fi - true -} diff --git a/storage/contrib/cirrus/setup.sh b/storage/contrib/cirrus/setup.sh deleted file mode 100755 index 970f2782bf..0000000000 --- a/storage/contrib/cirrus/setup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_vars GOSRC OS_RELEASE_ID OS_RELEASE_VER SHORT_APTGET TEST_DRIVER - -show_env_vars - -cd $GOSRC -msg "Setting up $OS_RELEASE_ID $OS_RELEASE_VER" -case "$OS_RELEASE_ID" in - fedora) - [[ -z "$RPMS_CONFLICTING" ]] || \ - $SHORT_DNFY remove $RPMS_CONFLICTING - ;; - debian) - [[ -z "$DEBS_CONFLICTING" ]] || \ - $SHORT_APTGET -q remove $DEBS_CONFLICTING - ;; - *) - bad_os_id_ver - ;; -esac diff --git a/storage/contrib/cirrus/timestamp.awk b/storage/contrib/cirrus/timestamp.awk deleted file mode 100644 index 95b312e515..0000000000 --- a/storage/contrib/cirrus/timestamp.awk +++ /dev/null @@ -1,20 +0,0 @@ - - -# This script is intended to be piped into by automation, in order to -# mark output lines with timing information. For example: -# /path/to/command |& awk --file timestamp.awk - -BEGIN { - STARTTIME=systime() - printf "[%s] START", strftime("%T") - printf " - All [+xxxx] lines that follow are relative to right now.\n" -} - -{ - printf "[%+05ds] %s\n", systime()-STARTTIME, $0 -} - -END { - printf "[%s] END", strftime("%T") - printf " - [%+05ds] total duration since START\n", systime()-STARTTIME -} diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index b50319707b..3ade1e7850 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -698,7 +698,7 @@ func SupportsNativeOverlay(home, runhome string) (bool, error) { if err != nil && !errors.Is(err, fs.ErrNotExist) { return false, err } - if err := os.WriteFile(getMountProgramFlagFile(home), []byte(fmt.Sprintf("%t", needsMountProgram)), 0o600); err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := os.WriteFile(getMountProgramFlagFile(home), fmt.Appendf(nil, "%t", needsMountProgram), 0o600); err != nil && !errors.Is(err, fs.ErrNotExist) { return false, err } if needsMountProgram { diff --git a/storage/internal/tempdir/tempdir_test.go b/storage/internal/tempdir/tempdir_test.go index afb37fd1eb..6a12c4fffc 100644 --- a/storage/internal/tempdir/tempdir_test.go +++ b/storage/internal/tempdir/tempdir_test.go @@ -49,7 +49,7 @@ func TestTempDirAddMultipleFiles(t *testing.T) { tempDir := t.TempDir() - for i := 0; i < 3; i++ { + for i := range 3 { testFile := filepath.Join(tempDir, fmt.Sprintf("testfile%d.txt", i)) err = os.WriteFile(testFile, fmt.Appendf(nil, "content %d", i), 0o644) require.NoError(t, err) diff --git a/storage/layers.go b/storage/layers.go index d176e456c1..29066f5c8b 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -847,8 +847,8 @@ func (r *layerStore) GarbageCollect() error { name := entry.Name() var id string var isDataDir bool - if strings.HasSuffix(name, tarSplitSuffix) { - id = strings.TrimSuffix(name, tarSplitSuffix) + if before, ok := strings.CutSuffix(name, tarSplitSuffix); ok { + id = before } else if stringid.ValidateID(name) == nil { id = name isDataDir = true