diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index b4abf7c274ab..6f89dc8bd045 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -1,27 +1,14 @@ name: 'Configure Docker' description: 'Set up Docker build driver and configure build cache args' -inputs: - cache-provider: - description: 'gha or cirrus cache provider' - required: true runs: using: 'composite' steps: - - name: Check inputs - shell: python - run: | - # We expect only gha or cirrus as inputs to cache-provider - if "${{ inputs.cache-provider }}" not in ("gha", "cirrus"): - print("::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}") - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - with: - # Use host network to allow access to cirrus gha cache running on the host - driver-opts: | - network=host - # This is required to allow buildkit to access the actions cache + # This is required when using the gha cache backend with a manual docker buildx invocation. + # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN + # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - name: Expose actions cache variables uses: actions/github-script@v8 with: @@ -37,24 +24,12 @@ runs: shell: bash run: | # Configure docker build cache backend - # - # On forks the gha cache will work but will use Github's cache backend. - # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN - # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - - # Use cirrus cache host - if [[ ${{ inputs.cache-provider }} == 'cirrus' ]]; then - url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}" - else - url_args="" - fi - # Always optimistically --cache‑from in case a cache blob exists - args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") + args=(--cache-from "type=gha,scope=${CONTAINER_NAME}") - # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch. - if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then - args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") + # Only add --cache-to when pushing to the default branch. + if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then + args+=(--cache-to "type=gha,mode=max,ignore-error=true,scope=${CONTAINER_NAME}") fi # Always `--load` into docker images (needed when using the `docker-container` build driver). diff --git a/.github/actions/restore-caches/action.yml b/.github/actions/restore-caches/action.yml index 21f2807f4c72..ee07d68b9070 100644 --- a/.github/actions/restore-caches/action.yml +++ b/.github/actions/restore-caches/action.yml @@ -5,7 +5,7 @@ runs: steps: - name: Restore Ccache cache id: ccache-cache - uses: cirruslabs/cache/restore@v5 + uses: actions/cache/restore@v5 with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} @@ -14,7 +14,7 @@ runs: - name: Restore depends sources cache id: depends-sources - uses: cirruslabs/cache/restore@v5 + uses: actions/cache/restore@v5 with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} @@ -23,7 +23,7 @@ runs: - name: Restore built depends cache id: depends-built - uses: cirruslabs/cache/restore@v5 + uses: actions/cache/restore@v5 with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} @@ -32,7 +32,7 @@ runs: - name: Restore previous releases cache id: previous-releases - uses: cirruslabs/cache/restore@v5 + uses: actions/cache/restore@v5 with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} diff --git a/.github/actions/save-caches/action.yml b/.github/actions/save-caches/action.yml index 3072ab3f224a..8667c91473df 100644 --- a/.github/actions/save-caches/action.yml +++ b/.github/actions/save-caches/action.yml @@ -11,28 +11,28 @@ runs: echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}" - name: Save Ccache cache - uses: cirruslabs/cache/save@v5 + uses: actions/cache/save@v5 if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }} with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} - name: Save depends sources cache - uses: cirruslabs/cache/save@v5 + uses: actions/cache/save@v5 if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }} with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} - name: Save built depends cache - uses: cirruslabs/cache/save@v5 + uses: actions/cache/save@v5 if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}} with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} - name: Save previous releases cache - uses: cirruslabs/cache/save@v5 + uses: actions/cache/save@v5 if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}} with: path: ${{ env.PREVIOUS_RELEASES_DIR }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 734b9f38c762..920c1f46047d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,7 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type. - REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners + REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners for this repo, instead of falling back to the slow GHA runners defaults: run: @@ -60,7 +59,7 @@ jobs: needs: runners runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' || 'ubuntu-24.04' }} env: - TEST_RUNNER_PORT_MIN: "14000" # Use a larger port, to avoid colliding with CIRRUS_CACHE_HOST port 12321. + TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. steps: @@ -327,8 +326,6 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker - with: - cache-provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Clear unnecessary files if: ${{ needs.runners.outputs.provider == 'gha' && true || false }} # Only needed on GHA runners @@ -370,8 +367,6 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker - with: - cache-provider: ${{ needs.runners.outputs.provider }} - name: CI script run: | diff --git a/ci/README.md b/ci/README.md index 293294a559c5..0f43b4f1f630 100644 --- a/ci/README.md +++ b/ci/README.md @@ -85,15 +85,15 @@ To configure the primary repository, follow these steps: 3. Enable organisation-level runners to be used in public repositories: 1. `Org settings -> Actions -> Runner Groups -> Default -> Allow public repos` 4. Permit the following actions to run: - 1. cirruslabs/cache/restore@\* - 1. cirruslabs/cache/save@\* + 1. actions/cache/restore@\* + 1. actions/cache/save@\* 1. docker/setup-buildx-action@\* 1. actions/github-script@\* ### Forked repositories When used in a fork the CI will run on GitHub's free hosted runners by default. -In this case, due to GitHub's 10GB-per-repo cache size limitations caches will be frequently evicted and missed, but the workflows will run (slowly). +In this case, GitHub's cache size limitations may cause caches to be frequently evicted and missed, but the workflows will run (slowly). It is also possible to use your own Cirrus Runners in your own fork with an appropriate patch to the `REPO_USE_CIRRUS_RUNNERS` variable in ../.github/workflows/ci.yml NB that Cirrus Runners only work at an organisation level, therefore in order to use your own Cirrus Runners, *the fork must be within your own organisation*. diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index fd794b512cd4..4824665c3ca5 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -set -o errexit -o xtrace +set -o errexit -o xtrace -o pipefail if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1" @@ -136,10 +136,30 @@ cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target $GOAL || ( ) ccache --version | head -n 1 && ccache --show-stats --verbose -hit_rate=$(ccache --show-stats | grep "Hits:" | head -1 | sed 's/.*(\(.*\)%).*/\1/') -if [ "${hit_rate%.*}" -lt 75 ]; then - echo "::notice title=low ccache hitrate::Ccache hit-rate in $CONTAINER_NAME was $hit_rate%" -fi +ccache --print-stats | python3 -c ' +import os +import sys + +for line in sys.stdin: + key, value = line.split("\t", 1) + # "primary storage" fallback only needed for ccache version 4.5.1 + if key in ("local_storage_hit", "primary_storage_hit"): + hits = int(value) + elif key in ("local_storage_miss", "primary_storage_miss"): + miss = int(value) + +calls = hits + miss +# codegen has no calls, so skip that here +if calls: + rate = (hits / calls * 100) + print(f"{rate:.2f}") + if rate < 75: + container = os.environ["CONTAINER_NAME"] + print( + "::notice title=low ccache hitrate::" + f"Ccache hit-rate in {container} was {rate:.2f}%" + ) +' du -sh "${DEPENDS_DIR}"/*/ du -sh "${PREVIOUS_RELEASES_DIR}" @@ -183,7 +203,7 @@ fi if [[ "${RUN_IWYU}" == true ]]; then # TODO: Consider enforcing IWYU across the entire codebase. - FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|univalue/(lib|test)|util|zmq)/.*|common/license_info|node/blockstorage|node/utxo_snapshot|clientversion|core_io|signet)\\.cpp)" + FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|univalue/(lib|test)|util|zmq)/.*|bench/(block_assemble|connectblock)|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" @@ -191,12 +211,14 @@ if [[ "${RUN_IWYU}" == true ]]; then run_iwyu() { mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json" - python3 "/include-what-you-use/iwyu_tool.py" \ + { + python3 "/include-what-you-use/iwyu_tool.py" \ -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ -Xiwyu --max_line_length=160 \ -Xiwyu --check_also="*/primitives/*.h" \ - 2>&1 | tee /tmp/iwyu_ci.out + 2>&1 || true + } | tee /tmp/iwyu_ci.out python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git diff -U1 | ./contrib/devtools/clang-format-diff.py -binary="clang-format-${IWYU_LLVM_V}" -p1 -i -v } diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index bef1c8412a7b..767ea9208f64 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -441,7 +441,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/contrib/guix/manifest_build.scm" \ --container \ --writable-root \ --pure \ diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 39291dfe9ceb..8c8682ddd0af 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -341,7 +341,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/contrib/guix/manifest_codesign.scm" \ --container \ --writable-root \ --pure \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest_build.scm similarity index 66% rename from contrib/guix/manifest.scm rename to contrib/guix/manifest_build.scm index 4fd901e57a5c..576021e60d7f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest_build.scm @@ -3,9 +3,8 @@ (gnu packages bison) ((gnu packages cmake) #:select (cmake-minimal)) (gnu packages commencement) - (gnu packages compression) + ((gnu packages compression) #:select (gzip xz zip)) (gnu packages cross-base) - ((gnu packages crypto) #:select (osslsigncode)) (gnu packages gawk) (gnu packages gcc) ((gnu packages installers) #:select (nsis-x86_64)) @@ -15,13 +14,8 @@ (gnu packages ninja) (gnu packages pkg-config) ((gnu packages python) #:select (python-minimal)) - ((gnu packages python-build) #:select (python-poetry-core)) - ((gnu packages python-crypto) #:select (python-asn1crypto python-oscrypto)) ((gnu packages python-xyz) #:select (python-lief)) - ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) - (guix build-system python) - (guix build-system pyproject) (guix build-system trivial) (guix download) (guix gexp) @@ -171,143 +165,6 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define-public python-elfesteem - (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) - (package - (name "python-elfesteem") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/LRGH/elfesteem") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) - (build-system python-build-system) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; PYTHONPATH problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/LRGH/elfesteem") - (synopsis "ELF/PE/Mach-O parsing library") - (description "elfesteem parses ELF, PE and Mach-O files.") - (license license:lgpl2.1)))) - -(define-public python-oscryptotests - (package (inherit python-oscrypto) - (name "python-oscryptotests") - (propagated-inputs - (list python-oscrypto)) - (arguments - `(#:tests? #f - #:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (chdir "tests") - #t))))))) - -(define-public python-certvalidator - (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) - (package - (name "python-certvalidator") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/certvalidator") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) - (build-system python-build-system) - (propagated-inputs - (list openssl - python-asn1crypto - python-oscrypto - python-oscryptotests)) ;; certvalidator tests import oscryptotests - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - (substitute* "tests/test_certificate_validator.py" - (("^(.*)class CertificateValidatorTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_crl_client.py" - (("^(.*)def test_fetch_crl" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_ocsp_client.py" - (("^(.*)def test_fetch_ocsp" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_registry.py" - (("^(.*)def test_build_paths" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_hard" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_soft" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/certvalidator") - (synopsis "Python library for validating X.509 certificates and paths") - (description "certvalidator is a Python library for validating X.509 -certificates or paths. Supports various options, including: validation at a -specific moment in time, whitelisting and revocation checks.") - (license license:expat)))) - -(define-public python-signapple - (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) - (package - (name "python-signapple") - (version (git-version "0.2.0" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/signapple") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) - (build-system pyproject-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-certvalidator - python-elfesteem)) - (native-inputs (list python-poetry-core)) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/achow101/signapple") - (synopsis "Mach-O binary signature tool") - (description "signapple is a Python tool for creating, verifying, and -inspecting signatures in Mach-O binaries.") - (license license:expat)))) - (define-public mingw-w64-base-gcc (package (inherit base-gcc) @@ -428,10 +285,9 @@ inspecting signatures in Mach-O binaries.") python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) - (list zip - (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (list (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 - osslsigncode)) + zip)) ((string-contains target "-linux-") (list bison pkg-config @@ -441,6 +297,5 @@ inspecting signatures in Mach-O binaries.") (list clang-toolchain-19 lld-19 (make-lld-wrapper lld-19 #:lld-as-ld? #t) - python-signapple zip)) (else '()))))) diff --git a/contrib/guix/manifest_codesign.scm b/contrib/guix/manifest_codesign.scm new file mode 100644 index 000000000000..652f40ac3f07 --- /dev/null +++ b/contrib/guix/manifest_codesign.scm @@ -0,0 +1,179 @@ +(use-modules ((gnu packages bash) #:select (bash-minimal)) + ((gnu packages compression) #:select (gzip zip)) + ((gnu packages crypto) #:select (osslsigncode)) + ((gnu packages python-build) #:select (python-poetry-core)) + ((gnu packages python-crypto) #:select (python-asn1crypto python-oscrypto)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) + (guix build-system python) + (guix build-system pyproject) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages)) + +(define-public python-elfesteem + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (propagated-inputs + (list python-oscrypto)) + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) + (build-system python-build-system) + (propagated-inputs + (list openssl + python-asn1crypto + python-oscrypto + python-oscryptotests)) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-signapple + (let ((commit "3fab3bb57f227f0dd31007b417683035f5204838")) + (package + (name "python-signapple") + (version (git-version "0.2.0" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "0qpr78bs50rw79dbihr9ifjq19y6819ih5pn9jd2rbjyifimzf7p")))) + (build-system pyproject-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-certvalidator + python-elfesteem)) + (native-inputs (list python-poetry-core)) + (arguments + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (list #:tests? #f + #:phases + #~(modify-phases %standard-phases + ;; Add a phase to inject OpenSSL paths for oscrypto. + (add-after 'wrap 'wrap-openssl-paths + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (wrap-program (string-append #$output "/bin/signapple") + `("SIGNAPPLE_OSCRYPTO_SSL_PATHS" = + (,(string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so")))))))))) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(packages->manifest + (append + (list ;; The Basics + bash-minimal + coreutils-minimal + ;; File(system) inspection + findutils + ;; Compression and archiving + tar + gzip + zip + ;; Git + git-minimal) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + (list osslsigncode)) + ((string-contains target "darwin") + (list python-signapple)) + (else '()))))) diff --git a/doc/developer-notes.md b/doc/developer-notes.md index fb85e5bf0888..3df109bf5865 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -42,7 +42,7 @@ code. - Constant names are all uppercase, and use `_` to separate words. - Enumerator constants may be `snake_case`, `PascalCase` or `ALL_CAPS`. This is a more tolerant policy than the [C++ Core - Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#renum-caps), which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. See [Internal interface @@ -55,16 +55,13 @@ code. - **Miscellaneous** - `++i` is preferred over `i++`. - - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. - Use a named cast or functional cast, not a C-Style cast. When casting between integer types, use functional casts such as `int(x)` or `int{x}` instead of `(int) x`. When casting between more complex types, use `static_cast`. Use `reinterpret_cast` and `const_cast` as appropriate. - - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-list) where possible. + - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#res-list) where possible. For example `int x{0};` instead of `int x = 0;` or `int x(0);` - - Recursion is checked by clang-tidy and thus must be made explicit. Use - `NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. For function calls a namespace should be specified explicitly, unless such functions have been declared within it. Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be @@ -140,7 +137,18 @@ public: non-optional in-out and output parameters should usually be references, as they cannot be null. -### Coding Style (C++ named arguments) +### Coding Style (clang-tidy rules) + +The clang-tidy tool is used to check some rules. Please refer to the [upstream +documentation](https://clang.llvm.org/extra/clang-tidy/checks/list.html) about +the details and rationale for each rule. + +#### C++ Recursive Functions + +Recursion is checked by clang-tidy and thus must be made explicit. Use +`NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. + +#### C++ named arguments When passing named arguments, use a format that clang-tidy understands. The argument names can otherwise not be verified by clang-tidy. @@ -156,7 +164,7 @@ int main() } ``` -### Running clang-tidy +#### Running clang-tidy To run clang-tidy on Ubuntu/Debian, install the dependencies: @@ -603,6 +611,21 @@ or using a graphical tool like [Hotspot](https://github.com/KDAB/hotspot). See the functional test documentation for how to invoke perf within tests. +### Valgrind + +Valgrind is a programming tool for memory debugging, memory leak detection, and +profiling. The repo contains a Valgrind suppressions file +([`valgrind.supp`](/test/sanitizer_suppressions/valgrind.supp)) +which includes known Valgrind warnings in our dependencies that cannot be fixed +in-tree. Example use: + +```shell +$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp build/bin/test_bitcoin +$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp --leak-check=full \ + --show-leak-kinds=all build/bin/test_bitcoin --log_level=test_suite +$ valgrind -v --leak-check=full build/bin/bitcoind -printtoconsole +$ ./build/test/functional/test_runner.py --valgrind +``` ### Sanitizers @@ -658,6 +681,11 @@ Additional resources: * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki) +# Development guidelines + +A few non-style-related recommendations for developers, as well as points to +pay attention to for reviewers of Bitcoin Core code. + ## Locking/mutex usage notes The code is multi-threaded and uses mutexes and the @@ -675,21 +703,21 @@ done by the components. ## Threads -- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#a0ddf1224851353fc92bfbff6f499fa97) +- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#main) : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and shutting down the application. -- [Init load (`b-initload`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) +- [Init load (`b-initload`)](https://doxygen.bitcoincore.org/init_8cpp.html#initload) : Performs various loading tasks that are part of init but shouldn't block the node from being started: external block import, reindex, reindex-chainstate, main chain activation, spawn indexes background sync threads and mempool load. -- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#a6e7fa51d3a25e7cb65446d4b50e6a987) +- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#checkqueue) : Parallel script validation threads for transactions in blocks. - [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) : Thread to listen for RPC and REST connections. -- [HTTP worker threads (`b-http_pool_x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#a2ad0a49dc9b5e8117c0dee98c24187d8) +- [HTTP worker threads (`b-http_pool_x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http_pool) : Threads to service RPC and REST requests. - [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#a14d2800815da93577858ea078aed1fba) @@ -701,33 +729,28 @@ done by the components. - Net threads: - - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#aacdbb7148575a31bb33bc345e2bf22a9) + - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#msghand) : Application level message handling (sending and receiving). Almost all net_processing and validation logic runs on this thread. - - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) + - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#dnsseed) : Loads addresses of peers from the DNS. - - ThreadMapPort (`b-mapport`) + - [ThreadMapPort (`b-mapport`)](https://doxygen.bitcoincore.org/mapport_8cpp.html#mapport) : Universal plug-and-play startup/shutdown. - - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) + - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#net) : Sends/Receives data from peers on port 8333. - - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a0b787caf95e52a346a2b31a580d60a62) + - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#addcon) : Opens network connections to added nodes. - - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) + - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#opencon) : Initiates new connections to peers. - - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#a57787b4f9ac847d24065fbb0dd6e70f8) + - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#i2paccept) : Listens for and accepts incoming I2P connections through the I2P SAM proxy. -# Development guidelines - -A few non-style-related recommendations for developers, as well as points to -pay attention to for reviewers of Bitcoin Core code. - ## General Bitcoin Core - New features should be exposed on RPC first, then can be made available in the GUI. @@ -780,7 +803,7 @@ Guidelines](https://isocpp.github.io/CppCoreGuidelines/). Common misconceptions are clarified in those sections: - Passing (non-)fundamental types in the [C++ Core - Guideline](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-conventional). + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#rf-conventional). - If you use the `.h`, you must link the `.cpp`. @@ -1343,6 +1366,46 @@ A few guidelines for modifying existing RPC interfaces: - *Rationale*: Changes in RPC JSON structure can break downstream application compatibility. Implementation of `deprecatedrpc` provides a grace period for downstream applications to migrate. Release notes provide notification to downstream users. +## Feature deprecation and removal process + +Bitcoin Core uses a structured process for deprecating and removing features to give +downstream users and applications time to migrate. + +### General principles + +- The minimum deprecation **grace period** for a feature that is going to be removed is one + major release. +- Any deprecation or removal must come with a release note. + +### RPC methods and fields + +- To deprecate an entire RPC method, gate the old behavior behind `-deprecatedrpc=`. + Deprecated features should remain accessible via this flag during the grace period so + downstream users are not immediately broken. +- The RPC help text must mention the deprecation and the `-deprecatedrpc=` flag + that re-enables it. For example: + ``` + "\nDeprecated in v25.0, use the `newfoo` RPC instead. Start bitcoind with" + " `-deprecatedrpc=foo` to continue using this RPC.\n" + ``` + +### Startup options + +- To deprecate a startup option, emit a warning via `LogWarning` or `InitWarning` when the + option is used, so users are notified at startup. +- Update the option's help text to indicate it is deprecated and, if applicable, will be + removed in a future release. + +### REST interface + +- Deprecated REST endpoints or behaviors should be documented in `doc/REST-interface.md` + with the version they were deprecated. + +### ZMQ + +- Deprecated ZMQ topics or behaviors should be documented in `doc/zmq.md` with the version + they were deprecated. + ## Internal interface guidelines Internal interfaces between parts of the codebase that are meant to be diff --git a/doc/release-notes-33966.md b/doc/release-notes-33966.md new file mode 100644 index 000000000000..7251a13d52d1 --- /dev/null +++ b/doc/release-notes-33966.md @@ -0,0 +1,9 @@ +Mining +------ + +- The IPC mining interface now rejects out-of-range block template options + instead of silently clamping them, such as oversized reserved block weight or + coinbase sigops limits. (#33966) + +- The `-blockmaxweight` startup option is now rejected when it is lower than + `-blockreservedweight`, instead of being silently clamped. (#33966) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a7ea788851e..93ef5b3c9e5d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -196,6 +196,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/mempool_persist.cpp node/mempool_persist_args.cpp node/miner.cpp + node/mining_args.cpp node/mini_miner.cpp node/minisketchwrapper.cpp node/peerman_args.cpp diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 84af166abf3a..191357fa6621 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -3,9 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include -#include +#include #include #include #include